Create an ENDPOINT enum type for use internally

We need it for the custom extensions API

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3139)

4 files changed, 41 insertions, 33 deletions

diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index ad73d8eee6..5ca7237fc5 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -1616,13 +1616,16 @@ struct cert_pkey_st {
 # define SSL_CERT_FLAGS_CHECK_TLS_STRICT \
         (SSL_CERT_FLAG_SUITEB_128_LOS|SSL_CERT_FLAG_TLS_STRICT)
 
+typedef enum {
+    ENDPOINT_CLIENT = 0,
+    ENDPOINT_SERVER,
+    ENDPOINT_BOTH
+} ENDPOINT;
+
+
 typedef struct {
     unsigned short ext_type;
-    /*
-     * Set to 1 if this is only for the server side, 0 if it is only for the
-     * client side, or -1 if it is either.
-     */
-    int server;
+    ENDPOINT role;
     /* The context which this extension applies to */
     unsigned int context;
     /*
@@ -2444,8 +2447,9 @@ __owur int srp_verify_server_param(SSL *s, int *al);
 
 /* statem/extensions_cust.c */
 
-custom_ext_method *custom_ext_find(const custom_ext_methods *exts, int server,
-                                   unsigned int ext_type, size_t *idx);
+custom_ext_method *custom_ext_find(const custom_ext_methods *exts,
+                                   ENDPOINT role, unsigned int ext_type,
+                                   size_t *idx);
 
 void custom_ext_init(custom_ext_methods *meths);
 
diff --git a/ssl/ssl_rsa.c b/ssl/ssl_rsa.c
index ecf2ff3e73..87be6460b5 100644
--- a/ssl/ssl_rsa.c
+++ b/ssl/ssl_rsa.c
@@ -798,7 +798,8 @@ static int serverinfo_process_buffer(const unsigned char *serverinfo,
         /* Register callbacks for extensions */
         ext_type = (serverinfo[0] << 8) + serverinfo[1];
         if (ctx != NULL
-                && custom_ext_find(&ctx->cert->custext, 1, ext_type, NULL)
+                && custom_ext_find(&ctx->cert->custext, ENDPOINT_SERVER,
+                                   ext_type, NULL)
                    == NULL
                 && !SSL_CTX_add_server_custom_ext(ctx, ext_type,
                                                   serverinfo_srv_add_cb,
diff --git a/ssl/statem/extensions.c b/ssl/statem/extensions.c
index 49a7156a2c..cd1d0bd3ec 100644
--- a/ssl/statem/extensions.c
+++ b/ssl/statem/extensions.c
@@ -373,15 +373,15 @@ static int verify_extension(SSL *s, unsigned int context, unsigned int type,
     /* Check the custom extensions */
     if (meths != NULL) {
         size_t offset = 0;
-        int server = -1;
+        ENDPOINT role = ENDPOINT_BOTH;
         custom_ext_method *meth = NULL;
 
         if ((context & SSL_EXT_CLIENT_HELLO) != 0)
-            server = 1;
+            role = ENDPOINT_SERVER;
         else if ((context & SSL_EXT_TLS1_2_SERVER_HELLO) != 0)
-            server = 0;
+            role = ENDPOINT_CLIENT;
 
-        meth = custom_ext_find(meths, server, type, &offset);
+        meth = custom_ext_find(meths, role, type, &offset);
         if (meth != NULL) {
             if (!validate_context(s, meth->context, context))
                 return 0;
diff --git a/ssl/statem/extensions_cust.c b/ssl/statem/extensions_cust.c
index 38bb03b390..ea79a98f17 100644
--- a/ssl/statem/extensions_cust.c
+++ b/ssl/statem/extensions_cust.c
@@ -69,22 +69,23 @@ static int custom_ext_parse_old_cb_wrap(SSL *s, unsigned int ext_type,
 }
 
 /*
- * Find a custom extension from the list. The |server| param is there to
+ * Find a custom extension from the list. The |role| param is there to
  * support the legacy API where custom extensions for client and server could
- * be set independently on the same SSL_CTX. It is set to 1 if we are trying
- * to find a method relevant to the server, 0 for the client, or -1 if we don't
- * care
+ * be set independently on the same SSL_CTX. It is set to ENDPOINT_SERVER if we
+ * are trying to find a method relevant to the server, ENDPOINT_CLIENT for the
+ * client, or ENDPOINT_BOTH for either
  */
-custom_ext_method *custom_ext_find(const custom_ext_methods *exts, int server,
-                                   unsigned int ext_type, size_t *idx)
+custom_ext_method *custom_ext_find(const custom_ext_methods *exts,
+                                   ENDPOINT role, unsigned int ext_type,
+                                   size_t *idx)
 {
     size_t i;
     custom_ext_method *meth = exts->meths;
 
     for (i = 0; i < exts->meths_count; i++, meth++) {
         if (ext_type == meth->ext_type
-                && (server == -1 || server == meth->server
-                    || meth->server == -1)) {
+                && (role == ENDPOINT_BOTH || role == meth->role
+                    || meth->role == ENDPOINT_BOTH)) {
             if (idx != NULL)
                 *idx = i;
             return meth;
@@ -112,12 +113,12 @@ int custom_ext_parse(SSL *s, unsigned int context, unsigned int ext_type,
 {
     custom_ext_methods *exts = &s->cert->custext;
     custom_ext_method *meth;
-    int server = -1;
+    ENDPOINT role = ENDPOINT_BOTH;
 
     if ((context & (SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO)) != 0)
-        server = s->server;
+        role = s->server ? ENDPOINT_SERVER : ENDPOINT_CLIENT;
 
-    meth = custom_ext_find(exts, server, ext_type, NULL);
+    meth = custom_ext_find(exts, role, ext_type, NULL);
     /* If not found return success */
     if (!meth)
         return 1;
@@ -297,10 +298,11 @@ void custom_exts_free(custom_ext_methods *exts)
 /* Return true if a client custom extension exists, false otherwise */
 int SSL_CTX_has_client_custom_ext(const SSL_CTX *ctx, unsigned int ext_type)
 {
-    return custom_ext_find(&ctx->cert->custext, 0, ext_type, NULL) != NULL;
+    return custom_ext_find(&ctx->cert->custext, ENDPOINT_CLIENT, ext_type,
+                           NULL) != NULL;
 }
 
-static int add_custom_ext_intern(SSL_CTX *ctx, int server,
+static int add_custom_ext_intern(SSL_CTX *ctx, ENDPOINT role,
                                  unsigned int ext_type,
                                  unsigned int context,
                                  SSL_custom_ext_add_cb_ex add_cb,
@@ -343,7 +345,7 @@ static int add_custom_ext_intern(SSL_CTX *ctx, int server,
     if (ext_type > 0xffff)
         return 0;
     /* Search for duplicate */
-    if (custom_ext_find(exts, server, ext_type, NULL))
+    if (custom_ext_find(exts, role, ext_type, NULL))
         return 0;
     tmp = OPENSSL_realloc(exts->meths,
                           (exts->meths_count + 1) * sizeof(custom_ext_method));
@@ -353,7 +355,7 @@ static int add_custom_ext_intern(SSL_CTX *ctx, int server,
     exts->meths = tmp;
     meth = exts->meths + exts->meths_count;
     memset(meth, 0, sizeof(*meth));
-    meth->server = server;
+    meth->role = role;
     meth->context = context;
     meth->parse_cb = parse_cb;
     meth->add_cb = add_cb;
@@ -365,7 +367,8 @@ static int add_custom_ext_intern(SSL_CTX *ctx, int server,
     return 1;
 }
 
-static int add_old_custom_ext(SSL_CTX *ctx, int server, unsigned int ext_type,
+static int add_old_custom_ext(SSL_CTX *ctx, ENDPOINT role,
+                              unsigned int ext_type,
                               unsigned int context,
                               custom_ext_add_cb add_cb,
                               custom_ext_free_cb free_cb,
@@ -395,7 +398,7 @@ static int add_old_custom_ext(SSL_CTX *ctx, int server, unsigned int ext_type,
      * client and server for the same type in the same SSL_CTX? We don't handle
      * that yet.
      */
-    ret = add_custom_ext_intern(ctx, server, ext_type,
+    ret = add_custom_ext_intern(ctx, role, ext_type,
                                 context,
                                 custom_ext_add_old_cb_wrap,
                                 custom_ext_free_old_cb_wrap,
@@ -418,7 +421,7 @@ int SSL_CTX_add_client_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
                                   void *add_arg,
                                   custom_ext_parse_cb parse_cb, void *parse_arg)
 {
-    return add_old_custom_ext(ctx, 0, ext_type,
+    return add_old_custom_ext(ctx, ENDPOINT_CLIENT, ext_type,
                               SSL_EXT_TLS1_2_AND_BELOW_ONLY
                               | SSL_EXT_CLIENT_HELLO
                               | SSL_EXT_TLS1_2_SERVER_HELLO
@@ -432,7 +435,7 @@ int SSL_CTX_add_server_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
                                   void *add_arg,
                                   custom_ext_parse_cb parse_cb, void *parse_arg)
 {
-    return add_old_custom_ext(ctx, 1, ext_type,
+    return add_old_custom_ext(ctx, ENDPOINT_SERVER, ext_type,
                               SSL_EXT_TLS1_2_AND_BELOW_ONLY
                               | SSL_EXT_CLIENT_HELLO
                               | SSL_EXT_TLS1_2_SERVER_HELLO
@@ -447,8 +450,8 @@ int SSL_CTX_add_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
                            void *add_arg,
                            SSL_custom_ext_parse_cb_ex parse_cb, void *parse_arg)
 {
-    return add_custom_ext_intern(ctx, -1, ext_type, context, add_cb, free_cb,
-                                 add_arg, parse_cb, parse_arg);
+    return add_custom_ext_intern(ctx, ENDPOINT_BOTH, ext_type, context, add_cb,
+                                 free_cb, add_arg, parse_cb, parse_arg);
 }
 
 int SSL_extension_supported(unsigned int ext_type)