Use cert tables instead of X509_certificate_type

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3858)

2 files changed, 18 insertions, 43 deletions

diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 6248882607..55ac4dd03e 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -3335,62 +3335,39 @@ int tls_construct_client_certificate(SSL *s, WPACKET *pkt)
     return 0;
 }
 
-#define has_bits(i,m)   (((i)&(m)) == (m))
-
 int ssl3_check_cert_and_algorithm(SSL *s)
 {
-    int i;
-#ifndef OPENSSL_NO_EC
-    int idx;
-#endif
+    const SSL_CERT_LOOKUP *clu;
+    size_t idx;
     long alg_k, alg_a;
-    EVP_PKEY *pkey = NULL;
     int al = SSL_AD_HANDSHAKE_FAILURE;
 
     alg_k = s->s3->tmp.new_cipher->algorithm_mkey;
     alg_a = s->s3->tmp.new_cipher->algorithm_auth;
 
     /* we don't have a certificate */
-    if ((alg_a & SSL_aNULL) || (alg_k & SSL_kPSK))
-        return (1);
+    if (!(alg_a & SSL_aCERT))
+        return 1;
 
     /* This is the passed certificate */
+    clu = ssl_cert_lookup_by_pkey(X509_get0_pubkey(s->session->peer), &idx);
 
-#ifndef OPENSSL_NO_EC
-    idx = s->session->peer_type;
-    if (idx == SSL_PKEY_ECC || idx == SSL_PKEY_ED25519) {
-        if (ssl_check_srvr_ecc_cert_and_alg(s->session->peer, s) == 0) {
-            /* check failed */
-            SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM, SSL_R_BAD_ECC_CERT);
-            goto f_err;
-        } else {
-            return 1;
-        }
-    } else if (alg_a & SSL_aECDSA) {
-        SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM,
-               SSL_R_MISSING_ECDSA_SIGNING_CERT);
+    /* Check certificate is recognised and suitable for cipher */
+    if (clu == NULL || (alg_a & clu->amask) == 0) {
+        SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM, SSL_R_MISSING_SIGNING_CERT);
         goto f_err;
     }
-#endif
-    pkey = X509_get0_pubkey(s->session->peer);
-    i = X509_certificate_type(s->session->peer, pkey);
 
-    /* Check that we have a certificate if we require one */
-    if ((alg_a & SSL_aRSA) && !has_bits(i, EVP_PK_RSA | EVP_PKT_SIGN)) {
-        SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM,
-               SSL_R_MISSING_RSA_SIGNING_CERT);
-        goto f_err;
-    }
-#ifndef OPENSSL_NO_DSA
-    else if ((alg_a & SSL_aDSS) && !has_bits(i, EVP_PK_DSA | EVP_PKT_SIGN)) {
-        SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM,
-               SSL_R_MISSING_DSA_SIGNING_CERT);
+#ifndef OPENSSL_NO_EC
+    if (clu->amask & SSL_aECDSA) {
+        if (ssl_check_srvr_ecc_cert_and_alg(s->session->peer, s))
+            return 1;
+        SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM, SSL_R_BAD_ECC_CERT);
         goto f_err;
     }
 #endif
 #ifndef OPENSSL_NO_RSA
-    if (alg_k & (SSL_kRSA | SSL_kRSAPSK) &&
-        !has_bits(i, EVP_PK_RSA | EVP_PKT_ENC)) {
+    if (alg_k & (SSL_kRSA | SSL_kRSAPSK) && idx != SSL_PKEY_RSA) {
         SSLerr(SSL_F_SSL3_CHECK_CERT_AND_ALGORITHM,
                SSL_R_MISSING_RSA_ENCRYPTING_CERT);
         goto f_err;
@@ -3404,10 +3381,10 @@ int ssl3_check_cert_and_algorithm(SSL *s)
     }
 #endif
 
-    return (1);
+    return 1;
  f_err:
     ssl3_send_alert(s, SSL3_AL_FATAL, al);
-    return (0);
+    return 0;
 }
 
 #ifndef OPENSSL_NO_NEXTPROTONEG
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index 31736c0663..a6baf2a5c9 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -316,7 +316,7 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
     unsigned char *gost_data = NULL;
 #endif
     int al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
-    int type = 0, j;
+    int j;
     unsigned int len;
     X509 *peer;
     const EVP_MD *md = NULL;
@@ -336,9 +336,7 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
     if (pkey == NULL)
         goto f_err;
 
-    type = X509_certificate_type(peer, pkey);
-
-    if (!(type & EVP_PKT_SIGN)) {
+    if (ssl_cert_lookup_by_pkey(pkey, NULL) == NULL) {
         SSLerr(SSL_F_TLS_PROCESS_CERT_VERIFY,
                SSL_R_SIGNATURE_FOR_NON_SIGNING_CERTIFICATE);
         al = SSL_AD_ILLEGAL_PARAMETER;