Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 43 insertions, 35 deletions

diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 63f90f1173..be8834d5f1 100644
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -38,7 +38,7 @@ static CMS_SignedData *cms_signed_data_init(CMS_ContentInfo *cms)
     if (cms->d.other == NULL) {
         cms->d.signedData = M_ASN1_new_of(CMS_SignedData);
         if (!cms->d.signedData) {
-            ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
             return NULL;
         }
         cms->d.signedData->version = 1;
@@ -349,8 +349,10 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
     if (!sd)
         goto err;
     si = M_ASN1_new_of(CMS_SignerInfo);
-    if (!si)
-        goto merr;
+    if (!si) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
+        goto err;
+    }
     /* Call for side-effect of computing hash and caching extensions */
     X509_check_purpose(signer, -1, -1);
 
@@ -364,7 +366,7 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
     si->pctx = NULL;
 
     if (si->mctx == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB);
         goto err;
     }
 
@@ -413,12 +415,15 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
     }
 
     if (i == sk_X509_ALGOR_num(sd->digestAlgorithms)) {
-        if ((alg = X509_ALGOR_new()) == NULL)
-            goto merr;
+        if ((alg = X509_ALGOR_new()) == NULL) {
+            ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
+            goto err;
+        }
         X509_ALGOR_set_md(alg, md);
         if (!sk_X509_ALGOR_push(sd->digestAlgorithms, alg)) {
             X509_ALGOR_free(alg);
-            goto merr;
+            ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB);
+            goto err;
         }
     }
 
@@ -431,8 +436,10 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
          */
         if (!si->signedAttrs) {
             si->signedAttrs = sk_X509_ATTRIBUTE_new_null();
-            if (!si->signedAttrs)
-                goto merr;
+            if (!si->signedAttrs) {
+                ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB);
+                goto err;
+            }
         }
 
         if (!(flags & CMS_NOSMIMECAP)) {
@@ -442,8 +449,10 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
             if (i)
                 i = CMS_add_smimecap(si, smcap);
             sk_X509_ALGOR_pop_free(smcap, X509_ALGOR_free);
-            if (!i)
-                goto merr;
+            if (!i) {
+                ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+                goto err;
+            }
         }
         if (flags & CMS_CADES) {
             ESS_SIGNING_CERT *sc = NULL;
@@ -479,8 +488,10 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
 
     if (!(flags & CMS_NOCERTS)) {
         /* NB ignore -1 return for duplicate cert */
-        if (!CMS_add1_cert(cms, signer))
-            goto merr;
+        if (!CMS_add1_cert(cms, signer)) {
+            ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+            goto err;
+        }
     }
 
     if (flags & CMS_KEY_PARAM) {
@@ -503,15 +514,15 @@ CMS_SignerInfo *CMS_add1_signer(CMS_ContentInfo *cms,
         }
     }
 
-    if (!sd->signerInfos)
+    if (sd->signerInfos == NULL)
         sd->signerInfos = sk_CMS_SignerInfo_new_null();
-    if (!sd->signerInfos || !sk_CMS_SignerInfo_push(sd->signerInfos, si))
-        goto merr;
+    if (sd->signerInfos == NULL || !sk_CMS_SignerInfo_push(sd->signerInfos, si)) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_CRYPTO_LIB);
+        goto err;
+    }
 
     return si;
 
- merr:
-    ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
  err:
     M_ASN1_free_of(si, CMS_SignerInfo);
     return NULL;
@@ -546,21 +557,22 @@ static int cms_add1_signingTime(CMS_SignerInfo *si, ASN1_TIME *t)
     else
         tt = X509_gmtime_adj(NULL, 0);
 
-    if (tt == NULL)
-        goto merr;
+    if (tt == NULL) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB);
+        goto err;
+    }
 
     if (CMS_signed_add1_attr_by_NID(si, NID_pkcs9_signingTime,
-                                    tt->type, tt, -1) <= 0)
-        goto merr;
+                                    tt->type, tt, -1) <= 0) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+        goto err;
+    }
 
     r = 1;
- merr:
+ err:
     if (t == NULL)
         ASN1_TIME_free(tt);
 
-    if (!r)
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
-
     return r;
 
 }
@@ -703,7 +715,7 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms,
     const CMS_CTX *ctx = ossl_cms_get0_cmsctx(cms);
 
     if (mctx == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
         return 0;
     }
 
@@ -751,10 +763,8 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms,
         }
         siglen = EVP_PKEY_get_size(si->pkey);
         sig = OPENSSL_malloc(siglen);
-        if (sig == NULL) {
-            ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        if (sig == NULL)
             goto err;
-        }
         if (EVP_PKEY_sign(pctx, sig, &siglen, md, mdlen) <= 0) {
             OPENSSL_free(sig);
             goto err;
@@ -769,10 +779,8 @@ static int cms_SignerInfo_content_sign(CMS_ContentInfo *cms,
             goto err;
         }
         sig = OPENSSL_malloc(EVP_PKEY_get_size(si->pkey));
-        if (sig == NULL) {
-            ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        if (sig == NULL)
             goto err;
-        }
         if (!EVP_SignFinal_ex(mctx, sig, &siglen, si->pkey,
                               ossl_cms_ctx_get0_libctx(ctx),
                               ossl_cms_ctx_get0_propq(ctx))) {
@@ -909,7 +917,7 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
     (void)ERR_pop_to_mark();
 
     if (si->mctx == NULL && (si->mctx = EVP_MD_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB);
         goto err;
     }
     mctx = si->mctx;
@@ -982,7 +990,7 @@ int CMS_SignerInfo_verify_content(CMS_SignerInfo *si, BIO *chain)
     unsigned int mlen;
 
     if (mctx == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_EVP_LIB);
         goto err;
     }
     /* If we have any signed attributes look for messageDigest value */