Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 60 insertions, 35 deletions

diff --git a/crypto/asn1/p5_scrypt.c b/crypto/asn1/p5_scrypt.c
index a02190d0dc..94b77fd3ab 100644
--- a/crypto/asn1/p5_scrypt.c
+++ b/crypto/asn1/p5_scrypt.c
@@ -67,16 +67,20 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher,
     }
 
     pbe2 = PBE2PARAM_new();
-    if (pbe2 == NULL)
-        goto merr;
+    if (pbe2 == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     /* Setup the AlgorithmIdentifier for the encryption scheme */
     scheme = pbe2->encryption;
 
     scheme->algorithm = OBJ_nid2obj(alg_nid);
     scheme->parameter = ASN1_TYPE_new();
-    if (scheme->parameter == NULL)
-        goto merr;
+    if (scheme->parameter == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     /* Create random IV */
     if (EVP_CIPHER_get_iv_length(cipher)) {
@@ -87,8 +91,10 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher,
     }
 
     ctx = EVP_CIPHER_CTX_new();
-    if (ctx == NULL)
-        goto merr;
+    if (ctx == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB);
+        goto err;
+    }
 
     /* Dummy cipherinit to just setup the IV */
     if (EVP_CipherInit_ex(ctx, cipher, NULL, NULL, iv, 0) == 0)
@@ -111,31 +117,34 @@ X509_ALGOR *PKCS5_pbe2_set_scrypt(const EVP_CIPHER *cipher,
 
     pbe2->keyfunc = pkcs5_scrypt_set(salt, saltlen, keylen, N, r, p);
 
-    if (pbe2->keyfunc == NULL)
-        goto merr;
+    if (pbe2->keyfunc == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     /* Now set up top level AlgorithmIdentifier */
 
     ret = X509_ALGOR_new();
-    if (ret == NULL)
-        goto merr;
+    if (ret == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     ret->algorithm = OBJ_nid2obj(NID_pbes2);
 
     /* Encode PBE2PARAM into parameter */
 
     if (ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(PBE2PARAM), pbe2,
-                                &ret->parameter) == NULL)
-        goto merr;
+                                &ret->parameter) == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     PBE2PARAM_free(pbe2);
     pbe2 = NULL;
 
     return ret;
 
- merr:
-    ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
-
  err:
     PBE2PARAM_free(pbe2);
     X509_ALGOR_free(ret);
@@ -151,57 +160,73 @@ static X509_ALGOR *pkcs5_scrypt_set(const unsigned char *salt, size_t saltlen,
     X509_ALGOR *keyfunc = NULL;
     SCRYPT_PARAMS *sparam = SCRYPT_PARAMS_new();
 
-    if (sparam == NULL)
-        goto merr;
+    if (sparam == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     if (!saltlen)
         saltlen = PKCS5_SALT_LEN;
 
     /* This will either copy salt or grow the buffer */
-    if (ASN1_STRING_set(sparam->salt, salt, saltlen) == 0)
-        goto merr;
+    if (ASN1_STRING_set(sparam->salt, salt, saltlen) == 0) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     if (salt == NULL && RAND_bytes(sparam->salt->data, saltlen) <= 0)
         goto err;
 
-    if (ASN1_INTEGER_set_uint64(sparam->costParameter, N) == 0)
-        goto merr;
+    if (ASN1_INTEGER_set_uint64(sparam->costParameter, N) == 0) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
-    if (ASN1_INTEGER_set_uint64(sparam->blockSize, r) == 0)
-        goto merr;
+    if (ASN1_INTEGER_set_uint64(sparam->blockSize, r) == 0) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
-    if (ASN1_INTEGER_set_uint64(sparam->parallelizationParameter, p) == 0)
-        goto merr;
+    if (ASN1_INTEGER_set_uint64(sparam->parallelizationParameter, p) == 0) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     /* If have a key len set it up */
 
     if (keylen > 0) {
         sparam->keyLength = ASN1_INTEGER_new();
-        if (sparam->keyLength == NULL)
-            goto merr;
-        if (ASN1_INTEGER_set_int64(sparam->keyLength, keylen) == 0)
-            goto merr;
+        if (sparam->keyLength == NULL) {
+            ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+            goto err;
+        }
+        if (ASN1_INTEGER_set_int64(sparam->keyLength, keylen) == 0) {
+            ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+            goto err;
+        }
     }
 
     /* Finally setup the keyfunc structure */
 
     keyfunc = X509_ALGOR_new();
-    if (keyfunc == NULL)
-        goto merr;
+    if (keyfunc == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     keyfunc->algorithm = OBJ_nid2obj(NID_id_scrypt);
 
     /* Encode SCRYPT_PARAMS into parameter of pbe2 */
 
     if (ASN1_TYPE_pack_sequence(ASN1_ITEM_rptr(SCRYPT_PARAMS), sparam,
-                                &keyfunc->parameter) == NULL)
-        goto merr;
+                                &keyfunc->parameter) == NULL) {
+        ERR_raise(ERR_LIB_ASN1, ERR_R_ASN1_LIB);
+        goto err;
+    }
 
     SCRYPT_PARAMS_free(sparam);
     return keyfunc;
 
- merr:
-    ERR_raise(ERR_LIB_ASN1, ERR_R_MALLOC_FAILURE);
  err:
     SCRYPT_PARAMS_free(sparam);
     X509_ALGOR_free(keyfunc);