Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

3 files changed, 9 insertions, 11 deletions

diff --git a/providers/implementations/storemgmt/file_store.c b/providers/implementations/storemgmt/file_store.c
index 7548a01fe7..3f1d1ff2de 100644
--- a/providers/implementations/storemgmt/file_store.c
+++ b/providers/implementations/storemgmt/file_store.c
@@ -155,7 +155,7 @@ static struct file_ctx_st *file_open_stream(BIO *source, const char *uri,
     struct file_ctx_st *ctx;
 
     if ((ctx = new_file_ctx(IS_FILE, uri, provctx)) == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB);
         goto err;
     }
 
@@ -172,7 +172,7 @@ static void *file_open_dir(const char *path, const char *uri, void *provctx)
     struct file_ctx_st *ctx;
 
     if ((ctx = new_file_ctx(IS_DIR, uri, provctx)) == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB);
         return NULL;
     }
 
@@ -422,7 +422,7 @@ static int file_setup_decoders(struct file_ctx_st *ctx)
     /* Setup for this session, so only if not already done */
     if (ctx->_.file.decoderctx == NULL) {
         if ((ctx->_.file.decoderctx = OSSL_DECODER_CTX_new()) == NULL) {
-            ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB);
             goto err;
         }
 
@@ -558,10 +558,8 @@ static char *file_name_to_uri(struct file_ctx_st *ctx, const char *name)
             + strlen(name) + 1 /* \0 */;
 
         data = OPENSSL_zalloc(calculated_length);
-        if (data == NULL) {
-            ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        if (data == NULL)
             return NULL;
-        }
 
         OPENSSL_strlcat(data, ctx->uri, calculated_length);
         OPENSSL_strlcat(data, pathsep, calculated_length);
diff --git a/providers/implementations/storemgmt/file_store_any2obj.c b/providers/implementations/storemgmt/file_store_any2obj.c
index 28601683bf..b0be1c4d22 100644
--- a/providers/implementations/storemgmt/file_store_any2obj.c
+++ b/providers/implementations/storemgmt/file_store_any2obj.c
@@ -125,7 +125,7 @@ static int msblob2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection,
     mem_want = 16;               /* The size of the MSBLOB header */
     if ((mem = BUF_MEM_new()) == NULL
         || !BUF_MEM_grow(mem, mem_want)) {
-        ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB);
         goto err;
     }
 
@@ -147,7 +147,7 @@ static int msblob2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection,
     ok = 0;
     mem_want = ossl_blob_length(bitlen, isdss, ispub);
     if (!BUF_MEM_grow(mem, mem_len + mem_want)) {
-        ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB);
         goto err;
     }
 
@@ -192,7 +192,7 @@ static int pvk2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection,
     mem_want = 24;               /* The size of the PVK header */
     if ((mem = BUF_MEM_new()) == NULL
         || !BUF_MEM_grow(mem, mem_want)) {
-        ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB);
         goto err;
     }
 
@@ -214,7 +214,7 @@ static int pvk2obj_decode(void *provctx, OSSL_CORE_BIO *cin, int selection,
     ok = 0;
     mem_want = saltlen + keylen;
     if (!BUF_MEM_grow(mem, mem_len + mem_want)) {
-        ERR_raise(ERR_LIB_PEM, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PEM, ERR_R_BUF_LIB);
         goto err;
     }
 
diff --git a/providers/implementations/storemgmt/winstore_store.c b/providers/implementations/storemgmt/winstore_store.c
index f686517f74..a7d041b697 100644
--- a/providers/implementations/storemgmt/winstore_store.c
+++ b/providers/implementations/storemgmt/winstore_store.c
@@ -186,7 +186,7 @@ static int setup_decoder(struct winstore_ctx_st *ctx)
 
     ctx->dctx = OSSL_DECODER_CTX_new();
     if (ctx->dctx == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_OSSL_DECODER_LIB);
         return 0;
     }