Fix CMS so that it still works with non fetchable algorithms.

Fixes #12633

For CMS the Gost engine still requires calls to EVP_get_digestbyname() and EVP_get_cipherbyname() when
EVP_MD_fetch() and EVP_CIPHER_fetch() return NULL.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/12689)

1 files changed, 17 insertions, 4 deletions

diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 4fac4e6182..c11d44487b 100644
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -817,7 +817,8 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
     unsigned char *abuf = NULL;
     int alen, r = -1;
     const char *name;
-    EVP_MD *md = NULL;
+    const EVP_MD *md;
+    EVP_MD *fetched_md = NULL;
     const CMS_CTX *ctx = si->cms_ctx;
 
     if (si->pkey == NULL) {
@@ -829,9 +830,21 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
         return -1;
 
     name = OBJ_nid2sn(OBJ_obj2nid(si->digestAlgorithm->algorithm));
-    md = EVP_MD_fetch(ctx->libctx, name, ctx->propq);
-    if (md == NULL)
+
+    (void)ERR_set_mark();
+    fetched_md = EVP_MD_fetch(ctx->libctx, name, ctx->propq);
+
+    if (fetched_md != NULL)
+        md = fetched_md;
+    else
+        md = EVP_get_digestbyobj(si->digestAlgorithm->algorithm);
+    if (md == NULL) {
+        (void)ERR_clear_last_mark();
+        CMSerr(0, CMS_R_UNKNOWN_DIGEST_ALGORITHM);
         return -1;
+    }
+    (void)ERR_pop_to_mark();
+
     if (si->mctx == NULL && (si->mctx = EVP_MD_CTX_new()) == NULL) {
         CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -860,7 +873,7 @@ int CMS_SignerInfo_verify(CMS_SignerInfo *si)
     if (r <= 0)
         CMSerr(CMS_F_CMS_SIGNERINFO_VERIFY, CMS_R_VERIFICATION_FAILURE);
  err:
-    EVP_MD_free(md);
+    EVP_MD_free(fetched_md);
     EVP_MD_CTX_reset(mctx);
     return r;
 }