Fix error handling in x509v3_cache_extensions and related functions

Basically we use EXFLAG_INVALID for all kinds of out of memory and
all kinds of parse errors in x509v3_cache_extensions.

[extended tests]

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/10755)

1 files changed, 6 insertions, 4 deletions

diff --git a/crypto/ts/ts_rsp_verify.c b/crypto/ts/ts_rsp_verify.c
index dfb068c8b6..1f20fb9c0e 100644
--- a/crypto/ts/ts_rsp_verify.c
+++ b/crypto/ts/ts_rsp_verify.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2006-2019 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -265,11 +265,12 @@ static int ts_find_cert(STACK_OF(ESS_CERT_ID) *cert_ids, X509 *cert)
     if (!cert_ids || !cert)
         return -1;
 
-    X509_digest(cert, EVP_sha1(), cert_sha1, NULL);
-
     /* Recompute SHA1 hash of certificate if necessary (side effect). */
     X509_check_purpose(cert, -1, 0);
 
+    if (!X509_digest(cert, EVP_sha1(), cert_sha1, NULL))
+        return -1;
+
     /* Look for cert in the cert_ids vector. */
     for (i = 0; i < sk_ESS_CERT_ID_num(cert_ids); ++i) {
         ESS_CERT_ID *cid = sk_ESS_CERT_ID_value(cert_ids, i);
@@ -302,7 +303,8 @@ static int ts_find_cert_v2(STACK_OF(ESS_CERT_ID_V2) *cert_ids, X509 *cert)
         else
             md = EVP_sha256();
 
-        X509_digest(cert, md, cert_digest, &len);
+        if (!X509_digest(cert, md, cert_digest, &len))
+            return -1;
         if (cid->hash->length != (int)len)
             return -1;