ENCODER & DECODER: Allow decoder implementations to specify "carry on"

So far, decoder implementations would return true (1) for a successful
decode all the way, including what the callback it called returned,
and false (0) in all other cases.

This construction didn't allow to stop to decoding process on fatal
errors, nor to choose what to report in the provider code.

This is now changed so that decoders implementations are made to
return false only on errors that should stop the decoding process from
carrying on with other implementations, and return true for all other
cases, even if that didn't result in a constructed object (EVP_PKEY
for example), essentially making it OK to return "empty handed".

The success of the decoding process is now all about successfully
constructing the final object, rather than about the return value of
the decoding chain.  If no construction is attempted, the central
decoding processing code concludes that whatever the input consisted
of, it's not supported by the available decoder implementations.

Fixes #14423

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14834)

3 files changed, 70 insertions, 24 deletions

diff --git a/crypto/encode_decode/decoder_err.c b/crypto/encode_decode/decoder_err.c
index cf68a4c7c5..1880c8f409 100644
--- a/crypto/encode_decode/decoder_err.c
+++ b/crypto/encode_decode/decoder_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -15,6 +15,8 @@
 #ifndef OPENSSL_NO_ERR
 
 static const ERR_STRING_DATA OSSL_DECODER_str_reasons[] = {
+    {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, OSSL_DECODER_R_COULD_NOT_DECODE_OBJECT),
+    "could not decode object"},
     {ERR_PACK(ERR_LIB_OSSL_DECODER, 0, OSSL_DECODER_R_MISSING_GET_PARAMS),
     "missing get params"},
     {0, NULL}
diff --git a/crypto/encode_decode/decoder_lib.c b/crypto/encode_decode/decoder_lib.c
index a644924aeb..e37989fec4 100644
--- a/crypto/encode_decode/decoder_lib.c
+++ b/crypto/encode_decode/decoder_lib.c
@@ -32,6 +32,12 @@ struct decoder_process_data_st {
     size_t current_decoder_inst_index;
     /* For tracing, count recursion level */
     size_t recursion;
+
+    /*-
+     * Flags
+     */
+    unsigned int flag_next_level_called : 1;
+    unsigned int flag_construct_called : 1;
 };
 
 static int decoder_process(const OSSL_PARAM params[], void *arg);
@@ -57,6 +63,29 @@ int OSSL_DECODER_from_bio(OSSL_DECODER_CTX *ctx, BIO *in)
 
     ok = decoder_process(NULL, &data);
 
+    if (!data.flag_construct_called) {
+        const char *spaces
+            = ctx->start_input_type != NULL && ctx->input_structure != NULL
+            ? " " : "";
+        const char *input_type_label
+            = ctx->start_input_type != NULL ? "Input type: " : "";
+        const char *input_structure_label
+            = ctx->input_structure != NULL ? "Input structure: " : "";
+        const char *comma
+            = ctx->start_input_type != NULL && ctx->input_structure != NULL
+            ? ", " : "";
+        const char *input_type
+            = ctx->start_input_type != NULL ? ctx->start_input_type : "";
+        const char *input_structure
+            = ctx->input_structure != NULL ? ctx->input_structure : "";
+
+        ERR_raise_data(ERR_LIB_OSSL_DECODER, ERR_R_UNSUPPORTED,
+                       "No supported for the data to decode.%s%s%s%s%s%s",
+                       spaces, input_type_label, input_type, comma,
+                       input_structure_label, input_structure);
+        ok = 0;
+    }
+
     /* Clear any internally cached passphrase */
     (void)ossl_pw_clear_passphrase_cache(&ctx->pwdata);
 
@@ -525,12 +554,18 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
     BIO *bio = data->bio;
     long loc;
     size_t i;
-    int err, lib, reason, ok = 0;
+    int ok = 0;
     /* For recursions */
     struct decoder_process_data_st new_data;
     const char *data_type = NULL;
     const char *data_structure = NULL;
 
+    /*
+     * This is an indicator up the call stack that something was indeed
+     * decoded, leading to a recursive call of this function.
+     */
+    data->flag_next_level_called = 1;
+
     memset(&new_data, 0, sizeof(new_data));
     new_data.ctx = data->ctx;
     new_data.recursion = data->recursion + 1;
@@ -562,10 +597,14 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
                                            data->current_decoder_inst_index);
         decoder = OSSL_DECODER_INSTANCE_get_decoder(decoder_inst);
 
-        if (ctx->construct != NULL
-            && ctx->construct(decoder_inst, params, ctx->construct_data)) {
-            ok = 1;
-            goto end;
+        data->flag_construct_called = 0;
+        if (ctx->construct != NULL) {
+            int rv = ctx->construct(decoder_inst, params, ctx->construct_data);
+
+            data->flag_construct_called = 1;
+            ok = (rv > 0);
+            if (ok)
+                goto end;
         }
 
         /* The constructor didn't return success */
@@ -746,6 +785,12 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
                        (void *)new_decoder_inst);
         } OSSL_TRACE_END(DECODER);
 
+        /*
+         * We only care about errors reported from decoder implementations
+         * if it returns false (i.e. there was a fatal error).
+         */
+        ERR_set_mark();
+
         new_data.current_decoder_inst_index = i;
         ok = new_decoder->decode(new_decoderctx, cbio,
                                  new_data.ctx->selection,
@@ -755,31 +800,29 @@ static int decoder_process(const OSSL_PARAM params[], void *arg)
 
         OSSL_TRACE_BEGIN(DECODER) {
             BIO_printf(trc_out,
-                       "(ctx %p) %s [%u] Running decoder instance %p => %d\n",
+                       "(ctx %p) %s [%u] Running decoder instance %p => %d"
+                       " (recursed further: %s, construct called: %s)\n",
                        (void *)new_data.ctx, LEVEL, (unsigned int)i,
-                       (void *)new_decoder_inst, ok);
+                       (void *)new_decoder_inst, ok,
+                       new_data.flag_next_level_called ? "yes" : "no",
+                       new_data.flag_construct_called ? "yes" : "no");
         } OSSL_TRACE_END(DECODER);
 
-        if (ok)
+        data->flag_construct_called = new_data.flag_construct_called;
+
+        /* Break on error or if we tried to construct an object already */
+        if (!ok || data->flag_construct_called) {
+            ERR_clear_last_mark();
             break;
+        }
+        ERR_pop_to_mark();
 
         /*
-         * These errors are assumed to come from ossl_store_handle_load_result()
-         * in crypto/store/store_result.c.  They are currently considered fatal
-         * errors, so we preserve them in the error queue and stop.
+         * Break if the decoder implementation that we called recursed, since
+         * that indicates that it successfully decoded something.
          */
-        err = ERR_peek_last_error();
-        lib = ERR_GET_LIB(err);
-        reason = ERR_GET_REASON(err);
-        if ((lib == ERR_LIB_EVP
-             && reason == EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM)
-#ifndef OPENSSL_NO_EC
-            || (lib == ERR_LIB_EC && reason == EC_R_UNKNOWN_GROUP)
-#endif
-            || (lib == ERR_LIB_X509 && reason == X509_R_UNSUPPORTED_ALGORITHM)
-            || (lib == ERR_LIB_PKCS12
-                && reason == PKCS12_R_PKCS12_CIPHERFINAL_ERROR))
-            goto end;
+        if (new_data.flag_next_level_called)
+            break;
     }
 
  end:
diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index eed0b71ada..81f9f1ef49 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -811,6 +811,7 @@ OCSP_R_STATUS_TOO_OLD:127:status too old
 OCSP_R_UNKNOWN_MESSAGE_DIGEST:119:unknown message digest
 OCSP_R_UNKNOWN_NID:120:unknown nid
 OCSP_R_UNSUPPORTED_REQUESTORNAME_TYPE:129:unsupported requestorname type
+OSSL_DECODER_R_COULD_NOT_DECODE_OBJECT:101:could not decode object
 OSSL_DECODER_R_MISSING_GET_PARAMS:100:missing get params
 OSSL_ENCODER_R_ENCODER_NOT_FOUND:101:encoder not found
 OSSL_ENCODER_R_INCORRECT_PROPERTY_QUERY:100:incorrect property query