Adapt libcrypto functionality to specify the desired output structure

This also modifies i2d_PublicKey() and i2d_KeyParams() to support
provided keys.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13167)

9 files changed, 142 insertions, 133 deletions

diff --git a/crypto/asn1/build.info b/crypto/asn1/build.info
index a66c3084ce..e10d631654 100644
--- a/crypto/asn1/build.info
+++ b/crypto/asn1/build.info
@@ -5,7 +5,7 @@ SOURCE[../../libcrypto]=\
         a_utf8.c a_sign.c a_digest.c a_verify.c a_mbstr.c a_strex.c \
         x_algor.c x_val.c x_sig.c x_bignum.c \
         x_int64.c x_info.c x_spki.c nsseq.c \
-        d2i_pu.c d2i_pr.c i2d_pu.c i2d_pr.c\
+        d2i_pu.c d2i_pr.c i2d_evp.c \
         t_pkey.c t_spki.c t_bitst.c \
         tasn_new.c tasn_fre.c tasn_enc.c tasn_dec.c tasn_utl.c tasn_typ.c \
         tasn_prn.c tasn_scn.c ameth_lib.c \
@@ -14,7 +14,7 @@ SOURCE[../../libcrypto]=\
         asn1_gen.c asn1_par.c asn1_lib.c asn1_err.c a_strnid.c \
         evp_asn1.c asn_pack.c p5_pbe.c p5_pbev2.c p5_scrypt.c p8_pkey.c \
         asn_moid.c asn_mstbl.c asn1_item_list.c \
-        d2i_param.c i2d_param.c
+        d2i_param.c
 IF[{- !$disabled{'rsa'} and !$disabled{'rc4'} -}]
   SOURCE[../../libcrypto]=n_pkey.c
 ENDIF
diff --git a/crypto/asn1/i2d_evp.c b/crypto/asn1/i2d_evp.c
new file mode 100644
index 0000000000..a81ae415fa
--- /dev/null
+++ b/crypto/asn1/i2d_evp.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/* We need to use some deprecated APIs to support the legacy bits */
+#define OPENSSL_SUPPRESS_DEPRECATED
+
+#include <stdio.h>
+#include "internal/cryptlib.h"
+#include <openssl/evp.h>
+#include <openssl/encoder.h>
+#include <openssl/buffer.h>
+#include <openssl/x509.h>
+#include <openssl/rsa.h>         /* For i2d_RSAPublicKey */
+#include <openssl/dsa.h>         /* For i2d_DSAPublicKey */
+#include <openssl/ec.h>          /* For i2o_ECPublicKey */
+#include "crypto/asn1.h"
+#include "crypto/evp.h"
+
+static int i2d_provided(const EVP_PKEY *a, int selection,
+                        const char *output_structures[],
+                        unsigned char **pp)
+{
+    OSSL_ENCODER_CTX *ctx = NULL;
+    int ret;
+
+    for (ret = -1;
+         ret == -1 && *output_structures != NULL;
+         output_structures++) {
+        /*
+         * The i2d_ calls don't take a boundary length for *pp.  However,
+         * OSSL_ENCODER_CTX_get_num_encoders() needs one, so we make one
+         * up.
+         */
+        size_t len = INT_MAX;
+
+        ctx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, selection, "DER",
+                                               *output_structures,
+                                               NULL, NULL);
+        if (ctx == NULL)
+            return -1;
+        if (OSSL_ENCODER_to_data(ctx, pp, &len))
+            ret = (int)len;
+        OSSL_ENCODER_CTX_free(ctx);
+        ctx = NULL;
+    }
+
+    if (ret == -1)
+        ERR_raise(ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_TYPE);
+    return ret;
+}
+
+int i2d_KeyParams(const EVP_PKEY *a, unsigned char **pp)
+{
+    if (evp_pkey_is_provided(a)) {
+        const char *output_structures[] = { "type-specific", NULL };
+
+        return i2d_provided(a, EVP_PKEY_KEY_PARAMETERS, output_structures, pp);
+    }
+    if (a->ameth != NULL && a->ameth->param_encode != NULL)
+        return a->ameth->param_encode(a, pp);
+    ERR_raise(ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_TYPE);
+    return -1;
+}
+
+int i2d_KeyParams_bio(BIO *bp, const EVP_PKEY *pkey)
+{
+    return ASN1_i2d_bio_of(EVP_PKEY, i2d_KeyParams, bp, pkey);
+}
+
+int i2d_PrivateKey(const EVP_PKEY *a, unsigned char **pp)
+{
+    if (evp_pkey_is_provided(a)) {
+        const char *output_structures[] = { "type-specific", "pkcs8", NULL };
+
+        return i2d_provided(a, EVP_PKEY_KEYPAIR, output_structures, pp);
+    }
+    if (a->ameth != NULL && a->ameth->old_priv_encode != NULL) {
+        return a->ameth->old_priv_encode(a, pp);
+    }
+    if (a->ameth != NULL && a->ameth->priv_encode != NULL) {
+        PKCS8_PRIV_KEY_INFO *p8 = EVP_PKEY2PKCS8(a);
+        int ret = 0;
+
+        if (p8 != NULL) {
+            ret = i2d_PKCS8_PRIV_KEY_INFO(p8, pp);
+            PKCS8_PRIV_KEY_INFO_free(p8);
+        }
+        return ret;
+    }
+    ERR_raise(ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
+    return -1;
+}
+
+int i2d_PublicKey(const EVP_PKEY *a, unsigned char **pp)
+{
+    if (evp_pkey_is_provided(a)) {
+        const char *output_structures[] = { "type-specific", NULL };
+
+        return i2d_provided(a, EVP_PKEY_PUBLIC_KEY, output_structures, pp);
+    }
+    switch (EVP_PKEY_id(a)) {
+#ifndef OPENSSL_NO_RSA
+    case EVP_PKEY_RSA:
+        return i2d_RSAPublicKey(EVP_PKEY_get0_RSA(a), pp);
+#endif
+#ifndef OPENSSL_NO_DSA
+    case EVP_PKEY_DSA:
+        return i2d_DSAPublicKey(EVP_PKEY_get0_DSA(a), pp);
+#endif
+#ifndef OPENSSL_NO_EC
+    case EVP_PKEY_EC:
+        return i2o_ECPublicKey(EVP_PKEY_get0_EC_KEY(a), pp);
+#endif
+    default:
+        ERR_raise(ERR_LIB_ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
+        return -1;
+    }
+}
diff --git a/crypto/asn1/i2d_param.c b/crypto/asn1/i2d_param.c
deleted file mode 100644
index 1e1ebc95b2..0000000000
--- a/crypto/asn1/i2d_param.c
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#include <stdio.h>
-#include "internal/cryptlib.h"
-#include <openssl/evp.h>
-#include <openssl/objects.h>
-#include <openssl/asn1.h>
-#include "crypto/asn1.h"
-#include "crypto/evp.h"
-
-int i2d_KeyParams(const EVP_PKEY *a, unsigned char **pp)
-{
-    if (a->ameth != NULL && a->ameth->param_encode != NULL)
-        return a->ameth->param_encode(a, pp);
-    ASN1err(ASN1_F_I2D_KEYPARAMS, ASN1_R_UNSUPPORTED_TYPE);
-    return -1;
-}
-
-int i2d_KeyParams_bio(BIO *bp, const EVP_PKEY *pkey)
-{
-    return ASN1_i2d_bio_of(EVP_PKEY, i2d_KeyParams, bp, pkey);
-}
-
diff --git a/crypto/asn1/i2d_pr.c b/crypto/asn1/i2d_pr.c
deleted file mode 100644
index 7185abef45..0000000000
--- a/crypto/asn1/i2d_pr.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#include <stdio.h>
-#include <limits.h>
-#include "internal/cryptlib.h"
-#include <openssl/evp.h>
-#include <openssl/encoder.h>
-#include <openssl/buffer.h>
-#include <openssl/x509.h>
-#include "crypto/asn1.h"
-#include "crypto/evp.h"
-
-int i2d_PrivateKey(const EVP_PKEY *a, unsigned char **pp)
-{
-    if (a->ameth && a->ameth->old_priv_encode) {
-        return a->ameth->old_priv_encode(a, pp);
-    }
-    if (a->ameth && a->ameth->priv_encode) {
-        PKCS8_PRIV_KEY_INFO *p8 = EVP_PKEY2PKCS8(a);
-        int ret = 0;
-        if (p8 != NULL) {
-            ret = i2d_PKCS8_PRIV_KEY_INFO(p8, pp);
-            PKCS8_PRIV_KEY_INFO_free(p8);
-        }
-        return ret;
-    }
-    if (evp_pkey_is_provided(a)) {
-        /* |*pp| is unbounded, so we need an upper limit */
-        size_t length = INT_MAX;
-        int selection = EVP_PKEY_KEYPAIR;
-        int ret = -1;
-        OSSL_ENCODER_CTX *ctx;
-
-        if ((ctx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, "DER", selection,
-                                                    NULL, NULL)) != NULL
-            && OSSL_ENCODER_CTX_get_num_encoders(ctx) != 0
-            && OSSL_ENCODER_to_data(ctx, pp, &length))
-            ret = (int)length;
-        OSSL_ENCODER_CTX_free(ctx);
-        return ret;
-    }
-    ASN1err(ASN1_F_I2D_PRIVATEKEY, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
-    return -1;
-}
diff --git a/crypto/asn1/i2d_pu.c b/crypto/asn1/i2d_pu.c
deleted file mode 100644
index d0151e5bd6..0000000000
--- a/crypto/asn1/i2d_pu.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the Apache License 2.0 (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-/*
- * DSA low level APIs are deprecated for public use, but still ok for
- * internal use.
- */
-#include "internal/deprecated.h"
-
-#include <stdio.h>
-#include "internal/cryptlib.h"
-#include <openssl/bn.h>
-#include <openssl/evp.h>
-#include <openssl/objects.h>
-#include <openssl/rsa.h>
-#include <openssl/dsa.h>
-#include <openssl/ec.h>
-
-int i2d_PublicKey(const EVP_PKEY *a, unsigned char **pp)
-{
-    switch (EVP_PKEY_id(a)) {
-#ifndef OPENSSL_NO_RSA
-    case EVP_PKEY_RSA:
-        return i2d_RSAPublicKey(EVP_PKEY_get0_RSA(a), pp);
-#endif
-#ifndef OPENSSL_NO_DSA
-    case EVP_PKEY_DSA:
-        return i2d_DSAPublicKey(EVP_PKEY_get0_DSA(a), pp);
-#endif
-#ifndef OPENSSL_NO_EC
-    case EVP_PKEY_EC:
-        return i2o_ECPublicKey(EVP_PKEY_get0_EC_KEY(a), pp);
-#endif
-    default:
-        ASN1err(ASN1_F_I2D_PUBLICKEY, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
-        return -1;
-    }
-}
diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 4eeb95e413..4b096ac17d 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -1186,7 +1186,7 @@ static int print_pkey(const EVP_PKEY *pkey, BIO *out, int indent,
     if (!print_set_indent(&out, &pop_f_prefix, &saved_indent, indent))
         return 0;
 
-    ctx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, "TEXT", selection,
+    ctx = OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, selection, "TEXT", NULL,
                                            libctx, propquery);
     if (OSSL_ENCODER_CTX_get_num_encoders(ctx) != 0)
         ret = OSSL_ENCODER_to_bio(ctx, out);
diff --git a/crypto/pem/pem_local.h b/crypto/pem/pem_local.h
index f9575d4988..10761b03d3 100644
--- a/crypto/pem/pem_local.h
+++ b/crypto/pem/pem_local.h
@@ -32,12 +32,20 @@
 # define PEM_SELECTION_PrivateKey       EVP_PKEY_KEYPAIR
 # define PEM_SELECTION_Parameters       EVP_PKEY_KEY_PARAMETERS
 
+/*
+ * Properties, named according to the ASN.1 names used throughout libcrypto.
+ */
+# define PEM_STRUCTURE_PUBKEY "SubjectPublicKeyInfo"
+# define PEM_STRUCTURE_PrivateKey "pkcs8"
+# define PEM_STRUCTURE_Parameters "type-specific"
+
 /* Alternative IMPLEMENT macros for provided encoders */
 
 # define IMPLEMENT_PEM_provided_write_body_vars(type, asn1)             \
     int ret = 0;                                                        \
     OSSL_ENCODER_CTX *ctx =                                             \
-        OSSL_ENCODER_CTX_new_by_##type(x, "PEM", PEM_SELECTION_##asn1,  \
+        OSSL_ENCODER_CTX_new_by_##type(x, PEM_SELECTION_##asn1,         \
+                                       "PEM", PEM_STRUCTURE_##asn1,     \
                                        NULL, NULL);                     \
                                                                         \
     if (OSSL_ENCODER_CTX_get_num_encoders(ctx) == 0) {                  \
diff --git a/crypto/pem/pem_pk8.c b/crypto/pem/pem_pk8.c
index 2abf687cbd..797c9881d8 100644
--- a/crypto/pem/pem_pk8.c
+++ b/crypto/pem/pem_pk8.c
@@ -74,8 +74,8 @@ static int do_pk8pkey(BIO *bp, const EVP_PKEY *x, int isder, int nid,
     int ret = 0;
     const char *outtype = isder ? "DER" : "PEM";
     OSSL_ENCODER_CTX *ctx =
-        OSSL_ENCODER_CTX_new_by_EVP_PKEY(x, outtype, OSSL_KEYMGMT_SELECT_ALL,
-                                         libctx, propq);
+        OSSL_ENCODER_CTX_new_by_EVP_PKEY(x, OSSL_KEYMGMT_SELECT_ALL,
+                                         outtype, "pkcs8", libctx, propq);
 
     if (ctx == NULL)
         return 0;
diff --git a/crypto/x509/x_pubkey.c b/crypto/x509/x_pubkey.c
index b7dd04838c..b24ed8ff46 100644
--- a/crypto/x509/x_pubkey.c
+++ b/crypto/x509/x_pubkey.c
@@ -104,7 +104,8 @@ int X509_PUBKEY_set(X509_PUBKEY **x, EVP_PKEY *pkey)
         unsigned char *der = NULL;
         size_t derlen = 0;
         OSSL_ENCODER_CTX *ectx =
-            OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, "DER", EVP_PKEY_PUBLIC_KEY,
+            OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, EVP_PKEY_PUBLIC_KEY,
+                                             "DER", "SubjectPublicKeyInfo",
                                              libctx, NULL);
 
         if (OSSL_ENCODER_to_data(ectx, &der, &derlen)) {
@@ -309,7 +310,8 @@ int i2d_PUBKEY(const EVP_PKEY *a, unsigned char **pp)
         const OSSL_PROVIDER *pkprov = EVP_KEYMGMT_provider(a->keymgmt);
         OSSL_LIB_CTX *libctx = ossl_provider_libctx(pkprov);
         OSSL_ENCODER_CTX *ctx =
-            OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, "DER", EVP_PKEY_PUBLIC_KEY,
+            OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, EVP_PKEY_PUBLIC_KEY,
+                                             "DER", "SubjectPublicKeyInfo",
                                              libctx, NULL);
         BIO *out = BIO_new(BIO_s_mem());
         BUF_MEM *buf = NULL;