Add EVP_PKEY_set1_engine() function.

Add an ENGINE to EVP_PKEY structure which can be used for cryptographic
operations: this will typically be used by an HSM key to redirect calls
to a custom EVP_PKEY_METHOD.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4503)

3 files changed, 26 insertions, 3 deletions

diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 74539375be..f0c30752bc 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -188,9 +188,11 @@ static int pkey_set_type(EVP_PKEY *pkey, int type, const char *str, int len)
         if ((type == pkey->save_type) && pkey->ameth)
             return 1;
 #ifndef OPENSSL_NO_ENGINE
-        /* If we have an ENGINE release it */
+        /* If we have ENGINEs release them */
         ENGINE_finish(pkey->engine);
         pkey->engine = NULL;
+        ENGINE_finish(pkey->pmeth_engine);
+        pkey->pmeth_engine = NULL;
 #endif
     }
     if (str)
@@ -224,7 +226,25 @@ int EVP_PKEY_set_type_str(EVP_PKEY *pkey, const char *str, int len)
 {
     return pkey_set_type(pkey, EVP_PKEY_NONE, str, len);
 }
-
+#ifndef OPENSSL_NO_ENGINE
+int EVP_PKEY_set1_engine(EVP_PKEY *pkey, ENGINE *e)
+{
+    if (e != NULL) {
+        if (!ENGINE_init(e)) {
+            EVPerr(EVP_F_EVP_PKEY_SET1_ENGINE, ERR_R_ENGINE_LIB);
+            return 0;
+        }
+        if (ENGINE_get_pkey_meth(e, pkey->type) == NULL) {
+            ENGINE_finish(e);
+            EVPerr(EVP_F_EVP_PKEY_SET1_ENGINE, EVP_R_UNSUPPORTED_ALGORITHM);
+            return 0;
+        }
+    }
+    ENGINE_finish(pkey->pmeth_engine);
+    pkey->pmeth_engine = e;
+    return 1;
+}
+#endif
 int EVP_PKEY_assign(EVP_PKEY *pkey, int type, void *key)
 {
     if (pkey == NULL || !EVP_PKEY_set_type(pkey, type))
@@ -443,6 +463,8 @@ static void EVP_PKEY_free_it(EVP_PKEY *x)
 #ifndef OPENSSL_NO_ENGINE
     ENGINE_finish(x->engine);
     x->engine = NULL;
+    ENGINE_finish(x->pmeth_engine);
+    x->pmeth_engine = NULL;
 #endif
 }
 
diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c
index 35e525b6ed..37c5e85257 100644
--- a/crypto/evp/pmeth_lib.c
+++ b/crypto/evp/pmeth_lib.c
@@ -106,7 +106,7 @@ static EVP_PKEY_CTX *int_ctx_new(EVP_PKEY *pkey, ENGINE *e, int id)
     }
 #ifndef OPENSSL_NO_ENGINE
     if (e == NULL && pkey != NULL)
-        e = pkey->engine;
+        e = pkey->pmeth_engine != NULL ? pkey->pmeth_engine : pkey->engine;
     /* Try to find an ENGINE which implements this method */
     if (e) {
         if (!ENGINE_init(e)) {
diff --git a/crypto/include/internal/evp_int.h b/crypto/include/internal/evp_int.h
index ccfa97ce35..f409400285 100644
--- a/crypto/include/internal/evp_int.h
+++ b/crypto/include/internal/evp_int.h
@@ -369,6 +369,7 @@ struct evp_pkey_st {
     CRYPTO_REF_COUNT references;
     const EVP_PKEY_ASN1_METHOD *ameth;
     ENGINE *engine;
+    ENGINE *pmeth_engine; /* If not NULL public key ENGINE to use */
     union {
         void *ptr;
 # ifndef OPENSSL_NO_RSA