pkey/ec: check private key validity with OpenSSL 3pr/565

The behavior of EVP_PKEY_public_check changed between OpenSSL 1.1.1
and 3.0 so that it no longer validates the private key. Instead, private
keys can be validated through EVP_PKEY_private_check and
EVP_PKEY_pairwise_check.

4 files changed, 46 insertions, 4 deletions

diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 06d59c2a..8c347e92 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -483,16 +483,38 @@ static VALUE ossl_ec_key_check_key(VALUE self)
 #ifdef HAVE_EVP_PKEY_CHECK
     EVP_PKEY *pkey;
     EVP_PKEY_CTX *pctx;
+    EC_KEY *ec;
+
     int ret;
 
     GetPKey(self, pkey);
     pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
     if (!pctx)
-        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
-    ret = EVP_PKEY_public_check(pctx);
-    EVP_PKEY_CTX_free(pctx);
-    if (ret != 1)
+        ossl_raise(eECError, "EVP_PKEY_CTX_new");
+
+    if (EVP_PKEY_public_check(pctx) != 1) {
+        EVP_PKEY_CTX_free(pctx);
         ossl_raise(eECError, "EVP_PKEY_public_check");
+    }
+
+#if (defined(OPENSSL_VERSION_MAJOR) && OPENSSL_VERSION_MAJOR >= 3)
+    GetEC(self, ec);
+    if (EC_KEY_get0_private_key(ec) == NULL)
+        goto skip_priv_key;
+
+    if (EVP_PKEY_private_check(pctx) != 1) {
+        EVP_PKEY_CTX_free(pctx);
+        ossl_raise(eECError, "EVP_PKEY_private_check");
+    }
+
+    if (EVP_PKEY_pairwise_check(pctx) != 1) {
+        EVP_PKEY_CTX_free(pctx);
+        ossl_raise(eECError, "EVP_PKEY_pairwise_check");
+    }
+    skip_priv_key:
+#endif
+
+    EVP_PKEY_CTX_free(pctx);
 #else
     EC_KEY *ec;
 
diff --git a/test/openssl/fixtures/pkey/p256_too_large.pem b/test/openssl/fixtures/pkey/p256_too_large.pem
new file mode 100644
index 00000000..59c486f1
--- /dev/null
+++ b/test/openssl/fixtures/pkey/p256_too_large.pem
@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIP+TT0V8Fndsnacji9tyf6hmhHywcOWTee9XkiBeJoVloAoGCCqGSM49               
+AwEHoUQDQgAEBkhhJIU/2/YdPSlY2I1k25xjK4trr5OXSgXvBC21PtY0HQ7lor7A               
+jzT0giJITqmcd81fwGw5+96zLcdxTF1hVQ==                                           
+-----END EC PRIVATE KEY-----
diff --git a/test/openssl/fixtures/pkey/p384_invalid.pem b/test/openssl/fixtures/pkey/p384_invalid.pem
new file mode 100644
index 00000000..d5cdc9a3
--- /dev/null
+++ b/test/openssl/fixtures/pkey/p384_invalid.pem
@@ -0,0 +1,6 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDDA1Tm0m7YhkfeVpFuarAJYVlHp2tQj+1fOBiLa10t9E8TiQO/hVfxB
+vGaVEQwOheWgBwYFK4EEACKhZANiAASyGqmryZGqdpsq5gEDIfNvgC3AwSJxiBCL
+XKHBTFRp+tCezLDOK/6V8KK/vVGBJlGFW6/I7ahyXprxS7xs7hPA9iz5YiuqXlu+
+lbrIpZOz7b73hyQQCkvbBO/Avg+hPAk=
+-----END EC PRIVATE KEY-----
diff --git a/test/openssl/test_pkey_ec.rb b/test/openssl/test_pkey_ec.rb
index 9a4818de..64d94f59 100644
--- a/test/openssl/test_pkey_ec.rb
+++ b/test/openssl/test_pkey_ec.rb
@@ -90,6 +90,15 @@ class OpenSSL::TestEC < OpenSSL::PKeyTestCase
     assert_equal(true, key2.public?)
     assert_equal(true, key2.check_key)
 
+    # Behavior of EVP_PKEY_public_check changes between OpenSSL 1.1.1 and 3.0
+    if openssl?(3, 0, 0)
+      key4 = Fixtures.pkey("p256_too_large")
+      assert_raise(OpenSSL::PKey::ECError) { key4.check_key }
+
+      key5 = Fixtures.pkey("p384_invalid")
+      assert_raise(OpenSSL::PKey::ECError) { key5.check_key }
+    end
+
     # EC#private_key= is deprecated in 3.0 and won't work on OpenSSL 3.0
     if !openssl?(3, 0, 0)
       key2.private_key += 1