Merge pull request #468 from rhenium/ky/openssl-3.0.0-part1

Miscellaneous changes for OpenSSL 3.0 support

12 files changed, 168 insertions, 150 deletions

diff --git a/ext/openssl/extconf.rb b/ext/openssl/extconf.rb
index 9a822f60..1d38b569 100644
--- a/ext/openssl/extconf.rb
+++ b/ext/openssl/extconf.rb
@@ -160,7 +160,7 @@ have_func("SSL_SESSION_get_protocol_version")
 have_func("TS_STATUS_INFO_get0_status")
 have_func("TS_STATUS_INFO_get0_text")
 have_func("TS_STATUS_INFO_get0_failure_info")
-have_func("TS_VERIFY_CTS_set_certs")
+have_func("TS_VERIFY_CTS_set_certs(NULL, NULL)", "openssl/ts.h")
 have_func("TS_VERIFY_CTX_set_store")
 have_func("TS_VERIFY_CTX_add_flags")
 have_func("TS_RESP_CTX_set_time_cb")
@@ -172,6 +172,13 @@ have_func("EVP_PKEY_check")
 
 # added in 3.0.0
 have_func("SSL_set0_tmp_dh_pkey")
+have_func("ERR_get_error_all")
+have_func("TS_VERIFY_CTX_set_certs(NULL, NULL)", "openssl/ts.h")
+have_func("SSL_CTX_load_verify_file")
+have_func("BN_check_prime")
+have_func("EVP_MD_CTX_get0_md")
+have_func("EVP_MD_CTX_get_pkey_ctx")
+have_func("EVP_PKEY_eq")
 
 Logging::message "=== Checking done. ===\n"
 
diff --git a/ext/openssl/openssl_missing.h b/ext/openssl/openssl_missing.h
index 4d9b8801..8629bfe5 100644
--- a/ext/openssl/openssl_missing.h
+++ b/ext/openssl/openssl_missing.h
@@ -21,10 +21,6 @@
 #  define EVP_MD_CTX_free EVP_MD_CTX_destroy
 #endif
 
-#if !defined(HAVE_EVP_MD_CTX_PKEY_CTX)
-#  define EVP_MD_CTX_pkey_ctx(x) (x)->pctx
-#endif
-
 #if !defined(HAVE_X509_STORE_GET_EX_DATA)
 #  define X509_STORE_get_ex_data(x, idx) \
 	CRYPTO_get_ex_data(&(x)->ex_data, (idx))
@@ -214,4 +210,29 @@ IMPL_PKEY_GETTER(EC_KEY, ec)
     } while (0)
 #endif
 
+/* added in 3.0.0 */
+#if !defined(HAVE_TS_VERIFY_CTX_SET_CERTS)
+#  define TS_VERIFY_CTX_set_certs(ctx, crts) TS_VERIFY_CTS_set_certs(ctx, crts)
+#endif
+
+#ifndef HAVE_EVP_MD_CTX_GET0_MD
+#  define EVP_MD_CTX_get0_md(ctx) EVP_MD_CTX_md(ctx)
+#endif
+
+/*
+ * OpenSSL 1.1.0 added EVP_MD_CTX_pkey_ctx(), and then it was renamed to
+ * EVP_MD_CTX_get_pkey_ctx(x) in OpenSSL 3.0.
+ */
+#ifndef HAVE_EVP_MD_CTX_GET_PKEY_CTX
+# ifdef HAVE_EVP_MD_CTX_PKEY_CTX
+#  define EVP_MD_CTX_get_pkey_ctx(x) EVP_MD_CTX_pkey_ctx(x)
+# else
+#  define EVP_MD_CTX_get_pkey_ctx(x) (x)->pctx
+# endif
+#endif
+
+#ifndef HAVE_EVP_PKEY_EQ
+#  define EVP_PKEY_eq(a, b) EVP_PKEY_cmp(a, b)
+#endif
+
 #endif /* _OSSL_OPENSSL_MISSING_H_ */
diff --git a/ext/openssl/ossl.c b/ext/openssl/ossl.c
index c7a755ce..6c532aca 100644
--- a/ext/openssl/ossl.c
+++ b/ext/openssl/ossl.c
@@ -313,27 +313,31 @@ void
 ossl_clear_error(void)
 {
     if (dOSSL == Qtrue) {
-	unsigned long e;
-	const char *file, *data, *errstr;
-	int line, flags;
-
-	while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
-	    errstr = ERR_error_string(e, NULL);
-	    if (!errstr)
-		errstr = "(null)";
-
-	    if (flags & ERR_TXT_STRING) {
-		if (!data)
-		    data = "(null)";
-		rb_warn("error on stack: %s (%s)", errstr, data);
-	    }
-	    else {
-		rb_warn("error on stack: %s", errstr);
-	    }
-	}
+        unsigned long e;
+        const char *file, *data, *func, *lib, *reason;
+        char append[256] = "";
+        int line, flags;
+
+#ifdef HAVE_ERR_GET_ERROR_ALL
+        while ((e = ERR_get_error_all(&file, &line, &func, &data, &flags))) {
+#else
+        while ((e = ERR_get_error_line_data(&file, &line, &data, &flags))) {
+            func = ERR_func_error_string(e);
+#endif
+            lib = ERR_lib_error_string(e);
+            reason = ERR_reason_error_string(e);
+
+            if (flags & ERR_TXT_STRING) {
+                if (!data)
+                    data = "(null)";
+                snprintf(append, sizeof(append), " (%s)", data);
+            }
+            rb_warn("error on stack: error:%08lX:%s:%s:%s%s", e, lib ? lib : "",
+                    func ? func : "", reason ? reason : "", append);
+        }
     }
     else {
-	ERR_clear_error();
+        ERR_clear_error();
     }
 }
 
diff --git a/ext/openssl/ossl.h b/ext/openssl/ossl.h
index be9dcbdf..3a0ab1e5 100644
--- a/ext/openssl/ossl.h
+++ b/ext/openssl/ossl.h
@@ -42,6 +42,18 @@
 #include <openssl/evp.h>
 #include <openssl/dh.h>
 
+#ifndef LIBRESSL_VERSION_NUMBER
+# define OSSL_IS_LIBRESSL 0
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) \
+      (OPENSSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) 0
+#else
+# define OSSL_IS_LIBRESSL 1
+# define OSSL_OPENSSL_PREREQ(maj, min, pat) 0
+# define OSSL_LIBRESSL_PREREQ(maj, min, pat) \
+      (LIBRESSL_VERSION_NUMBER >= (maj << 28) | (min << 20) | (pat << 12))
+#endif
+
 /*
  * Common Module
  */
diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c
index 8d0f63a9..56fa0ec3 100644
--- a/ext/openssl/ossl_bn.c
+++ b/ext/openssl/ossl_bn.c
@@ -792,78 +792,64 @@ BIGNUM_SELF_SHIFT(lshift)
  */
 BIGNUM_SELF_SHIFT(rshift)
 
-#define BIGNUM_RAND(func)					\
-    static VALUE						\
-    ossl_bn_s_##func(int argc, VALUE *argv, VALUE klass)	\
-    {								\
-	BIGNUM *result;						\
-	int bottom = 0, top = 0, b;				\
-	VALUE bits, fill, odd, obj;				\
-								\
-	switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) {	\
-	case 3:							\
-	    bottom = (odd == Qtrue) ? 1 : 0;			\
-	    /* FALLTHROUGH */					\
-	case 2:							\
-	    top = NUM2INT(fill);				\
-	}							\
-	b = NUM2INT(bits);					\
-	obj = NewBN(klass);					\
-	if (!(result = BN_new())) {				\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	if (BN_##func(result, b, top, bottom) <= 0) {		\
-	    BN_free(result);					\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	SetBN(obj, result);					\
-	return obj;						\
-    }
-
-/*
- * Document-method: OpenSSL::BN.rand
- *   BN.rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(rand)
-
 /*
- * Document-method: OpenSSL::BN.pseudo_rand
- *   BN.pseudo_rand(bits [, fill [, odd]]) -> aBN
- */
-BIGNUM_RAND(pseudo_rand)
-
-#define BIGNUM_RAND_RANGE(func)					\
-    static VALUE						\
-    ossl_bn_s_##func##_range(VALUE klass, VALUE range)		\
-    {								\
-	BIGNUM *bn = GetBNPtr(range), *result;			\
-	VALUE obj = NewBN(klass);				\
-	if (!(result = BN_new())) {				\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	if (BN_##func##_range(result, bn) <= 0) {		\
-	    BN_free(result);					\
-	    ossl_raise(eBNError, NULL);				\
-	}							\
-	SetBN(obj, result);					\
-	return obj;						\
-    }
-
-/*
- * Document-method: OpenSSL::BN.rand_range
  * call-seq:
- *   BN.rand_range(range) -> aBN
+ *    BN.rand(bits [, fill [, odd]]) -> aBN
+ *
+ * Generates a cryptographically strong pseudo-random number of +bits+.
  *
+ * See also the man page BN_rand(3).
  */
-BIGNUM_RAND_RANGE(rand)
+static VALUE
+ossl_bn_s_rand(int argc, VALUE *argv, VALUE klass)
+{
+    BIGNUM *result;
+    int bottom = 0, top = 0, b;
+    VALUE bits, fill, odd, obj;
+
+    switch (rb_scan_args(argc, argv, "12", &bits, &fill, &odd)) {
+      case 3:
+        bottom = (odd == Qtrue) ? 1 : 0;
+        /* FALLTHROUGH */
+      case 2:
+        top = NUM2INT(fill);
+    }
+    b = NUM2INT(bits);
+    obj = NewBN(klass);
+    if (!(result = BN_new())) {
+        ossl_raise(eBNError, "BN_new");
+    }
+    if (BN_rand(result, b, top, bottom) <= 0) {
+        BN_free(result);
+        ossl_raise(eBNError, "BN_rand");
+    }
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
- * Document-method: OpenSSL::BN.pseudo_rand_range
  * call-seq:
- *   BN.pseudo_rand_range(range) -> aBN
+ *    BN.rand_range(range) -> aBN
  *
+ * Generates a cryptographically strong pseudo-random number in the range
+ * 0...+range+.
+ *
+ * See also the man page BN_rand_range(3).
  */
-BIGNUM_RAND_RANGE(pseudo_rand)
+static VALUE
+ossl_bn_s_rand_range(VALUE klass, VALUE range)
+{
+    BIGNUM *bn = GetBNPtr(range), *result;
+    VALUE obj = NewBN(klass);
+    if (!(result = BN_new()))
+        ossl_raise(eBNError, "BN_new");
+    if (BN_rand_range(result, bn) <= 0) {
+        BN_free(result);
+        ossl_raise(eBNError, "BN_rand_range");
+    }
+    SetBN(obj, result);
+    return obj;
+}
 
 /*
  * call-seq:
@@ -1118,34 +1104,29 @@ ossl_bn_hash(VALUE self)
  *    bn.prime? => true | false
  *    bn.prime?(checks) => true | false
  *
- * Performs a Miller-Rabin probabilistic primality test with _checks_
- * iterations. If _checks_ is not specified, a number of iterations is used
- * that yields a false positive rate of at most 2^-80 for random input.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
+ * <b>+checks+ parameter is deprecated in version 3.0.</b> It has no effect.
  */
 static VALUE
 ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
 {
     BIGNUM *bn;
-    VALUE vchecks;
-    int checks = BN_prime_checks;
+    int ret;
 
-    if (rb_scan_args(argc, argv, "01", &vchecks) == 1) {
-	checks = NUM2INT(vchecks);
-    }
+    rb_check_arity(argc, 0, 1);
     GetBN(self, bn);
-    switch (BN_is_prime_ex(bn, checks, ossl_bn_ctx, NULL)) {
-    case 1:
-	return Qtrue;
-    case 0:
-	return Qfalse;
-    default:
-	ossl_raise(eBNError, NULL);
-    }
-    /* not reachable */
-    return Qnil;
+
+#ifdef HAVE_BN_CHECK_PRIME
+    ret = BN_check_prime(bn, ossl_bn_ctx, NULL);
+    if (ret < 0)
+        ossl_raise(eBNError, "BN_check_prime");
+#else
+    ret = BN_is_prime_fasttest_ex(bn, BN_prime_checks, ossl_bn_ctx, 1, NULL);
+    if (ret < 0)
+        ossl_raise(eBNError, "BN_is_prime_fasttest_ex");
+#endif
+    return ret ? Qtrue : Qfalse;
 }
 
 /*
@@ -1154,40 +1135,17 @@ ossl_bn_is_prime(int argc, VALUE *argv, VALUE self)
  *    bn.prime_fasttest?(checks) => true | false
  *    bn.prime_fasttest?(checks, trial_div) => true | false
  *
- * Performs a Miller-Rabin primality test. This is same as #prime? except this
- * first attempts trial divisions with some small primes.
+ * Performs a Miller-Rabin probabilistic primality test for +bn+.
  *
- * === Parameters
- * * _checks_ - integer
- * * _trial_div_ - boolean
+ * <b>Deprecated in version 3.0.</b> Use #prime? instead.
+ *
+ * +checks+ and +trial_div+ parameters no longer have any effect.
  */
 static VALUE
 ossl_bn_is_prime_fasttest(int argc, VALUE *argv, VALUE self)
 {
-    BIGNUM *bn;
-    VALUE vchecks, vtrivdiv;
-    int checks = BN_prime_checks, do_trial_division = 1;
-
-    rb_scan_args(argc, argv, "02", &vchecks, &vtrivdiv);
-
-    if (!NIL_P(vchecks)) {
-	checks = NUM2INT(vchecks);
-    }
-    GetBN(self, bn);
-    /* handle true/false */
-    if (vtrivdiv == Qfalse) {
-	do_trial_division = 0;
-    }
-    switch (BN_is_prime_fasttest_ex(bn, checks, ossl_bn_ctx, do_trial_division, NULL)) {
-    case 1:
-	return Qtrue;
-    case 0:
-	return Qfalse;
-    default:
-	ossl_raise(eBNError, NULL);
-    }
-    /* not reachable */
-    return Qnil;
+    rb_check_arity(argc, 0, 2);
+    return ossl_bn_is_prime(0, argv, self);
 }
 
 /*
@@ -1306,9 +1264,9 @@ Init_ossl_bn(void)
      * get_word */
 
     rb_define_singleton_method(cBN, "rand", ossl_bn_s_rand, -1);
-    rb_define_singleton_method(cBN, "pseudo_rand", ossl_bn_s_pseudo_rand, -1);
     rb_define_singleton_method(cBN, "rand_range", ossl_bn_s_rand_range, 1);
-    rb_define_singleton_method(cBN, "pseudo_rand_range", ossl_bn_s_pseudo_rand_range, 1);
+    rb_define_alias(rb_singleton_class(cBN), "pseudo_rand", "rand");
+    rb_define_alias(rb_singleton_class(cBN), "pseudo_rand_range", "rand_range");
 
     rb_define_singleton_method(cBN, "generate_prime", ossl_bn_s_generate_prime, -1);
     rb_define_method(cBN, "prime?", ossl_bn_is_prime, -1);
diff --git a/ext/openssl/ossl_digest.c b/ext/openssl/ossl_digest.c
index b2506de7..fc326ec1 100644
--- a/ext/openssl/ossl_digest.c
+++ b/ext/openssl/ossl_digest.c
@@ -63,7 +63,7 @@ ossl_evp_get_digestbyname(VALUE obj)
 
         GetDigest(obj, ctx);
 
-        md = EVP_MD_CTX_md(ctx);
+        md = EVP_MD_CTX_get0_md(ctx);
     }
 
     return md;
@@ -176,7 +176,7 @@ ossl_digest_reset(VALUE self)
     EVP_MD_CTX *ctx;
 
     GetDigest(self, ctx);
-    if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_md(ctx), NULL) != 1) {
+    if (EVP_DigestInit_ex(ctx, EVP_MD_CTX_get0_md(ctx), NULL) != 1) {
 	ossl_raise(eDigestError, "Digest initialization failed.");
     }
 
@@ -259,7 +259,7 @@ ossl_digest_name(VALUE self)
 
     GetDigest(self, ctx);
 
-    return rb_str_new2(EVP_MD_name(EVP_MD_CTX_md(ctx)));
+    return rb_str_new_cstr(EVP_MD_name(EVP_MD_CTX_get0_md(ctx)));
 }
 
 /*
diff --git a/ext/openssl/ossl_hmac.c b/ext/openssl/ossl_hmac.c
index a21db6c4..f89ff2f9 100644
--- a/ext/openssl/ossl_hmac.c
+++ b/ext/openssl/ossl_hmac.c
@@ -238,8 +238,8 @@ ossl_hmac_reset(VALUE self)
     EVP_PKEY *pkey;
 
     GetHMAC(self, ctx);
-    pkey = EVP_PKEY_CTX_get0_pkey(EVP_MD_CTX_pkey_ctx(ctx));
-    if (EVP_DigestSignInit(ctx, NULL, EVP_MD_CTX_md(ctx), NULL, pkey) != 1)
+    pkey = EVP_PKEY_CTX_get0_pkey(EVP_MD_CTX_get_pkey_ctx(ctx));
+    if (EVP_DigestSignInit(ctx, NULL, EVP_MD_CTX_get0_md(ctx), NULL, pkey) != 1)
         ossl_raise(eHMACError, "EVP_DigestSignInit");
 
     return self;
diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 95a2ea1e..f9f5162e 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -769,14 +769,14 @@ ossl_pkey_compare(VALUE self, VALUE other)
     if (EVP_PKEY_id(selfPKey) != EVP_PKEY_id(otherPKey))
         ossl_raise(rb_eTypeError, "cannot match different PKey types");
 
-    ret = EVP_PKEY_cmp(selfPKey, otherPKey);
+    ret = EVP_PKEY_eq(selfPKey, otherPKey);
 
     if (ret == 0)
         return Qfalse;
     else if (ret == 1)
         return Qtrue;
     else
-        ossl_raise(ePKeyError, "EVP_PKEY_cmp");
+        ossl_raise(ePKeyError, "EVP_PKEY_eq");
 }
 
 /*
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 47de7ccd..db80d112 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -479,7 +479,7 @@ static VALUE ossl_ec_key_check_key(VALUE self)
 static void
 ossl_ec_group_free(void *ptr)
 {
-    EC_GROUP_clear_free(ptr);
+    EC_GROUP_free(ptr);
 }
 
 static const rb_data_type_t ossl_ec_group_type = {
@@ -1252,6 +1252,8 @@ static VALUE ossl_ec_point_is_on_curve(VALUE self)
 /*
  * call-seq:
  *   point.make_affine! => self
+ *
+ * This method is deprecated and should not be used. This is a no-op.
  */
 static VALUE ossl_ec_point_make_affine(VALUE self)
 {
@@ -1261,8 +1263,11 @@ static VALUE ossl_ec_point_make_affine(VALUE self)
     GetECPoint(self, point);
     GetECPointGroup(self, group);
 
+    rb_warn("OpenSSL::PKey::EC::Point#make_affine! is deprecated");
+#if !OSSL_OPENSSL_PREREQ(3, 0, 0)
     if (EC_POINT_make_affine(group, point, ossl_bn_ctx) != 1)
         ossl_raise(cEC_POINT, "EC_POINT_make_affine");
+#endif
 
     return self;
 }
diff --git a/ext/openssl/ossl_ssl.c b/ext/openssl/ossl_ssl.c
index 1de0f989..28286574 100644
--- a/ext/openssl/ossl_ssl.c
+++ b/ext/openssl/ossl_ssl.c
@@ -828,10 +828,17 @@ ossl_sslctx_setup(VALUE self)
     ca_file = NIL_P(val) ? NULL : StringValueCStr(val);
     val = rb_attr_get(self, id_i_ca_path);
     ca_path = NIL_P(val) ? NULL : StringValueCStr(val);
+#ifdef HAVE_SSL_CTX_LOAD_VERIFY_FILE
+    if (ca_file && !SSL_CTX_load_verify_file(ctx, ca_file))
+        ossl_raise(eSSLError, "SSL_CTX_load_verify_file");
+    if (ca_path && !SSL_CTX_load_verify_dir(ctx, ca_path))
+        ossl_raise(eSSLError, "SSL_CTX_load_verify_dir");
+#else
     if(ca_file || ca_path){
 	if (!SSL_CTX_load_verify_locations(ctx, ca_file, ca_path))
 	    rb_warning("can't set verify locations");
     }
+#endif
 
     val = rb_attr_get(self, id_i_verify_mode);
     verify_mode = NIL_P(val) ? SSL_VERIFY_NONE : NUM2INT(val);
@@ -1222,7 +1229,7 @@ ossl_sslctx_add_certificate(int argc, VALUE *argv, VALUE self)
     EVP_PKEY_free(pub_pkey);
     if (!pub_pkey)
 	rb_raise(rb_eArgError, "certificate does not contain public key");
-    if (EVP_PKEY_cmp(pub_pkey, pkey) != 1)
+    if (EVP_PKEY_eq(pub_pkey, pkey) != 1)
 	rb_raise(rb_eArgError, "public key mismatch");
 
     if (argc >= 3)
@@ -1436,8 +1443,8 @@ ossl_sslctx_flush_sessions(int argc, VALUE *argv, VALUE self)
 static inline int
 ssl_started(SSL *ssl)
 {
-    /* the FD is set in ossl_ssl_setup(), called by #connect or #accept */
-    return SSL_get_fd(ssl) >= 0;
+    /* BIO is created through ossl_ssl_setup(), called by #connect or #accept */
+    return SSL_get_rbio(ssl) != NULL;
 }
 
 static void
diff --git a/ext/openssl/ossl_ts.c b/ext/openssl/ossl_ts.c
index b9a62fd9..b33ff10c 100644
--- a/ext/openssl/ossl_ts.c
+++ b/ext/openssl/ossl_ts.c
@@ -826,7 +826,7 @@ ossl_ts_resp_verify(int argc, VALUE *argv, VALUE self)
         X509_up_ref(cert);
     }
 
-    TS_VERIFY_CTS_set_certs(ctx, x509inter);
+    TS_VERIFY_CTX_set_certs(ctx, x509inter);
     TS_VERIFY_CTX_add_flags(ctx, TS_VFY_SIGNATURE);
     TS_VERIFY_CTX_set_store(ctx, x509st);
 
diff --git a/test/openssl/test_bn.rb b/test/openssl/test_bn.rb
index c36d6b89..3f0622f9 100644
--- a/test/openssl/test_bn.rb
+++ b/test/openssl/test_bn.rb
@@ -248,6 +248,10 @@ class OpenSSL::TestBN < OpenSSL::TestCase
       r5 = OpenSSL::BN.rand_range(256)
       assert_include(0..255, r5)
     }
+
+    # Aliases
+    assert_include(128..255, OpenSSL::BN.pseudo_rand(8))
+    assert_include(0..255, OpenSSL::BN.pseudo_rand_range(256))
   end
 
   begin