openssl: adapt OpenSSL::PKey to OpenSSL 1.1.0 opaque structs

* ext/openssl/openssl_missing.[ch]: Implement EVP_PKEY_get0_*() and
  {RSA,DSA,EC_KEY,DH}_get0_*() functions.
  OpenSSL 1.1.0 makes EVP_PKEY/RSA/DSA/DH opaque. We used to provide
  setter methods for each parameter of each PKey type, for example
  PKey::RSA#e=, but this is no longer possible because the new API
  RSA_set0_key() requires the 'n' at the same time. This commit adds
  deprecation warning to them and adds PKey::*#set_* methods as direct
  wrapper for those new APIs. For example, 'rsa.e = 3' now needs to be
  rewritten as 'rsa.set_key(rsa.n, 3, rsa.d)'.
  [ruby-core:75225] [Feature #12324]

* ext/openssl/ossl_pkey*.[ch]: Use the new accessor functions. Implement
  RSA#set_{key,factors,crt_params}, DSA#set_{key,pqg}, DH#set_{key,pqg}.
  Emit a warning with rb_warning() when old setter methods are used.

* test/drb/ut_array_drbssl.rb, test/drb/ut_drb_drbssl.rb,
  test/rubygems/test_gem_remote_fetcher.rb: Don't set a priv_key for DH
  object that are used in tmp_dh_callback. Generating a new key pair
  every time should be fine - actually the private exponent is ignored
  in OpenSSL >= 1.0.2f/1.0.1r even if we explicitly set.
  https://www.openssl.org/news/secadv/20160128.txt

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55285 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 72 insertions, 50 deletions

diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 704fad6d4d..a30eba85d7 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -7,19 +7,35 @@
  * This program is licensed under the same licence as Ruby.
  * (See the file 'LICENCE'.)
  */
-#if !defined(OPENSSL_NO_DSA)
-
 #include "ossl.h"
 
+#if !defined(OPENSSL_NO_DSA)
+
 #define GetPKeyDSA(obj, pkey) do { \
     GetPKey((obj), (pkey)); \
-    if (EVP_PKEY_type((pkey)->type) != EVP_PKEY_DSA) { /* PARANOIA? */ \
+    if (EVP_PKEY_base_id(pkey) != EVP_PKEY_DSA) { /* PARANOIA? */ \
 	ossl_raise(rb_eRuntimeError, "THIS IS NOT A DSA!"); \
     } \
 } while (0)
+#define GetDSA(obj, dsa) do { \
+    EVP_PKEY *_pkey; \
+    GetPKeyDSA((obj), _pkey); \
+    (dsa) = EVP_PKEY_get0_DSA(_pkey); \
+} while (0)
 
-#define DSA_HAS_PRIVATE(dsa) ((dsa)->priv_key)
-#define DSA_PRIVATE(obj,dsa) (DSA_HAS_PRIVATE(dsa)||OSSL_PKEY_IS_PRIVATE(obj))
+static inline int
+DSA_HAS_PRIVATE(DSA *dsa)
+{
+    BIGNUM *bn;
+    DSA_get0_key(dsa, NULL, &bn);
+    return !!bn;
+}
+
+static inline int
+DSA_PRIVATE(VALUE obj, DSA *dsa)
+{
+    return DSA_HAS_PRIVATE(dsa) || OSSL_PKEY_IS_PRIVATE(obj);
+}
 
 /*
  * Classes
@@ -61,7 +77,7 @@ ossl_dsa_new(EVP_PKEY *pkey)
 	obj = dsa_instance(cDSA, DSA_new());
     } else {
 	obj = NewPKey(cDSA);
-	if (EVP_PKEY_type(pkey->type) != EVP_PKEY_DSA) {
+	if (EVP_PKEY_base_id(pkey) != EVP_PKEY_DSA) {
 	    ossl_raise(rb_eTypeError, "Not a DSA key!");
 	}
 	SetPKey(obj, pkey);
@@ -259,11 +275,13 @@ ossl_dsa_initialize(int argc, VALUE *argv, VALUE self)
 static VALUE
 ossl_dsa_is_public(VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
+    BIGNUM *bn;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
+    DSA_get0_key(dsa, &bn, NULL);
 
-    return (pkey->pkey.dsa->pub_key) ? Qtrue : Qfalse;
+    return bn ? Qtrue : Qfalse;
 }
 
 /*
@@ -276,11 +294,11 @@ ossl_dsa_is_public(VALUE self)
 static VALUE
 ossl_dsa_is_private(VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
 
-    return (DSA_PRIVATE(self, pkey->pkey.dsa)) ? Qtrue : Qfalse;
+    return DSA_PRIVATE(self, dsa) ? Qtrue : Qfalse;
 }
 
 /*
@@ -303,12 +321,12 @@ ossl_dsa_is_private(VALUE self)
 static VALUE
 ossl_dsa_export(int argc, VALUE *argv, VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
     BIO *out;
     const EVP_CIPHER *ciph = NULL;
     VALUE cipher, pass, str;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
     rb_scan_args(argc, argv, "02", &cipher, &pass);
     if (!NIL_P(cipher)) {
 	ciph = GetCipherPtr(cipher);
@@ -317,14 +335,14 @@ ossl_dsa_export(int argc, VALUE *argv, VALUE self)
     if (!(out = BIO_new(BIO_s_mem()))) {
 	ossl_raise(eDSAError, NULL);
     }
-    if (DSA_HAS_PRIVATE(pkey->pkey.dsa)) {
-	if (!PEM_write_bio_DSAPrivateKey(out, pkey->pkey.dsa, ciph,
-					 NULL, 0, ossl_pem_passwd_cb, (void *)pass)){
+    if (DSA_HAS_PRIVATE(dsa)) {
+	if (!PEM_write_bio_DSAPrivateKey(out, dsa, ciph, NULL, 0,
+					 ossl_pem_passwd_cb, (void *)pass)){
 	    BIO_free(out);
 	    ossl_raise(eDSAError, NULL);
 	}
     } else {
-	if (!PEM_write_bio_DSA_PUBKEY(out, pkey->pkey.dsa)) {
+	if (!PEM_write_bio_DSA_PUBKEY(out, dsa)) {
 	    BIO_free(out);
 	    ossl_raise(eDSAError, NULL);
 	}
@@ -344,28 +362,29 @@ ossl_dsa_export(int argc, VALUE *argv, VALUE self)
 static VALUE
 ossl_dsa_to_der(VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
     int (*i2d_func)_((DSA*, unsigned char**));
     unsigned char *p;
     long len;
     VALUE str;
 
-    GetPKeyDSA(self, pkey);
-    if(DSA_HAS_PRIVATE(pkey->pkey.dsa))
+    GetDSA(self, dsa);
+    if(DSA_HAS_PRIVATE(dsa))
 	i2d_func = (int(*)_((DSA*,unsigned char**)))i2d_DSAPrivateKey;
     else
 	i2d_func = i2d_DSA_PUBKEY;
-    if((len = i2d_func(pkey->pkey.dsa, NULL)) <= 0)
+    if((len = i2d_func(dsa, NULL)) <= 0)
 	ossl_raise(eDSAError, NULL);
     str = rb_str_new(0, len);
     p = (unsigned char *)RSTRING_PTR(str);
-    if(i2d_func(pkey->pkey.dsa, &p) < 0)
+    if(i2d_func(dsa, &p) < 0)
 	ossl_raise(eDSAError, NULL);
     ossl_str_adjust(str, p);
 
     return str;
 }
 
+
 /*
  *  call-seq:
  *    dsa.params -> hash
@@ -377,18 +396,20 @@ ossl_dsa_to_der(VALUE self)
 static VALUE
 ossl_dsa_get_params(VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
     VALUE hash;
+    BIGNUM *p, *q, *g, *pub_key, *priv_key;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
+    DSA_get0_pqg(dsa, &p, &q, &g);
+    DSA_get0_key(dsa, &pub_key, &priv_key);
 
     hash = rb_hash_new();
-
-    rb_hash_aset(hash, rb_str_new2("p"), ossl_bn_new(pkey->pkey.dsa->p));
-    rb_hash_aset(hash, rb_str_new2("q"), ossl_bn_new(pkey->pkey.dsa->q));
-    rb_hash_aset(hash, rb_str_new2("g"), ossl_bn_new(pkey->pkey.dsa->g));
-    rb_hash_aset(hash, rb_str_new2("pub_key"), ossl_bn_new(pkey->pkey.dsa->pub_key));
-    rb_hash_aset(hash, rb_str_new2("priv_key"), ossl_bn_new(pkey->pkey.dsa->priv_key));
+    rb_hash_aset(hash, rb_str_new2("p"), ossl_bn_new(p));
+    rb_hash_aset(hash, rb_str_new2("q"), ossl_bn_new(q));
+    rb_hash_aset(hash, rb_str_new2("g"), ossl_bn_new(g));
+    rb_hash_aset(hash, rb_str_new2("pub_key"), ossl_bn_new(pub_key));
+    rb_hash_aset(hash, rb_str_new2("priv_key"), ossl_bn_new(priv_key));
 
     return hash;
 }
@@ -404,15 +425,15 @@ ossl_dsa_get_params(VALUE self)
 static VALUE
 ossl_dsa_to_text(VALUE self)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
     BIO *out;
     VALUE str;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
     if (!(out = BIO_new(BIO_s_mem()))) {
 	ossl_raise(eDSAError, NULL);
     }
-    if (!DSA_print(out, pkey->pkey.dsa, 0)) { /* offset = 0 */
+    if (!DSA_print(out, dsa, 0)) { /* offset = 0 */
 	BIO_free(out);
 	ossl_raise(eDSAError, NULL);
     }
@@ -449,7 +470,7 @@ ossl_dsa_to_public_key(VALUE self)
     /* err check performed by dsa_instance */
 #define DSAPublicKey_dup(dsa) (DSA *)ASN1_dup( \
 	(i2d_of_void *)i2d_DSAPublicKey, (d2i_of_void *)d2i_DSAPublicKey, (char *)(dsa))
-    dsa = DSAPublicKey_dup(pkey->pkey.dsa);
+    dsa = DSAPublicKey_dup(EVP_PKEY_get0_DSA(pkey));
 #undef DSAPublicKey_dup
     obj = dsa_instance(CLASS_OF(self), dsa);
     if (obj == Qfalse) {
@@ -459,7 +480,7 @@ ossl_dsa_to_public_key(VALUE self)
     return obj;
 }
 
-#define ossl_dsa_buf_size(pkey) (DSA_size((pkey)->pkey.dsa)+16)
+#define ossl_dsa_buf_size(dsa) (DSA_size(dsa) + 16)
 
 /*
  *  call-seq:
@@ -483,20 +504,22 @@ ossl_dsa_to_public_key(VALUE self)
 static VALUE
 ossl_dsa_sign(VALUE self, VALUE data)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
+    BIGNUM *dsa_q;
     unsigned int buf_len;
     VALUE str;
 
-    GetPKeyDSA(self, pkey);
-    if (!pkey->pkey.dsa->q)
+    GetDSA(self, dsa);
+    DSA_get0_pqg(dsa, NULL, &dsa_q, NULL);
+    if (!dsa_q)
 	ossl_raise(eDSAError, "incomplete DSA");
-    if (!DSA_PRIVATE(self, pkey->pkey.dsa))
+    if (!DSA_PRIVATE(self, dsa))
 	ossl_raise(eDSAError, "Private DSA key needed!");
     StringValue(data);
-    str = rb_str_new(0, ossl_dsa_buf_size(pkey));
+    str = rb_str_new(0, ossl_dsa_buf_size(dsa));
     if (!DSA_sign(0, (unsigned char *)RSTRING_PTR(data), RSTRING_LENINT(data),
 		  (unsigned char *)RSTRING_PTR(str),
-		  &buf_len, pkey->pkey.dsa)) { /* type is ignored (0) */
+		  &buf_len, dsa)) { /* type is ignored (0) */
 	ossl_raise(eDSAError, NULL);
     }
     rb_str_set_len(str, buf_len);
@@ -526,15 +549,15 @@ ossl_dsa_sign(VALUE self, VALUE data)
 static VALUE
 ossl_dsa_verify(VALUE self, VALUE digest, VALUE sig)
 {
-    EVP_PKEY *pkey;
+    DSA *dsa;
     int ret;
 
-    GetPKeyDSA(self, pkey);
+    GetDSA(self, dsa);
     StringValue(digest);
     StringValue(sig);
     /* type is ignored (0) */
     ret = DSA_verify(0, (unsigned char *)RSTRING_PTR(digest), RSTRING_LENINT(digest),
-		     (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), pkey->pkey.dsa);
+		     (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), dsa);
     if (ret < 0) {
 	ossl_raise(eDSAError, NULL);
     }
@@ -545,11 +568,8 @@ ossl_dsa_verify(VALUE self, VALUE digest, VALUE sig)
     return Qfalse;
 }
 
-OSSL_PKEY_BN(dsa, p)
-OSSL_PKEY_BN(dsa, q)
-OSSL_PKEY_BN(dsa, g)
-OSSL_PKEY_BN(dsa, pub_key)
-OSSL_PKEY_BN(dsa, priv_key)
+OSSL_PKEY_BN_DEF3(dsa, DSA, pqg, p, q, g)
+OSSL_PKEY_BN_DEF2(dsa, DSA, key, pub_key, priv_key)
 
 /*
  * INIT
@@ -603,6 +623,8 @@ Init_ossl_dsa(void)
     DEF_OSSL_PKEY_BN(cDSA, dsa, g);
     DEF_OSSL_PKEY_BN(cDSA, dsa, pub_key);
     DEF_OSSL_PKEY_BN(cDSA, dsa, priv_key);
+    rb_define_method(cDSA, "set_pqg", ossl_dsa_set_pqg, 3);
+    rb_define_method(cDSA, "set_key", ossl_dsa_set_key, 2);
 
     rb_define_method(cDSA, "params", ossl_dsa_get_params, 0);
 }