openssl: adapt OpenSSL::PKey to OpenSSL 1.1.0 opaque structs

* ext/openssl/openssl_missing.[ch]: Implement EVP_PKEY_get0_*() and
  {RSA,DSA,EC_KEY,DH}_get0_*() functions.
  OpenSSL 1.1.0 makes EVP_PKEY/RSA/DSA/DH opaque. We used to provide
  setter methods for each parameter of each PKey type, for example
  PKey::RSA#e=, but this is no longer possible because the new API
  RSA_set0_key() requires the 'n' at the same time. This commit adds
  deprecation warning to them and adds PKey::*#set_* methods as direct
  wrapper for those new APIs. For example, 'rsa.e = 3' now needs to be
  rewritten as 'rsa.set_key(rsa.n, 3, rsa.d)'.
  [ruby-core:75225] [Feature #12324]

* ext/openssl/ossl_pkey*.[ch]: Use the new accessor functions. Implement
  RSA#set_{key,factors,crt_params}, DSA#set_{key,pqg}, DH#set_{key,pqg}.
  Emit a warning with rb_warning() when old setter methods are used.

* test/drb/ut_array_drbssl.rb, test/drb/ut_drb_drbssl.rb,
  test/rubygems/test_gem_remote_fetcher.rb: Don't set a priv_key for DH
  object that are used in tmp_dh_callback. Generating a new key pair
  every time should be fine - actually the private exponent is ignored
  in OpenSSL >= 1.0.2f/1.0.1r even if we explicitly set.
  https://www.openssl.org/news/secadv/20160128.txt

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55285 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 58 insertions, 0 deletions

diff --git a/ext/openssl/openssl_missing.h b/ext/openssl/openssl_missing.h
index d81d6a74f7..22f16bf02f 100644
--- a/ext/openssl/openssl_missing.h
+++ b/ext/openssl/openssl_missing.h
@@ -11,6 +11,10 @@
 #define _OSSL_OPENSSL_MISSING_H_
 
 /* added in 1.0.0 */
+#if !defined(HAVE_EVP_PKEY_BASE_ID)
+#  define EVP_PKEY_base_id(pkey) EVP_PKEY_type((pkey)->type)
+#endif
+
 #if !defined(HAVE_EVP_CIPHER_CTX_COPY)
 int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in);
 #endif
@@ -70,4 +74,58 @@ int EC_curve_nist2nid(const char *);
 	CRYPTO_add(&(x)->references, 1, CRYPTO_LOCK_EVP_PKEY);
 #endif
 
+#if !defined(HAVE_OPAQUE_OPENSSL)
+#define IMPL_PKEY_GETTER(_type, _name) \
+static inline _type *EVP_PKEY_get0_##_type(EVP_PKEY *pkey) { \
+	return pkey->pkey._name; }
+#define IMPL_KEY_ACCESSOR2(_type, _group, a1, a2, _fail_cond) \
+static inline void _type##_get0_##_group(_type *obj, BIGNUM **a1, BIGNUM **a2) { \
+	if (a1) *a1 = obj->a1; \
+	if (a2) *a2 = obj->a2; } \
+static inline int _type##_set0_##_group(_type *obj, BIGNUM *a1, BIGNUM *a2) { \
+	if (_fail_cond) return 0; \
+	BN_clear_free(obj->a1); obj->a1 = a1; \
+	BN_clear_free(obj->a2); obj->a2 = a2; \
+	return 1; }
+#define IMPL_KEY_ACCESSOR3(_type, _group, a1, a2, a3, _fail_cond) \
+static inline void _type##_get0_##_group(_type *obj, BIGNUM **a1, BIGNUM **a2, BIGNUM **a3) { \
+	if (a1) *a1 = obj->a1; \
+	if (a2) *a2 = obj->a2; \
+	if (a3) *a3 = obj->a3; } \
+static inline int _type##_set0_##_group(_type *obj, BIGNUM *a1, BIGNUM *a2, BIGNUM *a3) { \
+	if (_fail_cond) return 0; \
+	BN_clear_free(obj->a1); obj->a1 = a1; \
+	BN_clear_free(obj->a2); obj->a2 = a2; \
+	BN_clear_free(obj->a3); obj->a3 = a3; \
+	return 1; }
+
+#if !defined(OPENSSL_NO_RSA)
+IMPL_PKEY_GETTER(RSA, rsa)
+IMPL_KEY_ACCESSOR3(RSA, key, n, e, d, (n == obj->n || e == obj->e || (obj->d && e == obj->d)))
+IMPL_KEY_ACCESSOR2(RSA, factors, p, q, (p == obj->p || q == obj->q))
+IMPL_KEY_ACCESSOR3(RSA, crt_params, dmp1, dmq1, iqmp, (dmp1 == obj->dmp1 || dmq1 == obj->dmq1 || iqmp == obj->iqmp))
+#endif
+
+#if !defined(OPENSSL_NO_DSA)
+IMPL_PKEY_GETTER(DSA, dsa)
+IMPL_KEY_ACCESSOR2(DSA, key, pub_key, priv_key, (pub_key == obj->pub_key || (obj->priv_key && priv_key == obj->priv_key)))
+IMPL_KEY_ACCESSOR3(DSA, pqg, p, q, g, (p == obj->p || q == obj->q || g == obj->g))
+#endif
+
+#if !defined(OPENSSL_NO_DH)
+IMPL_PKEY_GETTER(DH, dh)
+IMPL_KEY_ACCESSOR2(DH, key, pub_key, priv_key, (pub_key == obj->pub_key || (obj->priv_key && priv_key == obj->priv_key)))
+IMPL_KEY_ACCESSOR3(DH, pqg, p, q, g, (p == obj->p || q == obj->q || g == obj->g))
+static inline ENGINE *DH_get0_engine(DH *dh) { return dh->engine; }
+#endif
+
+#if !defined(OPENSSL_NO_EC)
+IMPL_PKEY_GETTER(EC_KEY, ec)
+#endif
+
+#undef IMPL_PKEY_GETTER
+#undef IMPL_KEY_ACCESSOR2
+#undef IMPL_KEY_ACCESSOR3
+#endif /* HAVE_OPAQUE_OPENSSL */
+
 #endif /* _OSSL_OPENSSL_MISSING_H_ */