pkey related fix

8 files changed, 92 insertions, 33 deletions

diff --git a/ext/openssl/extconf.rb b/ext/openssl/extconf.rb
index 25a0bb744b..a0d12a63bc 100644
--- a/ext/openssl/extconf.rb
+++ b/ext/openssl/extconf.rb
@@ -75,6 +75,7 @@ have_func("BN_rand_range")
 have_func("BN_is_prime_ex") # for 0.9.6
 have_func("BN_is_prime_fasttest_ex") # for 0.9.6
 have_func("BN_generate_prime_ex") # for 0.9.6
+have_func("BN_GENCB_new")
 have_func("CONF_get1_default_config_file")
 have_func("EVP_CIPHER_CTX_new")
 have_func("EVP_CIPHER_CTX_free")
@@ -89,6 +90,7 @@ have_func("EVP_MD_CTX_create") # for 0.9.6
 have_func("EVP_MD_CTX_free")
 have_func("EVP_MD_CTX_destroy") # for 0.9.6
 have_func("EVP_MD_CTX_init") # for 0.9.6
+have_func("EVP_PKEY_id")
 have_func("HMAC_CTX_new")
 have_func("HMAC_CTX_init") # for 0.9.6
 have_func("HMAC_CTX_free")
diff --git a/ext/openssl/openssl_missing.c b/ext/openssl/openssl_missing.c
index efb9d1c6c0..c569f1ec0a 100644
--- a/ext/openssl/openssl_missing.c
+++ b/ext/openssl/openssl_missing.c
@@ -344,6 +344,27 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
 }
 #endif
 
+#if !defined(HAVE_BN_GENCB_NEW)
+/* BN_GENCB_{new,free,get_arg} are new in 1.1.0 */
+BN_GENCB *
+BN_GENCB_new(void)
+{
+    return (BN_GENCB *)OPENSSL_malloc(sizeof(BN_GENCB));
+}
+
+void
+BN_GENCB_free(BN_GENCB *cb)
+{
+    OPENSSL_free(cb);
+}
+
+void *
+BN_GENCB_get_arg(BN_GENCB *cb)
+{
+    return cb->arg;
+}
+#endif
+
 #if !defined(HAVE_CONF_GET1_DEFAULT_CONFIG_FILE)
 #define OPENSSL_CONF "openssl.cnf"
 char *
@@ -419,9 +440,10 @@ ASN1_put_eoc(unsigned char **pp)
 #endif
 
 #if !defined(HAVE_OCSP_ID_GET0_INFO)
-int OCSP_id_get0_info(ASN1_OCTET_STRING **piNameHash, ASN1_OBJECT **pmd,
-		      ASN1_OCTET_STRING **pikeyHash,
-		      ASN1_INTEGER **pserial, OCSP_CERTID *cid)
+int
+OCSP_id_get0_info(ASN1_OCTET_STRING **piNameHash, ASN1_OBJECT **pmd,
+		  ASN1_OCTET_STRING **pikeyHash,
+		  ASN1_INTEGER **pserial, OCSP_CERTID *cid)
 {
     if (piNameHash || pmd || pikeyHash)
 	rb_bug("not supported");
@@ -430,3 +452,11 @@ int OCSP_id_get0_info(ASN1_OCTET_STRING **piNameHash, ASN1_OBJECT **pmd,
     return 1;
 }
 #endif
+
+#if !defined(HAVE_EVP_PKEY_id)
+int
+EVP_PKEY_id(const EVP_PKEY *pkey)
+{
+    return pkey->type;
+}
+#endif
diff --git a/ext/openssl/openssl_missing.h b/ext/openssl/openssl_missing.h
index 629b7423b4..e77ed979d6 100644
--- a/ext/openssl/openssl_missing.h
+++ b/ext/openssl/openssl_missing.h
@@ -176,6 +176,12 @@ int BN_rand_range(BIGNUM *r, BIGNUM *range);
 int BN_pseudo_rand_range(BIGNUM *r, BIGNUM *range);
 #endif
 
+#if !defined(HAVE_BN_GENCB_NEW)
+BN_GENCB *BN_GENCB_new(void);
+void BN_GENCB_free(BN_GENCB *cb);
+void *BN_GENCB_get_arg(BN_GENCB *cb);
+#endif
+
 #if !defined(HAVE_CONF_GET1_DEFAULT_CONFIG_FILE)
 char *CONF_get1_default_config_file(void);
 #endif
@@ -194,6 +200,10 @@ int OCSP_id_get0_info(ASN1_OCTET_STRING **piNameHash, ASN1_OBJECT **pmd,
 		      ASN1_INTEGER **pserial, OCSP_CERTID *cid);
 #endif
 
+#if !defined(HAVE_EVP_PKEY_id)
+int EVP_PKEY_id(const EVP_PKEY *pkey);
+#endif
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 7e3154afd9..8ead9cc472 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -41,7 +41,7 @@ ossl_generate_cb_2(int p, int n, BN_GENCB *cb)
     struct ossl_generate_cb_arg *arg;
     int state;
 
-    arg = (struct ossl_generate_cb_arg *)cb->arg;
+    arg = (struct ossl_generate_cb_arg *)BN_GENCB_get_arg(cb);
     if (arg->yield) {
 	ary = rb_ary_new2(2);
 	rb_ary_store(ary, 0, INT2NUM(p));
@@ -91,7 +91,7 @@ ossl_pkey_new(EVP_PKEY *pkey)
     if (!pkey) {
 	ossl_raise(ePKeyError, "Cannot make new key from NULL.");
     }
-    switch (EVP_PKEY_type(pkey->type)) {
+    switch (EVP_PKEY_type(EVP_PKEY_id(pkey))) {
 #if !defined(OPENSSL_NO_RSA)
     case EVP_PKEY_RSA:
 	return ossl_rsa_new(pkey);
@@ -286,7 +286,7 @@ static VALUE
 ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
 {
     EVP_PKEY *pkey;
-    EVP_MD_CTX ctx;
+    EVP_MD_CTX *ctx;
     unsigned int buf_len;
     VALUE str;
     int result;
@@ -295,12 +295,15 @@ ossl_pkey_sign(VALUE self, VALUE digest, VALUE data)
 	ossl_raise(rb_eArgError, "Private key is needed.");
     }
     GetPKey(self, pkey);
-    EVP_SignInit(&ctx, GetDigestPtr(digest));
+    ctx = EVP_MD_CTX_new();
+    if (!ctx)
+	ossl_raise(rb_eRuntimeError, "EVP_MD_CTX_new() failed");
+    EVP_SignInit(ctx, GetDigestPtr(digest));
     StringValue(data);
-    EVP_SignUpdate(&ctx, RSTRING_PTR(data), RSTRING_LEN(data));
+    EVP_SignUpdate(ctx, RSTRING_PTR(data), RSTRING_LEN(data));
     str = rb_str_new(0, EVP_PKEY_size(pkey)+16);
-    result = EVP_SignFinal(&ctx, (unsigned char *)RSTRING_PTR(str), &buf_len, pkey);
-    EVP_MD_CTX_cleanup(&ctx);
+    result = EVP_SignFinal(ctx, (unsigned char *)RSTRING_PTR(str), &buf_len, pkey);
+    EVP_MD_CTX_free(ctx);
     if (!result)
 	ossl_raise(ePKeyError, NULL);
     assert((long)buf_len <= RSTRING_LEN(str));
@@ -334,16 +337,19 @@ static VALUE
 ossl_pkey_verify(VALUE self, VALUE digest, VALUE sig, VALUE data)
 {
     EVP_PKEY *pkey;
-    EVP_MD_CTX ctx;
+    EVP_MD_CTX *ctx;
     int result;
 
     GetPKey(self, pkey);
     StringValue(sig);
     StringValue(data);
-    EVP_VerifyInit(&ctx, GetDigestPtr(digest));
-    EVP_VerifyUpdate(&ctx, RSTRING_PTR(data), RSTRING_LEN(data));
-    result = EVP_VerifyFinal(&ctx, (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), pkey);
-    EVP_MD_CTX_cleanup(&ctx);
+    ctx = EVP_MD_CTX_new();
+    if (!ctx)
+	ossl_raise(rb_eRuntimeError, "EVP_MD_CTX_new() failed");
+    EVP_VerifyInit(ctx, GetDigestPtr(digest));
+    EVP_VerifyUpdate(ctx, RSTRING_PTR(data), RSTRING_LEN(data));
+    result = EVP_VerifyFinal(ctx, (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), pkey);
+    EVP_MD_CTX_free(ctx);
     switch (result) {
     case 0:
 	return Qfalse;
diff --git a/ext/openssl/ossl_pkey_dh.c b/ext/openssl/ossl_pkey_dh.c
index 2f79bfb2f6..fb9ba36971 100644
--- a/ext/openssl/ossl_pkey_dh.c
+++ b/ext/openssl/ossl_pkey_dh.c
@@ -67,7 +67,7 @@ ossl_dh_new(EVP_PKEY *pkey)
 	obj = dh_instance(cDH, DH_new());
     } else {
 	obj = NewPKey(cDH);
-	if (EVP_PKEY_type(pkey->type) != EVP_PKEY_DH) {
+	if (EVP_PKEY_type(EVP_PKEY_id(pkey)) != EVP_PKEY_DH) {
 	    ossl_raise(rb_eTypeError, "Not a DH key!");
 	}
 	SetPKey(obj, pkey);
@@ -104,21 +104,25 @@ static DH *
 dh_generate(int size, int gen)
 {
 #if defined(HAVE_DH_GENERATE_PARAMETERS_EX) && HAVE_BN_GENCB
-    BN_GENCB cb;
     struct ossl_generate_cb_arg cb_arg;
     struct dh_blocking_gen_arg gen_arg;
     DH *dh = DH_new();
+    BN_GENCB *cb = BN_GENCB_new();
 
-    if (!dh) return 0;
+    if (!dh || !cb) {
+	if (dh) DH_free(e);
+	if (cb) BN_GENCB_free(cb);
+	return 0;
+    }
 
     memset(&cb_arg, 0, sizeof(struct ossl_generate_cb_arg));
     if (rb_block_given_p())
 	cb_arg.yield = 1;
-    BN_GENCB_set(&cb, ossl_generate_cb_2, &cb_arg);
+    BN_GENCB_set(cb, ossl_generate_cb_2, &cb_arg);
     gen_arg.dh = dh;
     gen_arg.size = size;
     gen_arg.gen = gen;
-    gen_arg.cb = &cb;
+    gen_arg.cb = cb;
     if (cb_arg.yield == 1) {
 	/* we cannot release GVL when callback proc is supplied */
 	dh_blocking_gen(&gen_arg);
@@ -127,6 +131,7 @@ dh_generate(int size, int gen)
 	rb_thread_call_without_gvl(dh_blocking_gen, &gen_arg, ossl_generate_cb_stop, &cb_arg);
     }
 
+    BN_GENCB_free(cb);
     if (!gen_arg.result) {
 	DH_free(dh);
 	if (cb_arg.state) rb_jump_tag(cb_arg.state);
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 04900cc649..2e6a734024 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -61,7 +61,7 @@ ossl_dsa_new(EVP_PKEY *pkey)
 	obj = dsa_instance(cDSA, DSA_new());
     } else {
 	obj = NewPKey(cDSA);
-	if (EVP_PKEY_type(pkey->type) != EVP_PKEY_DSA) {
+	if (EVP_PKEY_type(EVP_PKEY_id(pkey)) != EVP_PKEY_DSA) {
 	    ossl_raise(rb_eTypeError, "Not a DSA key!");
 	}
 	SetPKey(obj, pkey);
@@ -101,15 +101,19 @@ static DSA *
 dsa_generate(int size)
 {
 #if defined(HAVE_DSA_GENERATE_PARAMETERS_EX) && HAVE_BN_GENCB
-    BN_GENCB cb;
     struct ossl_generate_cb_arg cb_arg;
     struct dsa_blocking_gen_arg gen_arg;
     DSA *dsa = DSA_new();
+    BN_GENCB *cb = BN_GENCB_new();;
     unsigned char seed[20];
     int seed_len = 20, counter;
     unsigned long h;
 
-    if (!dsa) return 0;
+    if (!dsa || !cb) {
+	if (dsa) DSA_free(dsa);
+	if (cb) BN_GENCB_free(cb);
+	return 0;
+    }
     if (RAND_bytes(seed, seed_len) <= 0) {
 	DSA_free(dsa);
 	return 0;
@@ -118,14 +122,14 @@ dsa_generate(int size)
     memset(&cb_arg, 0, sizeof(struct ossl_generate_cb_arg));
     if (rb_block_given_p())
 	cb_arg.yield = 1;
-    BN_GENCB_set(&cb, ossl_generate_cb_2, &cb_arg);
+    BN_GENCB_set(cb, ossl_generate_cb_2, &cb_arg);
     gen_arg.dsa = dsa;
     gen_arg.size = size;
     gen_arg.seed = seed;
     gen_arg.seed_len = seed_len;
     gen_arg.counter = &counter;
     gen_arg.h = &h;
-    gen_arg.cb = &cb;
+    gen_arg.cb = cb;
     if (cb_arg.yield == 1) {
 	/* we cannot release GVL when callback proc is supplied */
 	dsa_blocking_gen(&gen_arg);
@@ -133,6 +137,7 @@ dsa_generate(int size)
 	/* there's a chance to unblock */
 	rb_thread_call_without_gvl(dsa_blocking_gen, &gen_arg, ossl_generate_cb_stop, &cb_arg);
     }
+    BN_GENCB_free(cb);
     if (!gen_arg.result) {
 	DSA_free(dsa);
 	if (cb_arg.state) rb_jump_tag(cb_arg.state);
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index c93e3cfb99..e723a38e06 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -137,7 +137,7 @@ VALUE ossl_ec_new(EVP_PKEY *pkey)
 	obj = ec_instance(cEC, EC_KEY_new());
     } else {
 	obj = NewPKey(cEC);
-	if (EVP_PKEY_type(pkey->type) != EVP_PKEY_EC) {
+	if (EVP_PKEY_type(EVP_PKEY_id(pkey)) != EVP_PKEY_EC) {
 	    ossl_raise(rb_eTypeError, "Not a EC key!");
 	}
 	SetPKey(obj, pkey);
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 20b993abb8..50e06535a7 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -62,7 +62,7 @@ ossl_rsa_new(EVP_PKEY *pkey)
     }
     else {
 	obj = NewPKey(cRSA);
-	if (EVP_PKEY_type(pkey->type) != EVP_PKEY_RSA) {
+	if (EVP_PKEY_type(EVP_PKEY_id(pkey)) != EVP_PKEY_RSA) {
 	    ossl_raise(rb_eTypeError, "Not a RSA key!");
 	}
 	SetPKey(obj, pkey);
@@ -100,15 +100,16 @@ rsa_generate(int size, unsigned long exp)
 {
 #if defined(HAVE_RSA_GENERATE_KEY_EX) && HAVE_BN_GENCB
     int i;
-    BN_GENCB cb;
     struct ossl_generate_cb_arg cb_arg;
     struct rsa_blocking_gen_arg gen_arg;
     RSA *rsa = RSA_new();
     BIGNUM *e = BN_new();
+    BN_GENCB *cb = BN_GENCB_new();
 
-    if (!rsa || !e) {
+    if (!rsa || !e || !cb) {
 	if (e) BN_free(e);
 	if (rsa) RSA_free(rsa);
+	if (cb) BN_GENCB_free(cb);
 	return 0;
     }
     for (i = 0; i < (int)sizeof(exp) * 8; ++i) {
@@ -124,11 +125,11 @@ rsa_generate(int size, unsigned long exp)
     memset(&cb_arg, 0, sizeof(struct ossl_generate_cb_arg));
     if (rb_block_given_p())
 	cb_arg.yield = 1;
-    BN_GENCB_set(&cb, ossl_generate_cb_2, &cb_arg);
+    BN_GENCB_set(cb, ossl_generate_cb_2, &cb_arg);
     gen_arg.rsa = rsa;
     gen_arg.e = e;
     gen_arg.size = size;
-    gen_arg.cb = &cb;
+    gen_arg.cb = cb;
     if (cb_arg.yield == 1) {
 	/* we cannot release GVL when callback proc is supplied */
 	rsa_blocking_gen(&gen_arg);
@@ -136,14 +137,14 @@ rsa_generate(int size, unsigned long exp)
 	/* there's a chance to unblock */
 	rb_thread_call_without_gvl(rsa_blocking_gen, &gen_arg, ossl_generate_cb_stop, &cb_arg);
     }
+    BN_free(e);
+    BN_GENCB_free(cb);
     if (!gen_arg.result) {
-	BN_free(e);
 	RSA_free(rsa);
 	if (cb_arg.state) rb_jump_tag(cb_arg.state);
 	return 0;
     }
 
-    BN_free(e);
     return rsa;
 #else
     return RSA_generate_key(size, exp, rb_block_given_p() ? ossl_generate_cb : NULL, NULL);