pkey: refactor #export/#to_pem and #to_derky/pkey-refactor-serialization

Add ossl_pkey_export_traditional() and ossl_pkey_export_spki() helper
functions, and use them. This reduces code duplication.

5 files changed, 114 insertions, 173 deletions

diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index 47ddd0f0..610a83fd 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -341,6 +341,52 @@ ossl_pkey_inspect(VALUE self)
                       OBJ_nid2sn(nid));
 }
 
+VALUE
+ossl_pkey_export_traditional(int argc, VALUE *argv, VALUE self, int to_der)
+{
+    EVP_PKEY *pkey;
+    VALUE cipher, pass;
+    const EVP_CIPHER *enc = NULL;
+    BIO *bio;
+
+    GetPKey(self, pkey);
+    rb_scan_args(argc, argv, "02", &cipher, &pass);
+    if (!NIL_P(cipher)) {
+	enc = ossl_evp_get_cipherbyname(cipher);
+	pass = ossl_pem_passwd_value(pass);
+    }
+
+    bio = BIO_new(BIO_s_mem());
+    if (!bio)
+	ossl_raise(ePKeyError, "BIO_new");
+    if (to_der) {
+	if (!i2d_PrivateKey_bio(bio, pkey)) {
+	    BIO_free(bio);
+	    ossl_raise(ePKeyError, "i2d_PrivateKey_bio");
+	}
+    }
+    else {
+#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)
+	if (!PEM_write_bio_PrivateKey_traditional(bio, pkey, enc, NULL, 0,
+						  ossl_pem_passwd_cb,
+						  (void *)pass)) {
+#else
+	char pem_str[80];
+	const char *aname;
+
+	EVP_PKEY_asn1_get0_info(NULL, NULL, NULL, NULL, &aname, pkey->ameth);
+	snprintf(pem_str, sizeof(pem_str), "%s PRIVATE KEY", aname);
+	if (!PEM_ASN1_write_bio((i2d_of_void *)i2d_PrivateKey, pem_str, bio,
+				pkey, enc, NULL, 0, ossl_pem_passwd_cb,
+				(void *)pass)) {
+#endif
+	    BIO_free(bio);
+	    ossl_raise(ePKeyError, "PEM_write_bio_PrivateKey_traditional");
+	}
+    }
+    return ossl_membio2str(bio);
+}
+
 static VALUE
 do_pkcs8_export(int argc, VALUE *argv, VALUE self, int to_der)
 {
@@ -410,8 +456,8 @@ ossl_pkey_private_to_pem(int argc, VALUE *argv, VALUE self)
     return do_pkcs8_export(argc, argv, self, 0);
 }
 
-static VALUE
-do_spki_export(VALUE self, int to_der)
+VALUE
+ossl_pkey_export_spki(VALUE self, int to_der)
 {
     EVP_PKEY *pkey;
     BIO *bio;
@@ -444,7 +490,7 @@ do_spki_export(VALUE self, int to_der)
 static VALUE
 ossl_pkey_public_to_der(VALUE self)
 {
-    return do_spki_export(self, 1);
+    return ossl_pkey_export_spki(self, 1);
 }
 
 /*
@@ -456,7 +502,7 @@ ossl_pkey_public_to_der(VALUE self)
 static VALUE
 ossl_pkey_public_to_pem(VALUE self)
 {
-    return do_spki_export(self, 0);
+    return ossl_pkey_export_spki(self, 0);
 }
 
 /*
diff --git a/ext/openssl/ossl_pkey.h b/ext/openssl/ossl_pkey.h
index 895927e3..7dbaed47 100644
--- a/ext/openssl/ossl_pkey.h
+++ b/ext/openssl/ossl_pkey.h
@@ -49,6 +49,20 @@ EVP_PKEY *ossl_pkey_read_generic(BIO *, VALUE);
 EVP_PKEY *GetPKeyPtr(VALUE);
 EVP_PKEY *DupPKeyPtr(VALUE);
 EVP_PKEY *GetPrivPKeyPtr(VALUE);
+
+/*
+ * Serializes _self_ in X.509 SubjectPublicKeyInfo format and returns the
+ * resulting String. Sub-classes use this when overriding #to_der.
+ */
+VALUE ossl_pkey_export_spki(VALUE self, int to_der);
+/*
+ * Serializes the private key _self_ in the traditional private key format
+ * and returns the resulting String. Sub-classes use this when overriding
+ * #to_der.
+ */
+VALUE ossl_pkey_export_traditional(int argc, VALUE *argv, VALUE self,
+				   int to_der);
+
 void Init_ossl_pkey(void);
 
 /*
diff --git a/ext/openssl/ossl_pkey_dsa.c b/ext/openssl/ossl_pkey_dsa.c
index 56f58559..0e68f7f2 100644
--- a/ext/openssl/ossl_pkey_dsa.c
+++ b/ext/openssl/ossl_pkey_dsa.c
@@ -296,34 +296,12 @@ static VALUE
 ossl_dsa_export(int argc, VALUE *argv, VALUE self)
 {
     DSA *dsa;
-    BIO *out;
-    const EVP_CIPHER *ciph = NULL;
-    VALUE cipher, pass, str;
 
     GetDSA(self, dsa);
-    rb_scan_args(argc, argv, "02", &cipher, &pass);
-    if (!NIL_P(cipher)) {
-	ciph = ossl_evp_get_cipherbyname(cipher);
-	pass = ossl_pem_passwd_value(pass);
-    }
-    if (!(out = BIO_new(BIO_s_mem()))) {
-	ossl_raise(eDSAError, NULL);
-    }
-    if (DSA_HAS_PRIVATE(dsa)) {
-	if (!PEM_write_bio_DSAPrivateKey(out, dsa, ciph, NULL, 0,
-					 ossl_pem_passwd_cb, (void *)pass)){
-	    BIO_free(out);
-	    ossl_raise(eDSAError, NULL);
-	}
-    } else {
-	if (!PEM_write_bio_DSA_PUBKEY(out, dsa)) {
-	    BIO_free(out);
-	    ossl_raise(eDSAError, NULL);
-	}
-    }
-    str = ossl_membio2str(out);
-
-    return str;
+    if (DSA_HAS_PRIVATE(dsa))
+        return ossl_pkey_export_traditional(argc, argv, self, 0);
+    else
+        return ossl_pkey_export_spki(self, 0);
 }
 
 /*
@@ -337,25 +315,12 @@ static VALUE
 ossl_dsa_to_der(VALUE self)
 {
     DSA *dsa;
-    int (*i2d_func)(DSA *, unsigned char **);
-    unsigned char *p;
-    long len;
-    VALUE str;
 
     GetDSA(self, dsa);
-    if(DSA_HAS_PRIVATE(dsa))
-	i2d_func = (int (*)(DSA *,unsigned char **))i2d_DSAPrivateKey;
+    if (DSA_HAS_PRIVATE(dsa))
+        return ossl_pkey_export_traditional(0, NULL, self, 1);
     else
-	i2d_func = i2d_DSA_PUBKEY;
-    if((len = i2d_func(dsa, NULL)) <= 0)
-	ossl_raise(eDSAError, NULL);
-    str = rb_str_new(0, len);
-    p = (unsigned char *)RSTRING_PTR(str);
-    if(i2d_func(dsa, &p) < 0)
-	ossl_raise(eDSAError, NULL);
-    ossl_str_adjust(str, p);
-
-    return str;
+        return ossl_pkey_export_spki(self, 1);
 }
 
 
diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index ca8f5c6e..6fe2533e 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -141,7 +141,7 @@ ossl_ec_key_s_generate(VALUE klass, VALUE arg)
 static VALUE ossl_ec_key_initialize(int argc, VALUE *argv, VALUE self)
 {
     EVP_PKEY *pkey;
-    EC_KEY *ec;
+    EC_KEY *ec = NULL;
     VALUE arg, pass;
 
     GetPKey(self, pkey);
@@ -378,66 +378,6 @@ static VALUE ossl_ec_key_is_private(VALUE self)
     return EC_KEY_get0_private_key(ec) ? Qtrue : Qfalse;
 }
 
-static VALUE ossl_ec_key_to_string(VALUE self, VALUE ciph, VALUE pass, int format)
-{
-    EC_KEY *ec;
-    BIO *out;
-    int i = -1;
-    int private = 0;
-    VALUE str;
-    const EVP_CIPHER *cipher = NULL;
-
-    GetEC(self, ec);
-
-    if (EC_KEY_get0_public_key(ec) == NULL)
-        ossl_raise(eECError, "can't export - no public key set");
-
-    if (EC_KEY_check_key(ec) != 1)
-	ossl_raise(eECError, "can't export - EC_KEY_check_key failed");
-
-    if (EC_KEY_get0_private_key(ec))
-        private = 1;
-
-    if (!NIL_P(ciph)) {
-	cipher = ossl_evp_get_cipherbyname(ciph);
-	pass = ossl_pem_passwd_value(pass);
-    }
-
-    if (!(out = BIO_new(BIO_s_mem())))
-        ossl_raise(eECError, "BIO_new(BIO_s_mem())");
-
-    switch(format) {
-    case EXPORT_PEM:
-    	if (private) {
-            i = PEM_write_bio_ECPrivateKey(out, ec, cipher, NULL, 0, ossl_pem_passwd_cb, (void *)pass);
-    	} else {
-            i = PEM_write_bio_EC_PUBKEY(out, ec);
-        }
-
-    	break;
-    case EXPORT_DER:
-        if (private) {
-            i = i2d_ECPrivateKey_bio(out, ec);
-        } else {
-            i = i2d_EC_PUBKEY_bio(out, ec);
-        }
-
-    	break;
-    default:
-        BIO_free(out);
-    	ossl_raise(rb_eRuntimeError, "unknown format (internal error)");
-    }
-
-    if (i != 1) {
-        BIO_free(out);
-        ossl_raise(eECError, "outlen=%d", i);
-    }
-
-    str = ossl_membio2str(out);
-
-    return str;
-}
-
 /*
  *  call-seq:
  *     key.export([cipher, pass_phrase]) => String
@@ -448,11 +388,16 @@ static VALUE ossl_ec_key_to_string(VALUE self, VALUE ciph, VALUE pass, int forma
  * instance. Note that encryption will only be effective for a private key,
  * public keys will always be encoded in plain text.
  */
-static VALUE ossl_ec_key_export(int argc, VALUE *argv, VALUE self)
+static VALUE
+ossl_ec_key_export(int argc, VALUE *argv, VALUE self)
 {
-    VALUE cipher, passwd;
-    rb_scan_args(argc, argv, "02", &cipher, &passwd);
-    return ossl_ec_key_to_string(self, cipher, passwd, EXPORT_PEM);
+    EC_KEY *ec;
+
+    GetEC(self, ec);
+    if (EC_KEY_get0_private_key(ec))
+        return ossl_pkey_export_traditional(argc, argv, self, 0);
+    else
+        return ossl_pkey_export_spki(self, 0);
 }
 
 /*
@@ -461,9 +406,16 @@ static VALUE ossl_ec_key_export(int argc, VALUE *argv, VALUE self)
  *
  *  See the OpenSSL documentation for i2d_ECPrivateKey_bio()
  */
-static VALUE ossl_ec_key_to_der(VALUE self)
+static VALUE
+ossl_ec_key_to_der(VALUE self)
 {
-    return ossl_ec_key_to_string(self, Qnil, Qnil, EXPORT_DER);
+    EC_KEY *ec;
+
+    GetEC(self, ec);
+    if (EC_KEY_get0_private_key(ec))
+        return ossl_pkey_export_traditional(0, NULL, self, 1);
+    else
+        return ossl_pkey_export_spki(self, 1);
 }
 
 /*
diff --git a/ext/openssl/ossl_pkey_rsa.c b/ext/openssl/ossl_pkey_rsa.c
index 8415121c..3c298a2a 100644
--- a/ext/openssl/ossl_pkey_rsa.c
+++ b/ext/openssl/ossl_pkey_rsa.c
@@ -173,8 +173,8 @@ ossl_rsa_s_generate(int argc, VALUE *argv, VALUE klass)
 static VALUE
 ossl_rsa_initialize(int argc, VALUE *argv, VALUE self)
 {
-    EVP_PKEY *pkey;
-    RSA *rsa;
+    EVP_PKEY *pkey, *tmp;
+    RSA *rsa = NULL;
     BIO *in;
     VALUE arg, pass;
 
@@ -279,6 +279,21 @@ ossl_rsa_is_private(VALUE self)
     return RSA_PRIVATE(self, rsa) ? Qtrue : Qfalse;
 }
 
+static int
+can_export_rsaprivatekey(VALUE self)
+{
+    RSA *rsa;
+    const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp;
+
+    GetRSA(self, rsa);
+
+    RSA_get0_key(rsa, &n, &e, &d);
+    RSA_get0_factors(rsa, &p, &q);
+    RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp);
+
+    return n && e && d && p && q && dmp1 && dmq1 && iqmp;
+}
+
 /*
  * call-seq:
  *   rsa.export([cipher, pass_phrase]) => PEM-format String
@@ -292,41 +307,10 @@ ossl_rsa_is_private(VALUE self)
 static VALUE
 ossl_rsa_export(int argc, VALUE *argv, VALUE self)
 {
-    RSA *rsa;
-    const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp;
-    BIO *out;
-    const EVP_CIPHER *ciph = NULL;
-    VALUE cipher, pass, str;
-
-    GetRSA(self, rsa);
-
-    rb_scan_args(argc, argv, "02", &cipher, &pass);
-
-    if (!NIL_P(cipher)) {
-	ciph = ossl_evp_get_cipherbyname(cipher);
-	pass = ossl_pem_passwd_value(pass);
-    }
-    if (!(out = BIO_new(BIO_s_mem()))) {
-	ossl_raise(eRSAError, NULL);
-    }
-    RSA_get0_key(rsa, &n, &e, &d);
-    RSA_get0_factors(rsa, &p, &q);
-    RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp);
-    if (n && e && d && p && q && dmp1 && dmq1 && iqmp) {
-	if (!PEM_write_bio_RSAPrivateKey(out, rsa, ciph, NULL, 0,
-					 ossl_pem_passwd_cb, (void *)pass)) {
-	    BIO_free(out);
-	    ossl_raise(eRSAError, NULL);
-	}
-    } else {
-	if (!PEM_write_bio_RSA_PUBKEY(out, rsa)) {
-	    BIO_free(out);
-	    ossl_raise(eRSAError, NULL);
-	}
-    }
-    str = ossl_membio2str(out);
-
-    return str;
+    if (can_export_rsaprivatekey(self))
+        return ossl_pkey_export_traditional(argc, argv, self, 0);
+    else
+        return ossl_pkey_export_spki(self, 0);
 }
 
 /*
@@ -338,30 +322,10 @@ ossl_rsa_export(int argc, VALUE *argv, VALUE self)
 static VALUE
 ossl_rsa_to_der(VALUE self)
 {
-    RSA *rsa;
-    const BIGNUM *n, *e, *d, *p, *q, *dmp1, *dmq1, *iqmp;
-    int (*i2d_func)(const RSA *, unsigned char **);
-    unsigned char *ptr;
-    long len;
-    VALUE str;
-
-    GetRSA(self, rsa);
-    RSA_get0_key(rsa, &n, &e, &d);
-    RSA_get0_factors(rsa, &p, &q);
-    RSA_get0_crt_params(rsa, &dmp1, &dmq1, &iqmp);
-    if (n && e && d && p && q && dmp1 && dmq1 && iqmp)
-	i2d_func = i2d_RSAPrivateKey;
+    if (can_export_rsaprivatekey(self))
+        return ossl_pkey_export_traditional(0, NULL, self, 1);
     else
-	i2d_func = (int (*)(const RSA *, unsigned char **))i2d_RSA_PUBKEY;
-    if((len = i2d_func(rsa, NULL)) <= 0)
-	ossl_raise(eRSAError, NULL);
-    str = rb_str_new(0, len);
-    ptr = (unsigned char *)RSTRING_PTR(str);
-    if(i2d_func(rsa, &ptr) < 0)
-	ossl_raise(eRSAError, NULL);
-    ossl_str_adjust(str, ptr);
-
-    return str;
+        return ossl_pkey_export_spki(self, 1);
 }
 
 /*