x509, ssl, pkcs7: try to parse as DER-encoding firstky/parse-der-then-pem

Methods that take both PEM-encoding and DER-encoding have not been
consistent in the order in which encoding to attempt to parse.

A DER-encoding may contain a valid PEM block ("\n-----BEGIN ..-----" to
"-----END ...-----") embedded within it. Also, the PEM-encoding parser
allows arbitrary data around the PEM block and silently skips it. As a
result, attempting to parse data in DER-encoding as PEM-encoding first
can incorrectly finds the embedded PEM block instead.

This commit ensures that DER encoding will always be attempted before
PEM encoding. OpenSSL::X509::Certificate is one of the updated classes.
With this, the following will always be true:

    # obj is an OpenSSL::X509::Certificate
    obj == OpenSSL::X509::Certificate.new(obj.to_der)
    obj == OpenSSL::X509::Certificate.new(obj.to_pem)

1 files changed, 9 insertions, 11 deletions

diff --git a/ext/openssl/ossl_pkcs7.c b/ext/openssl/ossl_pkcs7.c
index 0bcc76a9..dbe53476 100644
--- a/ext/openssl/ossl_pkcs7.c
+++ b/ext/openssl/ossl_pkcs7.c
@@ -330,7 +330,7 @@ ossl_pkcs7_alloc(VALUE klass)
 static VALUE
 ossl_pkcs7_initialize(int argc, VALUE *argv, VALUE self)
 {
-    PKCS7 *p7, *pkcs = DATA_PTR(self);
+    PKCS7 *p7, *p7_orig = RTYPEDDATA_DATA(self);
     BIO *in;
     VALUE arg;
 
@@ -338,19 +338,17 @@ ossl_pkcs7_initialize(int argc, VALUE *argv, VALUE self)
 	return self;
     arg = ossl_to_der_if_possible(arg);
     in = ossl_obj2bio(&arg);
-    p7 = PEM_read_bio_PKCS7(in, &pkcs, NULL, NULL);
+    p7 = d2i_PKCS7_bio(in, NULL);
     if (!p7) {
-	OSSL_BIO_reset(in);
-        p7 = d2i_PKCS7_bio(in, &pkcs);
-	if (!p7) {
-	    BIO_free(in);
-	    PKCS7_free(pkcs);
-	    DATA_PTR(self) = NULL;
-	    ossl_raise(rb_eArgError, "Could not parse the PKCS7");
-	}
+        OSSL_BIO_reset(in);
+        p7 = PEM_read_bio_PKCS7(in, NULL, NULL, NULL);
     }
-    DATA_PTR(self) = pkcs;
     BIO_free(in);
+    if (!p7)
+        ossl_raise(rb_eArgError, "Could not parse the PKCS7");
+
+    RTYPEDDATA_DATA(self) = p7;
+    PKCS7_free(p7_orig);
     ossl_pkcs7_set_data(self, Qnil);
     ossl_pkcs7_set_err_string(self, Qnil);