From 90a79d32d3ea2bc908e3f0eed029343e7e09c48f Mon Sep 17 00:00:00 2001
From: rhe <rhe@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Thu, 9 Jun 2016 12:42:08 +0000
Subject: openssl: use ASN1_ENUMERATED_to_BN() if needed

* ext/openssl/ossl_asn1.c (asn1integer_to_num): Use
  ASN1_ENUMERATED_to_BN() to convert an ASN1_ENUMERATED to a BN.
  Starting from OpenSSL 1.1.0, ASN1_INTEGER_to_BN() rejects
  non-ASN1_INTEGER objects. The format of INTEGER and ENUMERATED are
  almost identical so they behaved in the same way in OpenSSL <= 1.0.2.
  [ruby-core:75225] [Feature #12324]

* test/openssl/test_asn1.rb (test_decode_enumerated): Test that it
  works.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55344 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ext/openssl/ossl_asn1.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'ext/openssl')

diff --git a/ext/openssl/ossl_asn1.c b/ext/openssl/ossl_asn1.c
index 03822b5c26..cbedd59efa 100644
--- a/ext/openssl/ossl_asn1.c
+++ b/ext/openssl/ossl_asn1.c
@@ -125,9 +125,13 @@ asn1integer_to_num(ASN1_INTEGER *ai)
     if (!ai) {
 	ossl_raise(rb_eTypeError, "ASN1_INTEGER is NULL!");
     }
-    if (!(bn = ASN1_INTEGER_to_BN(ai, NULL))) {
+    if (ai->type == V_ASN1_ENUMERATED)
+	bn = ASN1_ENUMERATED_to_BN(ai, NULL);
+    else
+	bn = ASN1_INTEGER_to_BN(ai, NULL);
+
+    if (!bn)
 	ossl_raise(eOSSLError, NULL);
-    }
 #if DO_IT_VIA_RUBY
     if (!(txt = BN_bn2dec(bn))) {
 	BN_free(bn);
-- 
cgit v1.2.3