From ab67419b755af54aeee3319a1cddb62408f1dcc9 Mon Sep 17 00:00:00 2001 From: akr Date: Tue, 11 Feb 2014 14:06:51 +0000 Subject: * ext/openssl/ossl_bn.c (ossl_bn_initialize): Use rb_integer_pack. Fix SEGV by OpenSSL::BN.new(1 << (2**34)). git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@44909 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ext/openssl/ossl_bn.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'ext/openssl/ossl_bn.c') diff --git a/ext/openssl/ossl_bn.c b/ext/openssl/ossl_bn.c index 05473635ae..bdaf077b5f 100644 --- a/ext/openssl/ossl_bn.c +++ b/ext/openssl/ossl_bn.c @@ -140,26 +140,24 @@ ossl_bn_initialize(int argc, VALUE *argv, VALUE self) return self; } else if (RB_TYPE_P(str, T_BIGNUM)) { - int i, j, len = RBIGNUM_LENINT(str); - BDIGIT *ds = RBIGNUM_DIGITS(str); + size_t len = rb_absint_size(str, NULL); + unsigned char *bin; VALUE buf; - unsigned char *bin = (unsigned char*)ALLOCV_N(BDIGIT, buf, len); - - for (i = 0; len > i; i++) { - BDIGIT v = ds[i]; - for (j = SIZEOF_BDIGITS - 1; 0 <= j; j--) { - bin[(len-1-i)*SIZEOF_BDIGITS+j] = v&0xff; - v >>= 8; - } - } + int sign; + + if (INT_MAX < len) { + rb_raise(eBNError, "bignum too long"); + } + bin = (unsigned char*)ALLOCV_N(unsigned char, buf, len); + sign = rb_integer_pack(str, bin, len, 1, 0, INTEGER_PACK_BIG_ENDIAN); GetBN(self, bn); - if (!BN_bin2bn(bin, (int)SIZEOF_BDIGITS*len, bn)) { + if (!BN_bin2bn(bin, (int)len, bn)) { ALLOCV_END(buf); ossl_raise(eBNError, NULL); } ALLOCV_END(buf); - if (!RBIGNUM_SIGN(str)) BN_set_negative(bn, 1); + if (sign < 0) BN_set_negative(bn, 1); return self; } if (RTEST(rb_obj_is_kind_of(str, cBN))) { -- cgit v1.2.3