From e879a76b11eb9f1df5d1080647c5a0383b3e5e80 Mon Sep 17 00:00:00 2001
From: Kazuki Yamaguchi <k@rhe.jp>
Date: Thu, 15 Sep 2016 01:09:10 +0900
Subject: pack.c: avoid signed integer overflow

* pack.c (pack_pack, pack_unpack): Use strtol() instead of strtoul(), as
  the length must be in the range of long. This suppresses the implicit
  type conversion and let it raise a better exception for too large
  length.

* pack.c (pack_unpack): Fix 'b', 'B', 'h', 'H', 'u', 'm' format. Ensure
  that multiplying does not cause integer overflow.
---
 pack.c | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/pack.c b/pack.c
index 3fff3e65df..5e0f708537 100644
--- a/pack.c
+++ b/pack.c
@@ -353,7 +353,7 @@ pack_pack(VALUE ary, VALUE fmt)
 	}
 	else if (ISDIGIT(*p)) {
 	    errno = 0;
-	    len = STRTOUL(p, (char**)&p, 10);
+	    len = strtol(p, (char**)&p, 10);
 	    if (errno) {
 		rb_raise(rb_eRangeError, "pack length too big");
 	    }
@@ -1232,7 +1232,7 @@ pack_unpack(VALUE str, VALUE fmt)
 	}
 	else if (ISDIGIT(*p)) {
 	    errno = 0;
-	    len = STRTOUL(p, (char**)&p, 10);
+	    len = strtol(p, (char**)&p, 10);
 	    if (errno) {
 		rb_raise(rb_eRangeError, "pack length too big");
 	    }
@@ -1286,8 +1286,11 @@ pack_unpack(VALUE str, VALUE fmt)
 		int bits;
 		long i;
 
-		if (p[-1] == '*' || len > (send - s) * 8)
+		if (p[-1] == '*' || len / 8 >= send - s) {
+		    if (send - s > LONG_MAX / 8)
+			rb_raise(rb_eRuntimeError, "string size too big");
 		    len = (send - s) * 8;
+		}
 		bits = 0;
 		UNPACK_PUSH(bitstr = rb_usascii_str_new(0, len));
 		t = RSTRING_PTR(bitstr);
@@ -1306,8 +1309,11 @@ pack_unpack(VALUE str, VALUE fmt)
 		int bits;
 		long i;
 
-		if (p[-1] == '*' || len > (send - s) * 8)
+		if (p[-1] == '*' || len / 8 >= send - s) {
+		    if (send - s > LONG_MAX / 8)
+			rb_raise(rb_eRuntimeError, "string size too big");
 		    len = (send - s) * 8;
+		}
 		bits = 0;
 		UNPACK_PUSH(bitstr = rb_usascii_str_new(0, len));
 		t = RSTRING_PTR(bitstr);
@@ -1326,8 +1332,11 @@ pack_unpack(VALUE str, VALUE fmt)
 		int bits;
 		long i;
 
-		if (p[-1] == '*' || len > (send - s) * 2)
+		if (p[-1] == '*' || len / 2 >= send - s) {
+		    if (send - s > LONG_MAX / 2)
+			rb_raise(rb_eRuntimeError, "string size too big");
 		    len = (send - s) * 2;
+		}
 		bits = 0;
 		UNPACK_PUSH(bitstr = rb_usascii_str_new(0, len));
 		t = RSTRING_PTR(bitstr);
@@ -1348,8 +1357,11 @@ pack_unpack(VALUE str, VALUE fmt)
 		int bits;
 		long i;
 
-		if (p[-1] == '*' || len > (send - s) * 2)
+		if (p[-1] == '*' || len / 2 >= send - s) {
+		    if (send - s > LONG_MAX / 2)
+			rb_raise(rb_eRuntimeError, "string size too big");
 		    len = (send - s) * 2;
+		}
 		bits = 0;
 		UNPACK_PUSH(bitstr = rb_usascii_str_new(0, len));
 		t = RSTRING_PTR(bitstr);
@@ -1617,7 +1629,8 @@ pack_unpack(VALUE str, VALUE fmt)
 
 	  case 'm':
 	    {
-		VALUE buf = infected_str_new(0, (send - s + 3)*3/4, str); /* +3 is for skipping paddings */
+		/* + 2 is for truncated paddings */
+		VALUE buf = infected_str_new(0, (send - s) / 4 * 3 + 2, str);
 		char *ptr = RSTRING_PTR(buf);
 		int a = -1,b = -1,c = 0,d = 0;
 		static signed char b64_xtable[256];
-- 
cgit v1.2.3