avoid overflow in integer multiplication

This changeset basically replaces `ruby_xmalloc(x * y)` into
`ruby_xmalloc2(x, y)`.  Some convenient functions are also
provided for instance `rb_xmalloc_mul_add(x, y, z)` which allocates
x * y + z byes.

1 files changed, 8 insertions, 1 deletions

diff --git a/internal.h b/internal.h
index 3f57523bd3..6afd21d571 100644
--- a/internal.h
+++ b/internal.h
@@ -1627,7 +1627,7 @@ void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR
 void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_ALLOC_SIZE((2, 3));
 void ruby_sized_xfree(void *x, size_t size);
 RUBY_SYMBOL_EXPORT_END
-#define SIZED_REALLOC_N(var,type,n,old_n) ((var)=(type*)ruby_sized_xrealloc((char*)(var), (n) * sizeof(type), (old_n) * sizeof(type)))
+#define SIZED_REALLOC_N(var,type,n,old_n) ((var)=(type*)ruby_sized_xrealloc2((void*)(var), (n), sizeof(type), (old_n)))
 #endif
 
 /* optimized version of NEWOBJ() */
@@ -1647,6 +1647,13 @@ __attribute__((__alloc_align__(1)))
 void *rb_aligned_malloc(size_t, size_t) RUBY_ATTR_MALLOC RUBY_ATTR_ALLOC_SIZE((2));
 void rb_aligned_free(void *);
 
+size_t rb_size_mul_or_raise(size_t, size_t, VALUE); /* used in compile.c */
+size_t rb_size_mul_add_or_raise(size_t, size_t, size_t, VALUE); /* used in iseq.h */
+void *rb_xmalloc_mul_add(size_t, size_t, size_t) RUBY_ATTR_MALLOC;
+void *rb_xrealloc_mul_add(const void *, size_t, size_t, size_t);
+void *rb_xmalloc_mul_add_mul(size_t, size_t, size_t, size_t) RUBY_ATTR_MALLOC;
+void *rb_xcalloc_mul_add_mul(size_t, size_t, size_t, size_t) RUBY_ATTR_MALLOC;
+
 /* hash.c */
 #if RHASH_CONVERT_TABLE_DEBUG
 struct st_table *rb_hash_tbl_raw(VALUE hash, const char *file, int line);