aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 21:53:56 +0000
committerko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 21:53:56 +0000
commit312b105d0e263a36c129a14f2681fe16905c0244 (patch)
tree00bf11825e6f8a8cb12ba1f2c02c722f35090e0c
parent69b8ffcd5b8b28c56212368a3fe66f076b90e4c4 (diff)
downloadruby-312b105d0e263a36c129a14f2681fe16905c0244.tar.gz
introduce TransientHeap. [Bug #14858]
* transient_heap.c, transient_heap.h: implement TransientHeap (theap). theap is designed for Ruby's object system. theap is like Eden heap on generational GC terminology. theap allocation is very fast because it only needs to bump up pointer and deallocation is also fast because we don't do anything. However we need to evacuate (Copy GC terminology) if theap memory is long-lived. Evacuation logic is needed for each type. See [Bug #14858] for details. * array.c: Now, theap for T_ARRAY is supported. ary_heap_alloc() tries to allocate memory area from theap. If this trial sccesses, this array has theap ptr and RARRAY_TRANSIENT_FLAG is turned on. We don't need to free theap ptr. * ruby.h: RARRAY_CONST_PTR() returns malloc'ed memory area. It menas that if ary is allocated at theap, force evacuation to malloc'ed memory. It makes programs slow, but very compatible with current code because theap memory can be evacuated (theap memory will be recycled). If you want to get transient heap ptr, use RARRAY_CONST_PTR_TRANSIENT() instead of RARRAY_CONST_PTR(). If you can't understand when evacuation will occur, use RARRAY_CONST_PTR(). (re-commit of r65444) git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65449 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat
-rw-r--r--array.c398
-rw-r--r--common.mk10
-rw-r--r--compile.c6
-rw-r--r--debug_counter.h5
-rw-r--r--enum.c6
-rw-r--r--gc.c89
-rw-r--r--include/ruby/ruby.h30
-rw-r--r--inits.c1
-rw-r--r--insns.def2
-rw-r--r--internal.h25
-rw-r--r--string.c4
-rw-r--r--test/ruby/test_enum.rb15
-rw-r--r--transient_heap.c834
-rw-r--r--transient_heap.h38
-rw-r--r--vm_args.c30
-rw-r--r--vm_eval.c2
-rw-r--r--vm_insnhelper.c2
17 files changed, 1371 insertions, 126 deletions
diff --git a/array.c b/array.c
index 0b2be722ea..bbe11fee7f 100644
--- a/array.c
+++ b/array.c
@@ -14,12 +14,14 @@
#include "ruby/encoding.h"
#include "ruby/util.h"
#include "ruby/st.h"
-#include "internal.h"
#include "probes.h"
#include "id.h"
#include "debug_counter.h"
+#include "gc.h"
+#include "transient_heap.h"
+#include "internal.h"
-#ifndef ARRAY_DEBUG
+#if !ARRAY_DEBUG
# define NDEBUG
#endif
#include "ruby_assert.h"
@@ -42,17 +44,21 @@ VALUE rb_cArray;
#define ARY_HEAP_PTR(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.ptr)
#define ARY_HEAP_LEN(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.len)
+#define ARY_HEAP_CAPA(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.aux.capa)
+
#define ARY_EMBED_PTR(a) (assert(ARY_EMBED_P(a)), RARRAY(a)->as.ary)
#define ARY_EMBED_LEN(a) \
(assert(ARY_EMBED_P(a)), \
(long)((RBASIC(a)->flags >> RARRAY_EMBED_LEN_SHIFT) & \
(RARRAY_EMBED_LEN_MASK >> RARRAY_EMBED_LEN_SHIFT)))
-#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), RARRAY(a)->as.heap.aux.capa * sizeof(VALUE))
+#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), ARY_HEAP_CAPA(a) * sizeof(VALUE))
#define ARY_OWNS_HEAP_P(a) (!FL_TEST((a), ELTS_SHARED|RARRAY_EMBED_FLAG))
#define FL_SET_EMBED(a) do { \
assert(!ARY_SHARED_P(a)); \
FL_SET((a), RARRAY_EMBED_FLAG); \
+ FL_UNSET_RAW((a), RARRAY_TRANSIENT_FLAG); \
+ ary_verify(a); \
} while (0)
#define FL_UNSET_EMBED(ary) FL_UNSET((ary), RARRAY_EMBED_FLAG|RARRAY_EMBED_LEN_MASK)
#define FL_SET_SHARED(ary) do { \
@@ -102,7 +108,7 @@ VALUE rb_cArray;
} while (0)
#define ARY_CAPA(ary) (ARY_EMBED_P(ary) ? RARRAY_EMBED_LEN_MAX : \
- ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : RARRAY(ary)->as.heap.aux.capa)
+ ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : ARY_HEAP_CAPA(ary))
#define ARY_SET_CAPA(ary, n) do { \
assert(!ARY_EMBED_P(ary)); \
assert(!ARY_SHARED_P(ary)); \
@@ -130,11 +136,82 @@ VALUE rb_cArray;
} while (0)
#define FL_SET_SHARED_ROOT(ary) do { \
assert(!ARY_EMBED_P(ary)); \
+ assert(!RARRAY_TRANSIENT_P(ary)); \
FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \
} while (0)
#define ARY_SET(a, i, v) RARRAY_ASET((assert(!ARY_SHARED_P(a)), (a)), (i), (v))
+
+#if ARRAY_DEBUG
+#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__)
+
+static VALUE
+ary_verify_(VALUE ary, const char *file, int line)
+{
+ assert(RB_TYPE_P(ary, T_ARRAY));
+
+ if (FL_TEST(ary, ELTS_SHARED)) {
+ VALUE root = RARRAY(ary)->as.heap.aux.shared;
+ const VALUE *ptr = ARY_HEAP_PTR(ary);
+ const VALUE *root_ptr = RARRAY_CONST_PTR_TRANSIENT(root);
+ long len = ARY_HEAP_LEN(ary), root_len = RARRAY_LEN(root);
+ assert(FL_TEST(root, RARRAY_SHARED_ROOT_FLAG));
+ assert(root_ptr <= ptr && ptr + len <= root_ptr + root_len);
+ ary_verify(root);
+ }
+ else if (ARY_EMBED_P(ary)) {
+ assert(!RARRAY_TRANSIENT_P(ary));
+ assert(!ARY_SHARED_P(ary));
+ assert(RARRAY_LEN(ary) <= RARRAY_EMBED_LEN_MAX);
+ }
+ else {
+#if 1
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
+ long i, len = RARRAY_LEN(ary);
+ volatile VALUE v;
+ if (len > 1) len = 1; /* check only HEAD */
+ for (i=0; i<len; i++) {
+ v = ptr[i]; /* access check */
+ }
+ v = v;
+#endif
+ }
+
+ if (RARRAY_TRANSIENT_P(ary)) {
+ assert(rb_transient_heap_managed_ptr_p(RARRAY_CONST_PTR_TRANSIENT(ary)));
+ }
+
+ rb_transient_heap_verify();
+
+ return ary;
+}
+
+void
+rb_ary_verify(VALUE ary){
+ ary_verify(ary);
+}
+#else
+#define ary_verify(ary) ((void)0)
+#endif
+
+VALUE *
+rb_ary_ptr_use_start(VALUE ary)
+{
+#if ARRAY_DEBUG
+ FL_SET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
+#endif
+ return (VALUE *)RARRAY_CONST_PTR_TRANSIENT(ary);
+}
+
+void
+rb_ary_ptr_use_end(VALUE ary)
+{
+#if ARRAY_DEBUG
+ FL_UNSET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
+#endif
+}
+
void
rb_mem_clear(register VALUE *mem, register long size)
{
@@ -195,49 +272,167 @@ ary_memcpy(VALUE ary, long beg, long argc, const VALUE *argv)
ary_memcpy0(ary, beg, argc, argv, ary);
}
+static VALUE *
+ary_heap_alloc(VALUE ary, size_t capa)
+{
+ VALUE *ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * capa);
+
+ if (ptr != NULL) {
+ FL_SET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ ptr = ALLOC_N(VALUE, capa);
+ }
+
+ return ptr;
+}
+
+static void
+ary_heap_free_ptr(VALUE ary, const VALUE *ptr, long size)
+{
+ if (RARRAY_TRANSIENT_P(ary)) {
+ /* ignore it */
+ }
+ else {
+ ruby_sized_xfree((void *)ptr, size);
+ }
+}
+
+static void
+ary_heap_free(VALUE ary)
+{
+ if (RARRAY_TRANSIENT_P(ary)) {
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ }
+}
+
+static void
+ary_heap_realloc(VALUE ary, size_t new_capa)
+{
+ size_t old_capa = ARY_HEAP_CAPA(ary);
+
+ if (RARRAY_TRANSIENT_P(ary)) {
+ if (new_capa <= old_capa) {
+ /* do nothing */
+ }
+ else {
+ VALUE *new_ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * new_capa);
+
+ if (new_ptr == NULL) {
+ new_ptr = ALLOC_N(VALUE, new_capa);
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+
+ MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, old_capa);
+ ARY_SET_PTR(ary, new_ptr);
+ }
+ }
+ else {
+ SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, old_capa);
+ }
+ ary_verify(ary);
+}
+
+static inline void
+rb_ary_transient_heap_evacuate_(VALUE ary, int transient, int promote)
+{
+ if (transient) {
+ VALUE *new_ptr;
+ const VALUE *old_ptr = ARY_HEAP_PTR(ary);
+ long capa = ARY_HEAP_CAPA(ary);
+ long len = ARY_HEAP_LEN(ary);
+
+ if (ARY_SHARED_ROOT_P(ary)) {
+ capa = len;
+ }
+
+ assert(ARY_OWNS_HEAP_P(ary));
+ assert(RARRAY_TRANSIENT_P(ary));
+ assert(!ARY_PTR_USING_P(ary));
+
+ if (promote) {
+ new_ptr = ALLOC_N(VALUE, capa);
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ new_ptr = ary_heap_alloc(ary, capa);
+ }
+
+ MEMCPY(new_ptr, old_ptr, VALUE, capa);
+ /* do not use ARY_SET_PTR() because they assert !frozen */
+ RARRAY(ary)->as.heap.ptr = new_ptr;
+ }
+
+ ary_verify(ary);
+}
+
+void
+rb_ary_transient_heap_evacuate(VALUE ary, int promote)
+{
+ rb_ary_transient_heap_evacuate_(ary, RARRAY_TRANSIENT_P(ary), promote);
+}
+
+void
+rb_ary_detransient(VALUE ary)
+{
+ assert(RARRAY_TRANSIENT_P(ary));
+ rb_ary_transient_heap_evacuate_(ary, TRUE, TRUE);
+}
+
static void
ary_resize_capa(VALUE ary, long capacity)
{
assert(RARRAY_LEN(ary) <= capacity);
assert(!OBJ_FROZEN(ary));
assert(!ARY_SHARED_P(ary));
+
if (capacity > RARRAY_EMBED_LEN_MAX) {
if (ARY_EMBED_P(ary)) {
long len = ARY_EMBED_LEN(ary);
- VALUE *ptr = ALLOC_N(VALUE, (capacity));
+ VALUE *ptr = ary_heap_alloc(ary, capacity);
+
MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len);
FL_UNSET_EMBED(ary);
ARY_SET_PTR(ary, ptr);
ARY_SET_HEAP_LEN(ary, len);
}
else {
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, RARRAY(ary)->as.heap.aux.capa);
+ ary_heap_realloc(ary, capacity);
}
- ARY_SET_CAPA(ary, (capacity));
+ ARY_SET_CAPA(ary, capacity);
}
else {
if (!ARY_EMBED_P(ary)) {
- long len = RARRAY_LEN(ary);
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
+ long len = ARY_HEAP_LEN(ary);
+ long old_capa = ARY_HEAP_CAPA(ary);
+ const VALUE *ptr = ARY_HEAP_PTR(ary);
if (len > capacity) len = capacity;
MEMCPY((VALUE *)RARRAY(ary)->as.ary, ptr, VALUE, len);
+ ary_heap_free_ptr(ary, ptr, old_capa);
+
FL_SET_EMBED(ary);
ARY_SET_LEN(ary, len);
- ruby_sized_xfree((VALUE *)ptr, RARRAY(ary)->as.heap.aux.capa);
}
}
+
+ ary_verify(ary);
}
static inline void
ary_shrink_capa(VALUE ary)
{
long capacity = ARY_HEAP_LEN(ary);
- long old_capa = RARRAY(ary)->as.heap.aux.capa;
+ long old_capa = ARY_HEAP_CAPA(ary);
assert(!ARY_SHARED_P(ary));
assert(old_capa >= capacity);
- if (old_capa > capacity)
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, old_capa);
+ if (old_capa > capacity) ary_heap_realloc(ary, capacity);
+
+ ary_verify(ary);
}
static void
@@ -253,6 +448,8 @@ ary_double_capa(VALUE ary, long min)
}
new_capa += min;
ary_resize_capa(ary, new_capa);
+
+ ary_verify(ary);
}
static void
@@ -308,6 +505,7 @@ static inline void
rb_ary_modify_check(VALUE ary)
{
rb_check_frozen(ary);
+ ary_verify(ary);
}
void
@@ -317,6 +515,9 @@ rb_ary_modify(VALUE ary)
if (ARY_SHARED_P(ary)) {
long shared_len, len = RARRAY_LEN(ary);
VALUE shared = ARY_SHARED(ary);
+
+ ary_verify(shared);
+
if (len <= RARRAY_EMBED_LEN_MAX) {
const VALUE *ptr = ARY_HEAP_PTR(ary);
FL_UNSET_SHARED(ary);
@@ -326,9 +527,9 @@ rb_ary_modify(VALUE ary)
ARY_SET_EMBED_LEN(ary, len);
}
else if (ARY_SHARED_OCCUPIED(shared) && len > ((shared_len = RARRAY_LEN(shared))>>1)) {
- long shift = RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared);
+ long shift = RARRAY_CONST_PTR_TRANSIENT(ary) - RARRAY_CONST_PTR_TRANSIENT(shared);
FL_UNSET_SHARED(ary);
- ARY_SET_PTR(ary, RARRAY_CONST_PTR(shared));
+ ARY_SET_PTR(ary, RARRAY_CONST_PTR_TRANSIENT(shared));
ARY_SET_CAPA(ary, shared_len);
RARRAY_PTR_USE(ary, ptr, {
MEMMOVE(ptr, ptr+shift, VALUE, len);
@@ -337,8 +538,8 @@ rb_ary_modify(VALUE ary)
rb_ary_decrement_share(shared);
}
else {
- VALUE *ptr = ALLOC_N(VALUE, len);
- MEMCPY(ptr, RARRAY_CONST_PTR(ary), VALUE, len);
+ VALUE *ptr = ary_heap_alloc(ary, len);
+ MEMCPY(ptr, ARY_HEAP_PTR(ary), VALUE, len);
rb_ary_unshare(ary);
ARY_SET_CAPA(ary, len);
ARY_SET_PTR(ary, ptr);
@@ -346,6 +547,7 @@ rb_ary_modify(VALUE ary)
rb_gc_writebarrier_remember(ary);
}
+ ary_verify(ary);
}
static VALUE
@@ -362,8 +564,11 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
if (new_len > RARRAY_EMBED_LEN_MAX) {
VALUE shared = ARY_SHARED(ary);
if (ARY_SHARED_OCCUPIED(shared)) {
- if (RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared) + new_len <= RARRAY_LEN(shared)) {
+ if (ARY_HEAP_PTR(ary) - RARRAY_CONST_PTR_TRANSIENT(shared) + new_len <= RARRAY_LEN(shared)) {
rb_ary_modify_check(ary);
+
+ ary_verify(ary);
+ ary_verify(shared);
return shared;
}
else {
@@ -373,10 +578,12 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
if (new_len > capa - (capa >> 6)) {
ary_double_capa(ary, new_len);
}
+ ary_verify(ary);
return ary;
}
}
}
+ ary_verify(ary);
rb_ary_modify(ary);
}
else {
@@ -387,6 +594,7 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
ary_double_capa(ary, new_len);
}
+ ary_verify(ary);
return ary;
}
@@ -459,7 +667,7 @@ ary_new(VALUE klass, long capa)
ary = ary_alloc(klass);
if (capa > RARRAY_EMBED_LEN_MAX) {
- ptr = ALLOC_N(VALUE, capa);
+ ptr = ary_heap_alloc(ary, capa);
FL_UNSET_EMBED(ary);
ARY_SET_PTR(ary, ptr);
ARY_SET_CAPA(ary, capa);
@@ -523,7 +731,9 @@ rb_ary_new_from_values(long n, const VALUE *elts)
VALUE
rb_ary_tmp_new(long capa)
{
- return ary_new(0, capa);
+ VALUE ary = ary_new(0, capa);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
+ return ary;
}
VALUE
@@ -532,6 +742,7 @@ rb_ary_tmp_new_fill(long capa)
VALUE ary = ary_new(0, capa);
ary_memfill(ary, 0, capa, Qnil);
ARY_SET_LEN(ary, capa);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
return ary;
}
@@ -539,8 +750,13 @@ void
rb_ary_free(VALUE ary)
{
if (ARY_OWNS_HEAP_P(ary)) {
- RB_DEBUG_COUNTER_INC(obj_ary_ptr);
- ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ if (RARRAY_TRANSIENT_P(ary)) {
+ RB_DEBUG_COUNTER_INC(obj_ary_transient);
+ }
+ else {
+ RB_DEBUG_COUNTER_INC(obj_ary_ptr);
+ ary_heap_free(ary);
+ }
}
else {
RB_DEBUG_COUNTER_INC(obj_ary_embed);
@@ -563,13 +779,15 @@ ary_discard(VALUE ary)
{
rb_ary_free(ary);
RBASIC(ary)->flags |= RARRAY_EMBED_FLAG;
- RBASIC(ary)->flags &= ~RARRAY_EMBED_LEN_MASK;
+ RBASIC(ary)->flags &= ~(RARRAY_EMBED_LEN_MASK | RARRAY_TRANSIENT_FLAG);
}
static VALUE
ary_make_shared(VALUE ary)
{
assert(!ARY_EMBED_P(ary));
+ ary_verify(ary);
+
if (ARY_SHARED_P(ary)) {
return ARY_SHARED(ary);
}
@@ -577,6 +795,7 @@ ary_make_shared(VALUE ary)
return ary;
}
else if (OBJ_FROZEN(ary)) {
+ rb_ary_transient_heap_evacuate(ary, TRUE);
ary_shrink_capa(ary);
FL_SET_SHARED_ROOT(ary);
ARY_SET_SHARED_NUM(ary, 1);
@@ -584,17 +803,24 @@ ary_make_shared(VALUE ary)
}
else {
long capa = ARY_CAPA(ary), len = RARRAY_LEN(ary);
+ const VALUE *ptr;
NEWOBJ_OF(shared, struct RArray, 0, T_ARRAY | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0));
- FL_UNSET_EMBED(shared);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
+ ptr = ARY_HEAP_PTR(ary);
+
+ FL_UNSET_EMBED(shared);
ARY_SET_LEN((VALUE)shared, capa);
- ARY_SET_PTR((VALUE)shared, RARRAY_CONST_PTR(ary));
+ ARY_SET_PTR((VALUE)shared, ptr);
ary_mem_clear((VALUE)shared, len, capa - len);
FL_SET_SHARED_ROOT(shared);
ARY_SET_SHARED_NUM((VALUE)shared, 1);
FL_SET_SHARED(ary);
ARY_SET_SHARED(ary, (VALUE)shared);
OBJ_FREEZE(shared);
+
+ ary_verify((VALUE)shared);
+ ary_verify(ary);
return (VALUE)shared;
}
}
@@ -606,7 +832,7 @@ ary_make_substitution(VALUE ary)
if (len <= RARRAY_EMBED_LEN_MAX) {
VALUE subst = rb_ary_new2(len);
- ary_memcpy(subst, 0, len, RARRAY_CONST_PTR(ary));
+ ary_memcpy(subst, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_EMBED_LEN(subst, len);
return subst;
}
@@ -729,8 +955,8 @@ rb_ary_initialize(int argc, VALUE *argv, VALUE ary)
rb_ary_modify(ary);
if (argc == 0) {
- if (ARY_OWNS_HEAP_P(ary) && RARRAY_CONST_PTR(ary) != 0) {
- ruby_sized_xfree((void *)RARRAY_CONST_PTR(ary), ARY_HEAP_SIZE(ary));
+ if (ARY_OWNS_HEAP_P(ary) && ARY_HEAP_PTR(ary) != NULL) {
+ ary_heap_free(ary);
}
rb_ary_unshare_safe(ary);
FL_SET_EMBED(ary);
@@ -837,7 +1063,7 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len)
if (len <= RARRAY_EMBED_LEN_MAX) {
VALUE result = ary_alloc(klass);
- ary_memcpy(result, 0, len, RARRAY_CONST_PTR(ary) + offset);
+ ary_memcpy(result, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary) + offset);
ARY_SET_EMBED_LEN(result, len);
return result;
}
@@ -846,12 +1072,15 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len)
FL_UNSET_EMBED(result);
shared = ary_make_shared(ary);
- ARY_SET_PTR(result, RARRAY_CONST_PTR(ary));
+ ARY_SET_PTR(result, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_LEN(result, RARRAY_LEN(ary));
rb_ary_set_shared(result, shared);
ARY_INCREASE_PTR(result, offset);
ARY_SET_LEN(result, len);
+
+ ary_verify(shared);
+ ary_verify(result);
return result;
}
}
@@ -910,12 +1139,13 @@ ary_take_first_or_last(int argc, const VALUE *argv, VALUE ary, enum ary_take_pos
VALUE
rb_ary_push(VALUE ary, VALUE item)
{
- long idx = RARRAY_LEN(ary);
+ long idx = RARRAY_LEN((ary_verify(ary), ary));
VALUE target_ary = ary_ensure_room_for_push(ary, 1);
RARRAY_PTR_USE(ary, ptr, {
RB_OBJ_WRITE(target_ary, &ptr[idx], item);
});
ARY_SET_LEN(ary, idx + 1);
+ ary_verify(ary);
return ary;
}
@@ -967,6 +1197,7 @@ rb_ary_pop(VALUE ary)
}
--n;
ARY_SET_LEN(ary, n);
+ ary_verify(ary);
return RARRAY_AREF(ary, n);
}
@@ -1000,6 +1231,7 @@ rb_ary_pop_m(int argc, VALUE *argv, VALUE ary)
rb_ary_modify_check(ary);
result = ary_take_first_or_last(argc, argv, ary, ARY_TAKE_LAST);
ARY_INCREASE_LEN(ary, -RARRAY_LEN(result));
+ ary_verify(ary);
return result;
}
@@ -1018,6 +1250,7 @@ rb_ary_shift(VALUE ary)
MEMMOVE(ptr, ptr+1, VALUE, len-1);
}); /* WB: no new reference */
ARY_INCREASE_LEN(ary, -1);
+ ary_verify(ary);
return top;
}
assert(!ARY_EMBED_P(ary)); /* ARY_EMBED_LEN_MAX < ARY_DEFAULT_SIZE */
@@ -1031,6 +1264,8 @@ rb_ary_shift(VALUE ary)
ARY_INCREASE_PTR(ary, 1); /* shift ptr */
ARY_INCREASE_LEN(ary, -1);
+ ary_verify(ary);
+
return top;
}
@@ -1101,6 +1336,7 @@ rb_ary_behead(VALUE ary, long n)
}
ARY_INCREASE_LEN(ary, -n);
+ ary_verify(ary);
return ary;
}
@@ -1120,8 +1356,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
VALUE shared = ARY_SHARED(ary);
capa = RARRAY_LEN(shared);
if (ARY_SHARED_OCCUPIED(shared) && capa > new_len) {
- head = RARRAY_CONST_PTR(ary);
- sharedp = RARRAY_CONST_PTR(shared);
+ head = RARRAY_CONST_PTR_TRANSIENT(ary);
+ sharedp = RARRAY_CONST_PTR_TRANSIENT(shared);
goto makeroom_if_need;
}
}
@@ -1134,11 +1370,13 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
/* use shared array for big "queues" */
if (new_len > ARY_DEFAULT_SIZE * 4) {
+ ary_verify(ary);
+
/* make a room for unshifted items */
capa = ARY_CAPA(ary);
ary_make_shared(ary);
- head = sharedp = RARRAY_CONST_PTR(ary);
+ head = sharedp = RARRAY_CONST_PTR_TRANSIENT(ary);
goto makeroom;
makeroom_if_need:
if (head - sharedp < argc) {
@@ -1151,6 +1389,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
}
ARY_SET_PTR(ary, head - argc);
assert(ARY_SHARED_OCCUPIED(ARY_SHARED(ary)));
+
+ ary_verify(ary);
return ARY_SHARED(ary);
}
else {
@@ -1159,6 +1399,7 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
MEMMOVE(ptr + argc, ptr, VALUE, len);
});
+ ary_verify(ary);
return ary;
}
}
@@ -1576,7 +1817,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
}
{
- const VALUE *optr = RARRAY_CONST_PTR(ary);
+ const VALUE *optr = RARRAY_CONST_PTR_TRANSIENT(ary);
rofs = (rptr >= optr && rptr < optr + olen) ? rptr - optr : -1;
}
@@ -1589,7 +1830,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
len = beg + rlen;
ary_mem_clear(ary, olen, beg - olen);
if (rlen > 0) {
- if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs;
+ if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs;
ary_memcpy0(ary, beg, rlen, rptr, target_ary);
}
ARY_SET_LEN(ary, len);
@@ -1613,7 +1854,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
ARY_SET_LEN(ary, alen);
}
if (rlen > 0) {
- if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs;
+ if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs;
/* give up wb-protected ary */
MEMMOVE(RARRAY_PTR(ary) + beg, rptr, VALUE, rlen);
}
@@ -1673,11 +1914,12 @@ rb_ary_resize(VALUE ary, long len)
}
else {
if (olen > len + ARY_DEFAULT_SIZE) {
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, len, RARRAY(ary)->as.heap.aux.capa);
+ ary_heap_realloc(ary, len);
ARY_SET_CAPA(ary, len);
}
ARY_SET_HEAP_LEN(ary, len);
}
+ ary_verify(ary);
return ary;
}
@@ -1738,7 +1980,7 @@ rb_ary_aset(int argc, VALUE *argv, VALUE ary)
/* check if idx is Range */
range:
rpl = rb_ary_to_ary(argv[argc-1]);
- rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR(rpl), RARRAY_LEN(rpl));
+ rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR_TRANSIENT(rpl), RARRAY_LEN(rpl));
RB_GC_GUARD(rpl);
return argv[argc-1];
}
@@ -1820,7 +2062,7 @@ VALUE
rb_ary_each(VALUE ary)
{
long i;
-
+ ary_verify(ary);
RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length);
for (i=0; i<RARRAY_LEN(ary); i++) {
rb_yield(RARRAY_AREF(ary, i));
@@ -1930,15 +2172,18 @@ rb_ary_dup(VALUE ary)
{
long len = RARRAY_LEN(ary);
VALUE dup = rb_ary_new2(len);
- ary_memcpy(dup, 0, len, RARRAY_CONST_PTR(ary));
+ ary_memcpy(dup, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_LEN(dup, len);
+
+ ary_verify(ary);
+ ary_verify(dup);
return dup;
}
VALUE
rb_ary_resurrect(VALUE ary)
{
- return rb_ary_new4(RARRAY_LEN(ary), RARRAY_CONST_PTR(ary));
+ return rb_ary_new4(RARRAY_LEN(ary), RARRAY_CONST_PTR_TRANSIENT(ary));
}
extern VALUE rb_output_fs;
@@ -2279,8 +2524,8 @@ rb_ary_reverse_m(VALUE ary)
VALUE dup = rb_ary_new2(len);
if (len > 0) {
- const VALUE *p1 = RARRAY_CONST_PTR(ary);
- VALUE *p2 = (VALUE *)RARRAY_CONST_PTR(dup) + len - 1;
+ const VALUE *p1 = RARRAY_CONST_PTR_TRANSIENT(ary);
+ VALUE *p2 = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(dup) + len - 1;
do *p2-- = *p1++; while (--len > 0);
}
ARY_SET_LEN(dup, RARRAY_LEN(ary));
@@ -2382,7 +2627,7 @@ rb_ary_rotate_m(int argc, VALUE *argv, VALUE ary)
rotated = rb_ary_new2(len);
if (len > 0) {
cnt = rotate_count(cnt, len);
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
len -= cnt;
ary_memcpy(rotated, 0, len, ptr + cnt);
ary_memcpy(rotated, len, cnt, ptr);
@@ -2482,7 +2727,6 @@ rb_ary_sort_bang(VALUE ary)
VALUE tmp = ary_make_substitution(ary); /* only ary refers tmp */
struct ary_sort_data data;
long len = RARRAY_LEN(ary);
-
RBASIC_CLEAR_CLASS(tmp);
data.ary = tmp;
data.cmp_opt.opt_methods = 0;
@@ -2515,11 +2759,11 @@ rb_ary_sort_bang(VALUE ary)
rb_ary_unshare(ary);
}
else {
- ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ ary_heap_free(ary);
}
- ARY_SET_PTR(ary, RARRAY_CONST_PTR(tmp));
+ ARY_SET_PTR(ary, ARY_HEAP_PTR(tmp));
ARY_SET_HEAP_LEN(ary, len);
- ARY_SET_CAPA(ary, RARRAY_LEN(tmp));
+ ARY_SET_CAPA(ary, ARY_HEAP_LEN(tmp));
}
/* tmp was lost ownership for the ptr */
FL_UNSET(tmp, FL_FREEZE);
@@ -2530,6 +2774,7 @@ rb_ary_sort_bang(VALUE ary)
/* tmp will be GC'ed. */
RBASIC_SET_CLASS_RAW(tmp, rb_cArray); /* rb_cArray must be marked */
}
+ ary_verify(ary);
return ary;
}
@@ -2833,7 +3078,7 @@ append_values_at_single(VALUE result, VALUE ary, long olen, VALUE idx)
/* check if idx is Range */
else if (rb_range_beg_len(idx, &beg, &len, olen, 1)) {
if (len > 0) {
- const VALUE *const src = RARRAY_CONST_PTR(ary);
+ const VALUE *const src = RARRAY_CONST_PTR_TRANSIENT(ary);
const long end = beg + len;
const long prevlen = RARRAY_LEN(result);
if (beg < olen) {
@@ -3075,6 +3320,7 @@ rb_ary_delete(VALUE ary, VALUE item)
ary_resize_smaller(ary, i2);
+ ary_verify(ary);
return v;
}
@@ -3119,7 +3365,7 @@ rb_ary_delete_at(VALUE ary, long pos)
MEMMOVE(ptr+pos, ptr+pos+1, VALUE, len-pos-1);
});
ARY_INCREASE_LEN(ary, -1);
-
+ ary_verify(ary);
return del;
}
@@ -3187,7 +3433,7 @@ rb_ary_slice_bang(int argc, VALUE *argv, VALUE ary)
len = orig_len - pos;
}
if (len == 0) return rb_ary_new2(0);
- arg2 = rb_ary_new4(len, RARRAY_CONST_PTR(ary)+pos);
+ arg2 = rb_ary_new4(len, RARRAY_CONST_PTR_TRANSIENT(ary)+pos);
RBASIC_SET_CLASS(arg2, rb_obj_class(ary));
rb_ary_splice(ary, pos, len, 0, 0);
return arg2;
@@ -3223,6 +3469,7 @@ ary_reject(VALUE orig, VALUE result)
for (i = 0; i < RARRAY_LEN(orig); i++) {
VALUE v = RARRAY_AREF(orig, i);
+
if (!RTEST(rb_yield(v))) {
rb_ary_push(result, v);
}
@@ -3252,7 +3499,6 @@ static VALUE
ary_reject_bang(VALUE ary)
{
struct select_bang_arg args;
-
rb_ary_modify_check(ary);
args.ary = ary;
args.len[0] = args.len[1] = 0;
@@ -3326,6 +3572,7 @@ rb_ary_reject(VALUE ary)
static VALUE
rb_ary_delete_if(VALUE ary)
{
+ ary_verify(ary);
RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length);
ary_reject_bang(ary);
return ary;
@@ -3504,14 +3751,14 @@ rb_ary_replace(VALUE copy, VALUE orig)
VALUE shared = 0;
if (ARY_OWNS_HEAP_P(copy)) {
- RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy)));
+ ary_heap_free(copy);
}
else if (ARY_SHARED_P(copy)) {
shared = ARY_SHARED(copy);
FL_UNSET_SHARED(copy);
}
FL_SET_EMBED(copy);
- ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR(orig));
+ ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR_TRANSIENT(orig));
if (shared) {
rb_ary_decrement_share(shared);
}
@@ -3520,16 +3767,17 @@ rb_ary_replace(VALUE copy, VALUE orig)
else {
VALUE shared = ary_make_shared(orig);
if (ARY_OWNS_HEAP_P(copy)) {
- RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy)));
+ ary_heap_free(copy);
}
else {
rb_ary_unshare_safe(copy);
}
FL_UNSET_EMBED(copy);
- ARY_SET_PTR(copy, RARRAY_CONST_PTR(orig));
- ARY_SET_LEN(copy, RARRAY_LEN(orig));
+ ARY_SET_PTR(copy, ARY_HEAP_PTR(orig));
+ ARY_SET_LEN(copy, ARY_HEAP_LEN(orig));
rb_ary_set_shared(copy, shared);
}
+ ary_verify(copy);
return copy;
}
@@ -3547,16 +3795,20 @@ VALUE
rb_ary_clear(VALUE ary)
{
rb_ary_modify_check(ary);
- ARY_SET_LEN(ary, 0);
if (ARY_SHARED_P(ary)) {
if (!ARY_EMBED_P(ary)) {
rb_ary_unshare(ary);
FL_SET_EMBED(ary);
+ ARY_SET_EMBED_LEN(ary, 0);
}
}
- else if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) {
- ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2);
+ else {
+ ARY_SET_LEN(ary, 0);
+ if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) {
+ ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2);
+ }
}
+ ary_verify(ary);
return ary;
}
@@ -3689,8 +3941,8 @@ rb_ary_plus(VALUE x, VALUE y)
len = xlen + ylen;
z = rb_ary_new2(len);
- ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR(x));
- ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR(y));
+ ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR_TRANSIENT(x));
+ ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR_TRANSIENT(y));
ARY_SET_LEN(z, len);
return z;
}
@@ -3700,7 +3952,7 @@ ary_append(VALUE x, VALUE y)
{
long n = RARRAY_LEN(y);
if (n > 0) {
- rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR(y), n);
+ rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR_TRANSIENT(y), n);
}
return x;
}
@@ -3742,6 +3994,7 @@ rb_ary_concat_multi(int argc, VALUE *argv, VALUE ary)
ary_append(ary, args);
}
+ ary_verify(ary);
return ary;
}
@@ -3796,16 +4049,16 @@ rb_ary_times(VALUE ary, VALUE times)
ary2 = ary_new(rb_obj_class(ary), len);
ARY_SET_LEN(ary2, len);
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
t = RARRAY_LEN(ary);
if (0 < t) {
ary_memcpy(ary2, 0, t, ptr);
while (t <= len/2) {
- ary_memcpy(ary2, t, t, RARRAY_CONST_PTR(ary2));
+ ary_memcpy(ary2, t, t, RARRAY_CONST_PTR_TRANSIENT(ary2));
t *= 2;
}
if (t < len) {
- ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR(ary2));
+ ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR_TRANSIENT(ary2));
}
}
out:
@@ -3891,6 +4144,7 @@ recursive_equal(VALUE ary1, VALUE ary2, int recur)
if (recur) return Qtrue; /* Subtle! */
+ /* rb_equal() can evacuate ptrs */
p1 = RARRAY_CONST_PTR(ary1);
p2 = RARRAY_CONST_PTR(ary2);
len1 = RARRAY_LEN(ary1);
@@ -3941,7 +4195,7 @@ rb_ary_equal(VALUE ary1, VALUE ary2)
return rb_equal(ary2, ary1);
}
if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse;
- if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue;
+ if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue;
return rb_exec_recursive_paired(recursive_equal, ary1, ary2, ary2);
}
@@ -3972,7 +4226,7 @@ rb_ary_eql(VALUE ary1, VALUE ary2)
if (ary1 == ary2) return Qtrue;
if (!RB_TYPE_P(ary2, T_ARRAY)) return Qfalse;
if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse;
- if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue;
+ if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue;
return rb_exec_recursive_paired(recursive_eql, ary1, ary2, ary2);
}
@@ -4679,14 +4933,14 @@ rb_ary_compact_bang(VALUE ary)
long n;
rb_ary_modify(ary);
- p = t = (VALUE *)RARRAY_CONST_PTR(ary); /* WB: no new reference */
+ p = t = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(ary); /* WB: no new reference */
end = p + RARRAY_LEN(ary);
while (t < end) {
if (NIL_P(*t)) t++;
else *p++ = *t++;
}
- n = p - RARRAY_CONST_PTR(ary);
+ n = p - RARRAY_CONST_PTR_TRANSIENT(ary);
if (RARRAY_LEN(ary) == n) {
return Qnil;
}
@@ -4949,7 +5203,7 @@ rb_ary_shuffle_bang(int argc, VALUE *argv, VALUE ary)
while (i) {
long j = RAND_UPTO(i);
VALUE tmp;
- if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR(ary)) {
+ if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR_TRANSIENT(ary)) {
rb_raise(rb_eRuntimeError, "modified during shuffle");
}
tmp = ptr[--i];
diff --git a/common.mk b/common.mk
index 42584f65a2..b5f391d150 100644
--- a/common.mk
+++ b/common.mk
@@ -130,6 +130,7 @@ COMMONOBJS = array.$(OBJEXT) \
thread.$(OBJEXT) \
time.$(OBJEXT) \
transcode.$(OBJEXT) \
+ transient_heap.$(OBJEXT) \
util.$(OBJEXT) \
variable.$(OBJEXT) \
version.$(OBJEXT) \
@@ -2898,6 +2899,15 @@ transcode.$(OBJEXT): {$(VPATH)}st.h
transcode.$(OBJEXT): {$(VPATH)}subst.h
transcode.$(OBJEXT): {$(VPATH)}transcode.c
transcode.$(OBJEXT): {$(VPATH)}transcode_data.h
+transient_heap.$(OBJEXT): $(hdrdir)/ruby/ruby.h
+transient_heap.$(OBJEXT): $(top_srcdir)/include/ruby.h
+transient_heap.$(OBJEXT): {$(VPATH)}debug_counter.h
+transient_heap.$(OBJEXT): {$(VPATH)}gc.h
+transient_heap.$(OBJEXT): {$(VPATH)}internal.h
+transient_heap.$(OBJEXT): {$(VPATH)}ruby_assert.h
+transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c
+transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.h
+transient_heap.$(OBJEXT): {$(VPATH)}vm_debug.h
util.$(OBJEXT): $(hdrdir)/ruby/ruby.h
util.$(OBJEXT): $(top_srcdir)/include/ruby.h
util.$(OBJEXT): {$(VPATH)}config.h
diff --git a/compile.c b/compile.c
index 4f98ac7997..58575ec929 100644
--- a/compile.c
+++ b/compile.c
@@ -1649,7 +1649,7 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons
opt_table = ALLOC_N(VALUE, i+1);
- MEMCPY(opt_table, RARRAY_CONST_PTR(labels), VALUE, i+1);
+ MEMCPY(opt_table, RARRAY_CONST_PTR_TRANSIENT(labels), VALUE, i+1);
for (j = 0; j < i+1; j++) {
opt_table[j] &= ~1;
}
@@ -2297,14 +2297,14 @@ iseq_set_exception_table(rb_iseq_t *iseq)
struct iseq_catch_table_entry *entry;
tlen = (int)RARRAY_LEN(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
- tptr = RARRAY_CONST_PTR(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
+ tptr = RARRAY_CONST_PTR_TRANSIENT(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
if (tlen > 0) {
struct iseq_catch_table *table = xmalloc(iseq_catch_table_bytes(tlen));
table->size = tlen;
for (i = 0; i < table->size; i++) {
- ptr = RARRAY_CONST_PTR(tptr[i]);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(tptr[i]);
entry = &table->entries[i];
entry->type = (enum catch_type)(ptr[0] & 0xffff);
entry->start = label_get_position((LABEL *)(ptr[1] & ~1));
diff --git a/debug_counter.h b/debug_counter.h
index 3fcb562f65..c151e535b0 100644
--- a/debug_counter.h
+++ b/debug_counter.h
@@ -219,6 +219,11 @@ RB_DEBUG_COUNTER(heap_xmalloc)
RB_DEBUG_COUNTER(heap_xrealloc)
RB_DEBUG_COUNTER(heap_xfree)
+/* transient_heap */
+RB_DEBUG_COUNTER(theap_alloc)
+RB_DEBUG_COUNTER(theap_alloc_fail)
+RB_DEBUG_COUNTER(theap_evacuate)
+
/* load (not implemented yet) */
/*
RB_DEBUG_COUNTER(load_files)
diff --git a/enum.c b/enum.c
index 71e7a74d35..3be6941c3a 100644
--- a/enum.c
+++ b/enum.c
@@ -14,6 +14,7 @@
#include "ruby/util.h"
#include "id.h"
#include "symbol.h"
+#include "transient_heap.h"
#include <assert.h>
@@ -1171,9 +1172,10 @@ enum_sort_by(VALUE obj)
rb_ary_concat(ary, buf);
}
if (RARRAY_LEN(ary) > 2) {
+ rb_ary_transient_heap_evacuate(ary, TRUE); /* should be malloc heap */
RARRAY_PTR_USE(ary, ptr,
- ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
- sort_by_cmp, (void *)ary));
+ ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
+ sort_by_cmp, (void *)ary));
}
if (RBASIC(ary)->klass) {
rb_raise(rb_eRuntimeError, "sort_by reentered");
diff --git a/gc.c b/gc.c
index 00f51c9657..57619af9f3 100644
--- a/gc.c
+++ b/gc.c
@@ -35,6 +35,7 @@
#include <sys/types.h>
#include "ruby_assert.h"
#include "debug_counter.h"
+#include "transient_heap.h"
#include "mjit.h"
#undef rb_data_object_wrap
@@ -845,8 +846,6 @@ static void rb_objspace_call_finalizer(rb_objspace_t *objspace);
static VALUE define_final0(VALUE obj, VALUE block);
static void negative_size_allocation_error(const char *);
-static void *aligned_malloc(size_t, size_t);
-static void aligned_free(void *);
static void init_mark_stack(mark_stack_t *stack);
@@ -1190,6 +1189,7 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
+ rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
@@ -1486,7 +1486,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
- aligned_free(GET_PAGE_BODY(page->start));
+ rb_aligned_free(GET_PAGE_BODY(page->start));
free(page);
}
@@ -1524,7 +1524,7 @@ heap_page_allocate(rb_objspace_t *objspace)
int limit = HEAP_PAGE_OBJ_LIMIT;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
- page_body = (struct heap_page_body *)aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
+ page_body = (struct heap_page_body *)rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
if (page_body == 0) {
rb_memerror();
}
@@ -1532,7 +1532,7 @@ heap_page_allocate(rb_objspace_t *objspace)
/* assign heap_page entry */
page = (struct heap_page *)calloc(1, sizeof(struct heap_page));
if (page == 0) {
- aligned_free(page_body);
+ rb_aligned_free(page_body);
rb_memerror();
}
@@ -4515,6 +4515,7 @@ static void
gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
{
if (LIKELY(objspace->mark_func_data == NULL)) {
+ if (RB_TYPE_P(obj, T_NONE)) rb_bug("...");
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
gc_aging(objspace, obj);
@@ -4672,14 +4673,22 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
case T_ARRAY:
if (FL_TEST(obj, ELTS_SHARED)) {
- gc_mark(objspace, any->as.array.as.heap.aux.shared);
+ VALUE root = any->as.array.as.heap.aux.shared;
+ gc_mark(objspace, root);
}
else {
long i, len = RARRAY_LEN(obj);
- const VALUE *ptr = RARRAY_CONST_PTR(obj);
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(obj);
for (i=0; i < len; i++) {
- gc_mark(objspace, *ptr++);
+ gc_mark(objspace, ptr[i]);
}
+
+ if (objspace->mark_func_data == NULL) {
+ if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) &&
+ RARRAY_TRANSIENT_P(obj)) {
+ rb_transient_heap_mark(obj, ptr);
+ }
+ }
}
break;
@@ -5455,6 +5464,13 @@ rb_gc_verify_internal_consistency(void)
gc_verify_internal_consistency(Qnil);
}
+static VALUE
+gc_verify_transient_heap_internal_consistency(VALUE dmy)
+{
+ rb_transient_heap_verify();
+ return Qnil;
+}
+
/* marks */
static void
@@ -5671,6 +5687,8 @@ gc_marks_finish(rb_objspace_t *objspace)
#endif
}
+ rb_transient_heap_finish_marking();
+
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
return TRUE;
@@ -6562,6 +6580,7 @@ gc_start(rb_objspace_t *objspace, int reason)
objspace->profile.heap_used_at_gc_start = heap_allocated_pages;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace);
+ rb_transient_heap_start_marking(do_full_mark);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
GC_ASSERT(during_gc);
@@ -7812,8 +7831,8 @@ rb_memerror(void)
EC_JUMP_TAG(ec, TAG_RAISE);
}
-static void *
-aligned_malloc(size_t alignment, size_t size)
+void *
+rb_aligned_malloc(size_t alignment, size_t size)
{
void *res;
@@ -7846,8 +7865,8 @@ aligned_malloc(size_t alignment, size_t size)
return res;
}
-static void
-aligned_free(void *ptr)
+void
+rb_aligned_free(void *ptr)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
@@ -9551,13 +9570,21 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
#if USE_RGENGC
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
- snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
- (void *)obj, age,
- C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
- C(RVALUE_MARK_BITMAP(obj), "M"),
- C(RVALUE_MARKING_BITMAP(obj), "R"),
- C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
- obj_type_name(obj));
+ if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
+ snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
+ (void *)obj, age,
+ C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
+ C(RVALUE_MARK_BITMAP(obj), "M"),
+ C(RVALUE_MARKING_BITMAP(obj), "R"),
+ C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
+ obj_type_name(obj));
+ }
+ else {
+ /* fake */
+ snprintf(buff, buff_size, "%p [%dXXXX] %s",
+ (void *)obj, age,
+ obj_type_name(obj));
+ }
#else
snprintf(buff, buff_size, "%p [%s] %s",
(void *)obj,
@@ -9587,10 +9614,25 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
UNEXPECTED_NODE(rb_raw_obj_info);
break;
case T_ARRAY:
- snprintf(buff, buff_size, "%s [%s%s] len: %d", buff,
- C(ARY_EMBED_P(obj), "E"),
- C(ARY_SHARED_P(obj), "S"),
- (int)RARRAY_LEN(obj));
+ if (FL_TEST(obj, ELTS_SHARED)) {
+ snprintf(buff, buff_size, "%s shared -> %s", buff,
+ rb_obj_info(RARRAY(obj)->as.heap.aux.shared));
+ }
+ else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) {
+ snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff,
+ C(ARY_EMBED_P(obj), "E"),
+ C(ARY_SHARED_P(obj), "S"),
+ (int)RARRAY_LEN(obj));
+ }
+ else {
+ snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff,
+ C(ARY_EMBED_P(obj), "E"),
+ C(ARY_SHARED_P(obj), "S"),
+ C(RARRAY_TRANSIENT_P(obj), "T"),
+ (int)RARRAY_LEN(obj),
+ ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa,
+ RARRAY_CONST_PTR_TRANSIENT(obj));
+ }
break;
case T_STRING: {
snprintf(buff, buff_size, "%s %s", buff, RSTRING_PTR(obj));
@@ -9954,6 +9996,7 @@ Init_GC(void)
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0);
+ rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0);
#if MALLOC_ALLOCATED_SIZE
rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index a05651ac3b..1512e78179 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -1020,12 +1020,15 @@ enum ruby_rarray_flags {
RARRAY_EMBED_LEN_MASK = (RUBY_FL_USER4|RUBY_FL_USER3),
RARRAY_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+3),
+ RARRAY_TRANSIENT_FLAG = RUBY_FL_USER13,
+
RARRAY_ENUM_END
};
#define RARRAY_EMBED_FLAG (VALUE)RARRAY_EMBED_FLAG
#define RARRAY_EMBED_LEN_MASK (VALUE)RARRAY_EMBED_LEN_MASK
#define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX
#define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT
+#define RARRAY_TRANSIENT_FLAG RARRAY_TRANSIENT_FLAG
struct RArray {
struct RBasic basic;
union {
@@ -1046,9 +1049,14 @@ struct RArray {
#define RARRAY_LEN(a) rb_array_len(a)
#define RARRAY_LENINT(ary) rb_long2int(RARRAY_LEN(ary))
#define RARRAY_CONST_PTR(a) rb_array_const_ptr(a)
+#define RARRAY_CONST_PTR_TRANSIENT(a) rb_array_const_ptr_transient(a)
+#define RARRAY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG)
+
+VALUE *rb_ary_ptr_use_start(VALUE ary);
+void rb_ary_ptr_use_end(VALUE ary);
-#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR(a))
-#define RARRAY_PTR_USE_END(a) /* */
+#define RARRAY_PTR_USE_START(a) rb_ary_ptr_use_start(a)
+#define RARRAY_PTR_USE_END(a) rb_ary_ptr_use_end(a)
#define RARRAY_PTR_USE(ary, ptr_name, expr) do { \
const VALUE _ary = (ary); \
@@ -1057,11 +1065,12 @@ struct RArray {
RARRAY_PTR_USE_END(_ary); \
} while (0)
-#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR(a)[i])
+#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR_TRANSIENT(a)[i])
#define RARRAY_ASET(a, i, v) do { \
const VALUE _ary = (a); \
+ const VALUE _v = (v); \
VALUE *ptr = (VALUE *)RARRAY_PTR_USE_START(_ary); \
- RB_OBJ_WRITE(_ary, &ptr[i], (v)); \
+ RB_OBJ_WRITE(_ary, &ptr[i], _v); \
RARRAY_PTR_USE_END(_ary); \
} while (0)
@@ -2110,12 +2119,23 @@ rb_array_len(VALUE a)
#endif
static inline const VALUE *
-rb_array_const_ptr(VALUE a)
+rb_array_const_ptr_transient(VALUE a)
{
return FIX_CONST_VALUE_PTR((RBASIC(a)->flags & RARRAY_EMBED_FLAG) ?
RARRAY(a)->as.ary : RARRAY(a)->as.heap.ptr);
}
+void rb_ary_detransient(VALUE a);
+
+static inline const VALUE *
+rb_array_const_ptr(VALUE a)
+{
+ if (RARRAY_TRANSIENT_P(a)) {
+ rb_ary_detransient(a);
+ }
+ return rb_array_const_ptr_transient(a);
+}
+
#if defined(EXTLIB) && defined(USE_DLN_A_OUT)
/* hook for external modules */
static char *dln_libs_to_be_linked[] = { EXTLIB, 0 };
diff --git a/inits.c b/inits.c
index c9687de516..7eb543104f 100644
--- a/inits.c
+++ b/inits.c
@@ -16,6 +16,7 @@
void
rb_call_inits(void)
{
+ CALL(TransientHeap);
CALL(Method);
CALL(RandomSeedCore);
CALL(sym);
diff --git a/insns.def b/insns.def
index 132ce2f179..747e6ef8d4 100644
--- a/insns.def
+++ b/insns.def
@@ -524,7 +524,7 @@ newhashfromarray
{
VM_ASSERT(num * 2 == (rb_num_t)RARRAY_LEN(ary));
hash = rb_hash_new_with_size(num);
- rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR(ary), hash);
+ rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR_TRANSIENT(ary), hash);
}
/* put new Range object.(Range.new(low, high, flag)) */
diff --git a/internal.h b/internal.h
index 697a1196fa..ffe84528db 100644
--- a/internal.h
+++ b/internal.h
@@ -1073,6 +1073,26 @@ VALUE rb_gvar_set(struct rb_global_entry *, VALUE);
VALUE rb_gvar_defined(struct rb_global_entry *);
/* array.c */
+
+#ifndef ARRAY_DEBUG
+#define ARRAY_DEBUG 0
+#endif
+
+#ifdef ARRAY_DEBUG
+#define RARRAY_PTR_IN_USE_FLAG FL_USER14
+#define ARY_PTR_USING_P(ary) FL_TEST_RAW((ary), RARRAY_PTR_IN_USE_FLAG)
+
+#else
+
+/* disable debug function */
+#undef RARRAY_PTR_USE_START
+#undef RARRAY_PTR_USE_END
+#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR_TRANSIENT(a))
+#define RARRAY_PTR_USE_END(a)
+#define ARY_PTR_USING_P(ary) 0
+
+#endif
+
VALUE rb_ary_last(int, const VALUE *, VALUE);
void rb_ary_set_len(VALUE, long);
void rb_ary_delete_same(VALUE, VALUE);
@@ -1100,7 +1120,7 @@ static inline VALUE
rb_ary_entry_internal(VALUE ary, long offset)
{
long len = RARRAY_LEN(ary);
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
if (len == 0) return Qnil;
if (offset < 0) {
offset += len;
@@ -1337,6 +1357,9 @@ RUBY_SYMBOL_EXPORT_END
rb_wb_unprotected_newobj_of(klass, flags))
#define NEWOBJ_OF(obj,type,klass,flags) RB_NEWOBJ_OF(obj,type,klass,flags)
+void *rb_aligned_malloc(size_t, size_t);
+void rb_aligned_free(void *);
+
/* hash.c */
struct st_table *rb_hash_tbl_raw(VALUE hash);
VALUE rb_hash_new_with_size(st_index_t size);
diff --git a/string.c b/string.c
index 3449ce6b53..48718f93ac 100644
--- a/string.c
+++ b/string.c
@@ -2006,8 +2006,8 @@ rb_str_format_m(VALUE str, VALUE arg)
VALUE tmp = rb_check_array_type(arg);
if (!NIL_P(tmp)) {
- VALUE rv = rb_str_format(RARRAY_LENINT(tmp), RARRAY_CONST_PTR(tmp), str);
- RB_GC_GUARD(tmp);
+ const long len = RARRAY_LENINT(tmp);
+ VALUE rv = rb_str_format(len, RARRAY_CONST_PTR(tmp), str);
return rv;
}
return rb_str_format(1, &arg, str);
diff --git a/test/ruby/test_enum.rb b/test/ruby/test_enum.rb
index a4eace2d57..c56e280e06 100644
--- a/test/ruby/test_enum.rb
+++ b/test/ruby/test_enum.rb
@@ -1115,4 +1115,19 @@ class TestEnumerable < Test::Unit::TestCase
assert_equal([1, 2, 3, 4, 5, 10], (1..100).uniq{|x| (x**2) % 10 }.first(6))
assert_equal([1, [1, 2]], Foo.new.to_enum.uniq)
end
+
+ def test_transient_heap_sort_by
+ klass = Class.new do
+ include Comparable
+ attr_reader :i
+ def initialize e
+ @i = e
+ end
+ def <=> other
+ GC.start
+ i <=> other.i
+ end
+ end
+ assert_equal [1, 2, 3, 4, 5], (1..5).sort_by{|e| klass.new e}
+ end
end
diff --git a/transient_heap.c b/transient_heap.c
new file mode 100644
index 0000000000..e9a2460d88
--- /dev/null
+++ b/transient_heap.c
@@ -0,0 +1,834 @@
+/**********************************************************************
+
+ transient_heap.c - implement transient_heap.
+
+ Copyright (C) 2018 Koichi Sasada
+
+**********************************************************************/
+
+#include "ruby/ruby.h"
+#include "ruby/debug.h"
+#include "vm_debug.h"
+#include "gc.h"
+#include "internal.h"
+#include "ruby_assert.h"
+#include "transient_heap.h"
+#include "debug_counter.h"
+
+/*
+ * 1: enable assertions
+ * 2: enable verify all transient heaps
+ */
+#ifndef TRANSIENT_HEAP_CHECK_MODE
+#define TRANSIENT_HEAP_CHECK_MODE 0
+#endif
+#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
+
+/*
+ * 1: show events
+ * 2: show dump at events
+ * 3: show all operations
+ */
+#define TRANSIENT_HEAP_DEBUG 0
+
+/* For Debug: Provide blocks infinitely.
+ * This mode generates blocks unlimitedly
+ * and prohibit access free'ed blocks to check invalid access.
+ */
+#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+#include <sys/mman.h>
+#include <errno.h>
+#endif
+
+/* For Debug: Prohibit promoting to malloc space.
+ */
+#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
+
+/* size configuration */
+#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
+
+ /* K M */
+#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
+#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
+#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
+#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
+
+#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
+#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
+
+#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
+#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
+
+enum transient_heap_status {
+ transient_heap_none,
+ transient_heap_marking,
+ transient_heap_escaping
+};
+
+struct transient_heap_block {
+ struct transient_heap_block_header {
+ int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */
+ int16_t index;
+ int16_t last_marked_index;
+ int16_t objects;
+ struct transient_heap_block *next_block;
+ } info;
+ char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)];
+};
+
+struct transient_heap {
+ struct transient_heap_block *using_blocks;
+ struct transient_heap_block *marked_blocks;
+ struct transient_heap_block *free_blocks;
+ int total_objects;
+ int total_marked_objects;
+ int total_blocks;
+ enum transient_heap_status status;
+
+ VALUE *promoted_objects;
+ int promoted_objects_size;
+ int promoted_objects_index;
+
+ struct transient_heap_block *arena;
+ int arena_index; /* increment only */
+};
+
+struct transient_alloc_header {
+ uint16_t magic;
+ uint16_t size;
+ int16_t next_marked_index;
+ int16_t dummy;
+ VALUE obj;
+};
+
+static struct transient_heap global_transient_heap;
+
+static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
+static const void *transient_heap_ptr(VALUE obj, int error);
+static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
+
+#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
+
+static void
+transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
+{
+ int i=0, n=0;
+ struct transient_alloc_header *header = NULL;
+
+ while (i<block->info.index) {
+ header = (void *)&block->buff[i];
+ fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, header, header->size, header->next_marked_index, rb_obj_info(header->obj));
+ i += header->size;
+ n++;
+ }
+}
+
+static void
+transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
+{
+ while (block) {
+ fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
+ type_str, block, block->info.index, block->info.objects, block->info.last_marked_index, block->info.next_block);
+
+ transient_heap_block_dump(theap, block);
+ block = block->info.next_block;
+ }
+}
+
+static void
+transient_heap_dump(struct transient_heap* theap)
+{
+ fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
+ transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
+ transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
+ transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
+}
+
+/* Debug: dump all tarnsient_heap blocks */
+void
+rb_transient_heap_dump(void)
+{
+ transient_heap_dump(&global_transient_heap);
+}
+
+#if TRANSIENT_HEAP_CHECK_MODE >= 2
+static void
+transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
+{
+ if (obj != Qundef) {
+ const void *ptr = transient_heap_ptr(obj, FALSE);
+ TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
+ }
+}
+
+static int
+transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ int i=0, n=0;
+ struct transient_alloc_header *header;
+
+ while (i<block->info.index) {
+ header = (void *)&block->buff[i];
+ TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
+ transient_heap_ptr_check(theap, header->obj);
+ n ++;
+ i += header->size;
+ }
+ TH_ASSERT(block->info.objects == n);
+
+ return n;
+}
+
+static int
+transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
+{
+ int n = 0;
+ struct transient_heap_block *block = blocks;
+ while (block) {
+ n += transient_heap_block_verify(theap, block);
+ *block_num_ptr += 1;
+ block = block->info.next_block;
+ }
+
+ return n;
+}
+#endif
+
+static void
+transient_heap_verify(struct transient_heap *theap)
+{
+#if TRANSIENT_HEAP_CHECK_MODE >= 2
+ int n=0, block_num=0;
+
+ n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
+ n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
+
+ TH_ASSERT(n == theap->total_objects);
+ TH_ASSERT(n >= theap->total_marked_objects);
+ TH_ASSERT(block_num == theap->total_blocks);
+#endif
+}
+
+/* Debug: check assertions for all transient_heap blocks */
+void
+rb_transient_heap_verify(void)
+{
+ transient_heap_verify(&global_transient_heap);
+}
+
+static struct transient_heap*
+transient_heap_get(void)
+{
+ struct transient_heap* theap = &global_transient_heap;
+ transient_heap_verify(theap);
+ return theap;
+}
+
+static void
+reset_block(struct transient_heap_block *block)
+{
+ block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header);
+ block->info.index = 0;
+ block->info.objects = 0;
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+ block->info.next_block = NULL;
+}
+
+static void
+connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->free_blocks;
+ theap->free_blocks = block;
+}
+
+static void
+connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->using_blocks;
+ theap->using_blocks = block;
+}
+
+#if 0
+static void
+connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->marked_blocks;
+ theap->marked_blocks = block;
+}
+#endif
+
+static void
+append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
+{
+ if (theap->marked_blocks) {
+ struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
+ while (block) {
+ last_block = block;
+ block = block->info.next_block;
+ }
+
+ TH_ASSERT(last_block->info.next_block == NULL);
+ last_block->info.next_block = append_blocks;
+ }
+ else {
+ theap->marked_blocks = append_blocks;
+ }
+}
+
+static struct transient_heap_block *
+transient_heap_block_alloc(struct transient_heap* theap)
+{
+ struct transient_heap_block *block;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS,
+ -1, 0);
+ if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
+#else
+ if (theap->arena == NULL) {
+ theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
+ }
+
+ TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
+ block = &theap->arena[theap->arena_index++];
+ TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
+#endif
+ reset_block(block);
+
+ TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
+ if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, block);
+ return block;
+}
+
+
+static struct transient_heap_block *
+transient_heap_allocatable_block(struct transient_heap* theap)
+{
+ struct transient_heap_block *block;
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = transient_heap_block_alloc(theap);
+ theap->total_blocks++;
+#else
+ /* get one block from free_blocks */
+ block = theap->free_blocks;
+ if (block) {
+ theap->free_blocks = block->info.next_block;
+ block->info.next_block = NULL;
+ theap->total_blocks++;
+ }
+#endif
+
+ return block;
+}
+
+static struct transient_alloc_header *
+transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
+{
+ struct transient_heap_block *block = theap->using_blocks;
+
+ while (block) {
+ TH_ASSERT(block->info.size >= block->info.index);
+
+ if (block->info.size - block->info.index >= (int32_t)size) {
+ struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
+ block->info.index += size;
+ block->info.objects++;
+ return header;
+ }
+ else {
+ block = transient_heap_allocatable_block(theap);
+ if (block) connect_to_using_blocks(theap, block);
+ }
+ }
+
+ return NULL;
+}
+
+void *
+rb_transient_heap_alloc(VALUE obj, size_t req_size)
+{
+ struct transient_heap* theap = transient_heap_get();
+ size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
+
+ TH_ASSERT(RB_TYPE_P(obj, T_ARRAY)); /* supported types */
+
+ if (size > TRANSIENT_HEAP_ALLOC_MAX) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
+ return NULL;
+ }
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
+ else if (RB_OBJ_PROMOTED_RAW(obj)) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
+ return NULL;
+ }
+#else
+ else if (RBASIC_CLASS(obj) == 0) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
+ return NULL;
+ }
+#endif
+ else {
+ struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
+ if (header) {
+ void *ptr;
+
+ header->size = size;
+ header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
+ header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
+ header->obj = obj; /* TODO: can we eliminate it? */
+ ptr = header + 1;
+
+ theap->total_objects++; /* statistics */
+
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
+ if (RB_OBJ_PROMOTED_RAW(obj)) {
+ transient_heap_promote_add(theap, obj);
+ }
+#endif
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", header, ptr, (int)size, rb_obj_info(obj));
+
+ RB_DEBUG_COUNTER_INC(theap_alloc);
+ return ptr;
+ }
+ else {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
+ RB_DEBUG_COUNTER_INC(theap_alloc_fail);
+ return NULL;
+ }
+ }
+}
+
+void
+Init_TransientHeap(void)
+{
+ int i, block_num;
+ struct transient_heap* theap = transient_heap_get();
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block_num = 0;
+#else
+ TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
+ block_num = TRANSIENT_HEAP_BLOCK_NUM;
+#endif
+ for (i=0; i<block_num; i++) {
+ connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
+ }
+ theap->using_blocks = transient_heap_allocatable_block(theap);
+
+ theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
+ theap->promoted_objects_index = 0;
+ /* should not use ALLOC_N to be free from GC */
+ theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
+ if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
+}
+
+static struct transient_heap_block *
+blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block = blocks;
+
+ while (block) {
+ if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) {
+ return block;
+ }
+ block = block->info.next_block;
+ }
+
+ return NULL;
+}
+
+static struct transient_heap_block *
+alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block;
+
+ if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
+ return block;
+ }
+ else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
+ return block;
+ }
+ else {
+ return NULL;
+ }
+ return block;
+}
+
+static struct transient_alloc_header *
+ptr_to_alloc_header(const void *ptr)
+{
+ struct transient_alloc_header *header = (void *)ptr;
+ header -= 1;
+ return header;
+}
+
+static int
+transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
+{
+ if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
+ return TRUE;
+ }
+ else {
+ return FALSE;
+ }
+}
+
+
+int
+rb_transient_heap_managed_ptr_p(const void *ptr)
+{
+ return transient_header_managed_ptr_p(transient_heap_get(), ptr);
+}
+
+static struct transient_heap_block *
+alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = alloc_header_to_block_verbose(theap, header);
+ if (block == NULL) {
+ transient_heap_dump(theap);
+ rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
+ }
+#else
+ block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
+ TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
+#endif
+ return block;
+}
+
+void
+rb_transient_heap_mark(VALUE obj, const void *ptr)
+{
+ struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
+
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
+
+#if TRANSIENT_HEAP_CHECK_MODE > 0
+ {
+ struct transient_heap* theap = transient_heap_get();
+ TH_ASSERT(theap->status == transient_heap_marking);
+ TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
+
+ if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
+ transient_heap_dump(theap);
+ rb_bug("rb_transient_heap_mark: magic is broken");
+ }
+ else if (header->obj != obj) {
+ transient_heap_dump(theap);
+ rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
+ rb_obj_info(header->obj), rb_obj_info(obj));
+ }
+ }
+#endif
+
+ if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
+ /* already marked */
+ return;
+ }
+ else {
+ struct transient_heap* theap = transient_heap_get();
+ struct transient_heap_block *block = alloc_header_to_block(theap, header);
+ header->next_marked_index = block->info.last_marked_index;
+ block->info.last_marked_index = (int)((char *)header - block->buff);
+ theap->total_marked_objects++;
+
+ transient_heap_verify(theap);
+ }
+}
+
+static const void *
+transient_heap_ptr(VALUE obj, int error)
+{
+ const void *ptr;
+
+ switch (BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+ if (RARRAY_TRANSIENT_P(obj)) {
+ TH_ASSERT(!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG));
+ ptr = RARRAY(obj)->as.heap.ptr;
+ }
+ else {
+ ptr = NULL;
+ }
+ break;
+ default:
+ if (error) {
+ rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
+ }
+ else {
+ ptr = NULL;
+ }
+ }
+
+ return ptr;
+}
+
+static void
+transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
+{
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
+
+ if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
+ /* duplicate check */
+ int i;
+ for (i=0; i<theap->promoted_objects_index; i++) {
+ if (theap->promoted_objects[i] == obj) return;
+ }
+ }
+
+ if (theap->promoted_objects_size <= theap->promoted_objects_index) {
+ theap->promoted_objects_size *= 2;
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
+ theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
+ if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
+ }
+ theap->promoted_objects[theap->promoted_objects_index++] = obj;
+}
+
+void
+rb_transient_heap_promote(VALUE obj)
+{
+ if (transient_heap_ptr(obj, FALSE)) {
+ struct transient_heap* theap = transient_heap_get();
+ transient_heap_promote_add(theap, obj);
+ }
+ else {
+ /* ignore */
+ }
+}
+
+static struct transient_alloc_header *
+alloc_header(struct transient_heap_block* block, int index)
+{
+ return (void *)&block->buff[index];
+}
+
+static void
+transient_heap_reset(void)
+{
+ struct transient_heap* theap = transient_heap_get();
+ struct transient_heap_block* block;
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
+
+ block = theap->marked_blocks;
+ while (block) {
+ struct transient_heap_block *next_block = block->info.next_block;
+ theap->total_objects -= block->info.objects;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
+ rb_bug("madvise err:%d", errno);
+ }
+ if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
+ rb_bug("mprotect err:%d", errno);
+ }
+#else
+ reset_block(block);
+ connect_to_free_blocks(theap, block);
+#endif
+ theap->total_blocks--;
+ block = next_block;
+ }
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
+
+ theap->marked_blocks = NULL;
+ theap->total_marked_objects = 0;
+}
+
+static void
+transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
+{
+ int marked_index = block->info.last_marked_index;
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+
+ while (marked_index >= 0) {
+ struct transient_alloc_header *header = alloc_header(block, marked_index);
+ VALUE obj = header->obj;
+
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", header, rb_obj_info(obj));
+
+ if (obj != Qnil) {
+ RB_DEBUG_COUNTER_INC(theap_evacuate);
+
+ switch (BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
+ rb_ary_transient_heap_evacuate(obj, FALSE);
+#else
+ rb_ary_transient_heap_evacuate(obj, TRUE);
+#endif
+ break;
+ default:
+ rb_bug("unsupporeted");
+ }
+ header->obj = Qundef; /* for debug */
+ }
+ marked_index = header->next_marked_index;
+ }
+}
+
+static void
+transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
+{
+ TH_ASSERT(theap->status != status);
+ theap->status = status;
+}
+
+static void
+transient_heap_evacuate(void *dmy)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (theap->status == transient_heap_marking) {
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
+ }
+ else {
+ VALUE gc_disabled = rb_gc_disable();
+ struct transient_heap_block* block;
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) {
+ int i;
+ fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
+ if (TRANSIENT_HEAP_DEBUG >= 4) {
+ for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
+ }
+ }
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ TH_ASSERT(theap->status == transient_heap_none);
+ transient_heap_update_status(theap, transient_heap_escaping);
+
+ /* evacuate from marked blocks */
+ block = theap->marked_blocks;
+ while (block) {
+ transient_heap_block_evacuate(theap, block);
+ block = block->info.next_block;
+ }
+
+ /* evacuate from using blocks
+ only affect incremental marking */
+ block = theap->using_blocks;
+ while (block) {
+ transient_heap_block_evacuate(theap, block);
+ block = block->info.next_block;
+ }
+
+ /* all objects in marked_objects are escaped. */
+ transient_heap_reset();
+
+ if (TRANSIENT_HEAP_DEBUG > 0) {
+ fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
+ }
+
+ transient_heap_verify(theap);
+ transient_heap_update_status(theap, transient_heap_none);
+ if (gc_disabled != Qtrue) rb_gc_enable();
+ }
+}
+
+static void
+clear_marked_index(struct transient_heap_block* block)
+{
+ int marked_index = block->info.last_marked_index;
+
+ while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
+ struct transient_alloc_header *header = alloc_header(block, marked_index);
+ TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
+ if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", block, marked_index);
+
+ marked_index = header->next_marked_index;
+ header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
+ }
+
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+}
+
+static void
+blocks_clear_marked_index(struct transient_heap_block* block)
+{
+ while (block) {
+ clear_marked_index(block);
+ block = block->info.next_block;
+ }
+}
+
+void
+rb_transient_heap_start_marking(int full_marking)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promtoed:%d full_marking:%d\n",
+ theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ blocks_clear_marked_index(theap->marked_blocks);
+ blocks_clear_marked_index(theap->using_blocks);
+
+ if (theap->using_blocks) {
+ if (theap->using_blocks->info.objects > 0) {
+ append_to_marked_blocks(theap, theap->using_blocks);
+ theap->using_blocks = NULL;
+ }
+ else {
+ append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
+ theap->using_blocks->info.next_block = NULL;
+ }
+ }
+
+ if (theap->using_blocks == NULL) {
+ theap->using_blocks = transient_heap_allocatable_block(theap);
+ }
+
+ TH_ASSERT(theap->status == transient_heap_none);
+ transient_heap_update_status(theap, transient_heap_marking);
+ theap->total_marked_objects = 0;
+
+ if (full_marking) {
+ theap->promoted_objects_index = 0;
+ }
+ else { /* mark promoted objects */
+ int i;
+ for (i=0; i<theap->promoted_objects_index; i++) {
+ VALUE obj = theap->promoted_objects[i];
+ const void *ptr = transient_heap_ptr(obj, TRUE);
+ if (ptr) {
+ rb_transient_heap_mark(obj, ptr);
+ }
+ }
+ }
+
+ transient_heap_verify(theap);
+}
+
+void
+rb_transient_heap_finish_marking(void)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",
+ theap->total_objects,
+ theap->total_marked_objects);
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
+
+ TH_ASSERT(theap->status == transient_heap_marking);
+ transient_heap_update_status(theap, transient_heap_none);
+
+ if (theap->total_marked_objects > 0) {
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
+ rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
+ }
+ else {
+ transient_heap_reset();
+ }
+
+ transient_heap_verify(theap);
+}
diff --git a/transient_heap.h b/transient_heap.h
new file mode 100644
index 0000000000..ac0eaadb78
--- /dev/null
+++ b/transient_heap.h
@@ -0,0 +1,38 @@
+/**********************************************************************
+
+ transient_heap.h - declarations of transient_heap related APIs.
+
+ Copyright (C) 2018 Koichi Sasada
+
+**********************************************************************/
+
+#ifndef RUBY_TRANSIENT_HEAP_H
+#define RUBY_TRANSIENT_HEAP_H
+
+/* public API */
+
+/* Allocate req_size bytes from transient_heap.
+ Allocated memories are free-ed when next GC
+ if this memory is not marked by `rb_transient_heap_mark()`.
+ */
+void *rb_transient_heap_alloc(VALUE obj, size_t req_size);
+
+/* If `obj` uses a memory pointed by `ptr` from transient_heap,
+ you need to call `rb_transient_heap_mark(obj, ptr)`
+ to assert liveness of `obj` (and ptr). */
+void rb_transient_heap_mark(VALUE obj, const void *ptr);
+
+/* used by gc.c */
+void rb_transient_heap_promote(VALUE obj);
+void rb_transient_heap_start_marking(int full_marking);
+void rb_transient_heap_finish_marking(void);
+
+/* for debug API */
+void rb_transient_heap_dump(void);
+void rb_transient_heap_verify(void);
+int rb_transient_heap_managed_ptr_p(const void *ptr);
+
+/* evacuate functions */
+void rb_ary_transient_heap_evacuate(VALUE ary, int promote);
+
+#endif
diff --git a/vm_args.c b/vm_args.c
index f9f74b5b0b..16284c0f33 100644
--- a/vm_args.c
+++ b/vm_args.c
@@ -164,7 +164,7 @@ args_copy(struct args_info *args)
static inline const VALUE *
args_rest_argv(struct args_info *args)
{
- return RARRAY_CONST_PTR(args->rest) + args->rest_index;
+ return RARRAY_CONST_PTR_TRANSIENT(args->rest) + args->rest_index;
}
static inline VALUE
@@ -314,7 +314,7 @@ args_setup_post_parameters(struct args_info *args, int argc, VALUE *locals)
{
long len;
len = RARRAY_LEN(args->rest);
- MEMCPY(locals, RARRAY_CONST_PTR(args->rest) + len - argc, VALUE, argc);
+ MEMCPY(locals, RARRAY_CONST_PTR_TRANSIENT(args->rest) + len - argc, VALUE, argc);
rb_ary_resize(args->rest, len - argc);
}
@@ -334,13 +334,13 @@ args_setup_opt_parameters(struct args_info *args, int opt_max, VALUE *locals)
args->argc = 0;
if (args->rest) {
- int len = RARRAY_LENINT(args->rest);
- const VALUE *argv = RARRAY_CONST_PTR(args->rest);
+ int len = RARRAY_LENINT(args->rest);
+ const VALUE *argv = RARRAY_CONST_PTR_TRANSIENT(args->rest);
- for (; i<opt_max && args->rest_index < len; i++, args->rest_index++) {
- locals[i] = argv[args->rest_index];
- }
- }
+ for (; i<opt_max && args->rest_index < len; i++, args->rest_index++) {
+ locals[i] = argv[args->rest_index];
+ }
+ }
/* initialize by nil */
for (j=i; j<opt_max; j++) {
@@ -785,15 +785,15 @@ vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calli
cfp->sp--;
if (!NIL_P(ary)) {
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
- long len = RARRAY_LEN(ary), i;
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
+ long len = RARRAY_LEN(ary), i;
- CHECK_VM_STACK_OVERFLOW(cfp, len);
+ CHECK_VM_STACK_OVERFLOW(cfp, len);
- for (i = 0; i < len; i++) {
- *cfp->sp++ = ptr[i];
- }
- calling->argc += i - 1;
+ for (i = 0; i < len; i++) {
+ *cfp->sp++ = ptr[i];
+ }
+ calling->argc += i - 1;
}
}
diff --git a/vm_eval.c b/vm_eval.c
index 1176c494e1..a864b75712 100644
--- a/vm_eval.c
+++ b/vm_eval.c
@@ -764,7 +764,7 @@ rb_apply(VALUE recv, ID mid, VALUE args)
return ret;
}
argv = ALLOCA_N(VALUE, argc);
- MEMCPY(argv, RARRAY_CONST_PTR(args), VALUE, argc);
+ MEMCPY(argv, RARRAY_CONST_PTR_TRANSIENT(args), VALUE, argc);
return rb_call(recv, mid, argc, argv, CALL_FCALL);
}
diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index d16c318f2c..5904ce7233 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -1265,7 +1265,7 @@ vm_expandarray(VALUE *sp, VALUE ary, rb_num_t num, int flag)
len = 1;
}
else {
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
len = (rb_num_t)RARRAY_LEN(ary);
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-10 05:44:21 +0000