* include/ruby/ruby.h: use rb_gc_writebrrier() simply.

  For incremental GC, we need to get a pointer to the objspace.
  We can share this pointer for the following WB process.
  And considering icache hit ratio, prcess in the GC.
* gc.c (rb_gc_writebarrier): added.
* gc.c (gc_writebarrier_generational, gc_writebarrier_incremental):
  make them NOINLINE because inlining them into rb_gc_writebarrier()
  makes a prologue code of rb_gc_writebarrier() longer (storing callee
  save registers).
  This patch improve the performance of WB on micro-benchmarks.
  name                   ruby 2.1   trunk   modified
  vm1_gc_wb_ary*            0.511   0.632      0.532
  vm1_gc_wb_ary_promoted*   0.578   0.701      0.674
  vm1_gc_wb_obj*            0.419   0.575      0.492
  vm1_gc_wb_obj_promoted*   0.537   0.664      0.618
                                                 (sec)



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@49987 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 73 insertions, 48 deletions

diff --git a/ChangeLog b/ChangeLog
index 0d972bef79..eb0d665279 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+Tue Mar 17 18:59:16 2015  Koichi Sasada  <ko1@atdot.net>
+
+	* include/ruby/ruby.h: use rb_gc_writebrrier() simply.
+	  For incremental GC, we need to get a pointer to the objspace.
+	  We can share this pointer for the following WB process.
+	  And considering icache hit ratio, prcess in the GC.
+
+	* gc.c (rb_gc_writebarrier): added.
+
+	* gc.c (gc_writebarrier_generational, gc_writebarrier_incremental):
+	  make them NOINLINE because inlining them into rb_gc_writebarrier()
+	  makes a prologue code of rb_gc_writebarrier() longer (storing callee
+	  save registers).
+
+	  This patch improve the performance of WB on micro-benchmarks.
+
+	  name                   ruby 2.1   trunk   modified
+	  vm1_gc_wb_ary*            0.511   0.632      0.532
+	  vm1_gc_wb_ary_promoted*   0.578   0.701      0.674
+	  vm1_gc_wb_obj*            0.419   0.575      0.492
+	  vm1_gc_wb_obj_promoted*   0.537   0.664      0.618
+	                                                 (sec)
+
 Tue Mar 17 18:51:43 2015  Koichi Sasada  <ko1@atdot.net>
 
 	* benchmark/bm_vm1_gc_wb_ary(_promoted).rb: separate fastpath and
diff --git a/gc.c b/gc.c
index 9c9a268cd6..adb5ce442c 100644
--- a/gc.c
+++ b/gc.c
@@ -5458,11 +5458,11 @@ rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap)
 
 /* RGENGC: APIs */
 
-void
-rb_gc_writebarrier_generational(VALUE a, VALUE b)
-{
-    rb_objspace_t *objspace = &rb_objspace;
+NOINLINE(static void gc_writebarrier_generational(rb_objspace_t *objspace, VALUE a, VALUE b));
 
+static void
+gc_writebarrier_generational(rb_objspace_t *objspace, VALUE a, VALUE b)
+{
     if (RGENGC_CHECK_MODE) {
 	if (!RVALUE_OLD_P(a)) rb_bug("rb_gc_writebarrier_generational: %s is not an old object.", obj_info(a));
 	if ( RVALUE_OLD_P(b)) rb_bug("rb_gc_writebarrier_generational: %s is an old object.", obj_info(b));
@@ -5486,51 +5486,62 @@ gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
     gc_grey(objspace, obj);
 }
 
-int
-rb_gc_writebarrier_incremental(VALUE a, VALUE b)
-{
-    rb_objspace_t *objspace = &rb_objspace;
-
-    if (RGENGC_CHECK_MODE) {
-	if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const");
-	if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: a is special const");
-    }
+NOINLINE(static void gc_writebarrier_incremental(rb_objspace_t *objspace, VALUE a, VALUE b));
 
-    if (LIKELY(!is_incremental_marking(objspace))) {
-	return FALSE;
-    }
-    else {
-	gc_report(2, objspace, "rb_gc_writebarrier_incremental: [LG] %s -> %s\n", obj_info(a), obj_info(b));
+static void
+gc_writebarrier_incremental(rb_objspace_t *objspace, VALUE a, VALUE b)
+{
+    gc_report(2, objspace, "rb_gc_writebarrier_incremental: [LG] %s -> %s\n", obj_info(a), obj_info(b));
 
-	if (RVALUE_BLACK_P(a)) {
-	    if (RVALUE_WHITE_P(b)) {
-		if (!RVALUE_WB_UNPROTECTED(a)) {
-		    gc_report(2, objspace, "rb_gc_writebarrier_incremental: [IN] %s -> %s\n", obj_info(a), obj_info(b));
-		    gc_mark_from(objspace, b, a);
-		}
+    if (RVALUE_BLACK_P(a)) {
+	if (RVALUE_WHITE_P(b)) {
+	    if (!RVALUE_WB_UNPROTECTED(a)) {
+		gc_report(2, objspace, "rb_gc_writebarrier_incremental: [IN] %s -> %s\n", obj_info(a), obj_info(b));
+		gc_mark_from(objspace, b, a);
 	    }
-	    else if (RVALUE_OLD_P(a) && !RVALUE_OLD_P(b)) {
-		if (!RVALUE_WB_UNPROTECTED(b)) {
-		    gc_report(1, objspace, "rb_gc_writebarrier_incremental: [GN] %s -> %s\n", obj_info(a), obj_info(b));
-		    RVALUE_AGE_SET_OLD(objspace, b);
+	}
+	else if (RVALUE_OLD_P(a) && !RVALUE_OLD_P(b)) {
+	    if (!RVALUE_WB_UNPROTECTED(b)) {
+		gc_report(1, objspace, "rb_gc_writebarrier_incremental: [GN] %s -> %s\n", obj_info(a), obj_info(b));
+		RVALUE_AGE_SET_OLD(objspace, b);
 
-		    if (RVALUE_BLACK_P(b)) {
-			gc_grey(objspace, b);
-		    }
-		}
-		else {
-		    gc_report(1, objspace, "rb_gc_writebarrier_incremental: [LL] %s -> %s\n", obj_info(a), obj_info(b));
-		    gc_remember_unprotected(objspace, b);
+		if (RVALUE_BLACK_P(b)) {
+		    gc_grey(objspace, b);
 		}
 	    }
+	    else {
+		gc_report(1, objspace, "rb_gc_writebarrier_incremental: [LL] %s -> %s\n", obj_info(a), obj_info(b));
+		gc_remember_unprotected(objspace, b);
+	    }
 	}
-
-	return TRUE;
     }
 }
+#else
+#define gc_writebarrier_incremental(objspace, a, b)
 #endif
 
 void
+rb_gc_writebarrier(VALUE a, VALUE b)
+{
+    rb_objspace_t *objspace = &rb_objspace;
+
+    if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const");
+    if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const");
+
+    if (LIKELY(!is_incremental_marking(objspace))) {
+	if (!RVALUE_OLD_P(a) || RVALUE_OLD_P(b)) {
+	    return;
+	}
+	else {
+	    gc_writebarrier_generational(objspace, a, b);
+	}
+    }
+    else { /* slow path */
+	gc_writebarrier_incremental(objspace, a, b);
+    }
+}
+
+void
 rb_gc_writebarrier_unprotect(VALUE obj)
 {
     if (RVALUE_WB_UNPROTECTED(obj)) {
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index 351766a111..6a370bd118 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -1214,12 +1214,7 @@ rb_data_object_get_warning(VALUE obj)
 #define OBJ_PROMOTED(x)             (SPECIAL_CONST_P(x) ? 0 : OBJ_PROMOTED_RAW(x))
 #define OBJ_WB_UNPROTECT(x)         rb_obj_wb_unprotect(x, __FILE__, __LINE__)
 
-#if USE_RINCGC
-int rb_gc_writebarrier_incremental(VALUE a, VALUE b);
-#else
-#define rb_gc_writebarrier_incremental(a, b) 0
-#endif
-void rb_gc_writebarrier_generational(VALUE a, VALUE b);
+void rb_gc_writebarrier(VALUE a, VALUE b);
 void rb_gc_writebarrier_unprotect(VALUE obj);
 
 #else /* USE_RGENGC */
@@ -1271,11 +1266,7 @@ rb_obj_written(VALUE a, RB_UNUSED_VAR(VALUE oldv), VALUE b, RB_UNUSED_VAR(const
 
 #if USE_RGENGC
     if (!SPECIAL_CONST_P(b)) {
-	if (rb_gc_writebarrier_incremental(a, b) == 0) {
-	    if (OBJ_PROMOTED_RAW(a) && !OBJ_PROMOTED_RAW(b)) {
-		rb_gc_writebarrier_generational(a, b);
-	    }
-	}
+	rb_gc_writebarrier(a, b);
     }
 #endif