add new instruction attribute called leaf

An instruction is leaf if it has no rb_funcall inside.  In order to
check this property, we introduce stack canary which is a random
number collected at runtime.  Stack top is always filled with this
number and checked for stack smashing operations, when VM_CHECK_MODE.
[GH-1947]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@64677 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 35 insertions, 0 deletions

diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index f82b0e075b..702e22291b 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -3900,3 +3900,38 @@ vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *p
 	}
     }
 }
+
+#if VM_CHECK_MODE > 0
+static NORETURN( NOINLINE(
+#if GCC_VERSION_SINCE(4, 3, 0)
+__attribute__((__cold__))
+#endif
+void vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
+static VALUE vm_stack_canary;
+
+void
+Init_vm_stack_canary(void)
+{
+    /* This has to be called _after_ our PRNG is properly set up. */
+    int n = fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
+
+    VM_ASSERT(n == 0);
+}
+
+static void
+vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
+{
+    /* Because a method has already been called, why not call
+     * another one. */
+    const char *insn = rb_insns_name(i);
+    VALUE inspection = rb_inspect(c);
+    const char *str  = StringValueCStr(inspection);
+    VALUE message    = rb_sprintf("dead canary found at %s: %s", insn, str);
+    const char *msg  = StringValueCStr(message);
+
+    rb_bug(msg);
+}
+
+#elif !defined(MJIT_HEADER)
+void Init_vm_stack_canary(void) { /* nothing to do */ }
+#endif