add new instruction attribute called leaf

An instruction is leaf if it has no rb_funcall inside.  In order to
check this property, we introduce stack canary which is a random
number collected at runtime.  Stack top is always filled with this
number and checked for stack smashing operations, when VM_CHECK_MODE.
[GH-1947]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@64677 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 105 insertions, 0 deletions

diff --git a/tool/ruby_vm/views/_leaf_helpers.erb b/tool/ruby_vm/views/_leaf_helpers.erb
new file mode 100644
index 0000000000..00c2e3882a
--- /dev/null
+++ b/tool/ruby_vm/views/_leaf_helpers.erb
@@ -0,0 +1,105 @@
+%# -*- mode:c; style:ruby; coding: utf-8; indent-tabs-mode: nil -*-
+%# Copyright (c) 2018 Urabe, Shyouhei.  All rights reserved.
+%#
+%# This file is a part of  the programming language Ruby.  Permission is hereby
+%# granted, to either  redistribute and/or modify this file,  provided that the
+%# conditions mentioned  in the  file COPYING  are met.   Consult the  file for
+%# details.
+%;
+
+static bool
+leafness_of_getglobal(VALUE gentry)
+{
+    const struct rb_global_entry *e = (void *)gentry;
+
+    if (UNLIKELY(rb_gvar_is_traced(e))) {
+        return false;
+    }
+    else {
+        /* We cannot write this function using a switch() because a
+         * case label cannot be a function pointer. */
+        static rb_gvar_getter_t *const whitelist[] = {
+            rb_gvar_val_getter,
+            rb_gvar_var_getter,
+            rb_gvar_undef_getter,
+        };
+        rb_gvar_getter_t *f = rb_gvar_getter_function_of(e);
+        int i;
+
+        for (i = 0; i < numberof(whitelist); i++) {
+            if (f == whitelist[i]) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+
+static bool
+leafness_of_setglobal(VALUE gentry)
+{
+    const struct rb_global_entry *e = (void *)gentry;
+
+    if (UNLIKELY(rb_gvar_is_traced(e))) {
+        return false;
+    }
+    else {
+        /* We cannot write this function using a switch() because a
+         * case label cannot be a function pointer. */
+        static rb_gvar_setter_t *const whitelist[] = {
+            rb_gvar_val_setter,
+            rb_gvar_readonly_setter,
+            rb_gvar_var_setter,
+            rb_gvar_undef_setter,
+        };
+        rb_gvar_setter_t *f = rb_gvar_setter_function_of(e);
+        int i;
+
+        for (i = 0; i < numberof(whitelist); i++) {
+            if (f == whitelist[i]) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+
+#include "iseq.h"
+
+static bool
+leafness_of_defined(rb_num_t op_type)
+{
+    /* see also: vm_insnhelper.c:vm_defined() */
+    switch (op_type) {
+      case DEFINED_IVAR:
+      case DEFINED_IVAR2:
+      case DEFINED_GVAR:
+      case DEFINED_CVAR:
+      case DEFINED_YIELD:
+      case DEFINED_REF:
+      case DEFINED_ZSUPER:
+        return false;
+      case DEFINED_CONST:
+        /* has rb_autoload_load(); */
+        return false;
+      case DEFINED_FUNC:
+      case DEFINED_METHOD:
+        /* calls #respond_to_missing? */
+        return false;
+      default:
+        rb_bug("unknown operand %ld: blame @shyouhei.", op_type);
+    }
+}
+
+static bool
+leafness_of_checkmatch(rb_num_t flag)
+{
+    /* see also: vm_insnhelper.c:check_match() */
+    if (flag == VM_CHECKMATCH_TYPE_WHEN) {
+        return true;
+    }
+    else {
+        /* has rb_funcallv() */
+        return false;
+    }
+}