signal.c: improve stack overflow check

* signal.c (check_stack_overflow): [EXPERIMENTAL] consider the
  segfault a stack overflow when the fault address is between SP
  and BP.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56907 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 11 insertions, 1 deletions

diff --git a/signal.c b/signal.c
index 8ebb6b9c61..888c8eaa72 100644
--- a/signal.c
+++ b/signal.c
@@ -780,35 +780,45 @@ check_stack_overflow(const uintptr_t addr, const ucontext_t *ctx)
 # if defined __linux__
 #   if defined REG_RSP
     const greg_t sp = mctx->gregs[REG_RSP];
+    const greg_t bp = mctx->gregs[REG_RBP];
 #   else
     const greg_t sp = mctx->gregs[REG_ESP];
+    const greg_t bp = mctx->gregs[REG_EBP];
 #   endif
 # elif defined __APPLE__
 #   if defined(__LP64__)
     const uintptr_t sp = mctx->__ss.__rsp;
+    const uintptr_t bp = mctx->__ss.__rbp;
 #   else
     const uintptr_t sp = mctx->__ss.__esp;
+    const uintptr_t bp = mctx->__ss.__ebp;
 #   endif
 # elif defined __FreeBSD__
 #   if defined(__amd64__)
     const __register_t sp = mctx->mc_rsp;
+    const __register_t bp = mctx->mc_rbp;
 #   else
     const __register_t sp = mctx->mc_esp;
+    const __register_t bp = mctx->mc_ebp;
 #   endif
 # elif defined __HAIKU__
 #   if defined(__amd64__)
     const unsigned long sp = mctx->rsp;
+    const unsigned long bp = mctx->rbp;
 #   else
     const unsigned long sp = mctx->esp;
+    const unsigned long bp = mctx->ebp;
 #   endif
 # endif
     enum {pagesize = 4096};
     const uintptr_t sp_page = (uintptr_t)sp / pagesize;
+    const uintptr_t bp_page = (uintptr_t)bp / pagesize;
     const uintptr_t fault_page = addr / pagesize;
 
     /* SP in ucontext is not decremented yet when `push` failed, so
      * the fault page can be the next. */
-    if (sp_page == fault_page || sp_page == fault_page + 1) {
+    if (sp_page == fault_page || sp_page == fault_page + 1 ||
+	sp_page <= fault_page && fault_page <= bp_page) {
 	rb_thread_t *th = ruby_current_thread;
 	if ((uintptr_t)th->tag->buf / pagesize == sp_page) {
 	    /* drop the last tag if it is close to the fault,