Native implementation of coroutines to improve performance of fibers

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65834 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

10 files changed, 513 insertions, 0 deletions

diff --git a/coroutine/amd64/Context.h b/coroutine/amd64/Context.h
new file mode 100644
index 0000000000..fa5819e115
--- /dev/null
+++ b/coroutine/amd64/Context.h
@@ -0,0 +1,62 @@
+//
+//  amd64.h
+//  File file is part of the "Coroutine" project and released under the MIT License.
+//
+//  Created by Samuel Williams on 10/5/2018.
+//  Copyright, 2018, by Samuel Williams. All rights reserved.
+//
+
+#pragma once
+
+#include <assert.h>
+#include <string.h>
+
+#if __cplusplus
+extern "C" {
+#endif
+
+#define COROUTINE __attribute__((noreturn)) void
+
+const size_t COROUTINE_REGISTERS = 6;
+
+// The fiber context (stack pointer).
+typedef struct
+{
+	void **stack_pointer;
+} coroutine_context;
+
+// The initialization function.
+typedef void(* coroutine_start)(coroutine_context *from, coroutine_context *self);
+
+inline void coroutine_initialize(
+	coroutine_context *context,
+	coroutine_start start,
+	void *stack_pointer,
+	size_t stack_size
+) {
+	/* Force 16-byte alignment */
+	context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF);
+
+	if (!start) {
+		assert(!context->stack_pointer);
+		/* We are main coroutine for this thread */
+		return;
+	}
+
+	*--context->stack_pointer = NULL;
+	*--context->stack_pointer = (void*)start;
+
+	context->stack_pointer -= COROUTINE_REGISTERS;
+	memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+}
+
+coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target);
+
+inline void coroutine_destroy(coroutine_context * context)
+{
+	context->stack_pointer = NULL;
+}
+
+#if __cplusplus
+}
+#endif
diff --git a/coroutine/amd64/Context.s b/coroutine/amd64/Context.s
new file mode 100644
index 0000000000..980ba73202
--- /dev/null
+++ b/coroutine/amd64/Context.s
@@ -0,0 +1,44 @@
+##
+##  amd64.c
+##  File file is part of the "Coroutine" project and released under the MIT License.
+##
+##  Created by Samuel Williams on 10/5/2018.
+##  Copyright, 2018, by Samuel Williams. All rights reserved.
+##
+
+.text
+
+.globl coroutine_transfer
+coroutine_transfer:
+
+# For older linkers
+.globl _coroutine_transfer
+_coroutine_transfer:
+
+	# Save caller state
+	pushq %rbp
+	pushq %rbx
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	pushq %r15
+	
+	# Save caller stack pointer
+	movq %rsp, (%rdi)
+	
+	# Restore callee stack pointer
+	movq (%rsi), %rsp
+	
+	# Restore callee stack
+	popq %r15
+	popq %r14
+	popq %r13
+	popq %r12
+	popq %rbx
+	popq %rbp
+	
+	# Put the first argument into the return value
+	# movq %rdi, %rax
+	
+	# We pop the return address and jump to it
+	ret
diff --git a/coroutine/arm32/Context.h b/coroutine/arm32/Context.h
new file mode 100644
index 0000000000..1c730ed458
--- /dev/null
+++ b/coroutine/arm32/Context.h
@@ -0,0 +1,59 @@
+//
+//  amd64.h
+//  File file is part of the "Coroutine" project and released under the MIT License.
+//
+//  Created by Samuel Williams on 10/5/2018.
+//  Copyright, 2018, by Samuel Williams. All rights reserved.
+//
+
+#pragma once
+
+#include <assert.h>
+#include <string.h>
+
+#if __cplusplus
+extern "C" {
+#endif
+
+#define COROUTINE void
+
+const size_t COROUTINE_REGISTERS = 9;
+
+// The fiber context (stack pointer).
+typedef struct
+{
+	void **stack_pointer;
+} coroutine_context;
+
+// The initialization function.
+typedef void(* coroutine_start)(coroutine_context *from, coroutine_context *self);
+
+inline void coroutine_initialize(
+	coroutine_context *context,
+	coroutine_start start,
+	void *stack_pointer,
+	size_t stack_size
+) {
+	context->stack_pointer = (void**)stack_pointer;
+
+	if (!start) {
+		assert(!context->stack_pointer);
+		/* We are main coroutine for this thread */
+		return;
+	}
+
+	*--context->stack_pointer = (void*)start;
+
+	context->stack_pointer -= COROUTINE_REGISTERS;
+	memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+}
+
+coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target);
+
+inline void coroutine_destroy(coroutine_context * context)
+{
+}
+
+#if __cplusplus
+}
+#endif
diff --git a/coroutine/arm32/Context.s b/coroutine/arm32/Context.s
new file mode 100644
index 0000000000..91fc697c18
--- /dev/null
+++ b/coroutine/arm32/Context.s
@@ -0,0 +1,15 @@
+##
+##  arm.c
+##  File file is part of the "Coroutine" project and released under the MIT License.
+##
+##  Created by Samuel Williams on 10/5/2018.
+##  Copyright, 2018, by Samuel Williams. All rights reserved.
+##
+
+.text
+
+.globl coroutine_transfer
+coroutine_transfer:
+	stmia r1!, {r4-r11,sp,lr}
+	ldmia r0!, {r4-r11,sp,pc}
+	bx lr
\ No newline at end of file
diff --git a/coroutine/arm64/Context.h b/coroutine/arm64/Context.h
new file mode 100644
index 0000000000..8dc010d982
--- /dev/null
+++ b/coroutine/arm64/Context.h
@@ -0,0 +1,60 @@
+//
+//  amd64.h
+//  File file is part of the "Coroutine" project and released under the MIT License.
+//
+//  Created by Samuel Williams on 10/5/2018.
+//  Copyright, 2018, by Samuel Williams. All rights reserved.
+//
+
+#pragma once
+
+#include <assert.h>
+#include <string.h>
+
+#if __cplusplus
+extern "C" {
+#endif
+
+#define COROUTINE void
+
+const size_t COROUTINE_REGISTERS = 0xb0 / 8;
+
+// The fiber context (stack pointer).
+typedef struct
+{
+	void **stack_pointer;
+} coroutine_context;
+
+// The initialization function.
+typedef void(* coroutine_start)(coroutine_context *from, coroutine_context *self);
+
+inline void coroutine_initialize(
+	coroutine_context *context,
+	coroutine_start start,
+	void *stack_pointer,
+	size_t stack_size
+) {
+	/* Force 16-byte alignment */
+	context->stack_pointer = (void**)((uintptr_t)stack_pointer & ~0xF);
+
+	if (!start) {
+		assert(!context->stack_pointer);
+		/* We are main coroutine for this thread */
+		return;
+	}
+
+	context->stack_pointer -= COROUTINE_REGISTERS;
+	memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+
+	context->stack_pointer[0xa0 / 8] = (void*)start;
+}
+
+coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target);
+
+inline void coroutine_destroy(coroutine_context * context)
+{
+}
+
+#if __cplusplus
+}
+#endif
diff --git a/coroutine/arm64/Context.s b/coroutine/arm64/Context.s
new file mode 100644
index 0000000000..f8d76e82c4
--- /dev/null
+++ b/coroutine/arm64/Context.s
@@ -0,0 +1,58 @@
+##
+##  arm64.s
+##  File file is part of the "Coroutine" project and released under the MIT License.
+##
+##  Created by Samuel Williams on 10/5/2018.
+##  Copyright, 2018, by Samuel Williams. All rights reserved.
+##
+
+.text
+.align 2
+.global coroutine_transfer
+coroutine_transfer:
+	# Make space on the stack for caller registers
+	sub sp, sp, 0xb0
+	
+	# Save caller registers
+	stp d8, d9, [sp, 0x00]
+	stp d10, d11, [sp, 0x10]
+	stp d12, d13, [sp, 0x20]
+	stp d14, d15, [sp, 0x30]
+	stp x19, x20, [sp, 0x40]
+	stp x21, x22, [sp, 0x50]
+	stp x23, x24, [sp, 0x60]
+	stp x25, x26, [sp, 0x70]
+	stp x27, x28, [sp, 0x80]
+	stp x29, x30, [sp, 0x90]
+
+	# Save return address
+	str x30, [sp, 0xa0]
+
+	# Save stack pointer to x0 (first argument)
+	mov x2, sp
+	str x2, [x0, 0]
+
+	# Load stack pointer from x1 (second argument)
+	ldr x3, [x1, 0]
+	mov sp, x3
+
+	# Restore caller registers
+	ldp d8, d9, [sp, 0x00]
+	ldp d10, d11, [sp, 0x10]
+	ldp d12, d13, [sp, 0x20]
+	ldp d14, d15, [sp, 0x30]
+	ldp x19, x20, [sp, 0x40]
+	ldp x21, x22, [sp, 0x50]
+	ldp x23, x24, [sp, 0x60]
+	ldp x25, x26, [sp, 0x70]
+	ldp x27, x28, [sp, 0x80]
+	ldp x29, x30, [sp, 0x90]
+
+	# Load return address into x4
+	ldr x4, [sp, 0xa0]
+
+	# Pop stack frame
+	add sp, sp, 0xb0
+
+	# Jump to return address (in x4)
+	ret x4
diff --git a/coroutine/win32/Context.asm b/coroutine/win32/Context.asm
new file mode 100644
index 0000000000..aa27099cfe
--- /dev/null
+++ b/coroutine/win32/Context.asm
@@ -0,0 +1,43 @@
+;;
+;;  win32.asm
+;;  File file is part of the "Coroutine" project and released under the MIT License.
+;;
+;;  Created by Samuel Williams on 10/5/2018.
+;;  Copyright, 2018, by Samuel Williams. All rights reserved.
+;;
+
+.model flat
+
+.code
+
+; Using fastcall is a big win (and it's the same has how x64 works).
+; In coroutine transfer, the arguments are passed in ecx and edx. We don't need
+; to touch these in order to pass them to the destination coroutine.
+
+@coroutine_transfer@8 proc
+	; Save caller registers
+	push ebp
+	push ebx
+	push edi
+	push esi
+	
+	; Save caller stack pointer
+	mov dword ptr [ecx], esp
+	
+	; Restore callee stack pointer
+	mov esp, dword ptr [edx]
+
+	; Restore callee stack
+	pop esi
+	pop edi
+	pop ebx
+	pop ebp
+
+	; Save the first argument as the return value
+	mov eax, dword ptr ecx
+
+	; Jump to the address on the stack
+	ret
+@coroutine_transfer@8 endp
+
+end
diff --git a/coroutine/win32/Context.h b/coroutine/win32/Context.h
new file mode 100644
index 0000000000..e5283cd4b0
--- /dev/null
+++ b/coroutine/win32/Context.h
@@ -0,0 +1,64 @@
+//
+//  win32.h
+//  File file is part of the "Coroutine" project and released under the MIT License.
+//
+//  Created by Samuel Williams on 10/5/2018.
+//  Copyright, 2018, by Samuel Williams. All rights reserved.
+//
+
+#pragma once
+
+#include <assert.h>
+#include <string.h>
+
+#if __cplusplus
+extern "C" {
+#endif
+
+#define COROUTINE void __fastcall
+
+const size_t COROUTINE_REGISTERS = 4;
+
+// The fiber context (stack pointer).
+struct coroutine_context
+{
+	void **stack_pointer;
+};
+
+// The initialization function.
+typedef void(__fastcall * coroutine_start)(coroutine_context *from, coroutine_context *self);
+
+inline void coroutine_initialize(
+	coroutine_context *context,
+	coroutine_start start,
+	void *stack_pointer,
+	size_t stack_size
+) {
+	context->stack_pointer = (void**)stack_pointer;
+
+	if (!start) {
+		assert(!context->stack_pointer);
+		/* We are main coroutine for this thread */
+		return;
+	}
+
+	/* Windows Thread Information Block */
+	*--context->stack_pointer = 0;
+	*--context->stack_pointer = stack_pointer;
+	*--context->stack_pointer = (void*)stack_size;
+
+	*--context->stack_pointer = (void*)start;
+
+	context->stack_pointer -= COROUTINE_REGISTERS;
+	memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+}
+
+coroutine_context * __fastcall coroutine_transfer(coroutine_context * current, coroutine_context * target);
+
+inline void coroutine_destroy(coroutine_context * context)
+{
+}
+
+#if __cplusplus
+}
+#endif
\ No newline at end of file
diff --git a/coroutine/win64/Context.asm b/coroutine/win64/Context.asm
new file mode 100644
index 0000000000..f57d3fd387
--- /dev/null
+++ b/coroutine/win64/Context.asm
@@ -0,0 +1,44 @@
+;;
+;;  win64.asm
+;;  File file is part of the "Coroutine" project and released under the MIT License.
+;;
+;;  Created by Samuel Williams on 10/5/2018.
+;;  Copyright, 2018, by Samuel Williams. All rights reserved.
+;;
+
+.code
+
+coroutine_transfer proc
+	push rbp
+	push rbx
+	push rdi
+	push rsi
+	push r12
+	push r13
+	push r14
+	push r15
+	
+	; Save caller stack pointer
+	mov [rcx], rsp 
+	
+	; Restore callee stack pointer
+	mov rsp, [rdx]
+
+	; Restore callee stack
+	pop r15
+	pop r14
+	pop r13
+	pop r12
+	pop rsi
+	pop rdi
+	pop rbx
+	pop rbp
+
+	; Put the first argument into the return value
+	mov rax, rcx
+
+	; We pop the return address and jump to it
+	ret
+coroutine_transfer endp
+
+end
diff --git a/coroutine/win64/Context.h b/coroutine/win64/Context.h
new file mode 100644
index 0000000000..53d9d43ae6
--- /dev/null
+++ b/coroutine/win64/Context.h
@@ -0,0 +1,64 @@
+//
+//  win64.h
+//  File file is part of the "Coroutine" project and released under the MIT License.
+//
+//  Created by Samuel Williams on 10/5/2018.
+//  Copyright, 2018, by Samuel Williams. All rights reserved.
+//
+
+#pragma once
+
+#include <assert.h>
+#include <string.h>
+
+#if __cplusplus
+extern "C" {
+#endif
+
+#define COROUTINE void
+
+const size_t COROUTINE_REGISTERS = 8;
+
+// The fiber context (stack pointer).
+struct coroutine_context
+{
+	void **stack_pointer;
+};
+
+// The initialization function.
+typedef void(* coroutine_start)(coroutine_context *from, coroutine_context *self);
+
+inline void coroutine_initialize(
+	coroutine_context *context,
+	coroutine_start start,
+	void *stack_pointer,
+	size_t stack_size
+) {
+	context->stack_pointer = (void**)stack_pointer;
+
+	if (!start) {
+		assert(!context->stack_pointer);
+		/* We are main coroutine for this thread */
+		return;
+	}
+
+	/* Windows Thread Information Block */
+	*--context->stack_pointer = 0;
+	*--context->stack_pointer = stack_pointer;
+	*--context->stack_pointer = (void*)stack_size;
+
+	*--context->stack_pointer = (void*)start;
+
+	context->stack_pointer -= COROUTINE_REGISTERS;
+	memset(context->stack_pointer, 0, sizeof(void*) * COROUTINE_REGISTERS);
+}
+
+coroutine_context * coroutine_transfer(coroutine_context * current, coroutine_context * target);
+
+inline void coroutine_destroy(coroutine_context * context)
+{
+}
+
+#if __cplusplus
+}
+#endif
\ No newline at end of file