aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--gc.c8
-rw-r--r--vm_sync.c98
-rw-r--r--vm_sync.h13
3 files changed, 73 insertions, 46 deletions
diff --git a/gc.c b/gc.c
index 219c54882e..77229768bf 100644
--- a/gc.c
+++ b/gc.c
@@ -7740,7 +7740,7 @@ rb_gc_writebarrier(VALUE a, VALUE b)
// do nothing
}
else {
- RB_VM_LOCK_ENTER(); // can change GC state
+ RB_VM_LOCK_ENTER_NO_BARRIER(); // can change GC state
{
if (!is_incremental_marking(objspace)) {
if (!RVALUE_OLD_P(a) || RVALUE_OLD_P(b)) {
@@ -7754,11 +7754,11 @@ rb_gc_writebarrier(VALUE a, VALUE b)
retry = true;
}
}
- RB_VM_LOCK_LEAVE();
+ RB_VM_LOCK_LEAVE_NO_BARRIER();
}
}
else { /* slow path */
- RB_VM_LOCK_ENTER(); // can change GC state
+ RB_VM_LOCK_ENTER_NO_BARRIER(); // can change GC state
{
if (is_incremental_marking(objspace)) {
gc_writebarrier_incremental(a, b, objspace);
@@ -7767,7 +7767,7 @@ rb_gc_writebarrier(VALUE a, VALUE b)
retry = true;
}
}
- RB_VM_LOCK_LEAVE();
+ RB_VM_LOCK_LEAVE_NO_BARRIER();
}
if (retry) goto retry_;
diff --git a/vm_sync.c b/vm_sync.c
index 0a14d5394c..97848ada40 100644
--- a/vm_sync.c
+++ b/vm_sync.c
@@ -39,7 +39,7 @@ rb_vm_locked_p(void)
}
static void
-vm_lock_enter(rb_ractor_t *cr, rb_vm_t *vm, bool locked, unsigned int *lev APPEND_LOCATION_ARGS)
+vm_lock_enter(rb_ractor_t *cr, rb_vm_t *vm, bool locked, bool no_barrier, unsigned int *lev APPEND_LOCATION_ARGS)
{
RUBY_DEBUG_LOG2(file, line, "start locked:%d", locked);
@@ -57,43 +57,45 @@ vm_lock_enter(rb_ractor_t *cr, rb_vm_t *vm, bool locked, unsigned int *lev APPEN
VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
vm->ractor.sync.lock_owner = cr;
- // barrier
- while (vm->ractor.sync.barrier_waiting) {
- unsigned int barrier_cnt = vm->ractor.sync.barrier_cnt;
- rb_thread_t *th = GET_THREAD();
- bool running;
-
- RB_GC_SAVE_MACHINE_CONTEXT(th);
-
- if (rb_ractor_status_p(cr, ractor_running)) {
- rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__);
- running = true;
- }
- else {
- running = false;
- }
- VM_ASSERT(rb_ractor_status_p(cr, ractor_blocking));
-
- if (vm_barrier_finish_p(vm)) {
- RUBY_DEBUG_LOG("wakeup barrier owner", 0);
- rb_native_cond_signal(&vm->ractor.sync.barrier_cond);
- }
- else {
- RUBY_DEBUG_LOG("wait for barrier finish", 0);
- }
-
- // wait for restart
- while (barrier_cnt == vm->ractor.sync.barrier_cnt) {
- vm->ractor.sync.lock_owner = NULL;
- rb_native_cond_wait(&cr->barrier_wait_cond, &vm->ractor.sync.lock);
- VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
- vm->ractor.sync.lock_owner = cr;
- }
-
- RUBY_DEBUG_LOG("barrier is released. Acquire vm_lock", 0);
-
- if (running) {
- rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__);
+ if (!no_barrier) {
+ // barrier
+ while (vm->ractor.sync.barrier_waiting) {
+ unsigned int barrier_cnt = vm->ractor.sync.barrier_cnt;
+ rb_thread_t *th = GET_THREAD();
+ bool running;
+
+ RB_GC_SAVE_MACHINE_CONTEXT(th);
+
+ if (rb_ractor_status_p(cr, ractor_running)) {
+ rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__);
+ running = true;
+ }
+ else {
+ running = false;
+ }
+ VM_ASSERT(rb_ractor_status_p(cr, ractor_blocking));
+
+ if (vm_barrier_finish_p(vm)) {
+ RUBY_DEBUG_LOG("wakeup barrier owner", 0);
+ rb_native_cond_signal(&vm->ractor.sync.barrier_cond);
+ }
+ else {
+ RUBY_DEBUG_LOG("wait for barrier finish", 0);
+ }
+
+ // wait for restart
+ while (barrier_cnt == vm->ractor.sync.barrier_cnt) {
+ vm->ractor.sync.lock_owner = NULL;
+ rb_native_cond_wait(&cr->barrier_wait_cond, &vm->ractor.sync.lock);
+ VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
+ vm->ractor.sync.lock_owner = cr;
+ }
+
+ RUBY_DEBUG_LOG("barrier is released. Acquire vm_lock", 0);
+
+ if (running) {
+ rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__);
+ }
}
}
@@ -130,10 +132,22 @@ rb_vm_lock_enter_body(unsigned int *lev APPEND_LOCATION_ARGS)
{
rb_vm_t *vm = GET_VM();
if (vm_locked(vm)) {
- vm_lock_enter(NULL, vm, true, lev APPEND_LOCATION_PARAMS);
+ vm_lock_enter(NULL, vm, true, false, lev APPEND_LOCATION_PARAMS);
+ }
+ else {
+ vm_lock_enter(GET_RACTOR(), vm, false, false, lev APPEND_LOCATION_PARAMS);
+ }
+}
+
+MJIT_FUNC_EXPORTED void
+rb_vm_lock_enter_body_nb(unsigned int *lev APPEND_LOCATION_ARGS)
+{
+ rb_vm_t *vm = GET_VM();
+ if (vm_locked(vm)) {
+ vm_lock_enter(NULL, vm, true, true, lev APPEND_LOCATION_PARAMS);
}
else {
- vm_lock_enter(GET_RACTOR(), vm, false, lev APPEND_LOCATION_PARAMS);
+ vm_lock_enter(GET_RACTOR(), vm, false, true, lev APPEND_LOCATION_PARAMS);
}
}
@@ -141,7 +155,7 @@ MJIT_FUNC_EXPORTED void
rb_vm_lock_enter_body_cr(rb_ractor_t *cr, unsigned int *lev APPEND_LOCATION_ARGS)
{
rb_vm_t *vm = GET_VM();
- vm_lock_enter(cr, vm, vm_locked(vm), lev APPEND_LOCATION_PARAMS);
+ vm_lock_enter(cr, vm, vm_locked(vm), false, lev APPEND_LOCATION_PARAMS);
}
MJIT_FUNC_EXPORTED void
@@ -156,7 +170,7 @@ rb_vm_lock_body(LOCATION_ARGS)
rb_vm_t *vm = GET_VM();
ASSERT_vm_unlocking();
- vm_lock_enter(GET_RACTOR(), vm, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
+ vm_lock_enter(GET_RACTOR(), vm, false, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
}
void
diff --git a/vm_sync.h b/vm_sync.h
index 14b63f3017..8712e1a1ca 100644
--- a/vm_sync.h
+++ b/vm_sync.h
@@ -22,6 +22,7 @@ void rb_vm_unlock_body(LOCATION_ARGS);
struct rb_ractor_struct;
void rb_vm_lock_enter_body_cr(struct rb_ractor_struct *cr, unsigned int *lev APPEND_LOCATION_ARGS);
+void rb_vm_lock_enter_body_nb(unsigned int *lev APPEND_LOCATION_ARGS);
void rb_vm_lock_enter_body(unsigned int *lev APPEND_LOCATION_ARGS);
void rb_vm_lock_leave_body(unsigned int *lev APPEND_LOCATION_ARGS);
void rb_vm_barrier(void);
@@ -72,6 +73,14 @@ rb_vm_lock_enter(unsigned int *lev, const char *file, int line)
}
static inline void
+rb_vm_lock_enter_nb(unsigned int *lev, const char *file, int line)
+{
+ if (rb_multi_ractor_p()) {
+ rb_vm_lock_enter_body_nb(lev APPEND_LOCATION_PARAMS);
+ }
+}
+
+static inline void
rb_vm_lock_leave(unsigned int *lev, const char *file, int line)
{
if (rb_multi_ractor_p()) {
@@ -104,6 +113,10 @@ rb_vm_lock_leave_cr(struct rb_ractor_struct *cr, unsigned int *levp, const char
#define RB_VM_LOCK_ENTER() { unsigned int _lev; RB_VM_LOCK_ENTER_LEV(&_lev);
#define RB_VM_LOCK_LEAVE() RB_VM_LOCK_LEAVE_LEV(&_lev); }
+#define RB_VM_LOCK_ENTER_LEV_NB(levp) rb_vm_lock_enter_nb(levp, __FILE__, __LINE__)
+#define RB_VM_LOCK_ENTER_NO_BARRIER() { unsigned int _lev; RB_VM_LOCK_ENTER_LEV_NB(&_lev);
+#define RB_VM_LOCK_LEAVE_NO_BARRIER() RB_VM_LOCK_LEAVE_LEV(&_lev); }
+
#if RUBY_DEBUG > 0
void ASSERT_vm_locking(void);
void ASSERT_vm_unlocking(void);
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-22 15:23:30 +0000