From: Anna-Maria Gleixner Date: Mon, 27 May 2019 16:54:06 +0200 Subject: [PATCH] posix-timers: Add expiry lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2.10-rt5.tar.xz If a about to be removed posix timer is active then the code will retry the delete operation until it succeeds / the timer callback completes. Use hrtimer_grab_expiry_lock() for posix timers which use a hrtimer underneath to spin on a lock until the callback finished. Introduce cpu_timers_grab_expiry_lock() for the posix-cpu-timer. This will acquire the proper per-CPU spin_lock which is acquired by the CPU which is expirering the timer. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior --- fs/timerfd.c | 6 +++++- include/linux/hrtimer.h | 1 + include/linux/posix-timers.h | 1 + kernel/time/alarmtimer.c | 2 +- kernel/time/hrtimer.c | 2 +- kernel/time/itimer.c | 1 + kernel/time/posix-cpu-timers.c | 23 +++++++++++++++++++++++ kernel/time/posix-timers.c | 38 +++++++++++++++++++++++++++++--------- kernel/time/posix-timers.h | 2 ++ 9 files changed, 64 insertions(+), 12 deletions(-) --- a/fs/timerfd.c +++ b/fs/timerfd.c @@ -471,7 +471,11 @@ static int do_timerfd_settime(int ufd, i break; } spin_unlock_irq(&ctx->wqh.lock); - cpu_relax(); + + if (isalarm(ctx)) + hrtimer_grab_expiry_lock(&ctx->t.alarm.timer); + else + hrtimer_grab_expiry_lock(&ctx->t.tmr); } /* --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -400,6 +400,7 @@ static inline void hrtimer_start(struct extern int hrtimer_cancel(struct hrtimer *timer); extern int hrtimer_try_to_cancel(struct hrtimer *timer); +extern void hrtimer_grab_expiry_lock(const struct hrtimer *timer); static inline void hrtimer_start_expires(struct hrtimer *timer, enum hrtimer_mode mode) --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -15,6 +15,7 @@ struct cpu_timer_list { u64 expires; struct task_struct *task; int firing; + int firing_cpu; }; /* --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -433,7 +433,7 @@ int alarm_cancel(struct alarm *alarm) int ret = alarm_try_to_cancel(alarm); if (ret >= 0) return ret; - cpu_relax(); + hrtimer_grab_expiry_lock(&alarm->timer); } } EXPORT_SYMBOL_GPL(alarm_cancel); --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -930,7 +930,7 @@ u64 hrtimer_forward(struct hrtimer *time } EXPORT_SYMBOL_GPL(hrtimer_forward); -static void hrtimer_grab_expiry_lock(const struct hrtimer *timer) +void hrtimer_grab_expiry_lock(const struct hrtimer *timer) { struct hrtimer_clock_base *base = timer->base; --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -213,6 +213,7 @@ int do_setitimer(int which, struct itime /* We are sharing ->siglock with it_real_fn() */ if (hrtimer_try_to_cancel(timer) < 0) { spin_unlock_irq(&tsk->sighand->siglock); + hrtimer_grab_expiry_lock(timer); goto again; } expires = timeval_to_ktime(value->it_value); --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -785,6 +785,7 @@ check_timers_list(struct list_head *time return t->expires; t->firing = 1; + t->firing_cpu = smp_processor_id(); list_move_tail(&t->entry, firing); } @@ -1127,6 +1128,20 @@ static inline int fastpath_timer_check(s return 0; } +static DEFINE_PER_CPU(spinlock_t, cpu_timer_expiry_lock) = __SPIN_LOCK_UNLOCKED(cpu_timer_expiry_lock); + +void cpu_timers_grab_expiry_lock(struct k_itimer *timer) +{ + int cpu = timer->it.cpu.firing_cpu; + + if (cpu >= 0) { + spinlock_t *expiry_lock = per_cpu_ptr(&cpu_timer_expiry_lock, cpu); + + spin_lock_irq(expiry_lock); + spin_unlock_irq(expiry_lock); + } +} + /* * This is called from the timer interrupt handler. The irq handler has * already updated our counts. We need to check if any timers fire now. @@ -1137,6 +1152,7 @@ void run_posix_cpu_timers(struct task_st LIST_HEAD(firing); struct k_itimer *timer, *next; unsigned long flags; + spinlock_t *expiry_lock; lockdep_assert_irqs_disabled(); @@ -1147,6 +1163,9 @@ void run_posix_cpu_timers(struct task_st if (!fastpath_timer_check(tsk)) return; + expiry_lock = this_cpu_ptr(&cpu_timer_expiry_lock); + spin_lock(expiry_lock); + if (!lock_task_sighand(tsk, &flags)) return; /* @@ -1181,6 +1200,7 @@ void run_posix_cpu_timers(struct task_st list_del_init(&timer->it.cpu.entry); cpu_firing = timer->it.cpu.firing; timer->it.cpu.firing = 0; + timer->it.cpu.firing_cpu = -1; /* * The firing flag is -1 if we collided with a reset * of the timer, which already reported this @@ -1190,6 +1210,7 @@ void run_posix_cpu_timers(struct task_st cpu_timer_fire(timer); spin_unlock(&timer->it_lock); } + spin_unlock(expiry_lock); } /* @@ -1308,6 +1329,8 @@ static int do_cpu_nanosleep(const clocki spin_unlock_irq(&timer.it_lock); while (error == TIMER_RETRY) { + + cpu_timers_grab_expiry_lock(&timer); /* * We need to handle case when timer was or is in the * middle of firing. In other cases we already freed --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -805,6 +805,17 @@ static int common_hrtimer_try_to_cancel( return hrtimer_try_to_cancel(&timr->it.real.timer); } +static void timer_wait_for_callback(const struct k_clock *kc, struct k_itimer *timer) +{ + if (kc->timer_arm == common_hrtimer_arm) + hrtimer_grab_expiry_lock(&timer->it.real.timer); + else if (kc == &alarm_clock) + hrtimer_grab_expiry_lock(&timer->it.alarm.alarmtimer.timer); + else + /* posix-cpu-timers */ + cpu_timers_grab_expiry_lock(timer); +} + /* Set a POSIX.1b interval timer. */ int common_timer_set(struct k_itimer *timr, int flags, struct itimerspec64 *new_setting, @@ -870,11 +881,15 @@ static int do_timer_settime(timer_t time else error = kc->timer_set(timr, flags, new_spec64, old_spec64); - unlock_timer(timr, flag); if (error == TIMER_RETRY) { + rcu_read_lock(); + unlock_timer(timr, flag); + timer_wait_for_callback(kc, timr); + rcu_read_unlock(); old_spec64 = NULL; // We already got the old time... goto retry; } + unlock_timer(timr, flag); return error; } @@ -936,13 +951,21 @@ int common_timer_del(struct k_itimer *ti return 0; } -static inline int timer_delete_hook(struct k_itimer *timer) +static int timer_delete_hook(struct k_itimer *timer) { const struct k_clock *kc = timer->kclock; + int ret; if (WARN_ON_ONCE(!kc || !kc->timer_del)) return -EINVAL; - return kc->timer_del(timer); + ret = kc->timer_del(timer); + if (ret == TIMER_RETRY) { + rcu_read_lock(); + spin_unlock_irq(&timer->it_lock); + timer_wait_for_callback(kc, timer); + rcu_read_unlock(); + } + return ret; } /* Delete a POSIX.1b interval timer. */ @@ -956,10 +979,8 @@ SYSCALL_DEFINE1(timer_delete, timer_t, t if (!timer) return -EINVAL; - if (timer_delete_hook(timer) == TIMER_RETRY) { - unlock_timer(timer, flags); + if (timer_delete_hook(timer) == TIMER_RETRY) goto retry_delete; - } spin_lock(¤t->sighand->siglock); list_del(&timer->list); @@ -985,10 +1006,9 @@ static void itimer_delete(struct k_itime retry_delete: spin_lock_irqsave(&timer->it_lock, flags); - if (timer_delete_hook(timer) == TIMER_RETRY) { - unlock_timer(timer, flags); + if (timer_delete_hook(timer) == TIMER_RETRY) goto retry_delete; - } + list_del(&timer->list); /* * This keeps any tasks waiting on the spin lock from thinking --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -32,6 +32,8 @@ extern const struct k_clock clock_proces extern const struct k_clock clock_thread; extern const struct k_clock alarm_clock; +extern void cpu_timers_grab_expiry_lock(struct k_itimer *timer); + int posix_timer_event(struct k_itimer *timr, int si_private); void common_timer_get(struct k_itimer *timr, struct itimerspec64 *cur_setting);