From: Anna-Maria Gleixner Date: Mon, 27 May 2019 16:54:06 +0200 Subject: [PATCH] posix-timers: Add expiry lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.4/older/patches-5.4.3-rt1.tar.xz If a about to be removed posix timer is active then the code will retry the delete operation until it succeeds / the timer callback completes. Use hrtimer_grab_expiry_lock() for posix timers which use a hrtimer underneath to spin on a lock until the callback finished. Introduce cpu_timers_grab_expiry_lock() for the posix-cpu-timer. This will acquire the proper per-CPU spin_lock which is acquired by the CPU which is expirering the timer. Signed-off-by: Anna-Maria Gleixner [bigeasy: keep the posix-cpu timer bits, everything else got applied] Signed-off-by: Sebastian Andrzej Siewior --- include/linux/posix-timers.h | 1 + kernel/time/posix-cpu-timers.c | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -72,6 +72,7 @@ struct cpu_timer { struct task_struct *task; struct list_head elist; int firing; + int firing_cpu; }; static inline bool cpu_timer_enqueue(struct timerqueue_head *head, --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -446,6 +446,20 @@ static int posix_cpu_timer_del(struct k_ return ret; } +static DEFINE_PER_CPU(spinlock_t, cpu_timer_expiry_lock) = __SPIN_LOCK_UNLOCKED(cpu_timer_expiry_lock); + +static void posix_cpu_wait_running(struct k_itimer *timer) +{ + int cpu = timer->it.cpu.firing_cpu; + + if (cpu >= 0) { + spinlock_t *expiry_lock = per_cpu_ptr(&cpu_timer_expiry_lock, cpu); + + spin_lock_irq(expiry_lock); + spin_unlock_irq(expiry_lock); + } +} + static void cleanup_timerqueue(struct timerqueue_head *head) { struct timerqueue_node *node; @@ -783,6 +797,7 @@ static u64 collect_timerqueue(struct tim return expires; ctmr->firing = 1; + ctmr->firing_cpu = smp_processor_id(); cpu_timer_dequeue(ctmr); list_add_tail(&ctmr->elist, firing); } @@ -1121,6 +1136,7 @@ static void __run_posix_cpu_timers(struc { struct k_itimer *timer, *next; unsigned long flags; + spinlock_t *expiry_lock; LIST_HEAD(firing); /* @@ -1130,8 +1146,13 @@ static void __run_posix_cpu_timers(struc if (!fastpath_timer_check(tsk)) return; - if (!lock_task_sighand(tsk, &flags)) + expiry_lock = this_cpu_ptr(&cpu_timer_expiry_lock); + spin_lock(expiry_lock); + + if (!lock_task_sighand(tsk, &flags)) { + spin_unlock(expiry_lock); return; + } /* * Here we take off tsk->signal->cpu_timers[N] and * tsk->cpu_timers[N] all the timers that are firing, and @@ -1164,6 +1185,7 @@ static void __run_posix_cpu_timers(struc list_del_init(&timer->it.cpu.elist); cpu_firing = timer->it.cpu.firing; timer->it.cpu.firing = 0; + timer->it.cpu.firing_cpu = -1; /* * The firing flag is -1 if we collided with a reset * of the timer, which already reported this @@ -1173,6 +1195,7 @@ static void __run_posix_cpu_timers(struc cpu_timer_fire(timer); spin_unlock(&timer->it_lock); } + spin_unlock(expiry_lock); } #ifdef CONFIG_PREEMPT_RT @@ -1435,6 +1458,8 @@ static int do_cpu_nanosleep(const clocki spin_unlock_irq(&timer.it_lock); while (error == TIMER_RETRY) { + + posix_cpu_wait_running(&timer); /* * We need to handle case when timer was or is in the * middle of firing. In other cases we already freed @@ -1553,6 +1578,7 @@ const struct k_clock clock_posix_cpu = { .timer_del = posix_cpu_timer_del, .timer_get = posix_cpu_timer_get, .timer_rearm = posix_cpu_timer_rearm, + .timer_wait_running = posix_cpu_wait_running, }; const struct k_clock clock_process = {