From: Anna-Maria Gleixner Date: Thu, 10 Jan 2019 13:00:06 +0100 Subject: [PATCH] timers: Introduce expiry spin lock Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2.17-rt9.tar.xz When del_timer_sync() is called, it is possible, that the CPU has to spin, because the timer is marked as running. The function will repeatedly try to delete the timer until the timer callback completes and the function succeeds. On a virtual machine this spinning can waste CPU cycles if the vCPU invoking the timer callback is not scheduled by the host (and making no progress). The spinning and time wasting, could be prevented by using PARAVIRT_SPINLOCKS and introducing a per timer base spin lock for expiry. The lock is hold during expiring the timers of a base. When the deletion of a timer wasn't successful, because the timer is running at the moment, the expiry lock is trying to accessed instead of cpu_realax(). The lock is already held by the CPU expiring the timers, so the CPU could be scheduled out instead of spinning until the lock is released, because of the PARAVIRT_SPINLOCKS code. Thereby wasting time spinning around is prevented. The code isn't done conditionally on PARAVIRT_SPINLOCKS. The lock is taken only at two places. In one of them the lock is directly dropped after accessing it. So the probability for a slowpath when taking the lock is very low. But this keeps the code cleaner than introducing several CONFIG_PARAVIRT_SPINLOCKS dependend code paths and struct members. Signed-off-by: Anna-Maria Gleixner [bigeasy: Patch description reworded] Signed-off-by: Sebastian Andrzej Siewior --- kernel/time/timer.c | 57 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 15 deletions(-) --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -196,6 +196,7 @@ EXPORT_SYMBOL(jiffies_64); struct timer_base { raw_spinlock_t lock; struct timer_list *running_timer; + spinlock_t expiry_lock; unsigned long clk; unsigned long next_expiry; unsigned int cpu; @@ -1201,14 +1202,8 @@ int del_timer(struct timer_list *timer) } EXPORT_SYMBOL(del_timer); -/** - * try_to_del_timer_sync - Try to deactivate a timer - * @timer: timer to delete - * - * This function tries to deactivate a timer. Upon successful (ret >= 0) - * exit the timer is not queued and the handler is not running on any CPU. - */ -int try_to_del_timer_sync(struct timer_list *timer) +static int __try_to_del_timer_sync(struct timer_list *timer, + struct timer_base **basep) { struct timer_base *base; unsigned long flags; @@ -1216,7 +1211,7 @@ int try_to_del_timer_sync(struct timer_l debug_assert_init(timer); - base = lock_timer_base(timer, &flags); + *basep = base = lock_timer_base(timer, &flags); if (base->running_timer != timer) ret = detach_if_pending(timer, base, true); @@ -1225,9 +1220,42 @@ int try_to_del_timer_sync(struct timer_l return ret; } + +/** + * try_to_del_timer_sync - Try to deactivate a timer + * @timer: timer to delete + * + * This function tries to deactivate a timer. Upon successful (ret >= 0) + * exit the timer is not queued and the handler is not running on any CPU. + */ +int try_to_del_timer_sync(struct timer_list *timer) +{ + struct timer_base *base; + + return __try_to_del_timer_sync(timer, &base); +} EXPORT_SYMBOL(try_to_del_timer_sync); #ifdef CONFIG_SMP +static int __del_timer_sync(struct timer_list *timer) +{ + struct timer_base *base; + int ret; + + for (;;) { + ret = __try_to_del_timer_sync(timer, &base); + if (ret >= 0) + return ret; + + /* + * When accessing the lock, timers of base are no longer expired + * and so timer is no longer running. + */ + spin_lock(&base->expiry_lock); + spin_unlock(&base->expiry_lock); + } +} + /** * del_timer_sync - deactivate a timer and wait for the handler to finish. * @timer: the timer to be deactivated @@ -1283,12 +1311,8 @@ int del_timer_sync(struct timer_list *ti * could lead to deadlock. */ WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE)); - for (;;) { - int ret = try_to_del_timer_sync(timer); - if (ret >= 0) - return ret; - cpu_relax(); - } + + return __del_timer_sync(timer); } EXPORT_SYMBOL(del_timer_sync); #endif @@ -1658,6 +1682,7 @@ static inline void __run_timers(struct t if (!time_after_eq(jiffies, base->clk)) return; + spin_lock(&base->expiry_lock); raw_spin_lock_irq(&base->lock); /* @@ -1686,6 +1711,7 @@ static inline void __run_timers(struct t } base->running_timer = NULL; raw_spin_unlock_irq(&base->lock); + spin_unlock(&base->expiry_lock); } /* @@ -1930,6 +1956,7 @@ static void __init init_timer_cpu(int cp base->cpu = cpu; raw_spin_lock_init(&base->lock); base->clk = jiffies; + spin_lock_init(&base->expiry_lock); } }