1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
From: Anna-Maria Gleixner <anna-maria@linutronix.de>
Date: Thu, 10 Jan 2019 13:00:06 +0100
Subject: [PATCH] timers: Introduce expiry spin lock
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2.10-rt5.tar.xz
When del_timer_sync() is called, it is possible, that the CPU has to
spin, because the timer is marked as running. The function will
repeatedly try to delete the timer until the timer callback completes
and the function succeeds.
On a virtual machine this spinning can waste CPU cycles if the vCPU
invoking the timer callback is not scheduled by the host (and making no
progress).
The spinning and time wasting, could be prevented by using PARAVIRT_SPINLOCKS
and introducing a per timer base spin lock for expiry. The lock is hold during
expiring the timers of a base. When the deletion of a timer wasn't successful,
because the timer is running at the moment, the expiry lock is trying to
accessed instead of cpu_realax(). The lock is already held by the CPU expiring
the timers, so the CPU could be scheduled out instead of spinning until the lock
is released, because of the PARAVIRT_SPINLOCKS code. Thereby wasting time
spinning around is prevented.
The code isn't done conditionally on PARAVIRT_SPINLOCKS. The lock is taken only
at two places. In one of them the lock is directly dropped after accessing
it. So the probability for a slowpath when taking the lock is very low. But this
keeps the code cleaner than introducing several CONFIG_PARAVIRT_SPINLOCKS
dependend code paths and struct members.
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
[bigeasy: Patch description reworded]
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/time/timer.c | 57 ++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 42 insertions(+), 15 deletions(-)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -196,6 +196,7 @@ EXPORT_SYMBOL(jiffies_64);
struct timer_base {
raw_spinlock_t lock;
struct timer_list *running_timer;
+ spinlock_t expiry_lock;
unsigned long clk;
unsigned long next_expiry;
unsigned int cpu;
@@ -1201,14 +1202,8 @@ int del_timer(struct timer_list *timer)
}
EXPORT_SYMBOL(del_timer);
-/**
- * try_to_del_timer_sync - Try to deactivate a timer
- * @timer: timer to delete
- *
- * This function tries to deactivate a timer. Upon successful (ret >= 0)
- * exit the timer is not queued and the handler is not running on any CPU.
- */
-int try_to_del_timer_sync(struct timer_list *timer)
+static int __try_to_del_timer_sync(struct timer_list *timer,
+ struct timer_base **basep)
{
struct timer_base *base;
unsigned long flags;
@@ -1216,7 +1211,7 @@ int try_to_del_timer_sync(struct timer_l
debug_assert_init(timer);
- base = lock_timer_base(timer, &flags);
+ *basep = base = lock_timer_base(timer, &flags);
if (base->running_timer != timer)
ret = detach_if_pending(timer, base, true);
@@ -1225,9 +1220,42 @@ int try_to_del_timer_sync(struct timer_l
return ret;
}
+
+/**
+ * try_to_del_timer_sync - Try to deactivate a timer
+ * @timer: timer to delete
+ *
+ * This function tries to deactivate a timer. Upon successful (ret >= 0)
+ * exit the timer is not queued and the handler is not running on any CPU.
+ */
+int try_to_del_timer_sync(struct timer_list *timer)
+{
+ struct timer_base *base;
+
+ return __try_to_del_timer_sync(timer, &base);
+}
EXPORT_SYMBOL(try_to_del_timer_sync);
#ifdef CONFIG_SMP
+static int __del_timer_sync(struct timer_list *timer)
+{
+ struct timer_base *base;
+ int ret;
+
+ for (;;) {
+ ret = __try_to_del_timer_sync(timer, &base);
+ if (ret >= 0)
+ return ret;
+
+ /*
+ * When accessing the lock, timers of base are no longer expired
+ * and so timer is no longer running.
+ */
+ spin_lock(&base->expiry_lock);
+ spin_unlock(&base->expiry_lock);
+ }
+}
+
/**
* del_timer_sync - deactivate a timer and wait for the handler to finish.
* @timer: the timer to be deactivated
@@ -1283,12 +1311,8 @@ int del_timer_sync(struct timer_list *ti
* could lead to deadlock.
*/
WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
- for (;;) {
- int ret = try_to_del_timer_sync(timer);
- if (ret >= 0)
- return ret;
- cpu_relax();
- }
+
+ return __del_timer_sync(timer);
}
EXPORT_SYMBOL(del_timer_sync);
#endif
@@ -1658,6 +1682,7 @@ static inline void __run_timers(struct t
if (!time_after_eq(jiffies, base->clk))
return;
+ spin_lock(&base->expiry_lock);
raw_spin_lock_irq(&base->lock);
/*
@@ -1686,6 +1711,7 @@ static inline void __run_timers(struct t
}
base->running_timer = NULL;
raw_spin_unlock_irq(&base->lock);
+ spin_unlock(&base->expiry_lock);
}
/*
@@ -1930,6 +1956,7 @@ static void __init init_timer_cpu(int cp
base->cpu = cpu;
raw_spin_lock_init(&base->lock);
base->clk = jiffies;
+ spin_lock_init(&base->expiry_lock);
}
}
|
