1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
From: Anna-Maria Gleixner <anna-maria@linutronix.de>
Date: Mon, 27 May 2019 16:54:04 +0200
Subject: [PATCH] hrtimer: Introduce expiry spin lock
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2.10-rt5.tar.xz
When deleting a hrtimer, it is possible, that the CPU has to spin, because the
hrtimer is marked as running. This is done via cpu_relax() and repeating trying to
delete the timer. When doing this in a virtual machine, the CPU wastes vcpu time
because of spinning as long as the timer is no longer running.
The spinning and time wasting, could be prevented by using PARAVIRT_SPINLOCKS
and introducing a per timer base spin lock for expiry. The lock is hold during
expiring the timers of a base. When the deletion of a timer wasn't successful,
because the timer is running at the moment, the expiry lock is trying to
accessed instead of cpu_realax(). The lock is already held by the CPU expring
the timers, so the CPU could be scheduled out instead of spinning until the lock
is released, because of the PARAVIRT_SPINLOCKS code. Thereby wasting time
spinning around is prevented.
The code isn't done conditionally on PARAVIRT_SPINLOCKS. The lock is taken only
at two places. In one of them the lock is directly dropped after accessing
it. So the probability for a slowpath when taking the lock is very low. But this
keeps the code cleaner than introducing several CONFIG_PARAVIRT_SPINLOCKS
dependend code paths and struct members.
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/hrtimer.h | 3 +++
kernel/time/hrtimer.c | 15 ++++++++++++++-
2 files changed, 17 insertions(+), 1 deletion(-)
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -183,6 +183,8 @@ enum hrtimer_base_type {
* @nr_retries: Total number of hrtimer interrupt retries
* @nr_hangs: Total number of hrtimer interrupt hangs
* @max_hang_time: Maximum time spent in hrtimer_interrupt
+ * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are
+ * expired
* @expires_next: absolute time of the next event, is required for remote
* hrtimer enqueue; it is the total first expiry time (hard
* and soft hrtimer are taken into account)
@@ -210,6 +212,7 @@ struct hrtimer_cpu_base {
unsigned short nr_hangs;
unsigned int max_hang_time;
#endif
+ spinlock_t softirq_expiry_lock;
ktime_t expires_next;
struct hrtimer *next_timer;
ktime_t softirq_expires_next;
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -930,6 +930,16 @@ u64 hrtimer_forward(struct hrtimer *time
}
EXPORT_SYMBOL_GPL(hrtimer_forward);
+static void hrtimer_grab_expiry_lock(const struct hrtimer *timer)
+{
+ struct hrtimer_clock_base *base = timer->base;
+
+ if (base && base->cpu_base) {
+ spin_lock(&base->cpu_base->softirq_expiry_lock);
+ spin_unlock(&base->cpu_base->softirq_expiry_lock);
+ }
+}
+
/*
* enqueue_hrtimer - internal function to (re)start a timer
*
@@ -1162,7 +1172,7 @@ int hrtimer_cancel(struct hrtimer *timer
if (ret >= 0)
return ret;
- cpu_relax();
+ hrtimer_grab_expiry_lock(timer);
}
}
EXPORT_SYMBOL_GPL(hrtimer_cancel);
@@ -1459,6 +1469,7 @@ static __latent_entropy void hrtimer_run
unsigned long flags;
ktime_t now;
+ spin_lock(&cpu_base->softirq_expiry_lock);
raw_spin_lock_irqsave(&cpu_base->lock, flags);
now = hrtimer_update_base(cpu_base);
@@ -1468,6 +1479,7 @@ static __latent_entropy void hrtimer_run
hrtimer_update_softirq_timer(cpu_base, true);
raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
+ spin_unlock(&cpu_base->softirq_expiry_lock);
}
#ifdef CONFIG_HIGH_RES_TIMERS
@@ -1809,6 +1821,7 @@ int hrtimers_prepare_cpu(unsigned int cp
cpu_base->softirq_next_timer = NULL;
cpu_base->expires_next = KTIME_MAX;
cpu_base->softirq_expires_next = KTIME_MAX;
+ spin_lock_init(&cpu_base->softirq_expiry_lock);
return 0;
}
|
