1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
From: Anna-Maria Gleixner <anna-maria@linutronix.de>
Date: Mon, 27 May 2019 16:54:06 +0200
Subject: [PATCH] posix-timers: Add expiry lock
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.4/older/patches-5.4.3-rt1.tar.xz
If a about to be removed posix timer is active then the code will retry the
delete operation until it succeeds / the timer callback completes.
Use hrtimer_grab_expiry_lock() for posix timers which use a hrtimer underneath
to spin on a lock until the callback finished.
Introduce cpu_timers_grab_expiry_lock() for the posix-cpu-timer. This will
acquire the proper per-CPU spin_lock which is acquired by the CPU which is
expirering the timer.
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
[bigeasy: keep the posix-cpu timer bits, everything else got applied]
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/posix-timers.h | 1 +
kernel/time/posix-cpu-timers.c | 28 +++++++++++++++++++++++++++-
2 files changed, 28 insertions(+), 1 deletion(-)
--- a/include/linux/posix-timers.h
+++ b/include/linux/posix-timers.h
@@ -72,6 +72,7 @@ struct cpu_timer {
struct task_struct *task;
struct list_head elist;
int firing;
+ int firing_cpu;
};
static inline bool cpu_timer_enqueue(struct timerqueue_head *head,
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -446,6 +446,20 @@ static int posix_cpu_timer_del(struct k_
return ret;
}
+static DEFINE_PER_CPU(spinlock_t, cpu_timer_expiry_lock) = __SPIN_LOCK_UNLOCKED(cpu_timer_expiry_lock);
+
+static void posix_cpu_wait_running(struct k_itimer *timer)
+{
+ int cpu = timer->it.cpu.firing_cpu;
+
+ if (cpu >= 0) {
+ spinlock_t *expiry_lock = per_cpu_ptr(&cpu_timer_expiry_lock, cpu);
+
+ spin_lock_irq(expiry_lock);
+ spin_unlock_irq(expiry_lock);
+ }
+}
+
static void cleanup_timerqueue(struct timerqueue_head *head)
{
struct timerqueue_node *node;
@@ -783,6 +797,7 @@ static u64 collect_timerqueue(struct tim
return expires;
ctmr->firing = 1;
+ ctmr->firing_cpu = smp_processor_id();
cpu_timer_dequeue(ctmr);
list_add_tail(&ctmr->elist, firing);
}
@@ -1121,6 +1136,7 @@ static void __run_posix_cpu_timers(struc
{
struct k_itimer *timer, *next;
unsigned long flags;
+ spinlock_t *expiry_lock;
LIST_HEAD(firing);
/*
@@ -1130,8 +1146,13 @@ static void __run_posix_cpu_timers(struc
if (!fastpath_timer_check(tsk))
return;
- if (!lock_task_sighand(tsk, &flags))
+ expiry_lock = this_cpu_ptr(&cpu_timer_expiry_lock);
+ spin_lock(expiry_lock);
+
+ if (!lock_task_sighand(tsk, &flags)) {
+ spin_unlock(expiry_lock);
return;
+ }
/*
* Here we take off tsk->signal->cpu_timers[N] and
* tsk->cpu_timers[N] all the timers that are firing, and
@@ -1164,6 +1185,7 @@ static void __run_posix_cpu_timers(struc
list_del_init(&timer->it.cpu.elist);
cpu_firing = timer->it.cpu.firing;
timer->it.cpu.firing = 0;
+ timer->it.cpu.firing_cpu = -1;
/*
* The firing flag is -1 if we collided with a reset
* of the timer, which already reported this
@@ -1173,6 +1195,7 @@ static void __run_posix_cpu_timers(struc
cpu_timer_fire(timer);
spin_unlock(&timer->it_lock);
}
+ spin_unlock(expiry_lock);
}
#ifdef CONFIG_PREEMPT_RT
@@ -1435,6 +1458,8 @@ static int do_cpu_nanosleep(const clocki
spin_unlock_irq(&timer.it_lock);
while (error == TIMER_RETRY) {
+
+ posix_cpu_wait_running(&timer);
/*
* We need to handle case when timer was or is in the
* middle of firing. In other cases we already freed
@@ -1553,6 +1578,7 @@ const struct k_clock clock_posix_cpu = {
.timer_del = posix_cpu_timer_del,
.timer_get = posix_cpu_timer_get,
.timer_rearm = posix_cpu_timer_rearm,
+ .timer_wait_running = posix_cpu_wait_running,
};
const struct k_clock clock_process = {
|
