1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
From: Thomas Gleixner <tglx@linutronix.de>
Date: Wed, 26 Jun 2019 13:35:36 +0200
Subject: [PATCH] futex: Delay deallocation of pi_state
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2.9-rt3.tar.xz
On -RT we can't invoke kfree() in a non-preemptible context.
Defer the deallocation of pi_state to preemptible context.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/futex.c | 55 ++++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 44 insertions(+), 11 deletions(-)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -814,13 +814,13 @@ static void get_pi_state(struct futex_pi
* Drops a reference to the pi_state object and frees or caches it
* when the last reference is gone.
*/
-static void put_pi_state(struct futex_pi_state *pi_state)
+static struct futex_pi_state *__put_pi_state(struct futex_pi_state *pi_state)
{
if (!pi_state)
- return;
+ return NULL;
if (!refcount_dec_and_test(&pi_state->refcount))
- return;
+ return NULL;
/*
* If pi_state->owner is NULL, the owner is most probably dying
@@ -840,9 +840,7 @@ static void put_pi_state(struct futex_pi
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
}
- if (current->pi_state_cache) {
- kfree(pi_state);
- } else {
+ if (!current->pi_state_cache) {
/*
* pi_state->list is already empty.
* clear pi_state->owner.
@@ -851,6 +849,30 @@ static void put_pi_state(struct futex_pi
pi_state->owner = NULL;
refcount_set(&pi_state->refcount, 1);
current->pi_state_cache = pi_state;
+ pi_state = NULL;
+ }
+ return pi_state;
+}
+
+static void put_pi_state(struct futex_pi_state *pi_state)
+{
+ kfree(__put_pi_state(pi_state));
+}
+
+static void put_pi_state_atomic(struct futex_pi_state *pi_state,
+ struct list_head *to_free)
+{
+ if (__put_pi_state(pi_state))
+ list_add(&pi_state->list, to_free);
+}
+
+static void free_pi_state_list(struct list_head *to_free)
+{
+ struct futex_pi_state *p, *next;
+
+ list_for_each_entry_safe(p, next, to_free, list) {
+ list_del(&p->list);
+ kfree(p);
}
}
@@ -867,6 +889,7 @@ void exit_pi_state_list(struct task_stru
struct futex_pi_state *pi_state;
struct futex_hash_bucket *hb;
union futex_key key = FUTEX_KEY_INIT;
+ LIST_HEAD(to_free);
if (!futex_cmpxchg_enabled)
return;
@@ -911,7 +934,7 @@ void exit_pi_state_list(struct task_stru
/* retain curr->pi_lock for the loop invariant */
raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
raw_spin_unlock(&hb->lock);
- put_pi_state(pi_state);
+ put_pi_state_atomic(pi_state, &to_free);
continue;
}
@@ -930,6 +953,8 @@ void exit_pi_state_list(struct task_stru
raw_spin_lock_irq(&curr->pi_lock);
}
raw_spin_unlock_irq(&curr->pi_lock);
+
+ free_pi_state_list(&to_free);
}
#endif
@@ -1910,6 +1935,7 @@ static int futex_requeue(u32 __user *uad
struct futex_hash_bucket *hb1, *hb2;
struct futex_q *this, *next;
DEFINE_WAKE_Q(wake_q);
+ LIST_HEAD(to_free);
if (nr_wake < 0 || nr_requeue < 0)
return -EINVAL;
@@ -2147,7 +2173,7 @@ static int futex_requeue(u32 __user *uad
* object.
*/
this->pi_state = NULL;
- put_pi_state(pi_state);
+ put_pi_state_atomic(pi_state, &to_free);
/*
* We stop queueing more waiters and let user
* space deal with the mess.
@@ -2164,7 +2190,7 @@ static int futex_requeue(u32 __user *uad
* in futex_proxy_trylock_atomic() or in lookup_pi_state(). We
* need to drop it here again.
*/
- put_pi_state(pi_state);
+ put_pi_state_atomic(pi_state, &to_free);
out_unlock:
double_unlock_hb(hb1, hb2);
@@ -2185,6 +2211,7 @@ static int futex_requeue(u32 __user *uad
out_put_key1:
put_futex_key(&key1);
out:
+ free_pi_state_list(&to_free);
return ret ? ret : task_count;
}
@@ -2321,13 +2348,16 @@ static int unqueue_me(struct futex_q *q)
static void unqueue_me_pi(struct futex_q *q)
__releases(q->lock_ptr)
{
+ struct futex_pi_state *ps;
+
__unqueue_futex(q);
BUG_ON(!q->pi_state);
- put_pi_state(q->pi_state);
+ ps = __put_pi_state(q->pi_state);
q->pi_state = NULL;
raw_spin_unlock(q->lock_ptr);
+ kfree(ps);
}
static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
@@ -3279,6 +3309,8 @@ static int futex_wait_requeue_pi(u32 __u
* did a lock-steal - fix up the PI-state in that case.
*/
if (q.pi_state && (q.pi_state->owner != current)) {
+ struct futex_pi_state *ps_free;
+
raw_spin_lock(q.lock_ptr);
ret = fixup_pi_state_owner(uaddr2, &q, current);
if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
@@ -3289,8 +3321,9 @@ static int futex_wait_requeue_pi(u32 __u
* Drop the reference to the pi state which
* the requeue_pi() code acquired for us.
*/
- put_pi_state(q.pi_state);
+ ps_free = __put_pi_state(q.pi_state);
raw_spin_unlock(q.lock_ptr);
+ kfree(ps_free);
}
} else {
struct rt_mutex *pi_mutex;
|
