1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
|
From: Thomas Gleixner <tglx@linutronix.de>
Date: Thu, 12 Oct 2017 17:28:34 +0200
Subject: rtmutex: add rwsem implementation based on rtmutex
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.2/older/patches-5.2-rt1.tar.xz
The RT specific R/W semaphore implementation restricts the number of readers
to one because a writer cannot block on multiple readers and inherit its
priority or budget.
The single reader restricting is painful in various ways:
- Performance bottleneck for multi-threaded applications in the page fault
path (mmap sem)
- Progress blocker for drivers which are carefully crafted to avoid the
potential reader/writer deadlock in mainline.
The analysis of the writer code pathes shows, that properly written RT tasks
should not take them. Syscalls like mmap(), file access which take mmap sem
write locked have unbound latencies which are completely unrelated to mmap
sem. Other R/W sem users like graphics drivers are not suitable for RT tasks
either.
So there is little risk to hurt RT tasks when the RT rwsem implementation is
changed in the following way:
- Allow concurrent readers
- Make writers block until the last reader left the critical section. This
blocking is not subject to priority/budget inheritance.
- Readers blocked on a writer inherit their priority/budget in the normal
way.
There is a drawback with this scheme. R/W semaphores become writer unfair
though the applications which have triggered writer starvation (mostly on
mmap_sem) in the past are not really the typical workloads running on a RT
system. So while it's unlikely to hit writer starvation, it's possible. If
there are unexpected workloads on RT systems triggering it, we need to rethink
the approach.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/rwsem-rt.h | 68 ++++++++++
kernel/locking/rwsem-rt.c | 293 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 361 insertions(+)
create mode 100644 include/linux/rwsem-rt.h
create mode 100644 kernel/locking/rwsem-rt.c
--- /dev/null
+++ b/include/linux/rwsem-rt.h
@@ -0,0 +1,68 @@
+#ifndef _LINUX_RWSEM_RT_H
+#define _LINUX_RWSEM_RT_H
+
+#ifndef _LINUX_RWSEM_H
+#error "Include rwsem.h"
+#endif
+
+#include <linux/rtmutex.h>
+#include <linux/swait.h>
+
+#define READER_BIAS (1U << 31)
+#define WRITER_BIAS (1U << 30)
+
+struct rw_semaphore {
+ atomic_t readers;
+ struct rt_mutex rtmutex;
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+ struct lockdep_map dep_map;
+#endif
+};
+
+#define __RWSEM_INITIALIZER(name) \
+{ \
+ .readers = ATOMIC_INIT(READER_BIAS), \
+ .rtmutex = __RT_MUTEX_INITIALIZER(name.rtmutex), \
+ RW_DEP_MAP_INIT(name) \
+}
+
+#define DECLARE_RWSEM(lockname) \
+ struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname)
+
+extern void __rwsem_init(struct rw_semaphore *rwsem, const char *name,
+ struct lock_class_key *key);
+
+#define __init_rwsem(sem, name, key) \
+do { \
+ rt_mutex_init(&(sem)->rtmutex); \
+ __rwsem_init((sem), (name), (key)); \
+} while (0)
+
+#define init_rwsem(sem) \
+do { \
+ static struct lock_class_key __key; \
+ \
+ __init_rwsem((sem), #sem, &__key); \
+} while (0)
+
+static inline int rwsem_is_locked(struct rw_semaphore *sem)
+{
+ return atomic_read(&sem->readers) != READER_BIAS;
+}
+
+static inline int rwsem_is_contended(struct rw_semaphore *sem)
+{
+ return atomic_read(&sem->readers) > 0;
+}
+
+extern void __down_read(struct rw_semaphore *sem);
+extern int __down_read_killable(struct rw_semaphore *sem);
+extern int __down_read_trylock(struct rw_semaphore *sem);
+extern void __down_write(struct rw_semaphore *sem);
+extern int __must_check __down_write_killable(struct rw_semaphore *sem);
+extern int __down_write_trylock(struct rw_semaphore *sem);
+extern void __up_read(struct rw_semaphore *sem);
+extern void __up_write(struct rw_semaphore *sem);
+extern void __downgrade_write(struct rw_semaphore *sem);
+
+#endif
--- /dev/null
+++ b/kernel/locking/rwsem-rt.c
@@ -0,0 +1,293 @@
+/*
+ */
+#include <linux/rwsem.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/signal.h>
+#include <linux/export.h>
+
+#include "rtmutex_common.h"
+
+/*
+ * RT-specific reader/writer semaphores
+ *
+ * down_write()
+ * 1) Lock sem->rtmutex
+ * 2) Remove the reader BIAS to force readers into the slow path
+ * 3) Wait until all readers have left the critical region
+ * 4) Mark it write locked
+ *
+ * up_write()
+ * 1) Remove the write locked marker
+ * 2) Set the reader BIAS so readers can use the fast path again
+ * 3) Unlock sem->rtmutex to release blocked readers
+ *
+ * down_read()
+ * 1) Try fast path acquisition (reader BIAS is set)
+ * 2) Take sem->rtmutex.wait_lock which protects the writelocked flag
+ * 3) If !writelocked, acquire it for read
+ * 4) If writelocked, block on sem->rtmutex
+ * 5) unlock sem->rtmutex, goto 1)
+ *
+ * up_read()
+ * 1) Try fast path release (reader count != 1)
+ * 2) Wake the writer waiting in down_write()#3
+ *
+ * down_read()#3 has the consequence, that rw semaphores on RT are not writer
+ * fair, but writers, which should be avoided in RT tasks (think mmap_sem),
+ * are subject to the rtmutex priority/DL inheritance mechanism.
+ *
+ * It's possible to make the rw semaphores writer fair by keeping a list of
+ * active readers. A blocked writer would force all newly incoming readers to
+ * block on the rtmutex, but the rtmutex would have to be proxy locked for one
+ * reader after the other. We can't use multi-reader inheritance because there
+ * is no way to support that with SCHED_DEADLINE. Implementing the one by one
+ * reader boosting/handover mechanism is a major surgery for a very dubious
+ * value.
+ *
+ * The risk of writer starvation is there, but the pathological use cases
+ * which trigger it are not necessarily the typical RT workloads.
+ */
+
+void __rwsem_init(struct rw_semaphore *sem, const char *name,
+ struct lock_class_key *key)
+{
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+ /*
+ * Make sure we are not reinitializing a held semaphore:
+ */
+ debug_check_no_locks_freed((void *)sem, sizeof(*sem));
+ lockdep_init_map(&sem->dep_map, name, key, 0);
+#endif
+ atomic_set(&sem->readers, READER_BIAS);
+}
+EXPORT_SYMBOL(__rwsem_init);
+
+int __down_read_trylock(struct rw_semaphore *sem)
+{
+ int r, old;
+
+ /*
+ * Increment reader count, if sem->readers < 0, i.e. READER_BIAS is
+ * set.
+ */
+ for (r = atomic_read(&sem->readers); r < 0;) {
+ old = atomic_cmpxchg(&sem->readers, r, r + 1);
+ if (likely(old == r))
+ return 1;
+ r = old;
+ }
+ return 0;
+}
+
+static int __sched __down_read_common(struct rw_semaphore *sem, int state)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ struct rt_mutex_waiter waiter;
+ int ret;
+
+ if (__down_read_trylock(sem))
+ return 0;
+
+ might_sleep();
+ raw_spin_lock_irq(&m->wait_lock);
+ /*
+ * Allow readers as long as the writer has not completely
+ * acquired the semaphore for write.
+ */
+ if (atomic_read(&sem->readers) != WRITER_BIAS) {
+ atomic_inc(&sem->readers);
+ raw_spin_unlock_irq(&m->wait_lock);
+ return 0;
+ }
+
+ /*
+ * Call into the slow lock path with the rtmutex->wait_lock
+ * held, so this can't result in the following race:
+ *
+ * Reader1 Reader2 Writer
+ * down_read()
+ * down_write()
+ * rtmutex_lock(m)
+ * swait()
+ * down_read()
+ * unlock(m->wait_lock)
+ * up_read()
+ * swake()
+ * lock(m->wait_lock)
+ * sem->writelocked=true
+ * unlock(m->wait_lock)
+ *
+ * up_write()
+ * sem->writelocked=false
+ * rtmutex_unlock(m)
+ * down_read()
+ * down_write()
+ * rtmutex_lock(m)
+ * swait()
+ * rtmutex_lock(m)
+ *
+ * That would put Reader1 behind the writer waiting on
+ * Reader2 to call up_read() which might be unbound.
+ */
+ rt_mutex_init_waiter(&waiter, false);
+ ret = rt_mutex_slowlock_locked(m, state, NULL, RT_MUTEX_MIN_CHAINWALK,
+ &waiter);
+ /*
+ * The slowlock() above is guaranteed to return with the rtmutex (for
+ * ret = 0) is now held, so there can't be a writer active. Increment
+ * the reader count and immediately drop the rtmutex again.
+ * For ret != 0 we don't hold the rtmutex and need unlock the wait_lock.
+ * We don't own the lock then.
+ */
+ if (!ret)
+ atomic_inc(&sem->readers);
+ raw_spin_unlock_irq(&m->wait_lock);
+ if (!ret)
+ __rt_mutex_unlock(m);
+
+ debug_rt_mutex_free_waiter(&waiter);
+ return ret;
+}
+
+void __down_read(struct rw_semaphore *sem)
+{
+ int ret;
+
+ ret = __down_read_common(sem, TASK_UNINTERRUPTIBLE);
+ WARN_ON_ONCE(ret);
+}
+
+int __down_read_killable(struct rw_semaphore *sem)
+{
+ int ret;
+
+ ret = __down_read_common(sem, TASK_KILLABLE);
+ if (likely(!ret))
+ return ret;
+ WARN_ONCE(ret != -EINTR, "Unexpected state: %d\n", ret);
+ return -EINTR;
+}
+
+void __up_read(struct rw_semaphore *sem)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ struct task_struct *tsk;
+
+ /*
+ * sem->readers can only hit 0 when a writer is waiting for the
+ * active readers to leave the critical region.
+ */
+ if (!atomic_dec_and_test(&sem->readers))
+ return;
+
+ might_sleep();
+ raw_spin_lock_irq(&m->wait_lock);
+ /*
+ * Wake the writer, i.e. the rtmutex owner. It might release the
+ * rtmutex concurrently in the fast path (due to a signal), but to
+ * clean up the rwsem it needs to acquire m->wait_lock. The worst
+ * case which can happen is a spurious wakeup.
+ */
+ tsk = rt_mutex_owner(m);
+ if (tsk)
+ wake_up_process(tsk);
+
+ raw_spin_unlock_irq(&m->wait_lock);
+}
+
+static void __up_write_unlock(struct rw_semaphore *sem, int bias,
+ unsigned long flags)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+
+ atomic_add(READER_BIAS - bias, &sem->readers);
+ raw_spin_unlock_irqrestore(&m->wait_lock, flags);
+ __rt_mutex_unlock(m);
+}
+
+static int __sched __down_write_common(struct rw_semaphore *sem, int state)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ unsigned long flags;
+
+ /* Take the rtmutex as a first step */
+ if (__rt_mutex_lock_state(m, state))
+ return -EINTR;
+
+ /* Force readers into slow path */
+ atomic_sub(READER_BIAS, &sem->readers);
+ might_sleep();
+
+ set_current_state(state);
+ for (;;) {
+ raw_spin_lock_irqsave(&m->wait_lock, flags);
+ /* Have all readers left the critical region? */
+ if (!atomic_read(&sem->readers)) {
+ atomic_set(&sem->readers, WRITER_BIAS);
+ __set_current_state(TASK_RUNNING);
+ raw_spin_unlock_irqrestore(&m->wait_lock, flags);
+ return 0;
+ }
+
+ if (signal_pending_state(state, current)) {
+ __set_current_state(TASK_RUNNING);
+ __up_write_unlock(sem, 0, flags);
+ return -EINTR;
+ }
+ raw_spin_unlock_irqrestore(&m->wait_lock, flags);
+
+ if (atomic_read(&sem->readers) != 0) {
+ schedule();
+ set_current_state(state);
+ }
+ }
+}
+
+void __sched __down_write(struct rw_semaphore *sem)
+{
+ __down_write_common(sem, TASK_UNINTERRUPTIBLE);
+}
+
+int __sched __down_write_killable(struct rw_semaphore *sem)
+{
+ return __down_write_common(sem, TASK_KILLABLE);
+}
+
+int __down_write_trylock(struct rw_semaphore *sem)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ unsigned long flags;
+
+ if (!__rt_mutex_trylock(m))
+ return 0;
+
+ atomic_sub(READER_BIAS, &sem->readers);
+
+ raw_spin_lock_irqsave(&m->wait_lock, flags);
+ if (!atomic_read(&sem->readers)) {
+ atomic_set(&sem->readers, WRITER_BIAS);
+ raw_spin_unlock_irqrestore(&m->wait_lock, flags);
+ return 1;
+ }
+ __up_write_unlock(sem, 0, flags);
+ return 0;
+}
+
+void __up_write(struct rw_semaphore *sem)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&m->wait_lock, flags);
+ __up_write_unlock(sem, WRITER_BIAS, flags);
+}
+
+void __downgrade_write(struct rw_semaphore *sem)
+{
+ struct rt_mutex *m = &sem->rtmutex;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&m->wait_lock, flags);
+ /* Release it and account current as reader */
+ __up_write_unlock(sem, WRITER_BIAS - 1, flags);
+}
|