1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
From 21c812cf08d248a0fbe4bc403533bbc61156ba49 Mon Sep 17 00:00:00 2001
From: John Ogness <john.ogness@linutronix.de>
Date: Mon, 30 Nov 2020 01:42:05 +0106
Subject: [PATCH 010/158] printk: use seqcount_latch for console_seq
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/5.15/older/patches-5.15.10-rt24.tar.xz
In preparation for atomic printing, change @console_seq to use
seqcount_latch so that it can be read without requiring @console_sem.
Signed-off-by: John Ogness <john.ogness@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
kernel/printk/printk.c | 73 ++++++++++++++++++++++++------------------
1 file changed, 41 insertions(+), 32 deletions(-)
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index d403b1e91030..80eea212855c 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -362,9 +362,7 @@ static u64 syslog_seq;
static size_t syslog_partial;
static bool syslog_time;
-/* All 3 protected by @console_sem. */
-/* the next printk record to write to the console */
-static u64 console_seq;
+/* Both protected by @console_sem. */
static u64 exclusive_console_stop_seq;
static unsigned long console_dropped;
@@ -373,6 +371,17 @@ struct latched_seq {
u64 val[2];
};
+/*
+ * The next printk record to write to the console. There are two
+ * copies (updated with seqcount_latch) so that reads can locklessly
+ * access a valid value. Writers are synchronized by @console_sem.
+ */
+static struct latched_seq console_seq = {
+ .latch = SEQCNT_LATCH_ZERO(console_seq.latch),
+ .val[0] = 0,
+ .val[1] = 0,
+};
+
/*
* The next printk record to read after the last 'clear' command. There are
* two copies (updated with seqcount_latch) so that reads can locklessly
@@ -436,7 +445,7 @@ bool printk_percpu_data_ready(void)
return __printk_percpu_data_ready;
}
-/* Must be called under syslog_lock. */
+/* Must be called under associated write-protection lock. */
static void latched_seq_write(struct latched_seq *ls, u64 val)
{
raw_write_seqcount_latch(&ls->latch);
@@ -2278,9 +2287,9 @@ EXPORT_SYMBOL(_printk);
#define prb_read_valid(rb, seq, r) false
#define prb_first_valid_seq(rb) 0
+#define latched_seq_read_nolock(seq) 0
+#define latched_seq_write(dst, src)
-static u64 syslog_seq;
-static u64 console_seq;
static u64 exclusive_console_stop_seq;
static unsigned long console_dropped;
@@ -2608,7 +2617,7 @@ void console_unlock(void)
bool do_cond_resched, retry;
struct printk_info info;
struct printk_record r;
- u64 __maybe_unused next_seq;
+ u64 seq;
if (console_suspended) {
up_console_sem();
@@ -2652,12 +2661,14 @@ void console_unlock(void)
size_t len;
skip:
- if (!prb_read_valid(prb, console_seq, &r))
+ seq = latched_seq_read_nolock(&console_seq);
+ if (!prb_read_valid(prb, seq, &r))
break;
- if (console_seq != r.info->seq) {
- console_dropped += r.info->seq - console_seq;
- console_seq = r.info->seq;
+ if (seq != r.info->seq) {
+ console_dropped += r.info->seq - seq;
+ latched_seq_write(&console_seq, r.info->seq);
+ seq = r.info->seq;
}
if (suppress_message_printing(r.info->level)) {
@@ -2666,13 +2677,13 @@ void console_unlock(void)
* directly to the console when we received it, and
* record that has level above the console loglevel.
*/
- console_seq++;
+ latched_seq_write(&console_seq, seq + 1);
goto skip;
}
/* Output to all consoles once old messages replayed. */
if (unlikely(exclusive_console &&
- console_seq >= exclusive_console_stop_seq)) {
+ seq >= exclusive_console_stop_seq)) {
exclusive_console = NULL;
}
@@ -2693,7 +2704,7 @@ void console_unlock(void)
len = record_print_text(&r,
console_msg_format & MSG_FORMAT_SYSLOG,
printk_time);
- console_seq++;
+ latched_seq_write(&console_seq, seq + 1);
/*
* While actively printing out messages, if another printk()
@@ -2721,9 +2732,6 @@ void console_unlock(void)
cond_resched();
}
- /* Get consistent value of the next-to-be-used sequence number. */
- next_seq = console_seq;
-
console_locked = 0;
up_console_sem();
@@ -2733,7 +2741,7 @@ void console_unlock(void)
* there's a new owner and the console_unlock() from them will do the
* flush, no worries.
*/
- retry = prb_read_valid(prb, next_seq, NULL);
+ retry = prb_read_valid(prb, latched_seq_read_nolock(&console_seq), NULL);
if (retry && console_trylock())
goto again;
}
@@ -2785,18 +2793,19 @@ void console_unblank(void)
*/
void console_flush_on_panic(enum con_flush_mode mode)
{
- /*
- * If someone else is holding the console lock, trylock will fail
- * and may_schedule may be set. Ignore and proceed to unlock so
- * that messages are flushed out. As this can be called from any
- * context and we don't want to get preempted while flushing,
- * ensure may_schedule is cleared.
- */
- console_trylock();
- console_may_schedule = 0;
-
- if (mode == CONSOLE_REPLAY_ALL)
- console_seq = prb_first_valid_seq(prb);
+ if (console_trylock()) {
+ if (mode == CONSOLE_REPLAY_ALL)
+ latched_seq_write(&console_seq, prb_first_valid_seq(prb));
+ } else {
+ /*
+ * Another context is holding the console lock and
+ * @console_may_schedule may be set. Ignore and proceed to
+ * unlock so that messages are flushed out. As this can be
+ * called from any context and we don't want to get preempted
+ * while flushing, ensure @console_may_schedule is cleared.
+ */
+ console_may_schedule = 0;
+ }
console_unlock();
}
@@ -3032,11 +3041,11 @@ void register_console(struct console *newcon)
* ignores console_lock.
*/
exclusive_console = newcon;
- exclusive_console_stop_seq = console_seq;
+ exclusive_console_stop_seq = latched_seq_read_nolock(&console_seq);
/* Get a consistent copy of @syslog_seq. */
mutex_lock(&syslog_lock);
- console_seq = syslog_seq;
+ latched_seq_write(&console_seq, syslog_seq);
mutex_unlock(&syslog_lock);
}
console_unlock();
--
2.33.1
|
