1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
|
From ee1848356f32e31f001ca96cc68e06a445f72108 Mon Sep 17 00:00:00 2001
From: Brian Behlendorf <behlendorf1@llnl.gov>
Date: Fri, 12 Jul 2019 14:06:36 -0700
Subject: [PATCH] Linux 5.3 compat: retire rw_tryupgrade()
The Linux kernel's rwsem's have never provided an interface to
allow a reader to be upgraded to a writer. Historically, this
functionality has been implemented by a SPL wrapper function.
However, this approach depends on internal knowledge of the
rw_semaphore and is therefore rather brittle.
Since the ZFS code must always be able to fallback to rw_exit()
and rw_enter() when an rw_tryupgrade() fails; this functionality
isn't critical. Furthermore, the only potentially performance
sensitive consumer is dmu_zfetch() and no decrease in performance
was observed with this change applied. See the PR comments for
additional testing details.
Therefore, it is being retired to make the build more robust and
to simplify the rwlock implementation.
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Reviewed-by: Tomohiro Kusumi <kusumi.tomohiro@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #9029
---
include/spl/sys/rwlock.h | 60 +++--------------------
module/spl/spl-rwlock.c | 101 ---------------------------------------
2 files changed, 7 insertions(+), 154 deletions(-)
diff --git a/include/spl/sys/rwlock.h b/include/spl/sys/rwlock.h
index 5e052b532a4..89e02fa8f04 100644
--- a/include/spl/sys/rwlock.h
+++ b/include/spl/sys/rwlock.h
@@ -29,43 +29,6 @@
#include <linux/rwsem.h>
#include <linux/sched.h>
-/* Linux kernel compatibility */
-#if defined(CONFIG_PREEMPT_RT_FULL)
-#define SPL_RWSEM_SINGLE_READER_VALUE (1)
-#define SPL_RWSEM_SINGLE_WRITER_VALUE (0)
-#elif defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
-#define SPL_RWSEM_SINGLE_READER_VALUE (1)
-#define SPL_RWSEM_SINGLE_WRITER_VALUE (-1)
-#elif defined(RWSEM_ACTIVE_MASK)
-#define SPL_RWSEM_SINGLE_READER_VALUE (RWSEM_ACTIVE_READ_BIAS)
-#define SPL_RWSEM_SINGLE_WRITER_VALUE (RWSEM_ACTIVE_WRITE_BIAS)
-#endif
-
-/* Linux 3.16 changed activity to count for rwsem-spinlock */
-#if defined(CONFIG_PREEMPT_RT_FULL)
-#define RWSEM_COUNT(sem) sem->read_depth
-#elif defined(HAVE_RWSEM_ACTIVITY)
-#define RWSEM_COUNT(sem) sem->activity
-/* Linux 4.8 changed count to an atomic_long_t for !rwsem-spinlock */
-#elif defined(HAVE_RWSEM_ATOMIC_LONG_COUNT)
-#define RWSEM_COUNT(sem) atomic_long_read(&(sem)->count)
-#else
-#define RWSEM_COUNT(sem) sem->count
-#endif
-
-#if defined(RWSEM_SPINLOCK_IS_RAW)
-#define spl_rwsem_lock_irqsave(lk, fl) raw_spin_lock_irqsave(lk, fl)
-#define spl_rwsem_unlock_irqrestore(lk, fl) \
- raw_spin_unlock_irqrestore(lk, fl)
-#define spl_rwsem_trylock_irqsave(lk, fl) raw_spin_trylock_irqsave(lk, fl)
-#else
-#define spl_rwsem_lock_irqsave(lk, fl) spin_lock_irqsave(lk, fl)
-#define spl_rwsem_unlock_irqrestore(lk, fl) spin_unlock_irqrestore(lk, fl)
-#define spl_rwsem_trylock_irqsave(lk, fl) spin_trylock_irqsave(lk, fl)
-#endif /* RWSEM_SPINLOCK_IS_RAW */
-
-#define spl_rwsem_is_locked(rwsem) rwsem_is_locked(rwsem)
-
typedef enum {
RW_DRIVER = 2,
RW_DEFAULT = 4,
@@ -133,7 +96,7 @@ spl_rw_lockdep_on_maybe(krwlock_t *rwp) \
static inline int
RW_LOCK_HELD(krwlock_t *rwp)
{
- return (spl_rwsem_is_locked(SEM(rwp)));
+ return (rwsem_is_locked(SEM(rwp)));
}
static inline int
@@ -170,6 +133,12 @@ RW_READ_HELD(krwlock_t *rwp)
*/
#define rw_destroy(rwp) ((void) 0)
+/*
+ * Upgrading a rwsem from a reader to a writer is not supported by the
+ * Linux kernel. The lock must be dropped and reacquired as a writer.
+ */
+#define rw_tryupgrade(rwp) RW_WRITE_HELD(rwp)
+
#define rw_tryenter(rwp, rw) \
({ \
int _rc_ = 0; \
@@ -228,24 +197,9 @@ RW_READ_HELD(krwlock_t *rwp)
spl_rw_lockdep_on_maybe(rwp); \
})
-#define rw_tryupgrade(rwp) \
-({ \
- int _rc_ = 0; \
- \
- if (RW_WRITE_HELD(rwp)) { \
- _rc_ = 1; \
- } else { \
- spl_rw_lockdep_off_maybe(rwp); \
- if ((_rc_ = rwsem_tryupgrade(SEM(rwp)))) \
- spl_rw_set_owner(rwp); \
- spl_rw_lockdep_on_maybe(rwp); \
- } \
- _rc_; \
-})
/* END CSTYLED */
int spl_rw_init(void);
void spl_rw_fini(void);
-int rwsem_tryupgrade(struct rw_semaphore *rwsem);
#endif /* _SPL_RWLOCK_H */
diff --git a/module/spl/spl-rwlock.c b/module/spl/spl-rwlock.c
index 886e16924e6..10f7c38db4e 100644
--- a/module/spl/spl-rwlock.c
+++ b/module/spl/spl-rwlock.c
@@ -24,106 +24,5 @@
* Solaris Porting Layer (SPL) Reader/Writer Lock Implementation.
*/
-#include <sys/rwlock.h>
-#include <linux/module.h>
-
-#if defined(CONFIG_PREEMPT_RT_FULL)
-
-#include <linux/rtmutex.h>
-#define RT_MUTEX_OWNER_MASKALL 1UL
-
-static int
-__rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
-#if defined(READER_BIAS) && defined(WRITER_BIAS)
- /*
- * After the 4.9.20-rt16 kernel the realtime patch series lifted the
- * single reader restriction. While this could be accommodated by
- * adding additional compatibility code assume the rwsem can never
- * be upgraded. All caller must already cleanly handle this case.
- */
- return (0);
-#else
- ASSERT((struct task_struct *)
- ((unsigned long)rwsem->lock.owner & ~RT_MUTEX_OWNER_MASKALL) ==
- current);
-
- /*
- * Prior to 4.9.20-rt16 kernel the realtime patch series, rwsem is
- * implemented as a single mutex held by readers and writers alike.
- * However, this implementation would prevent a thread from taking
- * a read lock twice, as the mutex would already be locked on
- * the second attempt. Therefore the implementation allows a
- * single thread to take a rwsem as read lock multiple times
- * tracking that nesting as read_depth counter.
- */
- if (rwsem->read_depth <= 1) {
- /*
- * In case, the current thread has not taken the lock
- * more than once as read lock, we can allow an
- * upgrade to a write lock. rwsem_rt.h implements
- * write locks as read_depth == 0.
- */
- rwsem->read_depth = 0;
- return (1);
- }
- return (0);
-#endif
-}
-#elif defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
-static int
-__rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
- int ret = 0;
- unsigned long flags;
- spl_rwsem_lock_irqsave(&rwsem->wait_lock, flags);
- if (RWSEM_COUNT(rwsem) == SPL_RWSEM_SINGLE_READER_VALUE &&
- list_empty(&rwsem->wait_list)) {
- ret = 1;
- RWSEM_COUNT(rwsem) = SPL_RWSEM_SINGLE_WRITER_VALUE;
- }
- spl_rwsem_unlock_irqrestore(&rwsem->wait_lock, flags);
- return (ret);
-}
-#elif defined(RWSEM_ACTIVE_MASK)
-#if defined(HAVE_RWSEM_ATOMIC_LONG_COUNT)
-static int
-__rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
- long val;
- val = atomic_long_cmpxchg(&rwsem->count, SPL_RWSEM_SINGLE_READER_VALUE,
- SPL_RWSEM_SINGLE_WRITER_VALUE);
- return (val == SPL_RWSEM_SINGLE_READER_VALUE);
-}
-#else
-static int
-__rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
- typeof(rwsem->count) val;
- val = cmpxchg(&rwsem->count, SPL_RWSEM_SINGLE_READER_VALUE,
- SPL_RWSEM_SINGLE_WRITER_VALUE);
- return (val == SPL_RWSEM_SINGLE_READER_VALUE);
-}
-#endif
-#else
-static int
-__rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
- return (0);
-}
-#endif
-
-int
-rwsem_tryupgrade(struct rw_semaphore *rwsem)
-{
- if (__rwsem_tryupgrade(rwsem)) {
- rwsem_release(&rwsem->dep_map, 1, _RET_IP_);
- rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_);
- return (1);
- }
- return (0);
-}
-EXPORT_SYMBOL(rwsem_tryupgrade);
-
int spl_rw_init(void) { return 0; }
void spl_rw_fini(void) { }
|