1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* 2 /*
3 * Queued read/write locks 3 * Queued read/write locks
4 * 4 *
5 * (C) Copyright 2013-2014 Hewlett-Packard Dev 5 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
6 * 6 *
7 * Authors: Waiman Long <waiman.long@hp.com> 7 * Authors: Waiman Long <waiman.long@hp.com>
8 */ 8 */
9 #include <linux/smp.h> 9 #include <linux/smp.h>
10 #include <linux/bug.h> 10 #include <linux/bug.h>
11 #include <linux/cpumask.h> 11 #include <linux/cpumask.h>
12 #include <linux/percpu.h> 12 #include <linux/percpu.h>
13 #include <linux/hardirq.h> 13 #include <linux/hardirq.h>
14 #include <linux/spinlock.h> 14 #include <linux/spinlock.h>
15 #include <trace/events/lock.h> !! 15 #include <asm/qrwlock.h>
16 16
17 /** 17 /**
18 * queued_read_lock_slowpath - acquire read lo !! 18 * queued_read_lock_slowpath - acquire read lock of a queue rwlock
19 * @lock: Pointer to queued rwlock structure !! 19 * @lock: Pointer to queue rwlock structure
20 */ 20 */
21 void __lockfunc queued_read_lock_slowpath(stru !! 21 void queued_read_lock_slowpath(struct qrwlock *lock)
22 { 22 {
23 /* 23 /*
24 * Readers come here when they cannot 24 * Readers come here when they cannot get the lock without waiting
25 */ 25 */
26 if (unlikely(in_interrupt())) { 26 if (unlikely(in_interrupt())) {
27 /* 27 /*
28 * Readers in interrupt contex 28 * Readers in interrupt context will get the lock immediately
29 * if the writer is just waiti 29 * if the writer is just waiting (not holding the lock yet),
30 * so spin with ACQUIRE semant 30 * so spin with ACQUIRE semantics until the lock is available
31 * without waiting in the queu 31 * without waiting in the queue.
32 */ 32 */
33 atomic_cond_read_acquire(&lock 33 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
34 return; 34 return;
35 } 35 }
36 atomic_sub(_QR_BIAS, &lock->cnts); 36 atomic_sub(_QR_BIAS, &lock->cnts);
37 37
38 trace_contention_begin(lock, LCB_F_SPI <<
39 <<
40 /* 38 /*
41 * Put the reader into the wait queue 39 * Put the reader into the wait queue
42 */ 40 */
43 arch_spin_lock(&lock->wait_lock); 41 arch_spin_lock(&lock->wait_lock);
44 atomic_add(_QR_BIAS, &lock->cnts); 42 atomic_add(_QR_BIAS, &lock->cnts);
45 43
46 /* 44 /*
47 * The ACQUIRE semantics of the follow 45 * The ACQUIRE semantics of the following spinning code ensure
48 * that accesses can't leak upwards ou 46 * that accesses can't leak upwards out of our subsequent critical
49 * section in the case that the lock i 47 * section in the case that the lock is currently held for write.
50 */ 48 */
51 atomic_cond_read_acquire(&lock->cnts, 49 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
52 50
53 /* 51 /*
54 * Signal the next one in queue to bec 52 * Signal the next one in queue to become queue head
55 */ 53 */
56 arch_spin_unlock(&lock->wait_lock); 54 arch_spin_unlock(&lock->wait_lock);
57 <<
58 trace_contention_end(lock, 0); <<
59 } 55 }
60 EXPORT_SYMBOL(queued_read_lock_slowpath); 56 EXPORT_SYMBOL(queued_read_lock_slowpath);
61 57
62 /** 58 /**
63 * queued_write_lock_slowpath - acquire write !! 59 * queued_write_lock_slowpath - acquire write lock of a queue rwlock
64 * @lock : Pointer to queued rwlock structure !! 60 * @lock : Pointer to queue rwlock structure
65 */ 61 */
66 void __lockfunc queued_write_lock_slowpath(str !! 62 void queued_write_lock_slowpath(struct qrwlock *lock)
67 { 63 {
68 int cnts; <<
69 <<
70 trace_contention_begin(lock, LCB_F_SPI <<
71 <<
72 /* Put the writer into the wait queue 64 /* Put the writer into the wait queue */
73 arch_spin_lock(&lock->wait_lock); 65 arch_spin_lock(&lock->wait_lock);
74 66
75 /* Try to acquire the lock directly if 67 /* Try to acquire the lock directly if no reader is present */
76 if (!(cnts = atomic_read(&lock->cnts)) !! 68 if (!atomic_read(&lock->cnts) &&
77 atomic_try_cmpxchg_acquire(&lock-> !! 69 (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
78 goto unlock; 70 goto unlock;
79 71
80 /* Set the waiting flag to notify read 72 /* Set the waiting flag to notify readers that a writer is pending */
81 atomic_or(_QW_WAITING, &lock->cnts); !! 73 atomic_add(_QW_WAITING, &lock->cnts);
82 74
83 /* When no more readers or writers, se 75 /* When no more readers or writers, set the locked flag */
84 do { 76 do {
85 cnts = atomic_cond_read_relaxe !! 77 atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING);
86 } while (!atomic_try_cmpxchg_acquire(& !! 78 } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING,
>> 79 _QW_LOCKED) != _QW_WAITING);
87 unlock: 80 unlock:
88 arch_spin_unlock(&lock->wait_lock); 81 arch_spin_unlock(&lock->wait_lock);
89 <<
90 trace_contention_end(lock, 0); <<
91 } 82 }
92 EXPORT_SYMBOL(queued_write_lock_slowpath); 83 EXPORT_SYMBOL(queued_write_lock_slowpath);
93 84
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.