1 /* SPDX-License-Identifier: GPL-2.0-only */ 1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef _LINUX_RCUREF_H 2 #ifndef _LINUX_RCUREF_H
3 #define _LINUX_RCUREF_H 3 #define _LINUX_RCUREF_H
4 4
5 #include <linux/atomic.h> 5 #include <linux/atomic.h>
6 #include <linux/bug.h> 6 #include <linux/bug.h>
7 #include <linux/limits.h> 7 #include <linux/limits.h>
8 #include <linux/lockdep.h> 8 #include <linux/lockdep.h>
9 #include <linux/preempt.h> 9 #include <linux/preempt.h>
10 #include <linux/rcupdate.h> 10 #include <linux/rcupdate.h>
11 11
12 #define RCUREF_ONEREF 0x00000000U 12 #define RCUREF_ONEREF 0x00000000U
13 #define RCUREF_MAXREF 0x7FFFFFFFU 13 #define RCUREF_MAXREF 0x7FFFFFFFU
14 #define RCUREF_SATURATED 0xA0000000U 14 #define RCUREF_SATURATED 0xA0000000U
15 #define RCUREF_RELEASED 0xC0000000U 15 #define RCUREF_RELEASED 0xC0000000U
16 #define RCUREF_DEAD 0xE0000000U 16 #define RCUREF_DEAD 0xE0000000U
17 #define RCUREF_NOREF 0xFFFFFFFFU 17 #define RCUREF_NOREF 0xFFFFFFFFU
18 18
19 /** 19 /**
20 * rcuref_init - Initialize a rcuref reference 20 * rcuref_init - Initialize a rcuref reference count with the given reference count
21 * @ref: Pointer to the reference count 21 * @ref: Pointer to the reference count
22 * @cnt: The initial reference count ty 22 * @cnt: The initial reference count typically '1'
23 */ 23 */
24 static inline void rcuref_init(rcuref_t *ref, 24 static inline void rcuref_init(rcuref_t *ref, unsigned int cnt)
25 { 25 {
26 atomic_set(&ref->refcnt, cnt - 1); 26 atomic_set(&ref->refcnt, cnt - 1);
27 } 27 }
28 28
29 /** 29 /**
30 * rcuref_read - Read the number of held refer 30 * rcuref_read - Read the number of held reference counts of a rcuref
31 * @ref: Pointer to the reference count 31 * @ref: Pointer to the reference count
32 * 32 *
33 * Return: The number of held references (0 .. 33 * Return: The number of held references (0 ... N)
34 */ 34 */
35 static inline unsigned int rcuref_read(rcuref_ 35 static inline unsigned int rcuref_read(rcuref_t *ref)
36 { 36 {
37 unsigned int c = atomic_read(&ref->ref 37 unsigned int c = atomic_read(&ref->refcnt);
38 38
39 /* Return 0 if within the DEAD zone. * 39 /* Return 0 if within the DEAD zone. */
40 return c >= RCUREF_RELEASED ? 0 : c + 40 return c >= RCUREF_RELEASED ? 0 : c + 1;
41 } 41 }
42 42
43 extern __must_check bool rcuref_get_slowpath(r 43 extern __must_check bool rcuref_get_slowpath(rcuref_t *ref);
44 44
45 /** 45 /**
46 * rcuref_get - Acquire one reference on a rcu 46 * rcuref_get - Acquire one reference on a rcuref reference count
47 * @ref: Pointer to the reference count 47 * @ref: Pointer to the reference count
48 * 48 *
49 * Similar to atomic_inc_not_zero() but satura 49 * Similar to atomic_inc_not_zero() but saturates at RCUREF_MAXREF.
50 * 50 *
51 * Provides no memory ordering, it is assumed 51 * Provides no memory ordering, it is assumed the caller has guaranteed the
52 * object memory to be stable (RCU, etc.). It 52 * object memory to be stable (RCU, etc.). It does provide a control dependency
53 * and thereby orders future stores. See docum 53 * and thereby orders future stores. See documentation in lib/rcuref.c
54 * 54 *
55 * Return: 55 * Return:
56 * False if the attempt to acquire a refe 56 * False if the attempt to acquire a reference failed. This happens
57 * when the last reference has been put a 57 * when the last reference has been put already
58 * 58 *
59 * True if a reference was successfully a 59 * True if a reference was successfully acquired
60 */ 60 */
61 static inline __must_check bool rcuref_get(rcu 61 static inline __must_check bool rcuref_get(rcuref_t *ref)
62 { 62 {
63 /* 63 /*
64 * Unconditionally increase the refere 64 * Unconditionally increase the reference count. The saturation and
65 * dead zones provide enough tolerance 65 * dead zones provide enough tolerance for this.
66 */ 66 */
67 if (likely(!atomic_add_negative_relaxe 67 if (likely(!atomic_add_negative_relaxed(1, &ref->refcnt)))
68 return true; 68 return true;
69 69
70 /* Handle the cases inside the saturat 70 /* Handle the cases inside the saturation and dead zones */
71 return rcuref_get_slowpath(ref); 71 return rcuref_get_slowpath(ref);
72 } 72 }
73 73
74 extern __must_check bool rcuref_put_slowpath(r 74 extern __must_check bool rcuref_put_slowpath(rcuref_t *ref);
75 75
76 /* 76 /*
77 * Internal helper. Do not invoke directly. 77 * Internal helper. Do not invoke directly.
78 */ 78 */
79 static __always_inline __must_check bool __rcu 79 static __always_inline __must_check bool __rcuref_put(rcuref_t *ref)
80 { 80 {
81 RCU_LOCKDEP_WARN(!rcu_read_lock_held() 81 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && preemptible(),
82 "suspicious rcuref_pu 82 "suspicious rcuref_put_rcusafe() usage");
83 /* 83 /*
84 * Unconditionally decrease the refere 84 * Unconditionally decrease the reference count. The saturation and
85 * dead zones provide enough tolerance 85 * dead zones provide enough tolerance for this.
86 */ 86 */
87 if (likely(!atomic_add_negative_releas 87 if (likely(!atomic_add_negative_release(-1, &ref->refcnt)))
88 return false; 88 return false;
89 89
90 /* 90 /*
91 * Handle the last reference drop and 91 * Handle the last reference drop and cases inside the saturation
92 * and dead zones. 92 * and dead zones.
93 */ 93 */
94 return rcuref_put_slowpath(ref); 94 return rcuref_put_slowpath(ref);
95 } 95 }
96 96
97 /** 97 /**
98 * rcuref_put_rcusafe -- Release one reference 98 * rcuref_put_rcusafe -- Release one reference for a rcuref reference count RCU safe
99 * @ref: Pointer to the reference count 99 * @ref: Pointer to the reference count
100 * 100 *
101 * Provides release memory ordering, such that 101 * Provides release memory ordering, such that prior loads and stores are done
102 * before, and provides an acquire ordering on 102 * before, and provides an acquire ordering on success such that free()
103 * must come after. 103 * must come after.
104 * 104 *
105 * Can be invoked from contexts, which guarant 105 * Can be invoked from contexts, which guarantee that no grace period can
106 * happen which would free the object concurre 106 * happen which would free the object concurrently if the decrement drops
107 * the last reference and the slowpath races a 107 * the last reference and the slowpath races against a concurrent get() and
108 * put() pair. rcu_read_lock()'ed and atomic c 108 * put() pair. rcu_read_lock()'ed and atomic contexts qualify.
109 * 109 *
110 * Return: 110 * Return:
111 * True if this was the last reference wi 111 * True if this was the last reference with no future references
112 * possible. This signals the caller that 112 * possible. This signals the caller that it can safely release the
113 * object which is protected by the refer 113 * object which is protected by the reference counter.
114 * 114 *
115 * False if there are still active refere 115 * False if there are still active references or the put() raced
116 * with a concurrent get()/put() pair. Ca 116 * with a concurrent get()/put() pair. Caller is not allowed to
117 * release the protected object. 117 * release the protected object.
118 */ 118 */
119 static inline __must_check bool rcuref_put_rcu 119 static inline __must_check bool rcuref_put_rcusafe(rcuref_t *ref)
120 { 120 {
121 return __rcuref_put(ref); 121 return __rcuref_put(ref);
122 } 122 }
123 123
124 /** 124 /**
125 * rcuref_put -- Release one reference for a r 125 * rcuref_put -- Release one reference for a rcuref reference count
126 * @ref: Pointer to the reference count 126 * @ref: Pointer to the reference count
127 * 127 *
128 * Can be invoked from any context. 128 * Can be invoked from any context.
129 * 129 *
130 * Provides release memory ordering, such that 130 * Provides release memory ordering, such that prior loads and stores are done
131 * before, and provides an acquire ordering on 131 * before, and provides an acquire ordering on success such that free()
132 * must come after. 132 * must come after.
133 * 133 *
134 * Return: 134 * Return:
135 * 135 *
136 * True if this was the last reference wi 136 * True if this was the last reference with no future references
137 * possible. This signals the caller that 137 * possible. This signals the caller that it can safely schedule the
138 * object, which is protected by the refe 138 * object, which is protected by the reference counter, for
139 * deconstruction. 139 * deconstruction.
140 * 140 *
141 * False if there are still active refere 141 * False if there are still active references or the put() raced
142 * with a concurrent get()/put() pair. Ca 142 * with a concurrent get()/put() pair. Caller is not allowed to
143 * deconstruct the protected object. 143 * deconstruct the protected object.
144 */ 144 */
145 static inline __must_check bool rcuref_put(rcu 145 static inline __must_check bool rcuref_put(rcuref_t *ref)
146 { 146 {
147 bool released; 147 bool released;
148 148
149 preempt_disable(); 149 preempt_disable();
150 released = __rcuref_put(ref); 150 released = __rcuref_put(ref);
151 preempt_enable(); 151 preempt_enable();
152 return released; 152 return released;
153 } 153 }
154 154
155 #endif 155 #endif
156 156
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.