1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_NET_QUEUES_H 2 #ifndef _LINUX_NET_QUEUES_H
3 #define _LINUX_NET_QUEUES_H 3 #define _LINUX_NET_QUEUES_H
4 4
5 #include <linux/netdevice.h> 5 #include <linux/netdevice.h>
6 6
7 /* See the netdev.yaml spec for definition of 7 /* See the netdev.yaml spec for definition of each statistic */
8 struct netdev_queue_stats_rx { 8 struct netdev_queue_stats_rx {
9 u64 bytes; 9 u64 bytes;
10 u64 packets; 10 u64 packets;
11 u64 alloc_fail; 11 u64 alloc_fail;
12 12
13 u64 hw_drops; 13 u64 hw_drops;
14 u64 hw_drop_overruns; 14 u64 hw_drop_overruns;
15 15
16 u64 csum_unnecessary; 16 u64 csum_unnecessary;
17 u64 csum_none; 17 u64 csum_none;
18 u64 csum_bad; 18 u64 csum_bad;
19 19
20 u64 hw_gro_packets; 20 u64 hw_gro_packets;
21 u64 hw_gro_bytes; 21 u64 hw_gro_bytes;
22 u64 hw_gro_wire_packets; 22 u64 hw_gro_wire_packets;
23 u64 hw_gro_wire_bytes; 23 u64 hw_gro_wire_bytes;
24 24
25 u64 hw_drop_ratelimits; 25 u64 hw_drop_ratelimits;
26 }; 26 };
27 27
28 struct netdev_queue_stats_tx { 28 struct netdev_queue_stats_tx {
29 u64 bytes; 29 u64 bytes;
30 u64 packets; 30 u64 packets;
31 31
32 u64 hw_drops; 32 u64 hw_drops;
33 u64 hw_drop_errors; 33 u64 hw_drop_errors;
34 34
35 u64 csum_none; 35 u64 csum_none;
36 u64 needs_csum; 36 u64 needs_csum;
37 37
38 u64 hw_gso_packets; 38 u64 hw_gso_packets;
39 u64 hw_gso_bytes; 39 u64 hw_gso_bytes;
40 u64 hw_gso_wire_packets; 40 u64 hw_gso_wire_packets;
41 u64 hw_gso_wire_bytes; 41 u64 hw_gso_wire_bytes;
42 42
43 u64 hw_drop_ratelimits; 43 u64 hw_drop_ratelimits;
44 44
45 u64 stop; 45 u64 stop;
46 u64 wake; 46 u64 wake;
47 }; 47 };
48 48
49 /** 49 /**
50 * struct netdev_stat_ops - netdev ops for fin 50 * struct netdev_stat_ops - netdev ops for fine grained stats
51 * @get_queue_stats_rx: get stats for a given 51 * @get_queue_stats_rx: get stats for a given Rx queue
52 * @get_queue_stats_tx: get stats for a given 52 * @get_queue_stats_tx: get stats for a given Tx queue
53 * @get_base_stats: get base stats (not be 53 * @get_base_stats: get base stats (not belonging to any live instance)
54 * 54 *
55 * Query stats for a given object. The values 55 * Query stats for a given object. The values of the statistics are undefined
56 * on entry (specifically they are *not* zero- 56 * on entry (specifically they are *not* zero-initialized). Drivers should
57 * assign values only to the statistics they c 57 * assign values only to the statistics they collect. Statistics which are not
58 * collected must be left undefined. 58 * collected must be left undefined.
59 * 59 *
60 * Queue objects are not necessarily persisten 60 * Queue objects are not necessarily persistent, and only currently active
61 * queues are queried by the per-queue callbac 61 * queues are queried by the per-queue callbacks. This means that per-queue
62 * statistics will not generally add up to the 62 * statistics will not generally add up to the total number of events for
63 * the device. The @get_base_stats callback al 63 * the device. The @get_base_stats callback allows filling in the delta
64 * between events for currently live queues an 64 * between events for currently live queues and overall device history.
65 * @get_base_stats can also be used to report 65 * @get_base_stats can also be used to report any miscellaneous packets
66 * transferred outside of the main set of queu 66 * transferred outside of the main set of queues used by the networking stack.
67 * When the statistics for the entire device a 67 * When the statistics for the entire device are queried, first @get_base_stats
68 * is issued to collect the delta, and then a 68 * is issued to collect the delta, and then a series of per-queue callbacks.
69 * Only statistics which are set in @get_base_ 69 * Only statistics which are set in @get_base_stats will be reported
70 * at the device level, meaning that unlike in 70 * at the device level, meaning that unlike in queue callbacks, setting
71 * a statistic to zero in @get_base_stats is a 71 * a statistic to zero in @get_base_stats is a legitimate thing to do.
72 * This is because @get_base_stats has a secon 72 * This is because @get_base_stats has a second function of designating which
73 * statistics are in fact correct for the enti 73 * statistics are in fact correct for the entire device (e.g. when history
74 * for some of the events is not maintained, a 74 * for some of the events is not maintained, and reliable "total" cannot
75 * be provided). 75 * be provided).
76 * 76 *
77 * Device drivers can assume that when collect 77 * Device drivers can assume that when collecting total device stats,
78 * the @get_base_stats and subsequent per-queu 78 * the @get_base_stats and subsequent per-queue calls are performed
79 * "atomically" (without releasing the rtnl_lo 79 * "atomically" (without releasing the rtnl_lock).
80 * 80 *
81 * Device drivers are encouraged to reset the 81 * Device drivers are encouraged to reset the per-queue statistics when
82 * number of queues change. This is because th 82 * number of queues change. This is because the primary use case for
83 * per-queue statistics is currently to detect 83 * per-queue statistics is currently to detect traffic imbalance.
84 */ 84 */
85 struct netdev_stat_ops { 85 struct netdev_stat_ops {
86 void (*get_queue_stats_rx)(struct net_ 86 void (*get_queue_stats_rx)(struct net_device *dev, int idx,
87 struct netd 87 struct netdev_queue_stats_rx *stats);
88 void (*get_queue_stats_tx)(struct net_ 88 void (*get_queue_stats_tx)(struct net_device *dev, int idx,
89 struct netd 89 struct netdev_queue_stats_tx *stats);
90 void (*get_base_stats)(struct net_devi 90 void (*get_base_stats)(struct net_device *dev,
91 struct netdev_q 91 struct netdev_queue_stats_rx *rx,
92 struct netdev_q 92 struct netdev_queue_stats_tx *tx);
93 }; 93 };
94 94
95 /** 95 /**
96 * struct netdev_queue_mgmt_ops - netdev ops f 96 * struct netdev_queue_mgmt_ops - netdev ops for queue management
97 * 97 *
98 * @ndo_queue_mem_size: Size of the struct tha 98 * @ndo_queue_mem_size: Size of the struct that describes a queue's memory.
99 * 99 *
100 * @ndo_queue_mem_alloc: Allocate memory for a 100 * @ndo_queue_mem_alloc: Allocate memory for an RX queue at the specified index.
101 * The new memory is wri 101 * The new memory is written at the specified address.
102 * 102 *
103 * @ndo_queue_mem_free: Free memory from an RX 103 * @ndo_queue_mem_free: Free memory from an RX queue.
104 * 104 *
105 * @ndo_queue_start: Start an RX queue with 105 * @ndo_queue_start: Start an RX queue with the specified memory and at the
106 * specified index. 106 * specified index.
107 * 107 *
108 * @ndo_queue_stop: Stop the RX queue at t 108 * @ndo_queue_stop: Stop the RX queue at the specified index. The stopped
109 * queue's memory is writ 109 * queue's memory is written at the specified address.
110 */ 110 */
111 struct netdev_queue_mgmt_ops { 111 struct netdev_queue_mgmt_ops {
112 size_t ndo_queue_mem_ 112 size_t ndo_queue_mem_size;
113 int (*ndo_queue_me 113 int (*ndo_queue_mem_alloc)(struct net_device *dev,
114 114 void *per_queue_mem,
115 115 int idx);
116 void (*ndo_queue_me 116 void (*ndo_queue_mem_free)(struct net_device *dev,
117 117 void *per_queue_mem);
118 int (*ndo_queue_st 118 int (*ndo_queue_start)(struct net_device *dev,
119 119 void *per_queue_mem,
120 120 int idx);
121 int (*ndo_queue_st 121 int (*ndo_queue_stop)(struct net_device *dev,
122 122 void *per_queue_mem,
123 123 int idx);
124 }; 124 };
125 125
126 /** 126 /**
127 * DOC: Lockless queue stopping / waking helpe 127 * DOC: Lockless queue stopping / waking helpers.
128 * 128 *
129 * The netif_txq_maybe_stop() and __netif_txq_ 129 * The netif_txq_maybe_stop() and __netif_txq_completed_wake()
130 * macros are designed to safely implement sto 130 * macros are designed to safely implement stopping
131 * and waking netdev queues without full lock 131 * and waking netdev queues without full lock protection.
132 * 132 *
133 * We assume that there can be no concurrent s 133 * We assume that there can be no concurrent stop attempts and no concurrent
134 * wake attempts. The try-stop should happen f 134 * wake attempts. The try-stop should happen from the xmit handler,
135 * while wake up should be triggered from NAPI 135 * while wake up should be triggered from NAPI poll context.
136 * The two may run concurrently (single produc 136 * The two may run concurrently (single producer, single consumer).
137 * 137 *
138 * The try-stop side is expected to run from t 138 * The try-stop side is expected to run from the xmit handler and therefore
139 * it does not reschedule Tx (netif_tx_start_q 139 * it does not reschedule Tx (netif_tx_start_queue() instead of
140 * netif_tx_wake_queue()). Uses of the ``stop` 140 * netif_tx_wake_queue()). Uses of the ``stop`` macros outside of the xmit
141 * handler may lead to xmit queue being enable 141 * handler may lead to xmit queue being enabled but not run.
142 * The waking side does not have similar conte 142 * The waking side does not have similar context restrictions.
143 * 143 *
144 * The macros guarantee that rings will not re 144 * The macros guarantee that rings will not remain stopped if there's
145 * space available, but they do *not* prevent 145 * space available, but they do *not* prevent false wake ups when
146 * the ring is full! Drivers should check for 146 * the ring is full! Drivers should check for ring full at the start
147 * for the xmit handler. 147 * for the xmit handler.
148 * 148 *
149 * All descriptor ring indexes (and other rele 149 * All descriptor ring indexes (and other relevant shared state) must
150 * be updated before invoking the macros. 150 * be updated before invoking the macros.
151 */ 151 */
152 152
153 #define netif_txq_try_stop(txq, get_desc, star 153 #define netif_txq_try_stop(txq, get_desc, start_thrs) \
154 ({ 154 ({ \
155 int _res; 155 int _res; \
156 156 \
157 netif_tx_stop_queue(txq); 157 netif_tx_stop_queue(txq); \
158 /* Producer index and stop bit 158 /* Producer index and stop bit must be visible \
159 * to consumer before we reche 159 * to consumer before we recheck. \
160 * Pairs with a barrier in __n 160 * Pairs with a barrier in __netif_txq_completed_wake(). \
161 */ 161 */ \
162 smp_mb__after_atomic(); 162 smp_mb__after_atomic(); \
163 163 \
164 /* We need to check again in a 164 /* We need to check again in a case another \
165 * CPU has just made room avai 165 * CPU has just made room available. \
166 */ 166 */ \
167 _res = 0; 167 _res = 0; \
168 if (unlikely(get_desc >= start 168 if (unlikely(get_desc >= start_thrs)) { \
169 netif_tx_start_queue(t 169 netif_tx_start_queue(txq); \
170 _res = -1; 170 _res = -1; \
171 } 171 } \
172 _res; 172 _res; \
173 }) 173 }) \
174 174
175 /** 175 /**
176 * netif_txq_maybe_stop() - locklessly stop a 176 * netif_txq_maybe_stop() - locklessly stop a Tx queue, if needed
177 * @txq: struct netdev_queue to stop/st 177 * @txq: struct netdev_queue to stop/start
178 * @get_desc: get current number of free des 178 * @get_desc: get current number of free descriptors (see requirements below!)
179 * @stop_thrs: minimal number of available de 179 * @stop_thrs: minimal number of available descriptors for queue to be left
180 * enabled 180 * enabled
181 * @start_thrs: minimal number of descriptors 181 * @start_thrs: minimal number of descriptors to re-enable the queue, can be
182 * equal to @stop_thrs or higher 182 * equal to @stop_thrs or higher to avoid frequent waking
183 * 183 *
184 * All arguments may be evaluated multiple tim 184 * All arguments may be evaluated multiple times, beware of side effects.
185 * @get_desc must be a formula or a function c 185 * @get_desc must be a formula or a function call, it must always
186 * return up-to-date information when evaluate 186 * return up-to-date information when evaluated!
187 * Expected to be used from ndo_start_xmit, se 187 * Expected to be used from ndo_start_xmit, see the comment on top of the file.
188 * 188 *
189 * Returns: 189 * Returns:
190 * 0 if the queue was stopped 190 * 0 if the queue was stopped
191 * 1 if the queue was left enabled 191 * 1 if the queue was left enabled
192 * -1 if the queue was re-enabled (raced 192 * -1 if the queue was re-enabled (raced with waking)
193 */ 193 */
194 #define netif_txq_maybe_stop(txq, get_desc, st 194 #define netif_txq_maybe_stop(txq, get_desc, stop_thrs, start_thrs) \
195 ({ 195 ({ \
196 int _res; 196 int _res; \
197 197 \
198 _res = 1; 198 _res = 1; \
199 if (unlikely(get_desc < stop_t 199 if (unlikely(get_desc < stop_thrs)) \
200 _res = netif_txq_try_s 200 _res = netif_txq_try_stop(txq, get_desc, start_thrs); \
201 _res; 201 _res; \
202 }) 202 }) \
203 203
204 /* Variant of netdev_tx_completed_queue() whic 204 /* Variant of netdev_tx_completed_queue() which guarantees smp_mb() if
205 * @bytes != 0, regardless of kernel config. 205 * @bytes != 0, regardless of kernel config.
206 */ 206 */
207 static inline void 207 static inline void
208 netdev_txq_completed_mb(struct netdev_queue *d 208 netdev_txq_completed_mb(struct netdev_queue *dev_queue,
209 unsigned int pkts, uns 209 unsigned int pkts, unsigned int bytes)
210 { 210 {
211 if (IS_ENABLED(CONFIG_BQL)) 211 if (IS_ENABLED(CONFIG_BQL))
212 netdev_tx_completed_queue(dev_ 212 netdev_tx_completed_queue(dev_queue, pkts, bytes);
213 else if (bytes) 213 else if (bytes)
214 smp_mb(); 214 smp_mb();
215 } 215 }
216 216
217 /** 217 /**
218 * __netif_txq_completed_wake() - locklessly w 218 * __netif_txq_completed_wake() - locklessly wake a Tx queue, if needed
219 * @txq: struct netdev_queue to stop/st 219 * @txq: struct netdev_queue to stop/start
220 * @pkts: number of packets completed 220 * @pkts: number of packets completed
221 * @bytes: number of bytes completed 221 * @bytes: number of bytes completed
222 * @get_desc: get current number of free des 222 * @get_desc: get current number of free descriptors (see requirements below!)
223 * @start_thrs: minimal number of descriptors 223 * @start_thrs: minimal number of descriptors to re-enable the queue
224 * @down_cond: down condition, predicate indi 224 * @down_cond: down condition, predicate indicating that the queue should
225 * not be woken up even if descri 225 * not be woken up even if descriptors are available
226 * 226 *
227 * All arguments may be evaluated multiple tim 227 * All arguments may be evaluated multiple times.
228 * @get_desc must be a formula or a function c 228 * @get_desc must be a formula or a function call, it must always
229 * return up-to-date information when evaluate 229 * return up-to-date information when evaluated!
230 * Reports completed pkts/bytes to BQL. 230 * Reports completed pkts/bytes to BQL.
231 * 231 *
232 * Returns: 232 * Returns:
233 * 0 if the queue was woken up 233 * 0 if the queue was woken up
234 * 1 if the queue was already enabled (o 234 * 1 if the queue was already enabled (or disabled but @down_cond is true)
235 * -1 if the queue was left unchanged (@s 235 * -1 if the queue was left unchanged (@start_thrs not reached)
236 */ 236 */
237 #define __netif_txq_completed_wake(txq, pkts, 237 #define __netif_txq_completed_wake(txq, pkts, bytes, \
238 get_desc, s 238 get_desc, start_thrs, down_cond) \
239 ({ 239 ({ \
240 int _res; 240 int _res; \
241 241 \
242 /* Report to BQL and piggy bac 242 /* Report to BQL and piggy back on its barrier. \
243 * Barrier makes sure that any 243 * Barrier makes sure that anybody stopping the queue \
244 * after this point sees the n 244 * after this point sees the new consumer index. \
245 * Pairs with barrier in netif 245 * Pairs with barrier in netif_txq_try_stop(). \
246 */ 246 */ \
247 netdev_txq_completed_mb(txq, p 247 netdev_txq_completed_mb(txq, pkts, bytes); \
248 248 \
249 _res = -1; 249 _res = -1; \
250 if (pkts && likely(get_desc >= 250 if (pkts && likely(get_desc >= start_thrs)) { \
251 _res = 1; 251 _res = 1; \
252 if (unlikely(netif_tx_ 252 if (unlikely(netif_tx_queue_stopped(txq)) && \
253 !(down_cond)) { 253 !(down_cond)) { \
254 netif_tx_wake_ 254 netif_tx_wake_queue(txq); \
255 _res = 0; 255 _res = 0; \
256 } 256 } \
257 } 257 } \
258 _res; 258 _res; \
259 }) 259 })
260 260
261 #define netif_txq_completed_wake(txq, pkts, by 261 #define netif_txq_completed_wake(txq, pkts, bytes, get_desc, start_thrs) \
262 __netif_txq_completed_wake(txq, pkts, 262 __netif_txq_completed_wake(txq, pkts, bytes, get_desc, start_thrs, false)
263 263
264 /* subqueue variants follow */ 264 /* subqueue variants follow */
265 265
266 #define netif_subqueue_try_stop(dev, idx, get_ 266 #define netif_subqueue_try_stop(dev, idx, get_desc, start_thrs) \
267 ({ 267 ({ \
268 struct netdev_queue *txq; 268 struct netdev_queue *txq; \
269 269 \
270 txq = netdev_get_tx_queue(dev, 270 txq = netdev_get_tx_queue(dev, idx); \
271 netif_txq_try_stop(txq, get_de 271 netif_txq_try_stop(txq, get_desc, start_thrs); \
272 }) 272 })
273 273
274 #define netif_subqueue_maybe_stop(dev, idx, ge 274 #define netif_subqueue_maybe_stop(dev, idx, get_desc, stop_thrs, start_thrs) \
275 ({ 275 ({ \
276 struct netdev_queue *txq; 276 struct netdev_queue *txq; \
277 277 \
278 txq = netdev_get_tx_queue(dev, 278 txq = netdev_get_tx_queue(dev, idx); \
279 netif_txq_maybe_stop(txq, get_ 279 netif_txq_maybe_stop(txq, get_desc, stop_thrs, start_thrs); \
280 }) 280 })
281 281
282 #define netif_subqueue_completed_wake(dev, idx 282 #define netif_subqueue_completed_wake(dev, idx, pkts, bytes, \
283 get_desc 283 get_desc, start_thrs) \
284 ({ 284 ({ \
285 struct netdev_queue *txq; 285 struct netdev_queue *txq; \
286 286 \
287 txq = netdev_get_tx_queue(dev, 287 txq = netdev_get_tx_queue(dev, idx); \
288 netif_txq_completed_wake(txq, 288 netif_txq_completed_wake(txq, pkts, bytes, \
289 get_d 289 get_desc, start_thrs); \
290 }) 290 })
291 291
292 #endif 292 #endif
293 293
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.