1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* AF_XDP internal functions 2 /* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation. 3 * Copyright(c) 2018 Intel Corporation.
4 */ 4 */
5 5
6 #ifndef _LINUX_XDP_SOCK_H 6 #ifndef _LINUX_XDP_SOCK_H
7 #define _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H
8 8
9 #include <linux/bpf.h> 9 #include <linux/bpf.h>
10 #include <linux/workqueue.h> 10 #include <linux/workqueue.h>
11 #include <linux/if_xdp.h> 11 #include <linux/if_xdp.h>
12 #include <linux/mutex.h> 12 #include <linux/mutex.h>
13 #include <linux/spinlock.h> 13 #include <linux/spinlock.h>
14 #include <linux/mm.h> 14 #include <linux/mm.h>
15 #include <net/sock.h> 15 #include <net/sock.h>
16 16
17 #define XDP_UMEM_SG_FLAG (1 << 1) 17 #define XDP_UMEM_SG_FLAG (1 << 1)
18 18
19 struct net_device; 19 struct net_device;
20 struct xsk_queue; 20 struct xsk_queue;
21 struct xdp_buff; 21 struct xdp_buff;
22 22
23 struct xdp_umem { 23 struct xdp_umem {
24 void *addrs; 24 void *addrs;
25 u64 size; 25 u64 size;
26 u32 headroom; 26 u32 headroom;
27 u32 chunk_size; 27 u32 chunk_size;
28 u32 chunks; 28 u32 chunks;
29 u32 npgs; 29 u32 npgs;
30 struct user_struct *user; 30 struct user_struct *user;
31 refcount_t users; 31 refcount_t users;
32 u8 flags; 32 u8 flags;
33 u8 tx_metadata_len; <<
34 bool zc; 33 bool zc;
35 struct page **pgs; 34 struct page **pgs;
36 int id; 35 int id;
37 struct list_head xsk_dma_list; 36 struct list_head xsk_dma_list;
38 struct work_struct work; 37 struct work_struct work;
39 }; 38 };
40 39
41 struct xsk_map { 40 struct xsk_map {
42 struct bpf_map map; 41 struct bpf_map map;
43 spinlock_t lock; /* Synchronize map up 42 spinlock_t lock; /* Synchronize map updates */
44 atomic_t count; 43 atomic_t count;
45 struct xdp_sock __rcu *xsk_map[]; 44 struct xdp_sock __rcu *xsk_map[];
46 }; 45 };
47 46
48 struct xdp_sock { 47 struct xdp_sock {
49 /* struct sock must be the first membe 48 /* struct sock must be the first member of struct xdp_sock */
50 struct sock sk; 49 struct sock sk;
51 struct xsk_queue *rx ____cacheline_ali 50 struct xsk_queue *rx ____cacheline_aligned_in_smp;
52 struct net_device *dev; 51 struct net_device *dev;
53 struct xdp_umem *umem; 52 struct xdp_umem *umem;
54 struct list_head flush_node; 53 struct list_head flush_node;
55 struct xsk_buff_pool *pool; 54 struct xsk_buff_pool *pool;
56 u16 queue_id; 55 u16 queue_id;
57 bool zc; 56 bool zc;
58 bool sg; 57 bool sg;
59 enum { 58 enum {
60 XSK_READY = 0, 59 XSK_READY = 0,
61 XSK_BOUND, 60 XSK_BOUND,
62 XSK_UNBOUND, 61 XSK_UNBOUND,
63 } state; 62 } state;
64 63
65 struct xsk_queue *tx ____cacheline_ali 64 struct xsk_queue *tx ____cacheline_aligned_in_smp;
66 struct list_head tx_list; 65 struct list_head tx_list;
67 /* record the number of tx descriptors 66 /* record the number of tx descriptors sent by this xsk and
68 * when it exceeds MAX_PER_SOCKET_BUDG 67 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
69 * to be given to other xsks for sendi 68 * to be given to other xsks for sending tx descriptors, thereby
70 * preventing other XSKs from being st 69 * preventing other XSKs from being starved.
71 */ 70 */
72 u32 tx_budget_spent; 71 u32 tx_budget_spent;
73 72
74 /* Protects generic receive. */ 73 /* Protects generic receive. */
75 spinlock_t rx_lock; 74 spinlock_t rx_lock;
76 75
77 /* Statistics */ 76 /* Statistics */
78 u64 rx_dropped; 77 u64 rx_dropped;
79 u64 rx_queue_full; 78 u64 rx_queue_full;
80 79
81 /* When __xsk_generic_xmit() must retu 80 /* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
82 * packet, the partially built skb is 81 * packet, the partially built skb is saved here so that packet building can resume in next
83 * call of __xsk_generic_xmit(). 82 * call of __xsk_generic_xmit().
84 */ 83 */
85 struct sk_buff *skb; 84 struct sk_buff *skb;
86 85
87 struct list_head map_list; 86 struct list_head map_list;
88 /* Protects map_list */ 87 /* Protects map_list */
89 spinlock_t map_list_lock; 88 spinlock_t map_list_lock;
90 /* Protects multiple processes in the 89 /* Protects multiple processes in the control path */
91 struct mutex mutex; 90 struct mutex mutex;
92 struct xsk_queue *fq_tmp; /* Only as t 91 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
93 struct xsk_queue *cq_tmp; /* Only as t 92 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
94 }; 93 };
95 94
96 /* <<
97 * AF_XDP TX metadata hooks for network device <<
98 * The following hooks can be defined; unless <<
99 * optional and can be filled with a null poin <<
100 * <<
101 * void (*tmo_request_timestamp)(void *priv) <<
102 * Called when AF_XDP frame requested egre <<
103 * <<
104 * u64 (*tmo_fill_timestamp)(void *priv) <<
105 * Called when AF_XDP frame, that had requ <<
106 * received a completion. The hook needs t <<
107 * <<
108 * void (*tmo_request_checksum)(u16 csum_start <<
109 * Called when AF_XDP frame requested HW c <<
110 * indicates position where checksumming s <<
111 * csum_offset indicates position where ch <<
112 * <<
113 */ <<
114 struct xsk_tx_metadata_ops { <<
115 void (*tmo_request_timestamp)(void <<
116 u64 (*tmo_fill_timestamp)(void *pr <<
117 void (*tmo_request_checksum)(u16 cs <<
118 }; <<
119 <<
120 #ifdef CONFIG_XDP_SOCKETS 95 #ifdef CONFIG_XDP_SOCKETS
121 96
122 int xsk_generic_rcv(struct xdp_sock *xs, struc 97 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
123 int __xsk_map_redirect(struct xdp_sock *xs, st 98 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
124 void __xsk_map_flush(struct list_head *flush_l !! 99 void __xsk_map_flush(void);
125 <<
126 /** <<
127 * xsk_tx_metadata_to_compl - Save enough rel <<
128 * to perform tx completion in the future. <<
129 * @meta: pointer to AF_XDP metadata area <<
130 * @compl: pointer to output struct xsk_tx_me <<
131 * <<
132 * This function should be called by the netw <<
133 * it prepares AF_XDP egress packet. The valu <<
134 * and passed to xsk_tx_metadata_complete upo <<
135 */ <<
136 static inline void xsk_tx_metadata_to_compl(st <<
137 st <<
138 { <<
139 if (!meta) <<
140 return; <<
141 <<
142 if (meta->flags & XDP_TXMD_FLAGS_TIMES <<
143 compl->tx_timestamp = &meta->c <<
144 else <<
145 compl->tx_timestamp = NULL; <<
146 } <<
147 <<
148 /** <<
149 * xsk_tx_metadata_request - Evaluate AF_XDP <<
150 * and call appropriate xsk_tx_metadata_ops o <<
151 * @meta: pointer to AF_XDP metadata area <<
152 * @ops: pointer to struct xsk_tx_metadata_op <<
153 * @priv: pointer to driver-private aread <<
154 * <<
155 * This function should be called by the netw <<
156 * it prepares AF_XDP egress packet. <<
157 */ <<
158 static inline void xsk_tx_metadata_request(con <<
159 con <<
160 voi <<
161 { <<
162 if (!meta) <<
163 return; <<
164 <<
165 if (ops->tmo_request_timestamp) <<
166 if (meta->flags & XDP_TXMD_FLA <<
167 ops->tmo_request_times <<
168 <<
169 if (ops->tmo_request_checksum) <<
170 if (meta->flags & XDP_TXMD_FLA <<
171 ops->tmo_request_check <<
172 <<
173 } <<
174 <<
175 /** <<
176 * xsk_tx_metadata_complete - Evaluate AF_XDP <<
177 * and call appropriate xsk_tx_metadata_ops o <<
178 * @compl: pointer to completion metadata pro <<
179 * @ops: pointer to struct xsk_tx_metadata_op <<
180 * @priv: pointer to driver-private aread <<
181 * <<
182 * This function should be called by the netw <<
183 * AF_XDP egress completion. <<
184 */ <<
185 static inline void xsk_tx_metadata_complete(st <<
186 co <<
187 vo <<
188 { <<
189 if (!compl) <<
190 return; <<
191 if (!compl->tx_timestamp) <<
192 return; <<
193 <<
194 *compl->tx_timestamp = ops->tmo_fill_t <<
195 } <<
196 100
197 #else 101 #else
198 102
199 static inline int xsk_generic_rcv(struct xdp_s 103 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
200 { 104 {
201 return -ENOTSUPP; 105 return -ENOTSUPP;
202 } 106 }
203 107
204 static inline int __xsk_map_redirect(struct xd 108 static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
205 { 109 {
206 return -EOPNOTSUPP; 110 return -EOPNOTSUPP;
207 } 111 }
208 112
209 static inline void __xsk_map_flush(struct list !! 113 static inline void __xsk_map_flush(void)
210 { <<
211 } <<
212 <<
213 static inline void xsk_tx_metadata_to_compl(st <<
214 st <<
215 { 114 {
216 } 115 }
217 116
218 static inline void xsk_tx_metadata_request(str !! 117 #endif /* CONFIG_XDP_SOCKETS */
219 con <<
220 voi <<
221 { <<
222 } <<
223 118
224 static inline void xsk_tx_metadata_complete(st !! 119 #if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
225 co !! 120 bool xsk_map_check_flush(void);
226 vo !! 121 #else
>> 122 static inline bool xsk_map_check_flush(void)
227 { 123 {
>> 124 return false;
228 } 125 }
>> 126 #endif
229 127
230 #endif /* CONFIG_XDP_SOCKETS */ <<
231 #endif /* _LINUX_XDP_SOCK_H */ 128 #endif /* _LINUX_XDP_SOCK_H */
232 129
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.