1 /* SPDX-License-Identifier: GPL-2.0 */ 1
2 /* AF_XDP internal functions
3 * Copyright(c) 2018 Intel Corporation.
4 */
5
6 #ifndef _LINUX_XDP_SOCK_H
7 #define _LINUX_XDP_SOCK_H
8
9 #include <linux/bpf.h>
10 #include <linux/workqueue.h>
11 #include <linux/if_xdp.h>
12 #include <linux/mutex.h>
13 #include <linux/spinlock.h>
14 #include <linux/mm.h>
15 #include <net/sock.h>
16
17 #define XDP_UMEM_SG_FLAG (1 << 1)
18
19 struct net_device;
20 struct xsk_queue;
21 struct xdp_buff;
22
23 struct xdp_umem {
24 void *addrs;
25 u64 size;
26 u32 headroom;
27 u32 chunk_size;
28 u32 chunks;
29 u32 npgs;
30 struct user_struct *user;
31 refcount_t users;
32 u8 flags;
33 u8 tx_metadata_len;
34 bool zc;
35 struct page **pgs;
36 int id;
37 struct list_head xsk_dma_list;
38 struct work_struct work;
39 };
40
41 struct xsk_map {
42 struct bpf_map map;
43 spinlock_t lock; /* Synchronize map up
44 atomic_t count;
45 struct xdp_sock __rcu *xsk_map[];
46 };
47
48 struct xdp_sock {
49 /* struct sock must be the first membe
50 struct sock sk;
51 struct xsk_queue *rx ____cacheline_ali
52 struct net_device *dev;
53 struct xdp_umem *umem;
54 struct list_head flush_node;
55 struct xsk_buff_pool *pool;
56 u16 queue_id;
57 bool zc;
58 bool sg;
59 enum {
60 XSK_READY = 0,
61 XSK_BOUND,
62 XSK_UNBOUND,
63 } state;
64
65 struct xsk_queue *tx ____cacheline_ali
66 struct list_head tx_list;
67 /* record the number of tx descriptors
68 * when it exceeds MAX_PER_SOCKET_BUDG
69 * to be given to other xsks for sendi
70 * preventing other XSKs from being st
71 */
72 u32 tx_budget_spent;
73
74 /* Protects generic receive. */
75 spinlock_t rx_lock;
76
77 /* Statistics */
78 u64 rx_dropped;
79 u64 rx_queue_full;
80
81 /* When __xsk_generic_xmit() must retu
82 * packet, the partially built skb is
83 * call of __xsk_generic_xmit().
84 */
85 struct sk_buff *skb;
86
87 struct list_head map_list;
88 /* Protects map_list */
89 spinlock_t map_list_lock;
90 /* Protects multiple processes in the
91 struct mutex mutex;
92 struct xsk_queue *fq_tmp; /* Only as t
93 struct xsk_queue *cq_tmp; /* Only as t
94 };
95
96 /*
97 * AF_XDP TX metadata hooks for network device
98 * The following hooks can be defined; unless
99 * optional and can be filled with a null poin
100 *
101 * void (*tmo_request_timestamp)(void *priv)
102 * Called when AF_XDP frame requested egre
103 *
104 * u64 (*tmo_fill_timestamp)(void *priv)
105 * Called when AF_XDP frame, that had requ
106 * received a completion. The hook needs t
107 *
108 * void (*tmo_request_checksum)(u16 csum_start
109 * Called when AF_XDP frame requested HW c
110 * indicates position where checksumming s
111 * csum_offset indicates position where ch
112 *
113 */
114 struct xsk_tx_metadata_ops {
115 void (*tmo_request_timestamp)(void
116 u64 (*tmo_fill_timestamp)(void *pr
117 void (*tmo_request_checksum)(u16 cs
118 };
119
120 #ifdef CONFIG_XDP_SOCKETS
121
122 int xsk_generic_rcv(struct xdp_sock *xs, struc
123 int __xsk_map_redirect(struct xdp_sock *xs, st
124 void __xsk_map_flush(struct list_head *flush_l
125
126 /**
127 * xsk_tx_metadata_to_compl - Save enough rel
128 * to perform tx completion in the future.
129 * @meta: pointer to AF_XDP metadata area
130 * @compl: pointer to output struct xsk_tx_me
131 *
132 * This function should be called by the netw
133 * it prepares AF_XDP egress packet. The valu
134 * and passed to xsk_tx_metadata_complete upo
135 */
136 static inline void xsk_tx_metadata_to_compl(st
137 st
138 {
139 if (!meta)
140 return;
141
142 if (meta->flags & XDP_TXMD_FLAGS_TIMES
143 compl->tx_timestamp = &meta->c
144 else
145 compl->tx_timestamp = NULL;
146 }
147
148 /**
149 * xsk_tx_metadata_request - Evaluate AF_XDP
150 * and call appropriate xsk_tx_metadata_ops o
151 * @meta: pointer to AF_XDP metadata area
152 * @ops: pointer to struct xsk_tx_metadata_op
153 * @priv: pointer to driver-private aread
154 *
155 * This function should be called by the netw
156 * it prepares AF_XDP egress packet.
157 */
158 static inline void xsk_tx_metadata_request(con
159 con
160 voi
161 {
162 if (!meta)
163 return;
164
165 if (ops->tmo_request_timestamp)
166 if (meta->flags & XDP_TXMD_FLA
167 ops->tmo_request_times
168
169 if (ops->tmo_request_checksum)
170 if (meta->flags & XDP_TXMD_FLA
171 ops->tmo_request_check
172
173 }
174
175 /**
176 * xsk_tx_metadata_complete - Evaluate AF_XDP
177 * and call appropriate xsk_tx_metadata_ops o
178 * @compl: pointer to completion metadata pro
179 * @ops: pointer to struct xsk_tx_metadata_op
180 * @priv: pointer to driver-private aread
181 *
182 * This function should be called by the netw
183 * AF_XDP egress completion.
184 */
185 static inline void xsk_tx_metadata_complete(st
186 co
187 vo
188 {
189 if (!compl)
190 return;
191 if (!compl->tx_timestamp)
192 return;
193
194 *compl->tx_timestamp = ops->tmo_fill_t
195 }
196
197 #else
198
199 static inline int xsk_generic_rcv(struct xdp_s
200 {
201 return -ENOTSUPP;
202 }
203
204 static inline int __xsk_map_redirect(struct xd
205 {
206 return -EOPNOTSUPP;
207 }
208
209 static inline void __xsk_map_flush(struct list
210 {
211 }
212
213 static inline void xsk_tx_metadata_to_compl(st
214 st
215 {
216 }
217
218 static inline void xsk_tx_metadata_request(str
219 con
220 voi
221 {
222 }
223
224 static inline void xsk_tx_metadata_complete(st
225 co
226 vo
227 {
228 }
229
230 #endif /* CONFIG_XDP_SOCKETS */
231 #endif /* _LINUX_XDP_SOCK_H */
232
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.