Linux/include/net/xdp

Version: ~ [ linux-6.12-rc7 ] ~ [ linux-6.11.7 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.60 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.116 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.171 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.229 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.285 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.323 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.12 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* AF_XDP internal functions 2 /* AF_XDP internal functions 3 * Copyright(c) 2018 Intel Corporation. 3 * Copyright(c) 2018 Intel Corporation. 4 */ 4 */ 5 5 6 #ifndef _LINUX_XDP_SOCK_H 6 #ifndef _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H 8 8 9 #include <linux/bpf.h> << 10 #include <linux/workqueue.h> 9 #include <linux/workqueue.h> 11 #include <linux/if_xdp.h> 10 #include <linux/if_xdp.h> 12 #include <linux/mutex.h> 11 #include <linux/mutex.h> 13 #include <linux/spinlock.h> 12 #include <linux/spinlock.h> 14 #include <linux/mm.h> 13 #include <linux/mm.h> 15 #include <net/sock.h> 14 #include <net/sock.h> 16 15 17 #define XDP_UMEM_SG_FLAG (1 << 1) << 18 << 19 struct net_device; 16 struct net_device; 20 struct xsk_queue; 17 struct xsk_queue; 21 struct xdp_buff; 18 struct xdp_buff; 22 19 23 struct xdp_umem { 20 struct xdp_umem { 24 void *addrs; 21 void *addrs; 25 u64 size; 22 u64 size; 26 u32 headroom; 23 u32 headroom; 27 u32 chunk_size; 24 u32 chunk_size; 28 u32 chunks; 25 u32 chunks; 29 u32 npgs; 26 u32 npgs; 30 struct user_struct *user; 27 struct user_struct *user; 31 refcount_t users; 28 refcount_t users; 32 u8 flags; 29 u8 flags; 33 u8 tx_metadata_len; << 34 bool zc; 30 bool zc; 35 struct page **pgs; 31 struct page **pgs; 36 int id; 32 int id; 37 struct list_head xsk_dma_list; 33 struct list_head xsk_dma_list; 38 struct work_struct work; 34 struct work_struct work; 39 }; 35 }; 40 36 41 struct xsk_map { 37 struct xsk_map { 42 struct bpf_map map; 38 struct bpf_map map; 43 spinlock_t lock; /* Synchronize map up 39 spinlock_t lock; /* Synchronize map updates */ 44 atomic_t count; !! 40 struct xdp_sock *xsk_map[]; 45 struct xdp_sock __rcu *xsk_map[]; << 46 }; 41 }; 47 42 48 struct xdp_sock { 43 struct xdp_sock { 49 /* struct sock must be the first membe 44 /* struct sock must be the first member of struct xdp_sock */ 50 struct sock sk; 45 struct sock sk; 51 struct xsk_queue *rx ____cacheline_ali 46 struct xsk_queue *rx ____cacheline_aligned_in_smp; 52 struct net_device *dev; 47 struct net_device *dev; 53 struct xdp_umem *umem; 48 struct xdp_umem *umem; 54 struct list_head flush_node; 49 struct list_head flush_node; 55 struct xsk_buff_pool *pool; 50 struct xsk_buff_pool *pool; 56 u16 queue_id; 51 u16 queue_id; 57 bool zc; 52 bool zc; 58 bool sg; << 59 enum { 53 enum { 60 XSK_READY = 0, 54 XSK_READY = 0, 61 XSK_BOUND, 55 XSK_BOUND, 62 XSK_UNBOUND, 56 XSK_UNBOUND, 63 } state; 57 } state; 64 58 65 struct xsk_queue *tx ____cacheline_ali 59 struct xsk_queue *tx ____cacheline_aligned_in_smp; 66 struct list_head tx_list; 60 struct list_head tx_list; 67 /* record the number of tx descriptors << 68 * when it exceeds MAX_PER_SOCKET_BUDG << 69 * to be given to other xsks for sendi << 70 * preventing other XSKs from being st << 71 */ << 72 u32 tx_budget_spent; << 73 << 74 /* Protects generic receive. */ 61 /* Protects generic receive. */ 75 spinlock_t rx_lock; 62 spinlock_t rx_lock; 76 63 77 /* Statistics */ 64 /* Statistics */ 78 u64 rx_dropped; 65 u64 rx_dropped; 79 u64 rx_queue_full; 66 u64 rx_queue_full; 80 67 81 /* When __xsk_generic_xmit() must retu << 82 * packet, the partially built skb is << 83 * call of __xsk_generic_xmit(). << 84 */ << 85 struct sk_buff *skb; << 86 << 87 struct list_head map_list; 68 struct list_head map_list; 88 /* Protects map_list */ 69 /* Protects map_list */ 89 spinlock_t map_list_lock; 70 spinlock_t map_list_lock; 90 /* Protects multiple processes in the 71 /* Protects multiple processes in the control path */ 91 struct mutex mutex; 72 struct mutex mutex; 92 struct xsk_queue *fq_tmp; /* Only as t 73 struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */ 93 struct xsk_queue *cq_tmp; /* Only as t 74 struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */ 94 }; 75 }; 95 76 96 /* << 97 * AF_XDP TX metadata hooks for network device << 98 * The following hooks can be defined; unless << 99 * optional and can be filled with a null poin << 100 * << 101 * void (*tmo_request_timestamp)(void *priv) << 102 * Called when AF_XDP frame requested egre << 103 * << 104 * u64 (*tmo_fill_timestamp)(void *priv) << 105 * Called when AF_XDP frame, that had requ << 106 * received a completion. The hook needs t << 107 * << 108 * void (*tmo_request_checksum)(u16 csum_start << 109 * Called when AF_XDP frame requested HW c << 110 * indicates position where checksumming s << 111 * csum_offset indicates position where ch << 112 * << 113 */ << 114 struct xsk_tx_metadata_ops { << 115 void (*tmo_request_timestamp)(void << 116 u64 (*tmo_fill_timestamp)(void *pr << 117 void (*tmo_request_checksum)(u16 cs << 118 }; << 119 << 120 #ifdef CONFIG_XDP_SOCKETS 77 #ifdef CONFIG_XDP_SOCKETS 121 78 122 int xsk_generic_rcv(struct xdp_sock *xs, struc 79 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 123 int __xsk_map_redirect(struct xdp_sock *xs, st 80 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); 124 void __xsk_map_flush(struct list_head *flush_l !! 81 void __xsk_map_flush(void); 125 82 126 /** !! 83 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 127 * xsk_tx_metadata_to_compl - Save enough rel !! 84 u32 key) 128 * to perform tx completion in the future. << 129 * @meta: pointer to AF_XDP metadata area << 130 * @compl: pointer to output struct xsk_tx_me << 131 * << 132 * This function should be called by the netw << 133 * it prepares AF_XDP egress packet. The valu << 134 * and passed to xsk_tx_metadata_complete upo << 135 */ << 136 static inline void xsk_tx_metadata_to_compl(st << 137 st << 138 { 85 { 139 if (!meta) !! 86 struct xsk_map *m = container_of(map, struct xsk_map, map); 140 return; !! 87 struct xdp_sock *xs; 141 88 142 if (meta->flags & XDP_TXMD_FLAGS_TIMES !! 89 if (key >= map->max_entries) 143 compl->tx_timestamp = &meta->c !! 90 return NULL; 144 else << 145 compl->tx_timestamp = NULL; << 146 } << 147 << 148 /** << 149 * xsk_tx_metadata_request - Evaluate AF_XDP << 150 * and call appropriate xsk_tx_metadata_ops o << 151 * @meta: pointer to AF_XDP metadata area << 152 * @ops: pointer to struct xsk_tx_metadata_op << 153 * @priv: pointer to driver-private aread << 154 * << 155 * This function should be called by the netw << 156 * it prepares AF_XDP egress packet. << 157 */ << 158 static inline void xsk_tx_metadata_request(con << 159 con << 160 voi << 161 { << 162 if (!meta) << 163 return; << 164 91 165 if (ops->tmo_request_timestamp) !! 92 xs = READ_ONCE(m->xsk_map[key]); 166 if (meta->flags & XDP_TXMD_FLA !! 93 return xs; 167 ops->tmo_request_times << 168 << 169 if (ops->tmo_request_checksum) << 170 if (meta->flags & XDP_TXMD_FLA << 171 ops->tmo_request_check << 172 << 173 } << 174 << 175 /** << 176 * xsk_tx_metadata_complete - Evaluate AF_XDP << 177 * and call appropriate xsk_tx_metadata_ops o << 178 * @compl: pointer to completion metadata pro << 179 * @ops: pointer to struct xsk_tx_metadata_op << 180 * @priv: pointer to driver-private aread << 181 * << 182 * This function should be called by the netw << 183 * AF_XDP egress completion. << 184 */ << 185 static inline void xsk_tx_metadata_complete(st << 186 co << 187 vo << 188 { << 189 if (!compl) << 190 return; << 191 if (!compl->tx_timestamp) << 192 return; << 193 << 194 *compl->tx_timestamp = ops->tmo_fill_t << 195 } 94 } 196 95 197 #else 96 #else 198 97 199 static inline int xsk_generic_rcv(struct xdp_s 98 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 200 { 99 { 201 return -ENOTSUPP; 100 return -ENOTSUPP; 202 } 101 } 203 102 204 static inline int __xsk_map_redirect(struct xd 103 static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) 205 { 104 { 206 return -EOPNOTSUPP; 105 return -EOPNOTSUPP; 207 } 106 } 208 107 209 static inline void __xsk_map_flush(struct list !! 108 static inline void __xsk_map_flush(void) 210 { << 211 } << 212 << 213 static inline void xsk_tx_metadata_to_compl(st << 214 st << 215 { 109 { 216 } 110 } 217 111 218 static inline void xsk_tx_metadata_request(str !! 112 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 219 con !! 113 u32 key) 220 voi << 221 { << 222 } << 223 << 224 static inline void xsk_tx_metadata_complete(st << 225 co << 226 vo << 227 { 114 { >> 115 return NULL; 228 } 116 } 229 117 230 #endif /* CONFIG_XDP_SOCKETS */ 118 #endif /* CONFIG_XDP_SOCKETS */ >> 119 231 #endif /* _LINUX_XDP_SOCK_H */ 120 #endif /* _LINUX_XDP_SOCK_H */ 232 121

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

TOMOYO Linux Cross Reference
Linux/include/net/xdp_sock.h

Diff markup

Differences between /include/net/xdp_sock.h (Version linux-6.12-rc7) and /include/net/xdp_sock.h (Version linux-5.12.19)

TOMOYO Linux Cross Reference Linux/include/net/xdp_sock.h

Diff markup

Differences between /include/net/xdp_sock.h (Version linux-6.12-rc7) and /include/net/xdp_sock.h (Version linux-5.12.19)

TOMOYO Linux Cross Reference
Linux/include/net/xdp_sock.h