Linux/include/net/xdp

Version: ~ [ linux-6.12-rc7 ] ~ [ linux-6.11.7 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.60 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.116 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.171 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.229 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.285 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.323 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.12 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* AF_XDP internal functions 2 /* AF_XDP internal functions 3 * Copyright(c) 2018 Intel Corporation. 3 * Copyright(c) 2018 Intel Corporation. 4 */ 4 */ 5 5 6 #ifndef _LINUX_XDP_SOCK_H 6 #ifndef _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H 7 #define _LINUX_XDP_SOCK_H 8 8 9 #include <linux/bpf.h> << 10 #include <linux/workqueue.h> 9 #include <linux/workqueue.h> 11 #include <linux/if_xdp.h> 10 #include <linux/if_xdp.h> 12 #include <linux/mutex.h> 11 #include <linux/mutex.h> 13 #include <linux/spinlock.h> 12 #include <linux/spinlock.h> 14 #include <linux/mm.h> 13 #include <linux/mm.h> 15 #include <net/sock.h> 14 #include <net/sock.h> 16 15 17 #define XDP_UMEM_SG_FLAG (1 << 1) << 18 << 19 struct net_device; 16 struct net_device; 20 struct xsk_queue; 17 struct xsk_queue; 21 struct xdp_buff; 18 struct xdp_buff; 22 19 23 struct xdp_umem { 20 struct xdp_umem { 24 void *addrs; !! 21 struct xsk_queue *fq; >> 22 struct xsk_queue *cq; >> 23 struct xsk_buff_pool *pool; 25 u64 size; 24 u64 size; 26 u32 headroom; 25 u32 headroom; 27 u32 chunk_size; 26 u32 chunk_size; 28 u32 chunks; << 29 u32 npgs; << 30 struct user_struct *user; 27 struct user_struct *user; 31 refcount_t users; 28 refcount_t users; 32 u8 flags; !! 29 struct work_struct work; 33 u8 tx_metadata_len; << 34 bool zc; << 35 struct page **pgs; 30 struct page **pgs; >> 31 u32 npgs; >> 32 u16 queue_id; >> 33 u8 need_wakeup; >> 34 u8 flags; 36 int id; 35 int id; 37 struct list_head xsk_dma_list; !! 36 struct net_device *dev; 38 struct work_struct work; !! 37 bool zc; >> 38 spinlock_t xsk_tx_list_lock; >> 39 struct list_head xsk_tx_list; 39 }; 40 }; 40 41 41 struct xsk_map { 42 struct xsk_map { 42 struct bpf_map map; 43 struct bpf_map map; 43 spinlock_t lock; /* Synchronize map up 44 spinlock_t lock; /* Synchronize map updates */ 44 atomic_t count; !! 45 struct xdp_sock *xsk_map[]; 45 struct xdp_sock __rcu *xsk_map[]; << 46 }; 46 }; 47 47 48 struct xdp_sock { 48 struct xdp_sock { 49 /* struct sock must be the first membe 49 /* struct sock must be the first member of struct xdp_sock */ 50 struct sock sk; 50 struct sock sk; 51 struct xsk_queue *rx ____cacheline_ali !! 51 struct xsk_queue *rx; 52 struct net_device *dev; 52 struct net_device *dev; 53 struct xdp_umem *umem; 53 struct xdp_umem *umem; 54 struct list_head flush_node; 54 struct list_head flush_node; 55 struct xsk_buff_pool *pool; << 56 u16 queue_id; 55 u16 queue_id; 57 bool zc; 56 bool zc; 58 bool sg; << 59 enum { 57 enum { 60 XSK_READY = 0, 58 XSK_READY = 0, 61 XSK_BOUND, 59 XSK_BOUND, 62 XSK_UNBOUND, 60 XSK_UNBOUND, 63 } state; 61 } state; 64 !! 62 /* Protects multiple processes in the control path */ >> 63 struct mutex mutex; 65 struct xsk_queue *tx ____cacheline_ali 64 struct xsk_queue *tx ____cacheline_aligned_in_smp; 66 struct list_head tx_list; !! 65 struct list_head list; 67 /* record the number of tx descriptors !! 66 /* Mutual exclusion of NAPI TX thread and sendmsg error paths 68 * when it exceeds MAX_PER_SOCKET_BUDG !! 67 * in the SKB destructor callback. 69 * to be given to other xsks for sendi << 70 * preventing other XSKs from being st << 71 */ 68 */ 72 u32 tx_budget_spent; !! 69 spinlock_t tx_completion_lock; 73 << 74 /* Protects generic receive. */ 70 /* Protects generic receive. */ 75 spinlock_t rx_lock; 71 spinlock_t rx_lock; 76 72 77 /* Statistics */ 73 /* Statistics */ 78 u64 rx_dropped; 74 u64 rx_dropped; 79 u64 rx_queue_full; 75 u64 rx_queue_full; 80 76 81 /* When __xsk_generic_xmit() must retu << 82 * packet, the partially built skb is << 83 * call of __xsk_generic_xmit(). << 84 */ << 85 struct sk_buff *skb; << 86 << 87 struct list_head map_list; 77 struct list_head map_list; 88 /* Protects map_list */ 78 /* Protects map_list */ 89 spinlock_t map_list_lock; 79 spinlock_t map_list_lock; 90 /* Protects multiple processes in the << 91 struct mutex mutex; << 92 struct xsk_queue *fq_tmp; /* Only as t << 93 struct xsk_queue *cq_tmp; /* Only as t << 94 }; << 95 << 96 /* << 97 * AF_XDP TX metadata hooks for network device << 98 * The following hooks can be defined; unless << 99 * optional and can be filled with a null poin << 100 * << 101 * void (*tmo_request_timestamp)(void *priv) << 102 * Called when AF_XDP frame requested egre << 103 * << 104 * u64 (*tmo_fill_timestamp)(void *priv) << 105 * Called when AF_XDP frame, that had requ << 106 * received a completion. The hook needs t << 107 * << 108 * void (*tmo_request_checksum)(u16 csum_start << 109 * Called when AF_XDP frame requested HW c << 110 * indicates position where checksumming s << 111 * csum_offset indicates position where ch << 112 * << 113 */ << 114 struct xsk_tx_metadata_ops { << 115 void (*tmo_request_timestamp)(void << 116 u64 (*tmo_fill_timestamp)(void *pr << 117 void (*tmo_request_checksum)(u16 cs << 118 }; 80 }; 119 81 120 #ifdef CONFIG_XDP_SOCKETS 82 #ifdef CONFIG_XDP_SOCKETS 121 83 122 int xsk_generic_rcv(struct xdp_sock *xs, struc 84 int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp); 123 int __xsk_map_redirect(struct xdp_sock *xs, st 85 int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp); 124 void __xsk_map_flush(struct list_head *flush_l !! 86 void __xsk_map_flush(void); 125 87 126 /** !! 88 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 127 * xsk_tx_metadata_to_compl - Save enough rel !! 89 u32 key) 128 * to perform tx completion in the future. << 129 * @meta: pointer to AF_XDP metadata area << 130 * @compl: pointer to output struct xsk_tx_me << 131 * << 132 * This function should be called by the netw << 133 * it prepares AF_XDP egress packet. The valu << 134 * and passed to xsk_tx_metadata_complete upo << 135 */ << 136 static inline void xsk_tx_metadata_to_compl(st << 137 st << 138 { 90 { 139 if (!meta) !! 91 struct xsk_map *m = container_of(map, struct xsk_map, map); 140 return; !! 92 struct xdp_sock *xs; 141 << 142 if (meta->flags & XDP_TXMD_FLAGS_TIMES << 143 compl->tx_timestamp = &meta->c << 144 else << 145 compl->tx_timestamp = NULL; << 146 } << 147 << 148 /** << 149 * xsk_tx_metadata_request - Evaluate AF_XDP << 150 * and call appropriate xsk_tx_metadata_ops o << 151 * @meta: pointer to AF_XDP metadata area << 152 * @ops: pointer to struct xsk_tx_metadata_op << 153 * @priv: pointer to driver-private aread << 154 * << 155 * This function should be called by the netw << 156 * it prepares AF_XDP egress packet. << 157 */ << 158 static inline void xsk_tx_metadata_request(con << 159 con << 160 voi << 161 { << 162 if (!meta) << 163 return; << 164 << 165 if (ops->tmo_request_timestamp) << 166 if (meta->flags & XDP_TXMD_FLA << 167 ops->tmo_request_times << 168 << 169 if (ops->tmo_request_checksum) << 170 if (meta->flags & XDP_TXMD_FLA << 171 ops->tmo_request_check << 172 << 173 } << 174 93 175 /** !! 94 if (key >= map->max_entries) 176 * xsk_tx_metadata_complete - Evaluate AF_XDP !! 95 return NULL; 177 * and call appropriate xsk_tx_metadata_ops o << 178 * @compl: pointer to completion metadata pro << 179 * @ops: pointer to struct xsk_tx_metadata_op << 180 * @priv: pointer to driver-private aread << 181 * << 182 * This function should be called by the netw << 183 * AF_XDP egress completion. << 184 */ << 185 static inline void xsk_tx_metadata_complete(st << 186 co << 187 vo << 188 { << 189 if (!compl) << 190 return; << 191 if (!compl->tx_timestamp) << 192 return; << 193 96 194 *compl->tx_timestamp = ops->tmo_fill_t !! 97 xs = READ_ONCE(m->xsk_map[key]); >> 98 return xs; 195 } 99 } 196 100 197 #else 101 #else 198 102 199 static inline int xsk_generic_rcv(struct xdp_s 103 static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp) 200 { 104 { 201 return -ENOTSUPP; 105 return -ENOTSUPP; 202 } 106 } 203 107 204 static inline int __xsk_map_redirect(struct xd 108 static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp) 205 { 109 { 206 return -EOPNOTSUPP; 110 return -EOPNOTSUPP; 207 } 111 } 208 112 209 static inline void __xsk_map_flush(struct list !! 113 static inline void __xsk_map_flush(void) 210 { << 211 } << 212 << 213 static inline void xsk_tx_metadata_to_compl(st << 214 st << 215 { 114 { 216 } 115 } 217 116 218 static inline void xsk_tx_metadata_request(str !! 117 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 219 con !! 118 u32 key) 220 voi << 221 { << 222 } << 223 << 224 static inline void xsk_tx_metadata_complete(st << 225 co << 226 vo << 227 { 119 { >> 120 return NULL; 228 } 121 } 229 122 230 #endif /* CONFIG_XDP_SOCKETS */ 123 #endif /* CONFIG_XDP_SOCKETS */ >> 124 231 #endif /* _LINUX_XDP_SOCK_H */ 125 #endif /* _LINUX_XDP_SOCK_H */ 232 126

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

TOMOYO Linux Cross Reference
Linux/include/net/xdp_sock.h

Diff markup

Differences between /include/net/xdp_sock.h (Version linux-6.12-rc7) and /include/net/xdp_sock.h (Version linux-5.9.16)

TOMOYO Linux Cross Reference Linux/include/net/xdp_sock.h

Diff markup

Differences between /include/net/xdp_sock.h (Version linux-6.12-rc7) and /include/net/xdp_sock.h (Version linux-5.9.16)

TOMOYO Linux Cross Reference
Linux/include/net/xdp_sock.h