1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2023 Isovalent */
3 #ifndef __NET_TCX_H
4 #define __NET_TCX_H
5
6 #include <linux/bpf.h>
7 #include <linux/bpf_mprog.h>
8
9 #include <net/sch_generic.h>
10
11 struct mini_Qdisc;
12
13 struct tcx_entry {
14 struct mini_Qdisc __rcu *miniq;
15 struct bpf_mprog_bundle bundle;
16 u32 miniq_active;
17 struct rcu_head rcu;
18 };
19
20 struct tcx_link {
21 struct bpf_link link;
22 struct net_device *dev;
23 u32 location;
24 };
25
26 static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress)
27 {
28 #ifdef CONFIG_NET_XGRESS
29 skb->tc_at_ingress = ingress;
30 #endif
31 }
32
33 #ifdef CONFIG_NET_XGRESS
34 static inline struct tcx_entry *tcx_entry(struct bpf_mprog_entry *entry)
35 {
36 struct bpf_mprog_bundle *bundle = entry->parent;
37
38 return container_of(bundle, struct tcx_entry, bundle);
39 }
40
41 static inline struct tcx_link *tcx_link(const struct bpf_link *link)
42 {
43 return container_of(link, struct tcx_link, link);
44 }
45
46 void tcx_inc(void);
47 void tcx_dec(void);
48
49 static inline void tcx_entry_sync(void)
50 {
51 /* bpf_mprog_entry got a/b swapped, therefore ensure that
52 * there are no inflight users on the old one anymore.
53 */
54 synchronize_rcu();
55 }
56
57 static inline void
58 tcx_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry,
59 bool ingress)
60 {
61 ASSERT_RTNL();
62 if (ingress)
63 rcu_assign_pointer(dev->tcx_ingress, entry);
64 else
65 rcu_assign_pointer(dev->tcx_egress, entry);
66 }
67
68 static inline struct bpf_mprog_entry *
69 tcx_entry_fetch(struct net_device *dev, bool ingress)
70 {
71 ASSERT_RTNL();
72 if (ingress)
73 return rcu_dereference_rtnl(dev->tcx_ingress);
74 else
75 return rcu_dereference_rtnl(dev->tcx_egress);
76 }
77
78 static inline struct bpf_mprog_entry *tcx_entry_create_noprof(void)
79 {
80 struct tcx_entry *tcx = kzalloc_noprof(sizeof(*tcx), GFP_KERNEL);
81
82 if (tcx) {
83 bpf_mprog_bundle_init(&tcx->bundle);
84 return &tcx->bundle.a;
85 }
86 return NULL;
87 }
88 #define tcx_entry_create(...) alloc_hooks(tcx_entry_create_noprof(__VA_ARGS__))
89
90 static inline void tcx_entry_free(struct bpf_mprog_entry *entry)
91 {
92 kfree_rcu(tcx_entry(entry), rcu);
93 }
94
95 static inline struct bpf_mprog_entry *
96 tcx_entry_fetch_or_create(struct net_device *dev, bool ingress, bool *created)
97 {
98 struct bpf_mprog_entry *entry = tcx_entry_fetch(dev, ingress);
99
100 *created = false;
101 if (!entry) {
102 entry = tcx_entry_create();
103 if (!entry)
104 return NULL;
105 *created = true;
106 }
107 return entry;
108 }
109
110 static inline void tcx_skeys_inc(bool ingress)
111 {
112 tcx_inc();
113 if (ingress)
114 net_inc_ingress_queue();
115 else
116 net_inc_egress_queue();
117 }
118
119 static inline void tcx_skeys_dec(bool ingress)
120 {
121 if (ingress)
122 net_dec_ingress_queue();
123 else
124 net_dec_egress_queue();
125 tcx_dec();
126 }
127
128 static inline void tcx_miniq_inc(struct bpf_mprog_entry *entry)
129 {
130 ASSERT_RTNL();
131 tcx_entry(entry)->miniq_active++;
132 }
133
134 static inline void tcx_miniq_dec(struct bpf_mprog_entry *entry)
135 {
136 ASSERT_RTNL();
137 tcx_entry(entry)->miniq_active--;
138 }
139
140 static inline bool tcx_entry_is_active(struct bpf_mprog_entry *entry)
141 {
142 ASSERT_RTNL();
143 return bpf_mprog_total(entry) || tcx_entry(entry)->miniq_active;
144 }
145
146 static inline enum tcx_action_base tcx_action_code(struct sk_buff *skb,
147 int code)
148 {
149 switch (code) {
150 case TCX_PASS:
151 skb->tc_index = qdisc_skb_cb(skb)->tc_classid;
152 fallthrough;
153 case TCX_DROP:
154 case TCX_REDIRECT:
155 return code;
156 case TCX_NEXT:
157 default:
158 return TCX_NEXT;
159 }
160 }
161 #endif /* CONFIG_NET_XGRESS */
162
163 #if defined(CONFIG_NET_XGRESS) && defined(CONFIG_BPF_SYSCALL)
164 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog);
165 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
166 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog);
167 void tcx_uninstall(struct net_device *dev, bool ingress);
168
169 int tcx_prog_query(const union bpf_attr *attr,
170 union bpf_attr __user *uattr);
171
172 static inline void dev_tcx_uninstall(struct net_device *dev)
173 {
174 ASSERT_RTNL();
175 tcx_uninstall(dev, true);
176 tcx_uninstall(dev, false);
177 }
178 #else
179 static inline int tcx_prog_attach(const union bpf_attr *attr,
180 struct bpf_prog *prog)
181 {
182 return -EINVAL;
183 }
184
185 static inline int tcx_link_attach(const union bpf_attr *attr,
186 struct bpf_prog *prog)
187 {
188 return -EINVAL;
189 }
190
191 static inline int tcx_prog_detach(const union bpf_attr *attr,
192 struct bpf_prog *prog)
193 {
194 return -EINVAL;
195 }
196
197 static inline int tcx_prog_query(const union bpf_attr *attr,
198 union bpf_attr __user *uattr)
199 {
200 return -EINVAL;
201 }
202
203 static inline void dev_tcx_uninstall(struct net_device *dev)
204 {
205 }
206 #endif /* CONFIG_NET_XGRESS && CONFIG_BPF_SYSCALL */
207 #endif /* __NET_TCX_H */
208
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.