1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2023 Isovalent */
3
4 #include <linux/bpf.h>
5 #include <linux/bpf_mprog.h>
6 #include <linux/netdevice.h>
7
8 #include <net/tcx.h>
9
10 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog)
11 {
12 bool created, ingress = attr->attach_type == BPF_TCX_INGRESS;
13 struct net *net = current->nsproxy->net_ns;
14 struct bpf_mprog_entry *entry, *entry_new;
15 struct bpf_prog *replace_prog = NULL;
16 struct net_device *dev;
17 int ret;
18
19 rtnl_lock();
20 dev = __dev_get_by_index(net, attr->target_ifindex);
21 if (!dev) {
22 ret = -ENODEV;
23 goto out;
24 }
25 if (attr->attach_flags & BPF_F_REPLACE) {
26 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd,
27 prog->type);
28 if (IS_ERR(replace_prog)) {
29 ret = PTR_ERR(replace_prog);
30 replace_prog = NULL;
31 goto out;
32 }
33 }
34 entry = tcx_entry_fetch_or_create(dev, ingress, &created);
35 if (!entry) {
36 ret = -ENOMEM;
37 goto out;
38 }
39 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog,
40 attr->attach_flags, attr->relative_fd,
41 attr->expected_revision);
42 if (!ret) {
43 if (entry != entry_new) {
44 tcx_entry_update(dev, entry_new, ingress);
45 tcx_entry_sync();
46 tcx_skeys_inc(ingress);
47 }
48 bpf_mprog_commit(entry);
49 } else if (created) {
50 tcx_entry_free(entry);
51 }
52 out:
53 if (replace_prog)
54 bpf_prog_put(replace_prog);
55 rtnl_unlock();
56 return ret;
57 }
58
59 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog)
60 {
61 bool ingress = attr->attach_type == BPF_TCX_INGRESS;
62 struct net *net = current->nsproxy->net_ns;
63 struct bpf_mprog_entry *entry, *entry_new;
64 struct net_device *dev;
65 int ret;
66
67 rtnl_lock();
68 dev = __dev_get_by_index(net, attr->target_ifindex);
69 if (!dev) {
70 ret = -ENODEV;
71 goto out;
72 }
73 entry = tcx_entry_fetch(dev, ingress);
74 if (!entry) {
75 ret = -ENOENT;
76 goto out;
77 }
78 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags,
79 attr->relative_fd, attr->expected_revision);
80 if (!ret) {
81 if (!tcx_entry_is_active(entry_new))
82 entry_new = NULL;
83 tcx_entry_update(dev, entry_new, ingress);
84 tcx_entry_sync();
85 tcx_skeys_dec(ingress);
86 bpf_mprog_commit(entry);
87 if (!entry_new)
88 tcx_entry_free(entry);
89 }
90 out:
91 rtnl_unlock();
92 return ret;
93 }
94
95 void tcx_uninstall(struct net_device *dev, bool ingress)
96 {
97 struct bpf_mprog_entry *entry, *entry_new = NULL;
98 struct bpf_tuple tuple = {};
99 struct bpf_mprog_fp *fp;
100 struct bpf_mprog_cp *cp;
101 bool active;
102
103 entry = tcx_entry_fetch(dev, ingress);
104 if (!entry)
105 return;
106 active = tcx_entry(entry)->miniq_active;
107 if (active)
108 bpf_mprog_clear_all(entry, &entry_new);
109 tcx_entry_update(dev, entry_new, ingress);
110 tcx_entry_sync();
111 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) {
112 if (tuple.link)
113 tcx_link(tuple.link)->dev = NULL;
114 else
115 bpf_prog_put(tuple.prog);
116 tcx_skeys_dec(ingress);
117 }
118 if (!active)
119 tcx_entry_free(entry);
120 }
121
122 int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr)
123 {
124 bool ingress = attr->query.attach_type == BPF_TCX_INGRESS;
125 struct net *net = current->nsproxy->net_ns;
126 struct net_device *dev;
127 int ret;
128
129 rtnl_lock();
130 dev = __dev_get_by_index(net, attr->query.target_ifindex);
131 if (!dev) {
132 ret = -ENODEV;
133 goto out;
134 }
135 ret = bpf_mprog_query(attr, uattr, tcx_entry_fetch(dev, ingress));
136 out:
137 rtnl_unlock();
138 return ret;
139 }
140
141 static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
142 u64 revision)
143 {
144 struct tcx_link *tcx = tcx_link(link);
145 bool created, ingress = tcx->location == BPF_TCX_INGRESS;
146 struct bpf_mprog_entry *entry, *entry_new;
147 struct net_device *dev = tcx->dev;
148 int ret;
149
150 ASSERT_RTNL();
151 entry = tcx_entry_fetch_or_create(dev, ingress, &created);
152 if (!entry)
153 return -ENOMEM;
154 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags,
155 id_or_fd, revision);
156 if (!ret) {
157 if (entry != entry_new) {
158 tcx_entry_update(dev, entry_new, ingress);
159 tcx_entry_sync();
160 tcx_skeys_inc(ingress);
161 }
162 bpf_mprog_commit(entry);
163 } else if (created) {
164 tcx_entry_free(entry);
165 }
166 return ret;
167 }
168
169 static void tcx_link_release(struct bpf_link *link)
170 {
171 struct tcx_link *tcx = tcx_link(link);
172 bool ingress = tcx->location == BPF_TCX_INGRESS;
173 struct bpf_mprog_entry *entry, *entry_new;
174 struct net_device *dev;
175 int ret = 0;
176
177 rtnl_lock();
178 dev = tcx->dev;
179 if (!dev)
180 goto out;
181 entry = tcx_entry_fetch(dev, ingress);
182 if (!entry) {
183 ret = -ENOENT;
184 goto out;
185 }
186 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0);
187 if (!ret) {
188 if (!tcx_entry_is_active(entry_new))
189 entry_new = NULL;
190 tcx_entry_update(dev, entry_new, ingress);
191 tcx_entry_sync();
192 tcx_skeys_dec(ingress);
193 bpf_mprog_commit(entry);
194 if (!entry_new)
195 tcx_entry_free(entry);
196 tcx->dev = NULL;
197 }
198 out:
199 WARN_ON_ONCE(ret);
200 rtnl_unlock();
201 }
202
203 static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog,
204 struct bpf_prog *oprog)
205 {
206 struct tcx_link *tcx = tcx_link(link);
207 bool ingress = tcx->location == BPF_TCX_INGRESS;
208 struct bpf_mprog_entry *entry, *entry_new;
209 struct net_device *dev;
210 int ret = 0;
211
212 rtnl_lock();
213 dev = tcx->dev;
214 if (!dev) {
215 ret = -ENOLINK;
216 goto out;
217 }
218 if (oprog && link->prog != oprog) {
219 ret = -EPERM;
220 goto out;
221 }
222 oprog = link->prog;
223 if (oprog == nprog) {
224 bpf_prog_put(nprog);
225 goto out;
226 }
227 entry = tcx_entry_fetch(dev, ingress);
228 if (!entry) {
229 ret = -ENOENT;
230 goto out;
231 }
232 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog,
233 BPF_F_REPLACE | BPF_F_ID,
234 link->prog->aux->id, 0);
235 if (!ret) {
236 WARN_ON_ONCE(entry != entry_new);
237 oprog = xchg(&link->prog, nprog);
238 bpf_prog_put(oprog);
239 bpf_mprog_commit(entry);
240 }
241 out:
242 rtnl_unlock();
243 return ret;
244 }
245
246 static void tcx_link_dealloc(struct bpf_link *link)
247 {
248 kfree(tcx_link(link));
249 }
250
251 static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
252 {
253 const struct tcx_link *tcx = tcx_link(link);
254 u32 ifindex = 0;
255
256 rtnl_lock();
257 if (tcx->dev)
258 ifindex = tcx->dev->ifindex;
259 rtnl_unlock();
260
261 seq_printf(seq, "ifindex:\t%u\n", ifindex);
262 seq_printf(seq, "attach_type:\t%u (%s)\n",
263 tcx->location,
264 tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress");
265 }
266
267 static int tcx_link_fill_info(const struct bpf_link *link,
268 struct bpf_link_info *info)
269 {
270 const struct tcx_link *tcx = tcx_link(link);
271 u32 ifindex = 0;
272
273 rtnl_lock();
274 if (tcx->dev)
275 ifindex = tcx->dev->ifindex;
276 rtnl_unlock();
277
278 info->tcx.ifindex = ifindex;
279 info->tcx.attach_type = tcx->location;
280 return 0;
281 }
282
283 static int tcx_link_detach(struct bpf_link *link)
284 {
285 tcx_link_release(link);
286 return 0;
287 }
288
289 static const struct bpf_link_ops tcx_link_lops = {
290 .release = tcx_link_release,
291 .detach = tcx_link_detach,
292 .dealloc = tcx_link_dealloc,
293 .update_prog = tcx_link_update,
294 .show_fdinfo = tcx_link_fdinfo,
295 .fill_link_info = tcx_link_fill_info,
296 };
297
298 static int tcx_link_init(struct tcx_link *tcx,
299 struct bpf_link_primer *link_primer,
300 const union bpf_attr *attr,
301 struct net_device *dev,
302 struct bpf_prog *prog)
303 {
304 bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
305 tcx->location = attr->link_create.attach_type;
306 tcx->dev = dev;
307 return bpf_link_prime(&tcx->link, link_primer);
308 }
309
310 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
311 {
312 struct net *net = current->nsproxy->net_ns;
313 struct bpf_link_primer link_primer;
314 struct net_device *dev;
315 struct tcx_link *tcx;
316 int ret;
317
318 rtnl_lock();
319 dev = __dev_get_by_index(net, attr->link_create.target_ifindex);
320 if (!dev) {
321 ret = -ENODEV;
322 goto out;
323 }
324 tcx = kzalloc(sizeof(*tcx), GFP_USER);
325 if (!tcx) {
326 ret = -ENOMEM;
327 goto out;
328 }
329 ret = tcx_link_init(tcx, &link_primer, attr, dev, prog);
330 if (ret) {
331 kfree(tcx);
332 goto out;
333 }
334 ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags,
335 attr->link_create.tcx.relative_fd,
336 attr->link_create.tcx.expected_revision);
337 if (ret) {
338 tcx->dev = NULL;
339 bpf_link_cleanup(&link_primer);
340 goto out;
341 }
342 ret = bpf_link_settle(&link_primer);
343 out:
344 rtnl_unlock();
345 return ret;
346 }
347
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.