1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2023 Isovalent */ 2 /* Copyright (c) 2023 Isovalent */ 3 3 4 #include <linux/bpf.h> 4 #include <linux/bpf.h> 5 #include <linux/bpf_mprog.h> 5 #include <linux/bpf_mprog.h> 6 #include <linux/netdevice.h> 6 #include <linux/netdevice.h> 7 7 8 #include <net/tcx.h> 8 #include <net/tcx.h> 9 9 10 int tcx_prog_attach(const union bpf_attr *attr 10 int tcx_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog) 11 { 11 { 12 bool created, ingress = attr->attach_t 12 bool created, ingress = attr->attach_type == BPF_TCX_INGRESS; 13 struct net *net = current->nsproxy->ne 13 struct net *net = current->nsproxy->net_ns; 14 struct bpf_mprog_entry *entry, *entry_ 14 struct bpf_mprog_entry *entry, *entry_new; 15 struct bpf_prog *replace_prog = NULL; 15 struct bpf_prog *replace_prog = NULL; 16 struct net_device *dev; 16 struct net_device *dev; 17 int ret; 17 int ret; 18 18 19 rtnl_lock(); 19 rtnl_lock(); 20 dev = __dev_get_by_index(net, attr->ta 20 dev = __dev_get_by_index(net, attr->target_ifindex); 21 if (!dev) { 21 if (!dev) { 22 ret = -ENODEV; 22 ret = -ENODEV; 23 goto out; 23 goto out; 24 } 24 } 25 if (attr->attach_flags & BPF_F_REPLACE 25 if (attr->attach_flags & BPF_F_REPLACE) { 26 replace_prog = bpf_prog_get_ty 26 replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, 27 27 prog->type); 28 if (IS_ERR(replace_prog)) { 28 if (IS_ERR(replace_prog)) { 29 ret = PTR_ERR(replace_ 29 ret = PTR_ERR(replace_prog); 30 replace_prog = NULL; 30 replace_prog = NULL; 31 goto out; 31 goto out; 32 } 32 } 33 } 33 } 34 entry = tcx_entry_fetch_or_create(dev, 34 entry = tcx_entry_fetch_or_create(dev, ingress, &created); 35 if (!entry) { 35 if (!entry) { 36 ret = -ENOMEM; 36 ret = -ENOMEM; 37 goto out; 37 goto out; 38 } 38 } 39 ret = bpf_mprog_attach(entry, &entry_n 39 ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, 40 attr->attach_fl 40 attr->attach_flags, attr->relative_fd, 41 attr->expected_ 41 attr->expected_revision); 42 if (!ret) { 42 if (!ret) { 43 if (entry != entry_new) { 43 if (entry != entry_new) { 44 tcx_entry_update(dev, 44 tcx_entry_update(dev, entry_new, ingress); 45 tcx_entry_sync(); 45 tcx_entry_sync(); 46 tcx_skeys_inc(ingress) 46 tcx_skeys_inc(ingress); 47 } 47 } 48 bpf_mprog_commit(entry); 48 bpf_mprog_commit(entry); 49 } else if (created) { 49 } else if (created) { 50 tcx_entry_free(entry); 50 tcx_entry_free(entry); 51 } 51 } 52 out: 52 out: 53 if (replace_prog) 53 if (replace_prog) 54 bpf_prog_put(replace_prog); 54 bpf_prog_put(replace_prog); 55 rtnl_unlock(); 55 rtnl_unlock(); 56 return ret; 56 return ret; 57 } 57 } 58 58 59 int tcx_prog_detach(const union bpf_attr *attr 59 int tcx_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog) 60 { 60 { 61 bool ingress = attr->attach_type == BP 61 bool ingress = attr->attach_type == BPF_TCX_INGRESS; 62 struct net *net = current->nsproxy->ne 62 struct net *net = current->nsproxy->net_ns; 63 struct bpf_mprog_entry *entry, *entry_ 63 struct bpf_mprog_entry *entry, *entry_new; 64 struct net_device *dev; 64 struct net_device *dev; 65 int ret; 65 int ret; 66 66 67 rtnl_lock(); 67 rtnl_lock(); 68 dev = __dev_get_by_index(net, attr->ta 68 dev = __dev_get_by_index(net, attr->target_ifindex); 69 if (!dev) { 69 if (!dev) { 70 ret = -ENODEV; 70 ret = -ENODEV; 71 goto out; 71 goto out; 72 } 72 } 73 entry = tcx_entry_fetch(dev, ingress); 73 entry = tcx_entry_fetch(dev, ingress); 74 if (!entry) { 74 if (!entry) { 75 ret = -ENOENT; 75 ret = -ENOENT; 76 goto out; 76 goto out; 77 } 77 } 78 ret = bpf_mprog_detach(entry, &entry_n 78 ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, 79 attr->relative_ 79 attr->relative_fd, attr->expected_revision); 80 if (!ret) { 80 if (!ret) { 81 if (!tcx_entry_is_active(entry 81 if (!tcx_entry_is_active(entry_new)) 82 entry_new = NULL; 82 entry_new = NULL; 83 tcx_entry_update(dev, entry_ne 83 tcx_entry_update(dev, entry_new, ingress); 84 tcx_entry_sync(); 84 tcx_entry_sync(); 85 tcx_skeys_dec(ingress); 85 tcx_skeys_dec(ingress); 86 bpf_mprog_commit(entry); 86 bpf_mprog_commit(entry); 87 if (!entry_new) 87 if (!entry_new) 88 tcx_entry_free(entry); 88 tcx_entry_free(entry); 89 } 89 } 90 out: 90 out: 91 rtnl_unlock(); 91 rtnl_unlock(); 92 return ret; 92 return ret; 93 } 93 } 94 94 95 void tcx_uninstall(struct net_device *dev, boo 95 void tcx_uninstall(struct net_device *dev, bool ingress) 96 { 96 { 97 struct bpf_mprog_entry *entry, *entry_ 97 struct bpf_mprog_entry *entry, *entry_new = NULL; 98 struct bpf_tuple tuple = {}; 98 struct bpf_tuple tuple = {}; 99 struct bpf_mprog_fp *fp; 99 struct bpf_mprog_fp *fp; 100 struct bpf_mprog_cp *cp; 100 struct bpf_mprog_cp *cp; 101 bool active; 101 bool active; 102 102 103 entry = tcx_entry_fetch(dev, ingress); 103 entry = tcx_entry_fetch(dev, ingress); 104 if (!entry) 104 if (!entry) 105 return; 105 return; 106 active = tcx_entry(entry)->miniq_activ 106 active = tcx_entry(entry)->miniq_active; 107 if (active) 107 if (active) 108 bpf_mprog_clear_all(entry, &en 108 bpf_mprog_clear_all(entry, &entry_new); 109 tcx_entry_update(dev, entry_new, ingre 109 tcx_entry_update(dev, entry_new, ingress); 110 tcx_entry_sync(); 110 tcx_entry_sync(); 111 bpf_mprog_foreach_tuple(entry, fp, cp, 111 bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { 112 if (tuple.link) 112 if (tuple.link) 113 tcx_link(tuple.link)-> 113 tcx_link(tuple.link)->dev = NULL; 114 else 114 else 115 bpf_prog_put(tuple.pro 115 bpf_prog_put(tuple.prog); 116 tcx_skeys_dec(ingress); 116 tcx_skeys_dec(ingress); 117 } 117 } 118 if (!active) 118 if (!active) 119 tcx_entry_free(entry); 119 tcx_entry_free(entry); 120 } 120 } 121 121 122 int tcx_prog_query(const union bpf_attr *attr, 122 int tcx_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) 123 { 123 { 124 bool ingress = attr->query.attach_type 124 bool ingress = attr->query.attach_type == BPF_TCX_INGRESS; 125 struct net *net = current->nsproxy->ne 125 struct net *net = current->nsproxy->net_ns; 126 struct net_device *dev; 126 struct net_device *dev; 127 int ret; 127 int ret; 128 128 129 rtnl_lock(); 129 rtnl_lock(); 130 dev = __dev_get_by_index(net, attr->qu 130 dev = __dev_get_by_index(net, attr->query.target_ifindex); 131 if (!dev) { 131 if (!dev) { 132 ret = -ENODEV; 132 ret = -ENODEV; 133 goto out; 133 goto out; 134 } 134 } 135 ret = bpf_mprog_query(attr, uattr, tcx 135 ret = bpf_mprog_query(attr, uattr, tcx_entry_fetch(dev, ingress)); 136 out: 136 out: 137 rtnl_unlock(); 137 rtnl_unlock(); 138 return ret; 138 return ret; 139 } 139 } 140 140 141 static int tcx_link_prog_attach(struct bpf_lin 141 static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, 142 u64 revision) 142 u64 revision) 143 { 143 { 144 struct tcx_link *tcx = tcx_link(link); 144 struct tcx_link *tcx = tcx_link(link); 145 bool created, ingress = tcx->location 145 bool created, ingress = tcx->location == BPF_TCX_INGRESS; 146 struct bpf_mprog_entry *entry, *entry_ 146 struct bpf_mprog_entry *entry, *entry_new; 147 struct net_device *dev = tcx->dev; 147 struct net_device *dev = tcx->dev; 148 int ret; 148 int ret; 149 149 150 ASSERT_RTNL(); 150 ASSERT_RTNL(); 151 entry = tcx_entry_fetch_or_create(dev, 151 entry = tcx_entry_fetch_or_create(dev, ingress, &created); 152 if (!entry) 152 if (!entry) 153 return -ENOMEM; 153 return -ENOMEM; 154 ret = bpf_mprog_attach(entry, &entry_n 154 ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, 155 id_or_fd, revis 155 id_or_fd, revision); 156 if (!ret) { 156 if (!ret) { 157 if (entry != entry_new) { 157 if (entry != entry_new) { 158 tcx_entry_update(dev, 158 tcx_entry_update(dev, entry_new, ingress); 159 tcx_entry_sync(); 159 tcx_entry_sync(); 160 tcx_skeys_inc(ingress) 160 tcx_skeys_inc(ingress); 161 } 161 } 162 bpf_mprog_commit(entry); 162 bpf_mprog_commit(entry); 163 } else if (created) { 163 } else if (created) { 164 tcx_entry_free(entry); 164 tcx_entry_free(entry); 165 } 165 } 166 return ret; 166 return ret; 167 } 167 } 168 168 169 static void tcx_link_release(struct bpf_link * 169 static void tcx_link_release(struct bpf_link *link) 170 { 170 { 171 struct tcx_link *tcx = tcx_link(link); 171 struct tcx_link *tcx = tcx_link(link); 172 bool ingress = tcx->location == BPF_TC 172 bool ingress = tcx->location == BPF_TCX_INGRESS; 173 struct bpf_mprog_entry *entry, *entry_ 173 struct bpf_mprog_entry *entry, *entry_new; 174 struct net_device *dev; 174 struct net_device *dev; 175 int ret = 0; 175 int ret = 0; 176 176 177 rtnl_lock(); 177 rtnl_lock(); 178 dev = tcx->dev; 178 dev = tcx->dev; 179 if (!dev) 179 if (!dev) 180 goto out; 180 goto out; 181 entry = tcx_entry_fetch(dev, ingress); 181 entry = tcx_entry_fetch(dev, ingress); 182 if (!entry) { 182 if (!entry) { 183 ret = -ENOENT; 183 ret = -ENOENT; 184 goto out; 184 goto out; 185 } 185 } 186 ret = bpf_mprog_detach(entry, &entry_n 186 ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); 187 if (!ret) { 187 if (!ret) { 188 if (!tcx_entry_is_active(entry 188 if (!tcx_entry_is_active(entry_new)) 189 entry_new = NULL; 189 entry_new = NULL; 190 tcx_entry_update(dev, entry_ne 190 tcx_entry_update(dev, entry_new, ingress); 191 tcx_entry_sync(); 191 tcx_entry_sync(); 192 tcx_skeys_dec(ingress); 192 tcx_skeys_dec(ingress); 193 bpf_mprog_commit(entry); 193 bpf_mprog_commit(entry); 194 if (!entry_new) 194 if (!entry_new) 195 tcx_entry_free(entry); 195 tcx_entry_free(entry); 196 tcx->dev = NULL; 196 tcx->dev = NULL; 197 } 197 } 198 out: 198 out: 199 WARN_ON_ONCE(ret); 199 WARN_ON_ONCE(ret); 200 rtnl_unlock(); 200 rtnl_unlock(); 201 } 201 } 202 202 203 static int tcx_link_update(struct bpf_link *li 203 static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog, 204 struct bpf_prog *op 204 struct bpf_prog *oprog) 205 { 205 { 206 struct tcx_link *tcx = tcx_link(link); 206 struct tcx_link *tcx = tcx_link(link); 207 bool ingress = tcx->location == BPF_TC 207 bool ingress = tcx->location == BPF_TCX_INGRESS; 208 struct bpf_mprog_entry *entry, *entry_ 208 struct bpf_mprog_entry *entry, *entry_new; 209 struct net_device *dev; 209 struct net_device *dev; 210 int ret = 0; 210 int ret = 0; 211 211 212 rtnl_lock(); 212 rtnl_lock(); 213 dev = tcx->dev; 213 dev = tcx->dev; 214 if (!dev) { 214 if (!dev) { 215 ret = -ENOLINK; 215 ret = -ENOLINK; 216 goto out; 216 goto out; 217 } 217 } 218 if (oprog && link->prog != oprog) { 218 if (oprog && link->prog != oprog) { 219 ret = -EPERM; 219 ret = -EPERM; 220 goto out; 220 goto out; 221 } 221 } 222 oprog = link->prog; 222 oprog = link->prog; 223 if (oprog == nprog) { 223 if (oprog == nprog) { 224 bpf_prog_put(nprog); 224 bpf_prog_put(nprog); 225 goto out; 225 goto out; 226 } 226 } 227 entry = tcx_entry_fetch(dev, ingress); 227 entry = tcx_entry_fetch(dev, ingress); 228 if (!entry) { 228 if (!entry) { 229 ret = -ENOENT; 229 ret = -ENOENT; 230 goto out; 230 goto out; 231 } 231 } 232 ret = bpf_mprog_attach(entry, &entry_n 232 ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, 233 BPF_F_REPLACE | 233 BPF_F_REPLACE | BPF_F_ID, 234 link->prog->aux 234 link->prog->aux->id, 0); 235 if (!ret) { 235 if (!ret) { 236 WARN_ON_ONCE(entry != entry_ne 236 WARN_ON_ONCE(entry != entry_new); 237 oprog = xchg(&link->prog, npro 237 oprog = xchg(&link->prog, nprog); 238 bpf_prog_put(oprog); 238 bpf_prog_put(oprog); 239 bpf_mprog_commit(entry); 239 bpf_mprog_commit(entry); 240 } 240 } 241 out: 241 out: 242 rtnl_unlock(); 242 rtnl_unlock(); 243 return ret; 243 return ret; 244 } 244 } 245 245 246 static void tcx_link_dealloc(struct bpf_link * 246 static void tcx_link_dealloc(struct bpf_link *link) 247 { 247 { 248 kfree(tcx_link(link)); 248 kfree(tcx_link(link)); 249 } 249 } 250 250 251 static void tcx_link_fdinfo(const struct bpf_l 251 static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq) 252 { 252 { 253 const struct tcx_link *tcx = tcx_link( 253 const struct tcx_link *tcx = tcx_link(link); 254 u32 ifindex = 0; 254 u32 ifindex = 0; 255 255 256 rtnl_lock(); 256 rtnl_lock(); 257 if (tcx->dev) 257 if (tcx->dev) 258 ifindex = tcx->dev->ifindex; 258 ifindex = tcx->dev->ifindex; 259 rtnl_unlock(); 259 rtnl_unlock(); 260 260 261 seq_printf(seq, "ifindex:\t%u\n", ifin 261 seq_printf(seq, "ifindex:\t%u\n", ifindex); 262 seq_printf(seq, "attach_type:\t%u (%s) 262 seq_printf(seq, "attach_type:\t%u (%s)\n", 263 tcx->location, 263 tcx->location, 264 tcx->location == BPF_TCX_IN 264 tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress"); 265 } 265 } 266 266 267 static int tcx_link_fill_info(const struct bpf 267 static int tcx_link_fill_info(const struct bpf_link *link, 268 struct bpf_link_ 268 struct bpf_link_info *info) 269 { 269 { 270 const struct tcx_link *tcx = tcx_link( 270 const struct tcx_link *tcx = tcx_link(link); 271 u32 ifindex = 0; 271 u32 ifindex = 0; 272 272 273 rtnl_lock(); 273 rtnl_lock(); 274 if (tcx->dev) 274 if (tcx->dev) 275 ifindex = tcx->dev->ifindex; 275 ifindex = tcx->dev->ifindex; 276 rtnl_unlock(); 276 rtnl_unlock(); 277 277 278 info->tcx.ifindex = ifindex; 278 info->tcx.ifindex = ifindex; 279 info->tcx.attach_type = tcx->location; 279 info->tcx.attach_type = tcx->location; 280 return 0; 280 return 0; 281 } 281 } 282 282 283 static int tcx_link_detach(struct bpf_link *li 283 static int tcx_link_detach(struct bpf_link *link) 284 { 284 { 285 tcx_link_release(link); 285 tcx_link_release(link); 286 return 0; 286 return 0; 287 } 287 } 288 288 289 static const struct bpf_link_ops tcx_link_lops 289 static const struct bpf_link_ops tcx_link_lops = { 290 .release = tcx_link_release, 290 .release = tcx_link_release, 291 .detach = tcx_link_detach, 291 .detach = tcx_link_detach, 292 .dealloc = tcx_link_dealloc, 292 .dealloc = tcx_link_dealloc, 293 .update_prog = tcx_link_update, 293 .update_prog = tcx_link_update, 294 .show_fdinfo = tcx_link_fdinfo, 294 .show_fdinfo = tcx_link_fdinfo, 295 .fill_link_info = tcx_link_fill_info, 295 .fill_link_info = tcx_link_fill_info, 296 }; 296 }; 297 297 298 static int tcx_link_init(struct tcx_link *tcx, 298 static int tcx_link_init(struct tcx_link *tcx, 299 struct bpf_link_prime 299 struct bpf_link_primer *link_primer, 300 const union bpf_attr 300 const union bpf_attr *attr, 301 struct net_device *de 301 struct net_device *dev, 302 struct bpf_prog *prog 302 struct bpf_prog *prog) 303 { 303 { 304 bpf_link_init(&tcx->link, BPF_LINK_TYP 304 bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog); 305 tcx->location = attr->link_create.atta 305 tcx->location = attr->link_create.attach_type; 306 tcx->dev = dev; 306 tcx->dev = dev; 307 return bpf_link_prime(&tcx->link, link 307 return bpf_link_prime(&tcx->link, link_primer); 308 } 308 } 309 309 310 int tcx_link_attach(const union bpf_attr *attr 310 int tcx_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 311 { 311 { 312 struct net *net = current->nsproxy->ne 312 struct net *net = current->nsproxy->net_ns; 313 struct bpf_link_primer link_primer; 313 struct bpf_link_primer link_primer; 314 struct net_device *dev; 314 struct net_device *dev; 315 struct tcx_link *tcx; 315 struct tcx_link *tcx; 316 int ret; 316 int ret; 317 317 318 rtnl_lock(); 318 rtnl_lock(); 319 dev = __dev_get_by_index(net, attr->li 319 dev = __dev_get_by_index(net, attr->link_create.target_ifindex); 320 if (!dev) { 320 if (!dev) { 321 ret = -ENODEV; 321 ret = -ENODEV; 322 goto out; 322 goto out; 323 } 323 } 324 tcx = kzalloc(sizeof(*tcx), GFP_USER); 324 tcx = kzalloc(sizeof(*tcx), GFP_USER); 325 if (!tcx) { 325 if (!tcx) { 326 ret = -ENOMEM; 326 ret = -ENOMEM; 327 goto out; 327 goto out; 328 } 328 } 329 ret = tcx_link_init(tcx, &link_primer, 329 ret = tcx_link_init(tcx, &link_primer, attr, dev, prog); 330 if (ret) { 330 if (ret) { 331 kfree(tcx); 331 kfree(tcx); 332 goto out; 332 goto out; 333 } 333 } 334 ret = tcx_link_prog_attach(&tcx->link, 334 ret = tcx_link_prog_attach(&tcx->link, attr->link_create.flags, 335 attr->link_ 335 attr->link_create.tcx.relative_fd, 336 attr->link_ 336 attr->link_create.tcx.expected_revision); 337 if (ret) { 337 if (ret) { 338 tcx->dev = NULL; 338 tcx->dev = NULL; 339 bpf_link_cleanup(&link_primer) 339 bpf_link_cleanup(&link_primer); 340 goto out; 340 goto out; 341 } 341 } 342 ret = bpf_link_settle(&link_primer); 342 ret = bpf_link_settle(&link_primer); 343 out: 343 out: 344 rtnl_unlock(); 344 rtnl_unlock(); 345 return ret; 345 return ret; 346 } 346 } 347 347
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.