~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/bpf/bpf_task_storage.c

Version: ~ [ linux-6.11.5 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.58 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.114 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.169 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.228 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.284 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.322 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Copyright (c) 2020 Facebook
  4  * Copyright 2020 Google LLC.
  5  */
  6 
  7 #include <linux/pid.h>
  8 #include <linux/sched.h>
  9 #include <linux/rculist.h>
 10 #include <linux/list.h>
 11 #include <linux/hash.h>
 12 #include <linux/types.h>
 13 #include <linux/spinlock.h>
 14 #include <linux/bpf.h>
 15 #include <linux/bpf_local_storage.h>
 16 #include <linux/filter.h>
 17 #include <uapi/linux/btf.h>
 18 #include <linux/btf_ids.h>
 19 #include <linux/fdtable.h>
 20 #include <linux/rcupdate_trace.h>
 21 
 22 DEFINE_BPF_STORAGE_CACHE(task_cache);
 23 
 24 static DEFINE_PER_CPU(int, bpf_task_storage_busy);
 25 
 26 static void bpf_task_storage_lock(void)
 27 {
 28         migrate_disable();
 29         this_cpu_inc(bpf_task_storage_busy);
 30 }
 31 
 32 static void bpf_task_storage_unlock(void)
 33 {
 34         this_cpu_dec(bpf_task_storage_busy);
 35         migrate_enable();
 36 }
 37 
 38 static bool bpf_task_storage_trylock(void)
 39 {
 40         migrate_disable();
 41         if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
 42                 this_cpu_dec(bpf_task_storage_busy);
 43                 migrate_enable();
 44                 return false;
 45         }
 46         return true;
 47 }
 48 
 49 static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
 50 {
 51         struct task_struct *task = owner;
 52 
 53         return &task->bpf_storage;
 54 }
 55 
 56 static struct bpf_local_storage_data *
 57 task_storage_lookup(struct task_struct *task, struct bpf_map *map,
 58                     bool cacheit_lockit)
 59 {
 60         struct bpf_local_storage *task_storage;
 61         struct bpf_local_storage_map *smap;
 62 
 63         task_storage =
 64                 rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
 65         if (!task_storage)
 66                 return NULL;
 67 
 68         smap = (struct bpf_local_storage_map *)map;
 69         return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit);
 70 }
 71 
 72 void bpf_task_storage_free(struct task_struct *task)
 73 {
 74         struct bpf_local_storage *local_storage;
 75 
 76         rcu_read_lock();
 77 
 78         local_storage = rcu_dereference(task->bpf_storage);
 79         if (!local_storage) {
 80                 rcu_read_unlock();
 81                 return;
 82         }
 83 
 84         bpf_task_storage_lock();
 85         bpf_local_storage_destroy(local_storage);
 86         bpf_task_storage_unlock();
 87         rcu_read_unlock();
 88 }
 89 
 90 static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
 91 {
 92         struct bpf_local_storage_data *sdata;
 93         struct task_struct *task;
 94         unsigned int f_flags;
 95         struct pid *pid;
 96         int fd, err;
 97 
 98         fd = *(int *)key;
 99         pid = pidfd_get_pid(fd, &f_flags);
100         if (IS_ERR(pid))
101                 return ERR_CAST(pid);
102 
103         /* We should be in an RCU read side critical section, it should be safe
104          * to call pid_task.
105          */
106         WARN_ON_ONCE(!rcu_read_lock_held());
107         task = pid_task(pid, PIDTYPE_PID);
108         if (!task) {
109                 err = -ENOENT;
110                 goto out;
111         }
112 
113         bpf_task_storage_lock();
114         sdata = task_storage_lookup(task, map, true);
115         bpf_task_storage_unlock();
116         put_pid(pid);
117         return sdata ? sdata->data : NULL;
118 out:
119         put_pid(pid);
120         return ERR_PTR(err);
121 }
122 
123 static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
124                                              void *value, u64 map_flags)
125 {
126         struct bpf_local_storage_data *sdata;
127         struct task_struct *task;
128         unsigned int f_flags;
129         struct pid *pid;
130         int fd, err;
131 
132         fd = *(int *)key;
133         pid = pidfd_get_pid(fd, &f_flags);
134         if (IS_ERR(pid))
135                 return PTR_ERR(pid);
136 
137         /* We should be in an RCU read side critical section, it should be safe
138          * to call pid_task.
139          */
140         WARN_ON_ONCE(!rcu_read_lock_held());
141         task = pid_task(pid, PIDTYPE_PID);
142         if (!task) {
143                 err = -ENOENT;
144                 goto out;
145         }
146 
147         bpf_task_storage_lock();
148         sdata = bpf_local_storage_update(
149                 task, (struct bpf_local_storage_map *)map, value, map_flags,
150                 GFP_ATOMIC);
151         bpf_task_storage_unlock();
152 
153         err = PTR_ERR_OR_ZERO(sdata);
154 out:
155         put_pid(pid);
156         return err;
157 }
158 
159 static int task_storage_delete(struct task_struct *task, struct bpf_map *map,
160                                bool nobusy)
161 {
162         struct bpf_local_storage_data *sdata;
163 
164         sdata = task_storage_lookup(task, map, false);
165         if (!sdata)
166                 return -ENOENT;
167 
168         if (!nobusy)
169                 return -EBUSY;
170 
171         bpf_selem_unlink(SELEM(sdata), false);
172 
173         return 0;
174 }
175 
176 static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)
177 {
178         struct task_struct *task;
179         unsigned int f_flags;
180         struct pid *pid;
181         int fd, err;
182 
183         fd = *(int *)key;
184         pid = pidfd_get_pid(fd, &f_flags);
185         if (IS_ERR(pid))
186                 return PTR_ERR(pid);
187 
188         /* We should be in an RCU read side critical section, it should be safe
189          * to call pid_task.
190          */
191         WARN_ON_ONCE(!rcu_read_lock_held());
192         task = pid_task(pid, PIDTYPE_PID);
193         if (!task) {
194                 err = -ENOENT;
195                 goto out;
196         }
197 
198         bpf_task_storage_lock();
199         err = task_storage_delete(task, map, true);
200         bpf_task_storage_unlock();
201 out:
202         put_pid(pid);
203         return err;
204 }
205 
206 /* Called by bpf_task_storage_get*() helpers */
207 static void *__bpf_task_storage_get(struct bpf_map *map,
208                                     struct task_struct *task, void *value,
209                                     u64 flags, gfp_t gfp_flags, bool nobusy)
210 {
211         struct bpf_local_storage_data *sdata;
212 
213         sdata = task_storage_lookup(task, map, nobusy);
214         if (sdata)
215                 return sdata->data;
216 
217         /* only allocate new storage, when the task is refcounted */
218         if (refcount_read(&task->usage) &&
219             (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {
220                 sdata = bpf_local_storage_update(
221                         task, (struct bpf_local_storage_map *)map, value,
222                         BPF_NOEXIST, gfp_flags);
223                 return IS_ERR(sdata) ? NULL : sdata->data;
224         }
225 
226         return NULL;
227 }
228 
229 /* *gfp_flags* is a hidden argument provided by the verifier */
230 BPF_CALL_5(bpf_task_storage_get_recur, struct bpf_map *, map, struct task_struct *,
231            task, void *, value, u64, flags, gfp_t, gfp_flags)
232 {
233         bool nobusy;
234         void *data;
235 
236         WARN_ON_ONCE(!bpf_rcu_lock_held());
237         if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task)
238                 return (unsigned long)NULL;
239 
240         nobusy = bpf_task_storage_trylock();
241         data = __bpf_task_storage_get(map, task, value, flags,
242                                       gfp_flags, nobusy);
243         if (nobusy)
244                 bpf_task_storage_unlock();
245         return (unsigned long)data;
246 }
247 
248 /* *gfp_flags* is a hidden argument provided by the verifier */
249 BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
250            task, void *, value, u64, flags, gfp_t, gfp_flags)
251 {
252         void *data;
253 
254         WARN_ON_ONCE(!bpf_rcu_lock_held());
255         if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task)
256                 return (unsigned long)NULL;
257 
258         bpf_task_storage_lock();
259         data = __bpf_task_storage_get(map, task, value, flags,
260                                       gfp_flags, true);
261         bpf_task_storage_unlock();
262         return (unsigned long)data;
263 }
264 
265 BPF_CALL_2(bpf_task_storage_delete_recur, struct bpf_map *, map, struct task_struct *,
266            task)
267 {
268         bool nobusy;
269         int ret;
270 
271         WARN_ON_ONCE(!bpf_rcu_lock_held());
272         if (!task)
273                 return -EINVAL;
274 
275         nobusy = bpf_task_storage_trylock();
276         /* This helper must only be called from places where the lifetime of the task
277          * is guaranteed. Either by being refcounted or by being protected
278          * by an RCU read-side critical section.
279          */
280         ret = task_storage_delete(task, map, nobusy);
281         if (nobusy)
282                 bpf_task_storage_unlock();
283         return ret;
284 }
285 
286 BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *,
287            task)
288 {
289         int ret;
290 
291         WARN_ON_ONCE(!bpf_rcu_lock_held());
292         if (!task)
293                 return -EINVAL;
294 
295         bpf_task_storage_lock();
296         /* This helper must only be called from places where the lifetime of the task
297          * is guaranteed. Either by being refcounted or by being protected
298          * by an RCU read-side critical section.
299          */
300         ret = task_storage_delete(task, map, true);
301         bpf_task_storage_unlock();
302         return ret;
303 }
304 
305 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
306 {
307         return -ENOTSUPP;
308 }
309 
310 static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
311 {
312         return bpf_local_storage_map_alloc(attr, &task_cache, true);
313 }
314 
315 static void task_storage_map_free(struct bpf_map *map)
316 {
317         bpf_local_storage_map_free(map, &task_cache, &bpf_task_storage_busy);
318 }
319 
320 BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map)
321 const struct bpf_map_ops task_storage_map_ops = {
322         .map_meta_equal = bpf_map_meta_equal,
323         .map_alloc_check = bpf_local_storage_map_alloc_check,
324         .map_alloc = task_storage_map_alloc,
325         .map_free = task_storage_map_free,
326         .map_get_next_key = notsupp_get_next_key,
327         .map_lookup_elem = bpf_pid_task_storage_lookup_elem,
328         .map_update_elem = bpf_pid_task_storage_update_elem,
329         .map_delete_elem = bpf_pid_task_storage_delete_elem,
330         .map_check_btf = bpf_local_storage_map_check_btf,
331         .map_mem_usage = bpf_local_storage_map_mem_usage,
332         .map_btf_id = &bpf_local_storage_map_btf_id[0],
333         .map_owner_storage_ptr = task_storage_ptr,
334 };
335 
336 const struct bpf_func_proto bpf_task_storage_get_recur_proto = {
337         .func = bpf_task_storage_get_recur,
338         .gpl_only = false,
339         .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
340         .arg1_type = ARG_CONST_MAP_PTR,
341         .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
342         .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
343         .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
344         .arg4_type = ARG_ANYTHING,
345 };
346 
347 const struct bpf_func_proto bpf_task_storage_get_proto = {
348         .func = bpf_task_storage_get,
349         .gpl_only = false,
350         .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
351         .arg1_type = ARG_CONST_MAP_PTR,
352         .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
353         .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
354         .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
355         .arg4_type = ARG_ANYTHING,
356 };
357 
358 const struct bpf_func_proto bpf_task_storage_delete_recur_proto = {
359         .func = bpf_task_storage_delete_recur,
360         .gpl_only = false,
361         .ret_type = RET_INTEGER,
362         .arg1_type = ARG_CONST_MAP_PTR,
363         .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
364         .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
365 };
366 
367 const struct bpf_func_proto bpf_task_storage_delete_proto = {
368         .func = bpf_task_storage_delete,
369         .gpl_only = false,
370         .ret_type = RET_INTEGER,
371         .arg1_type = ARG_CONST_MAP_PTR,
372         .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
373         .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
374 };
375 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

sflogo.php