1 /* SPDX-License-Identifier: GPL-2.0 */ 1
2 /*
3 * BPF extensible scheduler class: Documentati
4 *
5 * Copyright (c) 2022 Meta Platforms, Inc. and
6 * Copyright (c) 2022 Tejun Heo <tj@kernel.org
7 * Copyright (c) 2022 David Vernet <dvernet@me
8 */
9 #ifndef _LINUX_SCHED_EXT_H
10 #define _LINUX_SCHED_EXT_H
11
12 #ifdef CONFIG_SCHED_CLASS_EXT
13
14 #include <linux/llist.h>
15 #include <linux/rhashtable-types.h>
16
17 enum scx_public_consts {
18 SCX_OPS_NAME_LEN = 128,
19
20 SCX_SLICE_DFL = 20 * 1000000
21 SCX_SLICE_INF = U64_MAX,
22 };
23
24 /*
25 * DSQ (dispatch queue) IDs are 64bit of the f
26 *
27 * Bits: [63] [62 .. 0]
28 * [ B] [ ID ]
29 *
30 * B: 1 for IDs for built-in DSQs, 0 for op
31 * ID: 63 bit ID
32 *
33 * Built-in IDs:
34 *
35 * Bits: [63] [62] [61..32] [31 .. 0]
36 * [ 1] [ L] [ R ] [ V ]
37 *
38 * 1: 1 for built-in DSQs.
39 * L: 1 for LOCAL_ON DSQ IDs, 0 for others
40 * V: For LOCAL_ON DSQ IDs, a CPU number. F
41 */
42 enum scx_dsq_id_flags {
43 SCX_DSQ_FLAG_BUILTIN = 1LLU << 63,
44 SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62,
45
46 SCX_DSQ_INVALID = SCX_DSQ_FLAG
47 SCX_DSQ_GLOBAL = SCX_DSQ_FLAG
48 SCX_DSQ_LOCAL = SCX_DSQ_FLAG
49 SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG
50 SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLL
51 };
52
53 /*
54 * A dispatch queue (DSQ) can be either a FIFO
55 * queue. A built-in DSQ is always a FIFO. The
56 * buffer between the scheduler core and the B
57 * documentation for more details.
58 */
59 struct scx_dispatch_q {
60 raw_spinlock_t lock;
61 struct list_head list; /* tas
62 struct rb_root priq; /* use
63 u32 nr;
64 u32 seq; /* use
65 u64 id;
66 struct rhash_head hash_node;
67 struct llist_node free_node;
68 struct rcu_head rcu;
69 };
70
71 /* scx_entity.flags */
72 enum scx_ent_flags {
73 SCX_TASK_QUEUED = 1 << 0, /* o
74 SCX_TASK_RESET_RUNNABLE_AT = 1 << 2, /
75 SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* l
76
77 SCX_TASK_STATE_SHIFT = 8, /* b
78 SCX_TASK_STATE_BITS = 2,
79 SCX_TASK_STATE_MASK = ((1 << SCX_T
80
81 SCX_TASK_CURSOR = 1 << 31, /*
82 };
83
84 /* scx_entity.flags & SCX_TASK_STATE_MASK */
85 enum scx_task_state {
86 SCX_TASK_NONE, /* ops.init_ta
87 SCX_TASK_INIT, /* ops.init_ta
88 SCX_TASK_READY, /* fully initi
89 SCX_TASK_ENABLED, /* fully initi
90
91 SCX_TASK_NR_STATES,
92 };
93
94 /* scx_entity.dsq_flags */
95 enum scx_ent_dsq_flags {
96 SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* t
97 };
98
99 /*
100 * Mask bits for scx_entity.kf_mask. Not all k
101 * everywhere and the following bits track whi
102 * allowed for %current. This simple per-task
103 * nest in a limited way. BPF will likely impl
104 * kfuncs depending on the calling context whi
105 * mechanism. See scx_kf_allow().
106 */
107 enum scx_kf_mask {
108 SCX_KF_UNLOCKED = 0, /* s
109 /* ENQUEUE and DISPATCH may be nested
110 SCX_KF_CPU_RELEASE = 1 << 0, /* o
111 /* ops.dequeue (in REST) may be nested
112 SCX_KF_DISPATCH = 1 << 1, /* o
113 SCX_KF_ENQUEUE = 1 << 2, /* o
114 SCX_KF_SELECT_CPU = 1 << 3, /* o
115 SCX_KF_REST = 1 << 4, /* o
116
117 __SCX_KF_RQ_LOCKED = SCX_KF_CPU_R
118 SCX_KF_ENQUE
119 __SCX_KF_TERMINAL = SCX_KF_ENQUE
120 };
121
122 enum scx_dsq_lnode_flags {
123 SCX_DSQ_LNODE_ITER_CURSOR = 1 << 0,
124
125 /* high 16 bits can be for iter cursor
126 __SCX_DSQ_LNODE_PRIV_SHIFT = 16,
127 };
128
129 struct scx_dsq_list_node {
130 struct list_head node;
131 u32 flags;
132 u32 priv;
133 };
134
135 /*
136 * The following is embedded in task_struct an
137 * for a task to be scheduled by SCX.
138 */
139 struct sched_ext_entity {
140 struct scx_dispatch_q *dsq;
141 struct scx_dsq_list_node dsq_list;
142 struct rb_node dsq_priq;
143 u32 dsq_seq;
144 u32 dsq_flags;
145 u32 flags;
146 u32 weight;
147 s32 sticky_cpu;
148 s32 holding_cpu;
149 u32 kf_mask;
150 struct task_struct *kf_tasks[2];
151 atomic_long_t ops_state;
152
153 struct list_head runnable_node;
154 unsigned long runnable_at;
155
156 #ifdef CONFIG_SCHED_CORE
157 u64 core_sched_at;
158 #endif
159 u64 ddsp_dsq_id;
160 u64 ddsp_enq_flags
161
162 /* BPF scheduler modifiable fields */
163
164 /*
165 * Runtime budget in nsecs. This is us
166 * scx_bpf_dispatch() but can also be
167 * scheduler. Automatically decreased
168 * depletion, a scheduling event is tr
169 *
170 * This value is cleared to zero if th
171 * %SCX_KICK_PREEMPT and shouldn't be
172 * task ran. Use p->se.sum_exec_runtim
173 */
174 u64 slice;
175
176 /*
177 * Used to order tasks when dispatchin
178 * queue of a dsq. This is usually set
179 * but can also be modified directly b
180 * while a task is queued on a dsq may
181 * recommended.
182 */
183 u64 dsq_vtime;
184
185 /*
186 * If set, reject future sched_setsche
187 * to %SCHED_EXT with -%EACCES.
188 *
189 * Can be set from ops.init_task() whi
190 * loaded (!scx_init_task_args->fork).
191 * already %SCHED_EXT, the task's poli
192 * reverted to %SCHED_NORMAL. The numb
193 * through /sys/kernel/debug/sched_ext
194 * during fork is not allowed.
195 */
196 bool disallow;
197
198 /* cold fields */
199 #ifdef CONFIG_EXT_GROUP_SCHED
200 struct cgroup *cgrp_moving_f
201 #endif
202 /* must be the last field, see init_sc
203 struct list_head tasks_node;
204 };
205
206 void sched_ext_free(struct task_struct *p);
207 void print_scx_info(const char *log_lvl, struc
208
209 #else /* !CONFIG_SCHED_CLASS_EXT */
210
211 static inline void sched_ext_free(struct task_
212 static inline void print_scx_info(const char *
213
214 #endif /* CONFIG_SCHED_CLASS_EXT */
215 #endif /* _LINUX_SCHED_EXT_H */
216
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.