1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * linux/init/main.c 3 * linux/init/main.c
4 * 4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 1991, 1992 Linus Torvalds
6 * 6 *
7 * GK 2/5/95 - Changed to support mounting 7 * GK 2/5/95 - Changed to support mounting root fs via NFS
8 * Added initrd & change_root: Werner Almesbe 8 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
9 * Moan early if gcc is old, avoiding bogus k 9 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
10 * Simplified starting of init: Michael A. G 10 * Simplified starting of init: Michael A. Griffith <grif@acm.org>
11 */ 11 */
12 12
13 #define DEBUG /* Enable initcall_deb 13 #define DEBUG /* Enable initcall_debug */
14 14
15 #include <linux/types.h> 15 #include <linux/types.h>
16 #include <linux/extable.h> 16 #include <linux/extable.h>
17 #include <linux/module.h> 17 #include <linux/module.h>
18 #include <linux/proc_fs.h> 18 #include <linux/proc_fs.h>
19 #include <linux/binfmts.h> 19 #include <linux/binfmts.h>
20 #include <linux/kernel.h> 20 #include <linux/kernel.h>
21 #include <linux/syscalls.h> 21 #include <linux/syscalls.h>
22 #include <linux/stackprotector.h> 22 #include <linux/stackprotector.h>
23 #include <linux/string.h> 23 #include <linux/string.h>
24 #include <linux/ctype.h> 24 #include <linux/ctype.h>
25 #include <linux/delay.h> 25 #include <linux/delay.h>
26 #include <linux/ioport.h> 26 #include <linux/ioport.h>
27 #include <linux/init.h> 27 #include <linux/init.h>
28 #include <linux/initrd.h> 28 #include <linux/initrd.h>
29 #include <linux/memblock.h> 29 #include <linux/memblock.h>
30 #include <linux/acpi.h> 30 #include <linux/acpi.h>
31 #include <linux/bootconfig.h> 31 #include <linux/bootconfig.h>
32 #include <linux/console.h> 32 #include <linux/console.h>
33 #include <linux/nmi.h> 33 #include <linux/nmi.h>
34 #include <linux/percpu.h> 34 #include <linux/percpu.h>
35 #include <linux/kmod.h> 35 #include <linux/kmod.h>
36 #include <linux/kprobes.h> 36 #include <linux/kprobes.h>
37 #include <linux/kmsan.h> 37 #include <linux/kmsan.h>
38 #include <linux/vmalloc.h> 38 #include <linux/vmalloc.h>
39 #include <linux/kernel_stat.h> 39 #include <linux/kernel_stat.h>
40 #include <linux/start_kernel.h> 40 #include <linux/start_kernel.h>
41 #include <linux/security.h> 41 #include <linux/security.h>
42 #include <linux/smp.h> 42 #include <linux/smp.h>
43 #include <linux/profile.h> 43 #include <linux/profile.h>
44 #include <linux/kfence.h> 44 #include <linux/kfence.h>
45 #include <linux/rcupdate.h> 45 #include <linux/rcupdate.h>
46 #include <linux/srcu.h> 46 #include <linux/srcu.h>
47 #include <linux/moduleparam.h> 47 #include <linux/moduleparam.h>
48 #include <linux/kallsyms.h> 48 #include <linux/kallsyms.h>
49 #include <linux/buildid.h> 49 #include <linux/buildid.h>
50 #include <linux/writeback.h> 50 #include <linux/writeback.h>
51 #include <linux/cpu.h> 51 #include <linux/cpu.h>
52 #include <linux/cpuset.h> 52 #include <linux/cpuset.h>
53 #include <linux/cgroup.h> 53 #include <linux/cgroup.h>
54 #include <linux/efi.h> 54 #include <linux/efi.h>
55 #include <linux/tick.h> 55 #include <linux/tick.h>
56 #include <linux/sched/isolation.h> 56 #include <linux/sched/isolation.h>
57 #include <linux/interrupt.h> 57 #include <linux/interrupt.h>
58 #include <linux/taskstats_kern.h> 58 #include <linux/taskstats_kern.h>
59 #include <linux/delayacct.h> 59 #include <linux/delayacct.h>
60 #include <linux/unistd.h> 60 #include <linux/unistd.h>
61 #include <linux/utsname.h> 61 #include <linux/utsname.h>
62 #include <linux/rmap.h> 62 #include <linux/rmap.h>
63 #include <linux/mempolicy.h> 63 #include <linux/mempolicy.h>
64 #include <linux/key.h> 64 #include <linux/key.h>
65 #include <linux/debug_locks.h> 65 #include <linux/debug_locks.h>
66 #include <linux/debugobjects.h> 66 #include <linux/debugobjects.h>
67 #include <linux/lockdep.h> 67 #include <linux/lockdep.h>
68 #include <linux/kmemleak.h> 68 #include <linux/kmemleak.h>
69 #include <linux/padata.h> 69 #include <linux/padata.h>
70 #include <linux/pid_namespace.h> 70 #include <linux/pid_namespace.h>
71 #include <linux/device/driver.h> 71 #include <linux/device/driver.h>
72 #include <linux/kthread.h> 72 #include <linux/kthread.h>
73 #include <linux/sched.h> 73 #include <linux/sched.h>
74 #include <linux/sched/init.h> 74 #include <linux/sched/init.h>
75 #include <linux/signal.h> 75 #include <linux/signal.h>
76 #include <linux/idr.h> 76 #include <linux/idr.h>
77 #include <linux/kgdb.h> 77 #include <linux/kgdb.h>
78 #include <linux/ftrace.h> 78 #include <linux/ftrace.h>
79 #include <linux/async.h> 79 #include <linux/async.h>
80 #include <linux/shmem_fs.h> 80 #include <linux/shmem_fs.h>
81 #include <linux/slab.h> 81 #include <linux/slab.h>
82 #include <linux/perf_event.h> 82 #include <linux/perf_event.h>
83 #include <linux/ptrace.h> 83 #include <linux/ptrace.h>
84 #include <linux/pti.h> 84 #include <linux/pti.h>
85 #include <linux/blkdev.h> 85 #include <linux/blkdev.h>
86 #include <linux/sched/clock.h> 86 #include <linux/sched/clock.h>
87 #include <linux/sched/task.h> 87 #include <linux/sched/task.h>
88 #include <linux/sched/task_stack.h> 88 #include <linux/sched/task_stack.h>
89 #include <linux/context_tracking.h> 89 #include <linux/context_tracking.h>
90 #include <linux/random.h> 90 #include <linux/random.h>
91 #include <linux/moduleloader.h> 91 #include <linux/moduleloader.h>
92 #include <linux/list.h> 92 #include <linux/list.h>
93 #include <linux/integrity.h> 93 #include <linux/integrity.h>
94 #include <linux/proc_ns.h> 94 #include <linux/proc_ns.h>
95 #include <linux/io.h> 95 #include <linux/io.h>
96 #include <linux/cache.h> 96 #include <linux/cache.h>
97 #include <linux/rodata_test.h> 97 #include <linux/rodata_test.h>
98 #include <linux/jump_label.h> 98 #include <linux/jump_label.h>
99 #include <linux/kcsan.h> 99 #include <linux/kcsan.h>
100 #include <linux/init_syscalls.h> 100 #include <linux/init_syscalls.h>
101 #include <linux/stackdepot.h> 101 #include <linux/stackdepot.h>
102 #include <linux/randomize_kstack.h> 102 #include <linux/randomize_kstack.h>
103 #include <linux/pidfs.h> 103 #include <linux/pidfs.h>
104 #include <linux/ptdump.h> 104 #include <linux/ptdump.h>
105 #include <net/net_namespace.h> 105 #include <net/net_namespace.h>
106 106
107 #include <asm/io.h> 107 #include <asm/io.h>
108 #include <asm/setup.h> 108 #include <asm/setup.h>
109 #include <asm/sections.h> 109 #include <asm/sections.h>
110 #include <asm/cacheflush.h> 110 #include <asm/cacheflush.h>
111 111
112 #define CREATE_TRACE_POINTS 112 #define CREATE_TRACE_POINTS
113 #include <trace/events/initcall.h> 113 #include <trace/events/initcall.h>
114 114
115 #include <kunit/test.h> 115 #include <kunit/test.h>
116 116
117 static int kernel_init(void *); 117 static int kernel_init(void *);
118 118
119 /* 119 /*
120 * Debug helper: via this flag we know that we 120 * Debug helper: via this flag we know that we are in 'early bootup code'
121 * where only the boot processor is running wi 121 * where only the boot processor is running with IRQ disabled. This means
122 * two things - IRQ must not be enabled before 122 * two things - IRQ must not be enabled before the flag is cleared and some
123 * operations which are not allowed with IRQ d 123 * operations which are not allowed with IRQ disabled are allowed while the
124 * flag is set. 124 * flag is set.
125 */ 125 */
126 bool early_boot_irqs_disabled __read_mostly; 126 bool early_boot_irqs_disabled __read_mostly;
127 127
128 enum system_states system_state __read_mostly; 128 enum system_states system_state __read_mostly;
129 EXPORT_SYMBOL(system_state); 129 EXPORT_SYMBOL(system_state);
130 130
131 /* 131 /*
132 * Boot command-line arguments 132 * Boot command-line arguments
133 */ 133 */
134 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMI 134 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
135 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMI 135 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
136 136
137 /* Default late time init is NULL. archs can o 137 /* Default late time init is NULL. archs can override this later. */
138 void (*__initdata late_time_init)(void); 138 void (*__initdata late_time_init)(void);
139 139
140 /* Untouched command line saved by arch-specif 140 /* Untouched command line saved by arch-specific code. */
141 char __initdata boot_command_line[COMMAND_LINE 141 char __initdata boot_command_line[COMMAND_LINE_SIZE];
142 /* Untouched saved command line (eg. for /proc 142 /* Untouched saved command line (eg. for /proc) */
143 char *saved_command_line __ro_after_init; 143 char *saved_command_line __ro_after_init;
144 unsigned int saved_command_line_len __ro_after 144 unsigned int saved_command_line_len __ro_after_init;
145 /* Command line for parameter parsing */ 145 /* Command line for parameter parsing */
146 static char *static_command_line; 146 static char *static_command_line;
147 /* Untouched extra command line */ 147 /* Untouched extra command line */
148 static char *extra_command_line; 148 static char *extra_command_line;
149 /* Extra init arguments */ 149 /* Extra init arguments */
150 static char *extra_init_args; 150 static char *extra_init_args;
151 151
152 #ifdef CONFIG_BOOT_CONFIG 152 #ifdef CONFIG_BOOT_CONFIG
153 /* Is bootconfig on command line? */ 153 /* Is bootconfig on command line? */
154 static bool bootconfig_found; 154 static bool bootconfig_found;
155 static size_t initargs_offs; 155 static size_t initargs_offs;
156 #else 156 #else
157 # define bootconfig_found false 157 # define bootconfig_found false
158 # define initargs_offs 0 158 # define initargs_offs 0
159 #endif 159 #endif
160 160
161 static char *execute_command; 161 static char *execute_command;
162 static char *ramdisk_execute_command = "/init" 162 static char *ramdisk_execute_command = "/init";
163 163
164 /* 164 /*
165 * Used to generate warnings if static_key man 165 * Used to generate warnings if static_key manipulation functions are used
166 * before jump_label_init is called. 166 * before jump_label_init is called.
167 */ 167 */
168 bool static_key_initialized __read_mostly; 168 bool static_key_initialized __read_mostly;
169 EXPORT_SYMBOL_GPL(static_key_initialized); 169 EXPORT_SYMBOL_GPL(static_key_initialized);
170 170
171 /* 171 /*
172 * If set, this is an indication to the driver 172 * If set, this is an indication to the drivers that reset the underlying
173 * device before going ahead with the initiali 173 * device before going ahead with the initialization otherwise driver might
174 * rely on the BIOS and skip the reset operati 174 * rely on the BIOS and skip the reset operation.
175 * 175 *
176 * This is useful if kernel is booting in an u 176 * This is useful if kernel is booting in an unreliable environment.
177 * For ex. kdump situation where previous kern 177 * For ex. kdump situation where previous kernel has crashed, BIOS has been
178 * skipped and devices will be in unknown stat 178 * skipped and devices will be in unknown state.
179 */ 179 */
180 unsigned int reset_devices; 180 unsigned int reset_devices;
181 EXPORT_SYMBOL(reset_devices); 181 EXPORT_SYMBOL(reset_devices);
182 182
183 static int __init set_reset_devices(char *str) 183 static int __init set_reset_devices(char *str)
184 { 184 {
185 reset_devices = 1; 185 reset_devices = 1;
186 return 1; 186 return 1;
187 } 187 }
188 188
189 __setup("reset_devices", set_reset_devices); 189 __setup("reset_devices", set_reset_devices);
190 190
191 static const char *argv_init[MAX_INIT_ARGS+2] 191 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
192 const char *envp_init[MAX_INIT_ENVS+2] = { "HO 192 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
193 static const char *panic_later, *panic_param; 193 static const char *panic_later, *panic_param;
194 194
195 static bool __init obsolete_checksetup(char *l 195 static bool __init obsolete_checksetup(char *line)
196 { 196 {
197 const struct obs_kernel_param *p; 197 const struct obs_kernel_param *p;
198 bool had_early_param = false; 198 bool had_early_param = false;
199 199
200 p = __setup_start; 200 p = __setup_start;
201 do { 201 do {
202 int n = strlen(p->str); 202 int n = strlen(p->str);
203 if (parameqn(line, p->str, n)) 203 if (parameqn(line, p->str, n)) {
204 if (p->early) { 204 if (p->early) {
205 /* Already don 205 /* Already done in parse_early_param?
206 * (Needs exac 206 * (Needs exact match on param part).
207 * Keep iterat 207 * Keep iterating, as we can have early
208 * params and 208 * params and __setups of same names 8( */
209 if (line[n] == 209 if (line[n] == '\0' || line[n] == '=')
210 had_ea 210 had_early_param = true;
211 } else if (!p->setup_f 211 } else if (!p->setup_func) {
212 pr_warn("Param 212 pr_warn("Parameter %s is obsolete, ignored\n",
213 p->str 213 p->str);
214 return true; 214 return true;
215 } else if (p->setup_fu 215 } else if (p->setup_func(line + n))
216 return true; 216 return true;
217 } 217 }
218 p++; 218 p++;
219 } while (p < __setup_end); 219 } while (p < __setup_end);
220 220
221 return had_early_param; 221 return had_early_param;
222 } 222 }
223 223
224 /* 224 /*
225 * This should be approx 2 Bo*oMips to start ( 225 * This should be approx 2 Bo*oMips to start (note initial shift), and will
226 * still work even if initially too large, it 226 * still work even if initially too large, it will just take slightly longer
227 */ 227 */
228 unsigned long loops_per_jiffy = (1<<12); 228 unsigned long loops_per_jiffy = (1<<12);
229 EXPORT_SYMBOL(loops_per_jiffy); 229 EXPORT_SYMBOL(loops_per_jiffy);
230 230
231 static int __init debug_kernel(char *str) 231 static int __init debug_kernel(char *str)
232 { 232 {
233 console_loglevel = CONSOLE_LOGLEVEL_DE 233 console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
234 return 0; 234 return 0;
235 } 235 }
236 236
237 static int __init quiet_kernel(char *str) 237 static int __init quiet_kernel(char *str)
238 { 238 {
239 console_loglevel = CONSOLE_LOGLEVEL_QU 239 console_loglevel = CONSOLE_LOGLEVEL_QUIET;
240 return 0; 240 return 0;
241 } 241 }
242 242
243 early_param("debug", debug_kernel); 243 early_param("debug", debug_kernel);
244 early_param("quiet", quiet_kernel); 244 early_param("quiet", quiet_kernel);
245 245
246 static int __init loglevel(char *str) 246 static int __init loglevel(char *str)
247 { 247 {
248 int newlevel; 248 int newlevel;
249 249
250 /* 250 /*
251 * Only update loglevel value when a c 251 * Only update loglevel value when a correct setting was passed,
252 * to prevent blind crashes (when logl 252 * to prevent blind crashes (when loglevel being set to 0) that
253 * are quite hard to debug 253 * are quite hard to debug
254 */ 254 */
255 if (get_option(&str, &newlevel)) { 255 if (get_option(&str, &newlevel)) {
256 console_loglevel = newlevel; 256 console_loglevel = newlevel;
257 return 0; 257 return 0;
258 } 258 }
259 259
260 return -EINVAL; 260 return -EINVAL;
261 } 261 }
262 262
263 early_param("loglevel", loglevel); 263 early_param("loglevel", loglevel);
264 264
265 #ifdef CONFIG_BLK_DEV_INITRD 265 #ifdef CONFIG_BLK_DEV_INITRD
266 static void * __init get_boot_config_from_init 266 static void * __init get_boot_config_from_initrd(size_t *_size)
267 { 267 {
268 u32 size, csum; 268 u32 size, csum;
269 char *data; 269 char *data;
270 u32 *hdr; 270 u32 *hdr;
271 int i; 271 int i;
272 272
273 if (!initrd_end) 273 if (!initrd_end)
274 return NULL; 274 return NULL;
275 275
276 data = (char *)initrd_end - BOOTCONFIG 276 data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
277 /* 277 /*
278 * Since Grub may align the size of in 278 * Since Grub may align the size of initrd to 4, we must
279 * check the preceding 3 bytes as well 279 * check the preceding 3 bytes as well.
280 */ 280 */
281 for (i = 0; i < 4; i++) { 281 for (i = 0; i < 4; i++) {
282 if (!memcmp(data, BOOTCONFIG_M 282 if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
283 goto found; 283 goto found;
284 data--; 284 data--;
285 } 285 }
286 return NULL; 286 return NULL;
287 287
288 found: 288 found:
289 hdr = (u32 *)(data - 8); 289 hdr = (u32 *)(data - 8);
290 size = le32_to_cpu(hdr[0]); 290 size = le32_to_cpu(hdr[0]);
291 csum = le32_to_cpu(hdr[1]); 291 csum = le32_to_cpu(hdr[1]);
292 292
293 data = ((void *)hdr) - size; 293 data = ((void *)hdr) - size;
294 if ((unsigned long)data < initrd_start 294 if ((unsigned long)data < initrd_start) {
295 pr_err("bootconfig size %d is 295 pr_err("bootconfig size %d is greater than initrd size %ld\n",
296 size, initrd_end - ini 296 size, initrd_end - initrd_start);
297 return NULL; 297 return NULL;
298 } 298 }
299 299
300 if (xbc_calc_checksum(data, size) != c 300 if (xbc_calc_checksum(data, size) != csum) {
301 pr_err("bootconfig checksum fa 301 pr_err("bootconfig checksum failed\n");
302 return NULL; 302 return NULL;
303 } 303 }
304 304
305 /* Remove bootconfig from initramfs/in 305 /* Remove bootconfig from initramfs/initrd */
306 initrd_end = (unsigned long)data; 306 initrd_end = (unsigned long)data;
307 if (_size) 307 if (_size)
308 *_size = size; 308 *_size = size;
309 309
310 return data; 310 return data;
311 } 311 }
312 #else 312 #else
313 static void * __init get_boot_config_from_init 313 static void * __init get_boot_config_from_initrd(size_t *_size)
314 { 314 {
315 return NULL; 315 return NULL;
316 } 316 }
317 #endif 317 #endif
318 318
319 #ifdef CONFIG_BOOT_CONFIG 319 #ifdef CONFIG_BOOT_CONFIG
320 320
321 static char xbc_namebuf[XBC_KEYLEN_MAX] __init 321 static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
322 322
323 #define rest(dst, end) ((end) > (dst) ? (end) 323 #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
324 324
325 static int __init xbc_snprint_cmdline(char *bu 325 static int __init xbc_snprint_cmdline(char *buf, size_t size,
326 struct x 326 struct xbc_node *root)
327 { 327 {
328 struct xbc_node *knode, *vnode; 328 struct xbc_node *knode, *vnode;
329 char *end = buf + size; 329 char *end = buf + size;
330 const char *val, *q; 330 const char *val, *q;
331 int ret; 331 int ret;
332 332
333 xbc_node_for_each_key_value(root, knod 333 xbc_node_for_each_key_value(root, knode, val) {
334 ret = xbc_node_compose_key_aft 334 ret = xbc_node_compose_key_after(root, knode,
335 xbc_na 335 xbc_namebuf, XBC_KEYLEN_MAX);
336 if (ret < 0) 336 if (ret < 0)
337 return ret; 337 return ret;
338 338
339 vnode = xbc_node_get_child(kno 339 vnode = xbc_node_get_child(knode);
340 if (!vnode) { 340 if (!vnode) {
341 ret = snprintf(buf, re 341 ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
342 if (ret < 0) 342 if (ret < 0)
343 return ret; 343 return ret;
344 buf += ret; 344 buf += ret;
345 continue; 345 continue;
346 } 346 }
347 xbc_array_for_each_value(vnode 347 xbc_array_for_each_value(vnode, val) {
348 /* 348 /*
349 * For prettier and mo 349 * For prettier and more readable /proc/cmdline, only
350 * quote the value whe 350 * quote the value when necessary, i.e. when it contains
351 * whitespace. 351 * whitespace.
352 */ 352 */
353 q = strpbrk(val, " \t\ 353 q = strpbrk(val, " \t\r\n") ? "\"" : "";
354 ret = snprintf(buf, re 354 ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ",
355 xbc_nam 355 xbc_namebuf, q, val, q);
356 if (ret < 0) 356 if (ret < 0)
357 return ret; 357 return ret;
358 buf += ret; 358 buf += ret;
359 } 359 }
360 } 360 }
361 361
362 return buf - (end - size); 362 return buf - (end - size);
363 } 363 }
364 #undef rest 364 #undef rest
365 365
366 /* Make an extra command line under given key 366 /* Make an extra command line under given key word */
367 static char * __init xbc_make_cmdline(const ch 367 static char * __init xbc_make_cmdline(const char *key)
368 { 368 {
369 struct xbc_node *root; 369 struct xbc_node *root;
370 char *new_cmdline; 370 char *new_cmdline;
371 int ret, len = 0; 371 int ret, len = 0;
372 372
373 root = xbc_find_node(key); 373 root = xbc_find_node(key);
374 if (!root) 374 if (!root)
375 return NULL; 375 return NULL;
376 376
377 /* Count required buffer size */ 377 /* Count required buffer size */
378 len = xbc_snprint_cmdline(NULL, 0, roo 378 len = xbc_snprint_cmdline(NULL, 0, root);
379 if (len <= 0) 379 if (len <= 0)
380 return NULL; 380 return NULL;
381 381
382 new_cmdline = memblock_alloc(len + 1, 382 new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
383 if (!new_cmdline) { 383 if (!new_cmdline) {
384 pr_err("Failed to allocate mem 384 pr_err("Failed to allocate memory for extra kernel cmdline.\n");
385 return NULL; 385 return NULL;
386 } 386 }
387 387
388 ret = xbc_snprint_cmdline(new_cmdline, 388 ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
389 if (ret < 0 || ret > len) { 389 if (ret < 0 || ret > len) {
390 pr_err("Failed to print extra 390 pr_err("Failed to print extra kernel cmdline.\n");
391 memblock_free(new_cmdline, len 391 memblock_free(new_cmdline, len + 1);
392 return NULL; 392 return NULL;
393 } 393 }
394 394
395 return new_cmdline; 395 return new_cmdline;
396 } 396 }
397 397
398 static int __init bootconfig_params(char *para 398 static int __init bootconfig_params(char *param, char *val,
399 const char 399 const char *unused, void *arg)
400 { 400 {
401 if (strcmp(param, "bootconfig") == 0) 401 if (strcmp(param, "bootconfig") == 0) {
402 bootconfig_found = true; 402 bootconfig_found = true;
403 } 403 }
404 return 0; 404 return 0;
405 } 405 }
406 406
407 static int __init warn_bootconfig(char *str) 407 static int __init warn_bootconfig(char *str)
408 { 408 {
409 /* The 'bootconfig' has been handled b 409 /* The 'bootconfig' has been handled by bootconfig_params(). */
410 return 0; 410 return 0;
411 } 411 }
412 412
413 static void __init setup_boot_config(void) 413 static void __init setup_boot_config(void)
414 { 414 {
415 static char tmp_cmdline[COMMAND_LINE_S 415 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
416 const char *msg, *data; 416 const char *msg, *data;
417 int pos, ret; 417 int pos, ret;
418 size_t size; 418 size_t size;
419 char *err; 419 char *err;
420 420
421 /* Cut out the bootconfig data even if 421 /* Cut out the bootconfig data even if we have no bootconfig option */
422 data = get_boot_config_from_initrd(&si 422 data = get_boot_config_from_initrd(&size);
423 /* If there is no bootconfig in initrd 423 /* If there is no bootconfig in initrd, try embedded one. */
424 if (!data) 424 if (!data)
425 data = xbc_get_embedded_bootco 425 data = xbc_get_embedded_bootconfig(&size);
426 426
427 strscpy(tmp_cmdline, boot_command_line 427 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
428 err = parse_args("bootconfig", tmp_cmd 428 err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
429 bootconfig_params); 429 bootconfig_params);
430 430
431 if (IS_ERR(err) || !(bootconfig_found 431 if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
432 return; 432 return;
433 433
434 /* parse_args() stops at the next para 434 /* parse_args() stops at the next param of '--' and returns an address */
435 if (err) 435 if (err)
436 initargs_offs = err - tmp_cmdl 436 initargs_offs = err - tmp_cmdline;
437 437
438 if (!data) { 438 if (!data) {
439 /* If user intended to use boo 439 /* If user intended to use bootconfig, show an error level message */
440 if (bootconfig_found) 440 if (bootconfig_found)
441 pr_err("'bootconfig' f 441 pr_err("'bootconfig' found on command line, but no bootconfig found\n");
442 else 442 else
443 pr_info("No bootconfig 443 pr_info("No bootconfig data provided, so skipping bootconfig");
444 return; 444 return;
445 } 445 }
446 446
447 if (size >= XBC_DATA_MAX) { 447 if (size >= XBC_DATA_MAX) {
448 pr_err("bootconfig size %ld gr 448 pr_err("bootconfig size %ld greater than max size %d\n",
449 (long)size, XBC_DATA_M 449 (long)size, XBC_DATA_MAX);
450 return; 450 return;
451 } 451 }
452 452
453 ret = xbc_init(data, size, &msg, &pos) 453 ret = xbc_init(data, size, &msg, &pos);
454 if (ret < 0) { 454 if (ret < 0) {
455 if (pos < 0) 455 if (pos < 0)
456 pr_err("Failed to init 456 pr_err("Failed to init bootconfig: %s.\n", msg);
457 else 457 else
458 pr_err("Failed to pars 458 pr_err("Failed to parse bootconfig: %s at %d.\n",
459 msg, pos); 459 msg, pos);
460 } else { 460 } else {
461 xbc_get_info(&ret, NULL); 461 xbc_get_info(&ret, NULL);
462 pr_info("Load bootconfig: %ld 462 pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
463 /* keys starting with "kernel. 463 /* keys starting with "kernel." are passed via cmdline */
464 extra_command_line = xbc_make_ 464 extra_command_line = xbc_make_cmdline("kernel");
465 /* Also, "init." keys are init 465 /* Also, "init." keys are init arguments */
466 extra_init_args = xbc_make_cmd 466 extra_init_args = xbc_make_cmdline("init");
467 } 467 }
468 return; 468 return;
469 } 469 }
470 470
471 static void __init exit_boot_config(void) 471 static void __init exit_boot_config(void)
472 { 472 {
473 xbc_exit(); 473 xbc_exit();
474 } 474 }
475 475
476 #else /* !CONFIG_BOOT_CONFIG */ 476 #else /* !CONFIG_BOOT_CONFIG */
477 477
478 static void __init setup_boot_config(void) 478 static void __init setup_boot_config(void)
479 { 479 {
480 /* Remove bootconfig data from initrd 480 /* Remove bootconfig data from initrd */
481 get_boot_config_from_initrd(NULL); 481 get_boot_config_from_initrd(NULL);
482 } 482 }
483 483
484 static int __init warn_bootconfig(char *str) 484 static int __init warn_bootconfig(char *str)
485 { 485 {
486 pr_warn("WARNING: 'bootconfig' found o 486 pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
487 return 0; 487 return 0;
488 } 488 }
489 489
490 #define exit_boot_config() do {} while (0 490 #define exit_boot_config() do {} while (0)
491 491
492 #endif /* CONFIG_BOOT_CONFIG */ 492 #endif /* CONFIG_BOOT_CONFIG */
493 493
494 early_param("bootconfig", warn_bootconfig); 494 early_param("bootconfig", warn_bootconfig);
495 495
496 bool __init cmdline_has_extra_options(void) 496 bool __init cmdline_has_extra_options(void)
497 { 497 {
498 return extra_command_line || extra_ini 498 return extra_command_line || extra_init_args;
499 } 499 }
500 500
501 /* Change NUL term back to "=", to make "param 501 /* Change NUL term back to "=", to make "param" the whole string. */
502 static void __init repair_env_string(char *par 502 static void __init repair_env_string(char *param, char *val)
503 { 503 {
504 if (val) { 504 if (val) {
505 /* param=val or param="val"? * 505 /* param=val or param="val"? */
506 if (val == param+strlen(param) 506 if (val == param+strlen(param)+1)
507 val[-1] = '='; 507 val[-1] = '=';
508 else if (val == param+strlen(p 508 else if (val == param+strlen(param)+2) {
509 val[-2] = '='; 509 val[-2] = '=';
510 memmove(val-1, val, st 510 memmove(val-1, val, strlen(val)+1);
511 } else 511 } else
512 BUG(); 512 BUG();
513 } 513 }
514 } 514 }
515 515
516 /* Anything after -- gets handed straight to i 516 /* Anything after -- gets handed straight to init. */
517 static int __init set_init_arg(char *param, ch 517 static int __init set_init_arg(char *param, char *val,
518 const char *unu 518 const char *unused, void *arg)
519 { 519 {
520 unsigned int i; 520 unsigned int i;
521 521
522 if (panic_later) 522 if (panic_later)
523 return 0; 523 return 0;
524 524
525 repair_env_string(param, val); 525 repair_env_string(param, val);
526 526
527 for (i = 0; argv_init[i]; i++) { 527 for (i = 0; argv_init[i]; i++) {
528 if (i == MAX_INIT_ARGS) { 528 if (i == MAX_INIT_ARGS) {
529 panic_later = "init"; 529 panic_later = "init";
530 panic_param = param; 530 panic_param = param;
531 return 0; 531 return 0;
532 } 532 }
533 } 533 }
534 argv_init[i] = param; 534 argv_init[i] = param;
535 return 0; 535 return 0;
536 } 536 }
537 537
538 /* 538 /*
539 * Unknown boot options get handed to init, un 539 * Unknown boot options get handed to init, unless they look like
540 * unused parameters (modprobe will find them 540 * unused parameters (modprobe will find them in /proc/cmdline).
541 */ 541 */
542 static int __init unknown_bootoption(char *par 542 static int __init unknown_bootoption(char *param, char *val,
543 const cha 543 const char *unused, void *arg)
544 { 544 {
545 size_t len = strlen(param); 545 size_t len = strlen(param);
546 546
547 /* Handle params aliased to sysctls */ 547 /* Handle params aliased to sysctls */
548 if (sysctl_is_alias(param)) 548 if (sysctl_is_alias(param))
549 return 0; 549 return 0;
550 550
551 repair_env_string(param, val); 551 repair_env_string(param, val);
552 552
553 /* Handle obsolete-style parameters */ 553 /* Handle obsolete-style parameters */
554 if (obsolete_checksetup(param)) 554 if (obsolete_checksetup(param))
555 return 0; 555 return 0;
556 556
557 /* Unused module parameter. */ 557 /* Unused module parameter. */
558 if (strnchr(param, len, '.')) 558 if (strnchr(param, len, '.'))
559 return 0; 559 return 0;
560 560
561 if (panic_later) 561 if (panic_later)
562 return 0; 562 return 0;
563 563
564 if (val) { 564 if (val) {
565 /* Environment option */ 565 /* Environment option */
566 unsigned int i; 566 unsigned int i;
567 for (i = 0; envp_init[i]; i++) 567 for (i = 0; envp_init[i]; i++) {
568 if (i == MAX_INIT_ENVS 568 if (i == MAX_INIT_ENVS) {
569 panic_later = 569 panic_later = "env";
570 panic_param = 570 panic_param = param;
571 } 571 }
572 if (!strncmp(param, en 572 if (!strncmp(param, envp_init[i], len+1))
573 break; 573 break;
574 } 574 }
575 envp_init[i] = param; 575 envp_init[i] = param;
576 } else { 576 } else {
577 /* Command line option */ 577 /* Command line option */
578 unsigned int i; 578 unsigned int i;
579 for (i = 0; argv_init[i]; i++) 579 for (i = 0; argv_init[i]; i++) {
580 if (i == MAX_INIT_ARGS 580 if (i == MAX_INIT_ARGS) {
581 panic_later = 581 panic_later = "init";
582 panic_param = 582 panic_param = param;
583 } 583 }
584 } 584 }
585 argv_init[i] = param; 585 argv_init[i] = param;
586 } 586 }
587 return 0; 587 return 0;
588 } 588 }
589 589
590 static int __init init_setup(char *str) 590 static int __init init_setup(char *str)
591 { 591 {
592 unsigned int i; 592 unsigned int i;
593 593
594 execute_command = str; 594 execute_command = str;
595 /* 595 /*
596 * In case LILO is going to boot us wi 596 * In case LILO is going to boot us with default command line,
597 * it prepends "auto" before the whole 597 * it prepends "auto" before the whole cmdline which makes
598 * the shell think it should execute a 598 * the shell think it should execute a script with such name.
599 * So we ignore all arguments entered 599 * So we ignore all arguments entered _before_ init=... [MJ]
600 */ 600 */
601 for (i = 1; i < MAX_INIT_ARGS; i++) 601 for (i = 1; i < MAX_INIT_ARGS; i++)
602 argv_init[i] = NULL; 602 argv_init[i] = NULL;
603 return 1; 603 return 1;
604 } 604 }
605 __setup("init=", init_setup); 605 __setup("init=", init_setup);
606 606
607 static int __init rdinit_setup(char *str) 607 static int __init rdinit_setup(char *str)
608 { 608 {
609 unsigned int i; 609 unsigned int i;
610 610
611 ramdisk_execute_command = str; 611 ramdisk_execute_command = str;
612 /* See "auto" comment in init_setup */ 612 /* See "auto" comment in init_setup */
613 for (i = 1; i < MAX_INIT_ARGS; i++) 613 for (i = 1; i < MAX_INIT_ARGS; i++)
614 argv_init[i] = NULL; 614 argv_init[i] = NULL;
615 return 1; 615 return 1;
616 } 616 }
617 __setup("rdinit=", rdinit_setup); 617 __setup("rdinit=", rdinit_setup);
618 618
619 #ifndef CONFIG_SMP 619 #ifndef CONFIG_SMP
620 static inline void setup_nr_cpu_ids(void) { } 620 static inline void setup_nr_cpu_ids(void) { }
621 static inline void smp_prepare_cpus(unsigned i 621 static inline void smp_prepare_cpus(unsigned int maxcpus) { }
622 #endif 622 #endif
623 623
624 /* 624 /*
625 * We need to store the untouched command line 625 * We need to store the untouched command line for future reference.
626 * We also need to store the touched command l 626 * We also need to store the touched command line since the parameter
627 * parsing is performed in place, and we shoul 627 * parsing is performed in place, and we should allow a component to
628 * store reference of name/value for future re 628 * store reference of name/value for future reference.
629 */ 629 */
630 static void __init setup_command_line(char *co 630 static void __init setup_command_line(char *command_line)
631 { 631 {
632 size_t len, xlen = 0, ilen = 0; 632 size_t len, xlen = 0, ilen = 0;
633 633
634 if (extra_command_line) 634 if (extra_command_line)
635 xlen = strlen(extra_command_li 635 xlen = strlen(extra_command_line);
636 if (extra_init_args) { 636 if (extra_init_args) {
637 extra_init_args = strim(extra_ 637 extra_init_args = strim(extra_init_args); /* remove trailing space */
638 ilen = strlen(extra_init_args) 638 ilen = strlen(extra_init_args) + 4; /* for " -- " */
639 } 639 }
640 640
641 len = xlen + strlen(boot_command_line) 641 len = xlen + strlen(boot_command_line) + ilen + 1;
642 642
643 saved_command_line = memblock_alloc(le 643 saved_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
644 if (!saved_command_line) 644 if (!saved_command_line)
645 panic("%s: Failed to allocate 645 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
646 646
647 len = xlen + strlen(command_line) + 1; 647 len = xlen + strlen(command_line) + 1;
648 648
649 static_command_line = memblock_alloc(l 649 static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
650 if (!static_command_line) 650 if (!static_command_line)
651 panic("%s: Failed to allocate 651 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
652 652
653 if (xlen) { 653 if (xlen) {
654 /* 654 /*
655 * We have to put extra_comman 655 * We have to put extra_command_line before boot command
656 * lines because there could b 656 * lines because there could be dashes (separator of init
657 * command line) in the comman 657 * command line) in the command lines.
658 */ 658 */
659 strcpy(saved_command_line, ext 659 strcpy(saved_command_line, extra_command_line);
660 strcpy(static_command_line, ex 660 strcpy(static_command_line, extra_command_line);
661 } 661 }
662 strcpy(saved_command_line + xlen, boot 662 strcpy(saved_command_line + xlen, boot_command_line);
663 strcpy(static_command_line + xlen, com 663 strcpy(static_command_line + xlen, command_line);
664 664
665 if (ilen) { 665 if (ilen) {
666 /* 666 /*
667 * Append supplemental init bo 667 * Append supplemental init boot args to saved_command_line
668 * so that user can check what 668 * so that user can check what command line options passed
669 * to init. 669 * to init.
670 * The order should always be 670 * The order should always be
671 * " -- "[bootconfig init-para 671 * " -- "[bootconfig init-param][cmdline init-param]
672 */ 672 */
673 if (initargs_offs) { 673 if (initargs_offs) {
674 len = xlen + initargs_ 674 len = xlen + initargs_offs;
675 strcpy(saved_command_l 675 strcpy(saved_command_line + len, extra_init_args);
676 len += ilen - 4; 676 len += ilen - 4; /* strlen(extra_init_args) */
677 strcpy(saved_command_l 677 strcpy(saved_command_line + len,
678 boot_command_l 678 boot_command_line + initargs_offs - 1);
679 } else { 679 } else {
680 len = strlen(saved_com 680 len = strlen(saved_command_line);
681 strcpy(saved_command_l 681 strcpy(saved_command_line + len, " -- ");
682 len += 4; 682 len += 4;
683 strcpy(saved_command_l 683 strcpy(saved_command_line + len, extra_init_args);
684 } 684 }
685 } 685 }
686 686
687 saved_command_line_len = strlen(saved_ 687 saved_command_line_len = strlen(saved_command_line);
688 } 688 }
689 689
690 /* 690 /*
691 * We need to finalize in a non-__init functio 691 * We need to finalize in a non-__init function or else race conditions
692 * between the root thread and the init thread 692 * between the root thread and the init thread may cause start_kernel to
693 * be reaped by free_initmem before the root t 693 * be reaped by free_initmem before the root thread has proceeded to
694 * cpu_idle. 694 * cpu_idle.
695 * 695 *
696 * gcc-3.4 accidentally inlines this function, 696 * gcc-3.4 accidentally inlines this function, so use noinline.
697 */ 697 */
698 698
699 static __initdata DECLARE_COMPLETION(kthreadd_ 699 static __initdata DECLARE_COMPLETION(kthreadd_done);
700 700
701 static noinline void __ref __noreturn rest_ini 701 static noinline void __ref __noreturn rest_init(void)
702 { 702 {
703 struct task_struct *tsk; 703 struct task_struct *tsk;
704 int pid; 704 int pid;
705 705
706 rcu_scheduler_starting(); 706 rcu_scheduler_starting();
707 /* 707 /*
708 * We need to spawn init first so that 708 * We need to spawn init first so that it obtains pid 1, however
709 * the init task will end up wanting t 709 * the init task will end up wanting to create kthreads, which, if
710 * we schedule it before we create kth 710 * we schedule it before we create kthreadd, will OOPS.
711 */ 711 */
712 pid = user_mode_thread(kernel_init, NU 712 pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
713 /* 713 /*
714 * Pin init on the boot CPU. Task migr 714 * Pin init on the boot CPU. Task migration is not properly working
715 * until sched_init_smp() has been run 715 * until sched_init_smp() has been run. It will set the allowed
716 * CPUs for init to the non isolated C 716 * CPUs for init to the non isolated CPUs.
717 */ 717 */
718 rcu_read_lock(); 718 rcu_read_lock();
719 tsk = find_task_by_pid_ns(pid, &init_p 719 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
720 tsk->flags |= PF_NO_SETAFFINITY; 720 tsk->flags |= PF_NO_SETAFFINITY;
721 set_cpus_allowed_ptr(tsk, cpumask_of(s 721 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
722 rcu_read_unlock(); 722 rcu_read_unlock();
723 723
724 numa_default_policy(); 724 numa_default_policy();
725 pid = kernel_thread(kthreadd, NULL, NU 725 pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES);
726 rcu_read_lock(); 726 rcu_read_lock();
727 kthreadd_task = find_task_by_pid_ns(pi 727 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
728 rcu_read_unlock(); 728 rcu_read_unlock();
729 729
730 /* 730 /*
731 * Enable might_sleep() and smp_proces 731 * Enable might_sleep() and smp_processor_id() checks.
732 * They cannot be enabled earlier beca 732 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
733 * kernel_thread() would trigger might 733 * kernel_thread() would trigger might_sleep() splats. With
734 * CONFIG_PREEMPT_VOLUNTARY=y the init 734 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
735 * already, but it's stuck on the kthr 735 * already, but it's stuck on the kthreadd_done completion.
736 */ 736 */
737 system_state = SYSTEM_SCHEDULING; 737 system_state = SYSTEM_SCHEDULING;
738 738
739 complete(&kthreadd_done); 739 complete(&kthreadd_done);
740 740
741 /* 741 /*
742 * The boot idle thread must execute s 742 * The boot idle thread must execute schedule()
743 * at least once to get things moving: 743 * at least once to get things moving:
744 */ 744 */
745 schedule_preempt_disabled(); 745 schedule_preempt_disabled();
746 /* Call into cpu_idle with preempt dis 746 /* Call into cpu_idle with preempt disabled */
747 cpu_startup_entry(CPUHP_ONLINE); 747 cpu_startup_entry(CPUHP_ONLINE);
748 } 748 }
749 749
750 /* Check for early params. */ 750 /* Check for early params. */
751 static int __init do_early_param(char *param, 751 static int __init do_early_param(char *param, char *val,
752 const char *u 752 const char *unused, void *arg)
753 { 753 {
754 const struct obs_kernel_param *p; 754 const struct obs_kernel_param *p;
755 755
756 for (p = __setup_start; p < __setup_en 756 for (p = __setup_start; p < __setup_end; p++) {
757 if ((p->early && parameq(param 757 if ((p->early && parameq(param, p->str)) ||
758 (strcmp(param, "console") 758 (strcmp(param, "console") == 0 &&
759 strcmp(p->str, "earlycon" 759 strcmp(p->str, "earlycon") == 0)
760 ) { 760 ) {
761 if (p->setup_func(val) 761 if (p->setup_func(val) != 0)
762 pr_warn("Malfo 762 pr_warn("Malformed early option '%s'\n", param);
763 } 763 }
764 } 764 }
765 /* We accept everything at this stage. 765 /* We accept everything at this stage. */
766 return 0; 766 return 0;
767 } 767 }
768 768
769 void __init parse_early_options(char *cmdline) 769 void __init parse_early_options(char *cmdline)
770 { 770 {
771 parse_args("early options", cmdline, N 771 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
772 do_early_param); 772 do_early_param);
773 } 773 }
774 774
775 /* Arch code calls this early on, or if not, j 775 /* Arch code calls this early on, or if not, just before other parsing. */
776 void __init parse_early_param(void) 776 void __init parse_early_param(void)
777 { 777 {
778 static int done __initdata; 778 static int done __initdata;
779 static char tmp_cmdline[COMMAND_LINE_S 779 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
780 780
781 if (done) 781 if (done)
782 return; 782 return;
783 783
784 /* All fall through to do_early_param. 784 /* All fall through to do_early_param. */
785 strscpy(tmp_cmdline, boot_command_line 785 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
786 parse_early_options(tmp_cmdline); 786 parse_early_options(tmp_cmdline);
787 done = 1; 787 done = 1;
788 } 788 }
789 789
790 void __init __weak arch_post_acpi_subsys_init( 790 void __init __weak arch_post_acpi_subsys_init(void) { }
791 791
792 void __init __weak smp_setup_processor_id(void 792 void __init __weak smp_setup_processor_id(void)
793 { 793 {
794 } 794 }
795 795
796 void __init __weak smp_prepare_boot_cpu(void) 796 void __init __weak smp_prepare_boot_cpu(void)
797 { 797 {
798 } 798 }
799 799
800 # if THREAD_SIZE >= PAGE_SIZE 800 # if THREAD_SIZE >= PAGE_SIZE
801 void __init __weak thread_stack_cache_init(voi 801 void __init __weak thread_stack_cache_init(void)
802 { 802 {
803 } 803 }
804 #endif 804 #endif
805 805
806 void __init __weak poking_init(void) { } 806 void __init __weak poking_init(void) { }
807 807
808 void __init __weak pgtable_cache_init(void) { 808 void __init __weak pgtable_cache_init(void) { }
809 809
810 void __init __weak trap_init(void) { } 810 void __init __weak trap_init(void) { }
811 811
812 bool initcall_debug; 812 bool initcall_debug;
813 core_param(initcall_debug, initcall_debug, boo 813 core_param(initcall_debug, initcall_debug, bool, 0644);
814 814
815 #ifdef TRACEPOINTS_ENABLED 815 #ifdef TRACEPOINTS_ENABLED
816 static void __init initcall_debug_enable(void) 816 static void __init initcall_debug_enable(void);
817 #else 817 #else
818 static inline void initcall_debug_enable(void) 818 static inline void initcall_debug_enable(void)
819 { 819 {
820 } 820 }
821 #endif 821 #endif
822 822
823 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET 823 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
824 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KS 824 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
825 randomize_kstack_of 825 randomize_kstack_offset);
826 DEFINE_PER_CPU(u32, kstack_offset); 826 DEFINE_PER_CPU(u32, kstack_offset);
827 827
828 static int __init early_randomize_kstack_offse 828 static int __init early_randomize_kstack_offset(char *buf)
829 { 829 {
830 int ret; 830 int ret;
831 bool bool_result; 831 bool bool_result;
832 832
833 ret = kstrtobool(buf, &bool_result); 833 ret = kstrtobool(buf, &bool_result);
834 if (ret) 834 if (ret)
835 return ret; 835 return ret;
836 836
837 if (bool_result) 837 if (bool_result)
838 static_branch_enable(&randomiz 838 static_branch_enable(&randomize_kstack_offset);
839 else 839 else
840 static_branch_disable(&randomi 840 static_branch_disable(&randomize_kstack_offset);
841 return 0; 841 return 0;
842 } 842 }
843 early_param("randomize_kstack_offset", early_r 843 early_param("randomize_kstack_offset", early_randomize_kstack_offset);
844 #endif 844 #endif
845 845
846 static void __init print_unknown_bootoptions(v 846 static void __init print_unknown_bootoptions(void)
847 { 847 {
848 char *unknown_options; 848 char *unknown_options;
849 char *end; 849 char *end;
850 const char *const *p; 850 const char *const *p;
851 size_t len; 851 size_t len;
852 852
853 if (panic_later || (!argv_init[1] && ! 853 if (panic_later || (!argv_init[1] && !envp_init[2]))
854 return; 854 return;
855 855
856 /* 856 /*
857 * Determine how many options we have 857 * Determine how many options we have to print out, plus a space
858 * before each 858 * before each
859 */ 859 */
860 len = 1; /* null terminator */ 860 len = 1; /* null terminator */
861 for (p = &argv_init[1]; *p; p++) { 861 for (p = &argv_init[1]; *p; p++) {
862 len++; 862 len++;
863 len += strlen(*p); 863 len += strlen(*p);
864 } 864 }
865 for (p = &envp_init[2]; *p; p++) { 865 for (p = &envp_init[2]; *p; p++) {
866 len++; 866 len++;
867 len += strlen(*p); 867 len += strlen(*p);
868 } 868 }
869 869
870 unknown_options = memblock_alloc(len, 870 unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
871 if (!unknown_options) { 871 if (!unknown_options) {
872 pr_err("%s: Failed to allocate 872 pr_err("%s: Failed to allocate %zu bytes\n",
873 __func__, len); 873 __func__, len);
874 return; 874 return;
875 } 875 }
876 end = unknown_options; 876 end = unknown_options;
877 877
878 for (p = &argv_init[1]; *p; p++) 878 for (p = &argv_init[1]; *p; p++)
879 end += sprintf(end, " %s", *p) 879 end += sprintf(end, " %s", *p);
880 for (p = &envp_init[2]; *p; p++) 880 for (p = &envp_init[2]; *p; p++)
881 end += sprintf(end, " %s", *p) 881 end += sprintf(end, " %s", *p);
882 882
883 /* Start at unknown_options[1] to skip 883 /* Start at unknown_options[1] to skip the initial space */
884 pr_notice("Unknown kernel command line 884 pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
885 &unknown_options[1]); 885 &unknown_options[1]);
886 memblock_free(unknown_options, len); 886 memblock_free(unknown_options, len);
887 } 887 }
888 888
889 static void __init early_numa_node_init(void) 889 static void __init early_numa_node_init(void)
890 { 890 {
891 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID 891 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
892 #ifndef cpu_to_node 892 #ifndef cpu_to_node
893 int cpu; 893 int cpu;
894 894
895 /* The early_cpu_to_node() should be r 895 /* The early_cpu_to_node() should be ready here. */
896 for_each_possible_cpu(cpu) 896 for_each_possible_cpu(cpu)
897 set_cpu_numa_node(cpu, early_c 897 set_cpu_numa_node(cpu, early_cpu_to_node(cpu));
898 #endif 898 #endif
899 #endif 899 #endif
900 } 900 }
901 901
902 asmlinkage __visible __init __no_sanitize_addr 902 asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector
903 void start_kernel(void) 903 void start_kernel(void)
904 { 904 {
905 char *command_line; 905 char *command_line;
906 char *after_dashes; 906 char *after_dashes;
907 907
908 set_task_stack_end_magic(&init_task); 908 set_task_stack_end_magic(&init_task);
909 smp_setup_processor_id(); 909 smp_setup_processor_id();
910 debug_objects_early_init(); 910 debug_objects_early_init();
911 init_vmlinux_build_id(); 911 init_vmlinux_build_id();
912 912
913 cgroup_init_early(); 913 cgroup_init_early();
914 914
915 local_irq_disable(); 915 local_irq_disable();
916 early_boot_irqs_disabled = true; 916 early_boot_irqs_disabled = true;
917 917
918 /* 918 /*
919 * Interrupts are still disabled. Do n 919 * Interrupts are still disabled. Do necessary setups, then
920 * enable them. 920 * enable them.
921 */ 921 */
922 boot_cpu_init(); 922 boot_cpu_init();
923 page_address_init(); 923 page_address_init();
924 pr_notice("%s", linux_banner); 924 pr_notice("%s", linux_banner);
925 setup_arch(&command_line); 925 setup_arch(&command_line);
926 /* Static keys and static calls are ne 926 /* Static keys and static calls are needed by LSMs */
927 jump_label_init(); 927 jump_label_init();
928 static_call_init(); 928 static_call_init();
929 early_security_init(); 929 early_security_init();
930 setup_boot_config(); 930 setup_boot_config();
931 setup_command_line(command_line); 931 setup_command_line(command_line);
932 setup_nr_cpu_ids(); 932 setup_nr_cpu_ids();
933 setup_per_cpu_areas(); 933 setup_per_cpu_areas();
934 smp_prepare_boot_cpu(); /* arch-specif 934 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
935 early_numa_node_init(); 935 early_numa_node_init();
936 boot_cpu_hotplug_init(); 936 boot_cpu_hotplug_init();
937 937
938 pr_notice("Kernel command line: %s\n", 938 pr_notice("Kernel command line: %s\n", saved_command_line);
939 /* parameters may set static keys */ 939 /* parameters may set static keys */
940 parse_early_param(); 940 parse_early_param();
941 after_dashes = parse_args("Booting ker 941 after_dashes = parse_args("Booting kernel",
942 static_comma 942 static_command_line, __start___param,
943 __stop___par 943 __stop___param - __start___param,
944 -1, -1, NULL 944 -1, -1, NULL, &unknown_bootoption);
945 print_unknown_bootoptions(); 945 print_unknown_bootoptions();
946 if (!IS_ERR_OR_NULL(after_dashes)) 946 if (!IS_ERR_OR_NULL(after_dashes))
947 parse_args("Setting init args" 947 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
948 NULL, set_init_arg) 948 NULL, set_init_arg);
949 if (extra_init_args) 949 if (extra_init_args)
950 parse_args("Setting extra init 950 parse_args("Setting extra init args", extra_init_args,
951 NULL, 0, -1, -1, NU 951 NULL, 0, -1, -1, NULL, set_init_arg);
952 952
953 /* Architectural and non-timekeeping r 953 /* Architectural and non-timekeeping rng init, before allocator init */
954 random_init_early(command_line); 954 random_init_early(command_line);
955 955
956 /* 956 /*
957 * These use large bootmem allocations 957 * These use large bootmem allocations and must precede
958 * initalization of page allocator 958 * initalization of page allocator
959 */ 959 */
960 setup_log_buf(0); 960 setup_log_buf(0);
961 vfs_caches_init_early(); 961 vfs_caches_init_early();
962 sort_main_extable(); 962 sort_main_extable();
963 trap_init(); 963 trap_init();
964 mm_core_init(); 964 mm_core_init();
965 poking_init(); 965 poking_init();
966 ftrace_init(); 966 ftrace_init();
967 967
968 /* trace_printk can be enabled here */ 968 /* trace_printk can be enabled here */
969 early_trace_init(); 969 early_trace_init();
970 970
971 /* 971 /*
972 * Set up the scheduler prior starting 972 * Set up the scheduler prior starting any interrupts (such as the
973 * timer interrupt). Full topology set 973 * timer interrupt). Full topology setup happens at smp_init()
974 * time - but meanwhile we still have 974 * time - but meanwhile we still have a functioning scheduler.
975 */ 975 */
976 sched_init(); 976 sched_init();
977 977
978 if (WARN(!irqs_disabled(), 978 if (WARN(!irqs_disabled(),
979 "Interrupts were enabled *ver 979 "Interrupts were enabled *very* early, fixing it\n"))
980 local_irq_disable(); 980 local_irq_disable();
981 radix_tree_init(); 981 radix_tree_init();
982 maple_tree_init(); 982 maple_tree_init();
983 983
984 /* 984 /*
985 * Set up housekeeping before setting 985 * Set up housekeeping before setting up workqueues to allow the unbound
986 * workqueue to take non-housekeeping 986 * workqueue to take non-housekeeping into account.
987 */ 987 */
988 housekeeping_init(); 988 housekeeping_init();
989 989
990 /* 990 /*
991 * Allow workqueue creation and work i 991 * Allow workqueue creation and work item queueing/cancelling
992 * early. Work item execution depends 992 * early. Work item execution depends on kthreads and starts after
993 * workqueue_init(). 993 * workqueue_init().
994 */ 994 */
995 workqueue_init_early(); 995 workqueue_init_early();
996 996
997 rcu_init(); 997 rcu_init();
998 998
999 /* Trace events are available after th 999 /* Trace events are available after this */
1000 trace_init(); 1000 trace_init();
1001 1001
1002 if (initcall_debug) 1002 if (initcall_debug)
1003 initcall_debug_enable(); 1003 initcall_debug_enable();
1004 1004
1005 context_tracking_init(); 1005 context_tracking_init();
1006 /* init some links before init_ISA_ir 1006 /* init some links before init_ISA_irqs() */
1007 early_irq_init(); 1007 early_irq_init();
1008 init_IRQ(); 1008 init_IRQ();
1009 tick_init(); 1009 tick_init();
1010 rcu_init_nohz(); 1010 rcu_init_nohz();
1011 init_timers(); 1011 init_timers();
1012 srcu_init(); 1012 srcu_init();
1013 hrtimers_init(); 1013 hrtimers_init();
1014 softirq_init(); 1014 softirq_init();
1015 timekeeping_init(); 1015 timekeeping_init();
1016 time_init(); 1016 time_init();
1017 1017
1018 /* This must be after timekeeping is 1018 /* This must be after timekeeping is initialized */
1019 random_init(); 1019 random_init();
1020 1020
1021 /* These make use of the fully initia 1021 /* These make use of the fully initialized rng */
1022 kfence_init(); 1022 kfence_init();
1023 boot_init_stack_canary(); 1023 boot_init_stack_canary();
1024 1024
1025 perf_event_init(); 1025 perf_event_init();
1026 profile_init(); 1026 profile_init();
1027 call_function_init(); 1027 call_function_init();
1028 WARN(!irqs_disabled(), "Interrupts we 1028 WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1029 1029
1030 early_boot_irqs_disabled = false; 1030 early_boot_irqs_disabled = false;
1031 local_irq_enable(); 1031 local_irq_enable();
1032 1032
1033 kmem_cache_init_late(); 1033 kmem_cache_init_late();
1034 1034
1035 /* 1035 /*
1036 * HACK ALERT! This is early. We're e 1036 * HACK ALERT! This is early. We're enabling the console before
1037 * we've done PCI setups etc, and con 1037 * we've done PCI setups etc, and console_init() must be aware of
1038 * this. But we do want output early, 1038 * this. But we do want output early, in case something goes wrong.
1039 */ 1039 */
1040 console_init(); 1040 console_init();
1041 if (panic_later) 1041 if (panic_later)
1042 panic("Too many boot %s vars 1042 panic("Too many boot %s vars at `%s'", panic_later,
1043 panic_param); 1043 panic_param);
1044 1044
1045 lockdep_init(); 1045 lockdep_init();
1046 1046
1047 /* 1047 /*
1048 * Need to run this when irqs are ena 1048 * Need to run this when irqs are enabled, because it wants
1049 * to self-test [hard/soft]-irqs on/o 1049 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1050 * too: 1050 * too:
1051 */ 1051 */
1052 locking_selftest(); 1052 locking_selftest();
1053 1053
1054 #ifdef CONFIG_BLK_DEV_INITRD 1054 #ifdef CONFIG_BLK_DEV_INITRD
1055 if (initrd_start && !initrd_below_sta 1055 if (initrd_start && !initrd_below_start_ok &&
1056 page_to_pfn(virt_to_page((void *) 1056 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1057 pr_crit("initrd overwritten ( 1057 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1058 page_to_pfn(virt_to_page( 1058 page_to_pfn(virt_to_page((void *)initrd_start)),
1059 min_low_pfn); 1059 min_low_pfn);
1060 initrd_start = 0; 1060 initrd_start = 0;
1061 } 1061 }
1062 #endif 1062 #endif
1063 setup_per_cpu_pageset(); 1063 setup_per_cpu_pageset();
1064 numa_policy_init(); 1064 numa_policy_init();
1065 acpi_early_init(); 1065 acpi_early_init();
1066 if (late_time_init) 1066 if (late_time_init)
1067 late_time_init(); 1067 late_time_init();
1068 sched_clock_init(); 1068 sched_clock_init();
1069 calibrate_delay(); 1069 calibrate_delay();
1070 1070
1071 arch_cpu_finalize_init(); 1071 arch_cpu_finalize_init();
1072 1072
1073 pid_idr_init(); 1073 pid_idr_init();
1074 anon_vma_init(); 1074 anon_vma_init();
1075 #ifdef CONFIG_X86 1075 #ifdef CONFIG_X86
1076 if (efi_enabled(EFI_RUNTIME_SERVICES) 1076 if (efi_enabled(EFI_RUNTIME_SERVICES))
1077 efi_enter_virtual_mode(); 1077 efi_enter_virtual_mode();
1078 #endif 1078 #endif
1079 thread_stack_cache_init(); 1079 thread_stack_cache_init();
1080 cred_init(); 1080 cred_init();
1081 fork_init(); 1081 fork_init();
1082 proc_caches_init(); 1082 proc_caches_init();
1083 uts_ns_init(); 1083 uts_ns_init();
1084 key_init(); 1084 key_init();
1085 security_init(); 1085 security_init();
1086 dbg_late_init(); 1086 dbg_late_init();
1087 net_ns_init(); 1087 net_ns_init();
1088 vfs_caches_init(); 1088 vfs_caches_init();
1089 pagecache_init(); 1089 pagecache_init();
1090 signals_init(); 1090 signals_init();
1091 seq_file_init(); 1091 seq_file_init();
1092 proc_root_init(); 1092 proc_root_init();
1093 nsfs_init(); 1093 nsfs_init();
1094 pidfs_init(); 1094 pidfs_init();
1095 cpuset_init(); 1095 cpuset_init();
1096 cgroup_init(); 1096 cgroup_init();
1097 taskstats_init_early(); 1097 taskstats_init_early();
1098 delayacct_init(); 1098 delayacct_init();
1099 1099
1100 acpi_subsystem_init(); 1100 acpi_subsystem_init();
1101 arch_post_acpi_subsys_init(); 1101 arch_post_acpi_subsys_init();
1102 kcsan_init(); 1102 kcsan_init();
1103 1103
1104 /* Do the rest non-__init'ed, we're n 1104 /* Do the rest non-__init'ed, we're now alive */
1105 rest_init(); 1105 rest_init();
1106 1106
1107 /* 1107 /*
1108 * Avoid stack canaries in callers of 1108 * Avoid stack canaries in callers of boot_init_stack_canary for gcc-10
1109 * and older. 1109 * and older.
1110 */ 1110 */
1111 #if !__has_attribute(__no_stack_protector__) 1111 #if !__has_attribute(__no_stack_protector__)
1112 prevent_tail_call_optimization(); 1112 prevent_tail_call_optimization();
1113 #endif 1113 #endif
1114 } 1114 }
1115 1115
1116 /* Call all constructor functions linked into 1116 /* Call all constructor functions linked into the kernel. */
1117 static void __init do_ctors(void) 1117 static void __init do_ctors(void)
1118 { 1118 {
1119 /* 1119 /*
1120 * For UML, the constructors have already bee 1120 * For UML, the constructors have already been called by the
1121 * normal setup code as it's just a normal EL 1121 * normal setup code as it's just a normal ELF binary, so we
1122 * cannot do it again - but we do need CONFIG 1122 * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1123 * even on UML for modules. 1123 * even on UML for modules.
1124 */ 1124 */
1125 #if defined(CONFIG_CONSTRUCTORS) && !defined( 1125 #if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1126 ctor_fn_t *fn = (ctor_fn_t *) __ctors 1126 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1127 1127
1128 for (; fn < (ctor_fn_t *) __ctors_end 1128 for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1129 (*fn)(); 1129 (*fn)();
1130 #endif 1130 #endif
1131 } 1131 }
1132 1132
1133 #ifdef CONFIG_KALLSYMS 1133 #ifdef CONFIG_KALLSYMS
1134 struct blacklist_entry { 1134 struct blacklist_entry {
1135 struct list_head next; 1135 struct list_head next;
1136 char *buf; 1136 char *buf;
1137 }; 1137 };
1138 1138
1139 static __initdata_or_module LIST_HEAD(blackli 1139 static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1140 1140
1141 static int __init initcall_blacklist(char *st 1141 static int __init initcall_blacklist(char *str)
1142 { 1142 {
1143 char *str_entry; 1143 char *str_entry;
1144 struct blacklist_entry *entry; 1144 struct blacklist_entry *entry;
1145 1145
1146 /* str argument is a comma-separated 1146 /* str argument is a comma-separated list of functions */
1147 do { 1147 do {
1148 str_entry = strsep(&str, ",") 1148 str_entry = strsep(&str, ",");
1149 if (str_entry) { 1149 if (str_entry) {
1150 pr_debug("blacklistin 1150 pr_debug("blacklisting initcall %s\n", str_entry);
1151 entry = memblock_allo 1151 entry = memblock_alloc(sizeof(*entry),
1152 1152 SMP_CACHE_BYTES);
1153 if (!entry) 1153 if (!entry)
1154 panic("%s: Fa 1154 panic("%s: Failed to allocate %zu bytes\n",
1155 __func_ 1155 __func__, sizeof(*entry));
1156 entry->buf = memblock 1156 entry->buf = memblock_alloc(strlen(str_entry) + 1,
1157 1157 SMP_CACHE_BYTES);
1158 if (!entry->buf) 1158 if (!entry->buf)
1159 panic("%s: Fa 1159 panic("%s: Failed to allocate %zu bytes\n",
1160 __func_ 1160 __func__, strlen(str_entry) + 1);
1161 strcpy(entry->buf, st 1161 strcpy(entry->buf, str_entry);
1162 list_add(&entry->next 1162 list_add(&entry->next, &blacklisted_initcalls);
1163 } 1163 }
1164 } while (str_entry); 1164 } while (str_entry);
1165 1165
1166 return 1; 1166 return 1;
1167 } 1167 }
1168 1168
1169 static bool __init_or_module initcall_blackli 1169 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1170 { 1170 {
1171 struct blacklist_entry *entry; 1171 struct blacklist_entry *entry;
1172 char fn_name[KSYM_SYMBOL_LEN]; 1172 char fn_name[KSYM_SYMBOL_LEN];
1173 unsigned long addr; 1173 unsigned long addr;
1174 1174
1175 if (list_empty(&blacklisted_initcalls 1175 if (list_empty(&blacklisted_initcalls))
1176 return false; 1176 return false;
1177 1177
1178 addr = (unsigned long) dereference_fu 1178 addr = (unsigned long) dereference_function_descriptor(fn);
1179 sprint_symbol_no_offset(fn_name, addr 1179 sprint_symbol_no_offset(fn_name, addr);
1180 1180
1181 /* 1181 /*
1182 * fn will be "function_name [module_ 1182 * fn will be "function_name [module_name]" where [module_name] is not
1183 * displayed for built-in init functi 1183 * displayed for built-in init functions. Strip off the [module_name].
1184 */ 1184 */
1185 strreplace(fn_name, ' ', '\0'); 1185 strreplace(fn_name, ' ', '\0');
1186 1186
1187 list_for_each_entry(entry, &blacklist 1187 list_for_each_entry(entry, &blacklisted_initcalls, next) {
1188 if (!strcmp(fn_name, entry->b 1188 if (!strcmp(fn_name, entry->buf)) {
1189 pr_debug("initcall %s 1189 pr_debug("initcall %s blacklisted\n", fn_name);
1190 return true; 1190 return true;
1191 } 1191 }
1192 } 1192 }
1193 1193
1194 return false; 1194 return false;
1195 } 1195 }
1196 #else 1196 #else
1197 static int __init initcall_blacklist(char *st 1197 static int __init initcall_blacklist(char *str)
1198 { 1198 {
1199 pr_warn("initcall_blacklist requires 1199 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1200 return 0; 1200 return 0;
1201 } 1201 }
1202 1202
1203 static bool __init_or_module initcall_blackli 1203 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1204 { 1204 {
1205 return false; 1205 return false;
1206 } 1206 }
1207 #endif 1207 #endif
1208 __setup("initcall_blacklist=", initcall_black 1208 __setup("initcall_blacklist=", initcall_blacklist);
1209 1209
1210 static __init_or_module void 1210 static __init_or_module void
1211 trace_initcall_start_cb(void *data, initcall_ 1211 trace_initcall_start_cb(void *data, initcall_t fn)
1212 { 1212 {
1213 ktime_t *calltime = data; 1213 ktime_t *calltime = data;
1214 1214
1215 printk(KERN_DEBUG "calling %pS @ %i\ 1215 printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current));
1216 *calltime = ktime_get(); 1216 *calltime = ktime_get();
1217 } 1217 }
1218 1218
1219 static __init_or_module void 1219 static __init_or_module void
1220 trace_initcall_finish_cb(void *data, initcall 1220 trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1221 { 1221 {
1222 ktime_t rettime, *calltime = data; 1222 ktime_t rettime, *calltime = data;
1223 1223
1224 rettime = ktime_get(); 1224 rettime = ktime_get();
1225 printk(KERN_DEBUG "initcall %pS retur 1225 printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1226 fn, ret, (unsigned long long 1226 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));
1227 } 1227 }
1228 1228
1229 static ktime_t initcall_calltime; 1229 static ktime_t initcall_calltime;
1230 1230
1231 #ifdef TRACEPOINTS_ENABLED 1231 #ifdef TRACEPOINTS_ENABLED
1232 static void __init initcall_debug_enable(void 1232 static void __init initcall_debug_enable(void)
1233 { 1233 {
1234 int ret; 1234 int ret;
1235 1235
1236 ret = register_trace_initcall_start(t 1236 ret = register_trace_initcall_start(trace_initcall_start_cb,
1237 & 1237 &initcall_calltime);
1238 ret |= register_trace_initcall_finish 1238 ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1239 1239 &initcall_calltime);
1240 WARN(ret, "Failed to register initcal 1240 WARN(ret, "Failed to register initcall tracepoints\n");
1241 } 1241 }
1242 # define do_trace_initcall_start trace 1242 # define do_trace_initcall_start trace_initcall_start
1243 # define do_trace_initcall_finish trace 1243 # define do_trace_initcall_finish trace_initcall_finish
1244 #else 1244 #else
1245 static inline void do_trace_initcall_start(in 1245 static inline void do_trace_initcall_start(initcall_t fn)
1246 { 1246 {
1247 if (!initcall_debug) 1247 if (!initcall_debug)
1248 return; 1248 return;
1249 trace_initcall_start_cb(&initcall_cal 1249 trace_initcall_start_cb(&initcall_calltime, fn);
1250 } 1250 }
1251 static inline void do_trace_initcall_finish(i 1251 static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1252 { 1252 {
1253 if (!initcall_debug) 1253 if (!initcall_debug)
1254 return; 1254 return;
1255 trace_initcall_finish_cb(&initcall_ca 1255 trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1256 } 1256 }
1257 #endif /* !TRACEPOINTS_ENABLED */ 1257 #endif /* !TRACEPOINTS_ENABLED */
1258 1258
1259 int __init_or_module do_one_initcall(initcall 1259 int __init_or_module do_one_initcall(initcall_t fn)
1260 { 1260 {
1261 int count = preempt_count(); 1261 int count = preempt_count();
1262 char msgbuf[64]; 1262 char msgbuf[64];
1263 int ret; 1263 int ret;
1264 1264
1265 if (initcall_blacklisted(fn)) 1265 if (initcall_blacklisted(fn))
1266 return -EPERM; 1266 return -EPERM;
1267 1267
1268 do_trace_initcall_start(fn); 1268 do_trace_initcall_start(fn);
1269 ret = fn(); 1269 ret = fn();
1270 do_trace_initcall_finish(fn, ret); 1270 do_trace_initcall_finish(fn, ret);
1271 1271
1272 msgbuf[0] = 0; 1272 msgbuf[0] = 0;
1273 1273
1274 if (preempt_count() != count) { 1274 if (preempt_count() != count) {
1275 sprintf(msgbuf, "preemption i 1275 sprintf(msgbuf, "preemption imbalance ");
1276 preempt_count_set(count); 1276 preempt_count_set(count);
1277 } 1277 }
1278 if (irqs_disabled()) { 1278 if (irqs_disabled()) {
1279 strlcat(msgbuf, "disabled int 1279 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1280 local_irq_enable(); 1280 local_irq_enable();
1281 } 1281 }
1282 WARN(msgbuf[0], "initcall %pS returne 1282 WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1283 1283
1284 add_latent_entropy(); 1284 add_latent_entropy();
1285 return ret; 1285 return ret;
1286 } 1286 }
1287 1287
1288 1288
1289 static initcall_entry_t *initcall_levels[] __ 1289 static initcall_entry_t *initcall_levels[] __initdata = {
1290 __initcall0_start, 1290 __initcall0_start,
1291 __initcall1_start, 1291 __initcall1_start,
1292 __initcall2_start, 1292 __initcall2_start,
1293 __initcall3_start, 1293 __initcall3_start,
1294 __initcall4_start, 1294 __initcall4_start,
1295 __initcall5_start, 1295 __initcall5_start,
1296 __initcall6_start, 1296 __initcall6_start,
1297 __initcall7_start, 1297 __initcall7_start,
1298 __initcall_end, 1298 __initcall_end,
1299 }; 1299 };
1300 1300
1301 /* Keep these in sync with initcalls in inclu 1301 /* Keep these in sync with initcalls in include/linux/init.h */
1302 static const char *initcall_level_names[] __i 1302 static const char *initcall_level_names[] __initdata = {
1303 "pure", 1303 "pure",
1304 "core", 1304 "core",
1305 "postcore", 1305 "postcore",
1306 "arch", 1306 "arch",
1307 "subsys", 1307 "subsys",
1308 "fs", 1308 "fs",
1309 "device", 1309 "device",
1310 "late", 1310 "late",
1311 }; 1311 };
1312 1312
1313 static int __init ignore_unknown_bootoption(c 1313 static int __init ignore_unknown_bootoption(char *param, char *val,
1314 const char *un 1314 const char *unused, void *arg)
1315 { 1315 {
1316 return 0; 1316 return 0;
1317 } 1317 }
1318 1318
1319 static void __init do_initcall_level(int leve 1319 static void __init do_initcall_level(int level, char *command_line)
1320 { 1320 {
1321 initcall_entry_t *fn; 1321 initcall_entry_t *fn;
1322 1322
1323 parse_args(initcall_level_names[level 1323 parse_args(initcall_level_names[level],
1324 command_line, __start___pa 1324 command_line, __start___param,
1325 __stop___param - __start__ 1325 __stop___param - __start___param,
1326 level, level, 1326 level, level,
1327 NULL, ignore_unknown_booto 1327 NULL, ignore_unknown_bootoption);
1328 1328
1329 trace_initcall_level(initcall_level_n 1329 trace_initcall_level(initcall_level_names[level]);
1330 for (fn = initcall_levels[level]; fn 1330 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1331 do_one_initcall(initcall_from 1331 do_one_initcall(initcall_from_entry(fn));
1332 } 1332 }
1333 1333
1334 static void __init do_initcalls(void) 1334 static void __init do_initcalls(void)
1335 { 1335 {
1336 int level; 1336 int level;
1337 size_t len = saved_command_line_len + 1337 size_t len = saved_command_line_len + 1;
1338 char *command_line; 1338 char *command_line;
1339 1339
1340 command_line = kzalloc(len, GFP_KERNE 1340 command_line = kzalloc(len, GFP_KERNEL);
1341 if (!command_line) 1341 if (!command_line)
1342 panic("%s: Failed to allocate 1342 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1343 1343
1344 for (level = 0; level < ARRAY_SIZE(in 1344 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1345 /* Parser modifies command_li 1345 /* Parser modifies command_line, restore it each time */
1346 strcpy(command_line, saved_co 1346 strcpy(command_line, saved_command_line);
1347 do_initcall_level(level, comm 1347 do_initcall_level(level, command_line);
1348 } 1348 }
1349 1349
1350 kfree(command_line); 1350 kfree(command_line);
1351 } 1351 }
1352 1352
1353 /* 1353 /*
1354 * Ok, the machine is now initialized. None o 1354 * Ok, the machine is now initialized. None of the devices
1355 * have been touched yet, but the CPU subsyst 1355 * have been touched yet, but the CPU subsystem is up and
1356 * running, and memory and process management 1356 * running, and memory and process management works.
1357 * 1357 *
1358 * Now we can finally start doing some real w 1358 * Now we can finally start doing some real work..
1359 */ 1359 */
1360 static void __init do_basic_setup(void) 1360 static void __init do_basic_setup(void)
1361 { 1361 {
1362 cpuset_init_smp(); 1362 cpuset_init_smp();
1363 driver_init(); 1363 driver_init();
1364 init_irq_proc(); 1364 init_irq_proc();
1365 do_ctors(); 1365 do_ctors();
1366 do_initcalls(); 1366 do_initcalls();
1367 } 1367 }
1368 1368
1369 static void __init do_pre_smp_initcalls(void) 1369 static void __init do_pre_smp_initcalls(void)
1370 { 1370 {
1371 initcall_entry_t *fn; 1371 initcall_entry_t *fn;
1372 1372
1373 trace_initcall_level("early"); 1373 trace_initcall_level("early");
1374 for (fn = __initcall_start; fn < __in 1374 for (fn = __initcall_start; fn < __initcall0_start; fn++)
1375 do_one_initcall(initcall_from 1375 do_one_initcall(initcall_from_entry(fn));
1376 } 1376 }
1377 1377
1378 static int run_init_process(const char *init_ 1378 static int run_init_process(const char *init_filename)
1379 { 1379 {
1380 const char *const *p; 1380 const char *const *p;
1381 1381
1382 argv_init[0] = init_filename; 1382 argv_init[0] = init_filename;
1383 pr_info("Run %s as init process\n", i 1383 pr_info("Run %s as init process\n", init_filename);
1384 pr_debug(" with arguments:\n"); 1384 pr_debug(" with arguments:\n");
1385 for (p = argv_init; *p; p++) 1385 for (p = argv_init; *p; p++)
1386 pr_debug(" %s\n", *p); 1386 pr_debug(" %s\n", *p);
1387 pr_debug(" with environment:\n"); 1387 pr_debug(" with environment:\n");
1388 for (p = envp_init; *p; p++) 1388 for (p = envp_init; *p; p++)
1389 pr_debug(" %s\n", *p); 1389 pr_debug(" %s\n", *p);
1390 return kernel_execve(init_filename, a 1390 return kernel_execve(init_filename, argv_init, envp_init);
1391 } 1391 }
1392 1392
1393 static int try_to_run_init_process(const char 1393 static int try_to_run_init_process(const char *init_filename)
1394 { 1394 {
1395 int ret; 1395 int ret;
1396 1396
1397 ret = run_init_process(init_filename) 1397 ret = run_init_process(init_filename);
1398 1398
1399 if (ret && ret != -ENOENT) { 1399 if (ret && ret != -ENOENT) {
1400 pr_err("Starting init: %s exi 1400 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1401 init_filename, ret); 1401 init_filename, ret);
1402 } 1402 }
1403 1403
1404 return ret; 1404 return ret;
1405 } 1405 }
1406 1406
1407 static noinline void __init kernel_init_freea 1407 static noinline void __init kernel_init_freeable(void);
1408 1408
1409 #if defined(CONFIG_STRICT_KERNEL_RWX) || defi 1409 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1410 bool rodata_enabled __ro_after_init = true; 1410 bool rodata_enabled __ro_after_init = true;
1411 1411
1412 #ifndef arch_parse_debug_rodata 1412 #ifndef arch_parse_debug_rodata
1413 static inline bool arch_parse_debug_rodata(ch 1413 static inline bool arch_parse_debug_rodata(char *str) { return false; }
1414 #endif 1414 #endif
1415 1415
1416 static int __init set_debug_rodata(char *str) 1416 static int __init set_debug_rodata(char *str)
1417 { 1417 {
1418 if (arch_parse_debug_rodata(str)) 1418 if (arch_parse_debug_rodata(str))
1419 return 0; 1419 return 0;
1420 1420
1421 if (str && !strcmp(str, "on")) 1421 if (str && !strcmp(str, "on"))
1422 rodata_enabled = true; 1422 rodata_enabled = true;
1423 else if (str && !strcmp(str, "off")) 1423 else if (str && !strcmp(str, "off"))
1424 rodata_enabled = false; 1424 rodata_enabled = false;
1425 else 1425 else
1426 pr_warn("Invalid option strin 1426 pr_warn("Invalid option string for rodata: '%s'\n", str);
1427 return 0; 1427 return 0;
1428 } 1428 }
1429 early_param("rodata", set_debug_rodata); 1429 early_param("rodata", set_debug_rodata);
1430 #endif 1430 #endif
1431 1431
1432 static void mark_readonly(void) 1432 static void mark_readonly(void)
1433 { 1433 {
1434 if (IS_ENABLED(CONFIG_STRICT_KERNEL_R 1434 if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX) && rodata_enabled) {
1435 /* 1435 /*
1436 * load_module() results in W 1436 * load_module() results in W+X mappings, which are cleaned
1437 * up with init_free_wq. Let' 1437 * up with init_free_wq. Let's make sure that queued work is
1438 * flushed so that we don't h 1438 * flushed so that we don't hit false positives looking for
1439 * insecure pages which are W 1439 * insecure pages which are W+X.
1440 */ 1440 */
1441 flush_module_init_free_work() 1441 flush_module_init_free_work();
1442 jump_label_init_ro(); 1442 jump_label_init_ro();
1443 mark_rodata_ro(); 1443 mark_rodata_ro();
1444 debug_checkwx(); 1444 debug_checkwx();
1445 rodata_test(); 1445 rodata_test();
1446 } else if (IS_ENABLED(CONFIG_STRICT_K 1446 } else if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) {
1447 pr_info("Kernel memory protec 1447 pr_info("Kernel memory protection disabled.\n");
1448 } else if (IS_ENABLED(CONFIG_ARCH_HAS 1448 } else if (IS_ENABLED(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)) {
1449 pr_warn("Kernel memory protec 1449 pr_warn("Kernel memory protection not selected by kernel config.\n");
1450 } else { 1450 } else {
1451 pr_warn("This architecture do 1451 pr_warn("This architecture does not have kernel memory protection.\n");
1452 } 1452 }
1453 } 1453 }
1454 1454
1455 void __weak free_initmem(void) 1455 void __weak free_initmem(void)
1456 { 1456 {
1457 free_initmem_default(POISON_FREE_INIT 1457 free_initmem_default(POISON_FREE_INITMEM);
1458 } 1458 }
1459 1459
1460 static int __ref kernel_init(void *unused) 1460 static int __ref kernel_init(void *unused)
1461 { 1461 {
1462 int ret; 1462 int ret;
1463 1463
1464 /* 1464 /*
1465 * Wait until kthreadd is all set-up. 1465 * Wait until kthreadd is all set-up.
1466 */ 1466 */
1467 wait_for_completion(&kthreadd_done); 1467 wait_for_completion(&kthreadd_done);
1468 1468
1469 kernel_init_freeable(); 1469 kernel_init_freeable();
1470 /* need to finish all async __init co 1470 /* need to finish all async __init code before freeing the memory */
1471 async_synchronize_full(); 1471 async_synchronize_full();
1472 1472
1473 system_state = SYSTEM_FREEING_INITMEM 1473 system_state = SYSTEM_FREEING_INITMEM;
1474 kprobe_free_init_mem(); 1474 kprobe_free_init_mem();
1475 ftrace_free_init_mem(); 1475 ftrace_free_init_mem();
1476 kgdb_free_init_mem(); 1476 kgdb_free_init_mem();
1477 exit_boot_config(); 1477 exit_boot_config();
1478 free_initmem(); 1478 free_initmem();
1479 mark_readonly(); 1479 mark_readonly();
1480 1480
1481 /* 1481 /*
1482 * Kernel mappings are now finalized 1482 * Kernel mappings are now finalized - update the userspace page-table
1483 * to finalize PTI. 1483 * to finalize PTI.
1484 */ 1484 */
1485 pti_finalize(); 1485 pti_finalize();
1486 1486
1487 system_state = SYSTEM_RUNNING; 1487 system_state = SYSTEM_RUNNING;
1488 numa_default_policy(); 1488 numa_default_policy();
1489 1489
1490 rcu_end_inkernel_boot(); 1490 rcu_end_inkernel_boot();
1491 1491
1492 do_sysctl_args(); 1492 do_sysctl_args();
1493 1493
1494 if (ramdisk_execute_command) { 1494 if (ramdisk_execute_command) {
1495 ret = run_init_process(ramdis 1495 ret = run_init_process(ramdisk_execute_command);
1496 if (!ret) 1496 if (!ret)
1497 return 0; 1497 return 0;
1498 pr_err("Failed to execute %s 1498 pr_err("Failed to execute %s (error %d)\n",
1499 ramdisk_execute_comman 1499 ramdisk_execute_command, ret);
1500 } 1500 }
1501 1501
1502 /* 1502 /*
1503 * We try each of these until one suc 1503 * We try each of these until one succeeds.
1504 * 1504 *
1505 * The Bourne shell can be used inste 1505 * The Bourne shell can be used instead of init if we are
1506 * trying to recover a really broken 1506 * trying to recover a really broken machine.
1507 */ 1507 */
1508 if (execute_command) { 1508 if (execute_command) {
1509 ret = run_init_process(execut 1509 ret = run_init_process(execute_command);
1510 if (!ret) 1510 if (!ret)
1511 return 0; 1511 return 0;
1512 panic("Requested init %s fail 1512 panic("Requested init %s failed (error %d).",
1513 execute_command, ret); 1513 execute_command, ret);
1514 } 1514 }
1515 1515
1516 if (CONFIG_DEFAULT_INIT[0] != '\0') { 1516 if (CONFIG_DEFAULT_INIT[0] != '\0') {
1517 ret = run_init_process(CONFIG 1517 ret = run_init_process(CONFIG_DEFAULT_INIT);
1518 if (ret) 1518 if (ret)
1519 pr_err("Default init 1519 pr_err("Default init %s failed (error %d)\n",
1520 CONFIG_DEFAULT 1520 CONFIG_DEFAULT_INIT, ret);
1521 else 1521 else
1522 return 0; 1522 return 0;
1523 } 1523 }
1524 1524
1525 if (!try_to_run_init_process("/sbin/i 1525 if (!try_to_run_init_process("/sbin/init") ||
1526 !try_to_run_init_process("/etc/in 1526 !try_to_run_init_process("/etc/init") ||
1527 !try_to_run_init_process("/bin/in 1527 !try_to_run_init_process("/bin/init") ||
1528 !try_to_run_init_process("/bin/sh 1528 !try_to_run_init_process("/bin/sh"))
1529 return 0; 1529 return 0;
1530 1530
1531 panic("No working init found. Try pa 1531 panic("No working init found. Try passing init= option to kernel. "
1532 "See Linux Documentation/admin- 1532 "See Linux Documentation/admin-guide/init.rst for guidance.");
1533 } 1533 }
1534 1534
1535 /* Open /dev/console, for stdin/stdout/stderr 1535 /* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1536 void __init console_on_rootfs(void) 1536 void __init console_on_rootfs(void)
1537 { 1537 {
1538 struct file *file = filp_open("/dev/c 1538 struct file *file = filp_open("/dev/console", O_RDWR, 0);
1539 1539
1540 if (IS_ERR(file)) { 1540 if (IS_ERR(file)) {
1541 pr_err("Warning: unable to op 1541 pr_err("Warning: unable to open an initial console.\n");
1542 return; 1542 return;
1543 } 1543 }
1544 init_dup(file); 1544 init_dup(file);
1545 init_dup(file); 1545 init_dup(file);
1546 init_dup(file); 1546 init_dup(file);
1547 fput(file); 1547 fput(file);
1548 } 1548 }
1549 1549
1550 static noinline void __init kernel_init_freea 1550 static noinline void __init kernel_init_freeable(void)
1551 { 1551 {
1552 /* Now the scheduler is fully set up 1552 /* Now the scheduler is fully set up and can do blocking allocations */
1553 gfp_allowed_mask = __GFP_BITS_MASK; 1553 gfp_allowed_mask = __GFP_BITS_MASK;
1554 1554
1555 /* 1555 /*
1556 * init can allocate pages on any nod 1556 * init can allocate pages on any node
1557 */ 1557 */
1558 set_mems_allowed(node_states[N_MEMORY 1558 set_mems_allowed(node_states[N_MEMORY]);
1559 1559
1560 cad_pid = get_pid(task_pid(current)); 1560 cad_pid = get_pid(task_pid(current));
1561 1561
1562 smp_prepare_cpus(setup_max_cpus); 1562 smp_prepare_cpus(setup_max_cpus);
1563 1563
1564 workqueue_init(); 1564 workqueue_init();
1565 1565
1566 init_mm_internals(); 1566 init_mm_internals();
1567 1567
1568 rcu_init_tasks_generic(); 1568 rcu_init_tasks_generic();
1569 do_pre_smp_initcalls(); 1569 do_pre_smp_initcalls();
1570 lockup_detector_init(); 1570 lockup_detector_init();
1571 1571
1572 smp_init(); 1572 smp_init();
1573 sched_init_smp(); 1573 sched_init_smp();
1574 1574
1575 workqueue_init_topology(); 1575 workqueue_init_topology();
1576 async_init(); 1576 async_init();
1577 padata_init(); 1577 padata_init();
1578 page_alloc_init_late(); 1578 page_alloc_init_late();
1579 1579
1580 do_basic_setup(); 1580 do_basic_setup();
1581 1581
1582 kunit_run_all_tests(); 1582 kunit_run_all_tests();
1583 1583
1584 wait_for_initramfs(); 1584 wait_for_initramfs();
1585 console_on_rootfs(); 1585 console_on_rootfs();
1586 1586
1587 /* 1587 /*
1588 * check if there is an early userspa 1588 * check if there is an early userspace init. If yes, let it do all
1589 * the work 1589 * the work
1590 */ 1590 */
1591 if (init_eaccess(ramdisk_execute_comm 1591 if (init_eaccess(ramdisk_execute_command) != 0) {
1592 ramdisk_execute_command = NUL 1592 ramdisk_execute_command = NULL;
1593 prepare_namespace(); 1593 prepare_namespace();
1594 } 1594 }
1595 1595
1596 /* 1596 /*
1597 * Ok, we have completed the initial 1597 * Ok, we have completed the initial bootup, and
1598 * we're essentially up and running. 1598 * we're essentially up and running. Get rid of the
1599 * initmem segments and start the use 1599 * initmem segments and start the user-mode stuff..
1600 * 1600 *
1601 * rootfs is available now, try loadi 1601 * rootfs is available now, try loading the public keys
1602 * and default modules 1602 * and default modules
1603 */ 1603 */
1604 1604
1605 integrity_load_keys(); 1605 integrity_load_keys();
1606 } 1606 }
1607 1607
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.