1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * linux/init/main.c 3 * linux/init/main.c
4 * 4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 1991, 1992 Linus Torvalds
6 * 6 *
7 * GK 2/5/95 - Changed to support mounting 7 * GK 2/5/95 - Changed to support mounting root fs via NFS
8 * Added initrd & change_root: Werner Almesbe 8 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
9 * Moan early if gcc is old, avoiding bogus k 9 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
10 * Simplified starting of init: Michael A. G 10 * Simplified starting of init: Michael A. Griffith <grif@acm.org>
11 */ 11 */
12 12
13 #define DEBUG /* Enable initcall_deb 13 #define DEBUG /* Enable initcall_debug */
14 14
15 #include <linux/types.h> 15 #include <linux/types.h>
16 #include <linux/extable.h> 16 #include <linux/extable.h>
17 #include <linux/module.h> 17 #include <linux/module.h>
18 #include <linux/proc_fs.h> 18 #include <linux/proc_fs.h>
19 #include <linux/binfmts.h> 19 #include <linux/binfmts.h>
20 #include <linux/kernel.h> 20 #include <linux/kernel.h>
21 #include <linux/syscalls.h> 21 #include <linux/syscalls.h>
22 #include <linux/stackprotector.h> 22 #include <linux/stackprotector.h>
23 #include <linux/string.h> 23 #include <linux/string.h>
24 #include <linux/ctype.h> 24 #include <linux/ctype.h>
25 #include <linux/delay.h> 25 #include <linux/delay.h>
26 #include <linux/ioport.h> 26 #include <linux/ioport.h>
27 #include <linux/init.h> 27 #include <linux/init.h>
28 #include <linux/initrd.h> 28 #include <linux/initrd.h>
29 #include <linux/memblock.h> 29 #include <linux/memblock.h>
30 #include <linux/acpi.h> 30 #include <linux/acpi.h>
31 #include <linux/bootconfig.h> 31 #include <linux/bootconfig.h>
32 #include <linux/console.h> 32 #include <linux/console.h>
33 #include <linux/nmi.h> 33 #include <linux/nmi.h>
34 #include <linux/percpu.h> 34 #include <linux/percpu.h>
35 #include <linux/kmod.h> 35 #include <linux/kmod.h>
36 #include <linux/kprobes.h> 36 #include <linux/kprobes.h>
37 #include <linux/kmsan.h> 37 #include <linux/kmsan.h>
38 #include <linux/vmalloc.h> 38 #include <linux/vmalloc.h>
39 #include <linux/kernel_stat.h> 39 #include <linux/kernel_stat.h>
40 #include <linux/start_kernel.h> 40 #include <linux/start_kernel.h>
41 #include <linux/security.h> 41 #include <linux/security.h>
42 #include <linux/smp.h> 42 #include <linux/smp.h>
43 #include <linux/profile.h> 43 #include <linux/profile.h>
44 #include <linux/kfence.h> 44 #include <linux/kfence.h>
45 #include <linux/rcupdate.h> 45 #include <linux/rcupdate.h>
46 #include <linux/srcu.h> 46 #include <linux/srcu.h>
47 #include <linux/moduleparam.h> 47 #include <linux/moduleparam.h>
48 #include <linux/kallsyms.h> 48 #include <linux/kallsyms.h>
49 #include <linux/buildid.h> 49 #include <linux/buildid.h>
50 #include <linux/writeback.h> 50 #include <linux/writeback.h>
51 #include <linux/cpu.h> 51 #include <linux/cpu.h>
52 #include <linux/cpuset.h> 52 #include <linux/cpuset.h>
53 #include <linux/cgroup.h> 53 #include <linux/cgroup.h>
54 #include <linux/efi.h> 54 #include <linux/efi.h>
55 #include <linux/tick.h> 55 #include <linux/tick.h>
56 #include <linux/sched/isolation.h> 56 #include <linux/sched/isolation.h>
57 #include <linux/interrupt.h> 57 #include <linux/interrupt.h>
58 #include <linux/taskstats_kern.h> 58 #include <linux/taskstats_kern.h>
59 #include <linux/delayacct.h> 59 #include <linux/delayacct.h>
60 #include <linux/unistd.h> 60 #include <linux/unistd.h>
61 #include <linux/utsname.h> 61 #include <linux/utsname.h>
62 #include <linux/rmap.h> 62 #include <linux/rmap.h>
63 #include <linux/mempolicy.h> 63 #include <linux/mempolicy.h>
64 #include <linux/key.h> 64 #include <linux/key.h>
65 #include <linux/debug_locks.h> 65 #include <linux/debug_locks.h>
66 #include <linux/debugobjects.h> 66 #include <linux/debugobjects.h>
67 #include <linux/lockdep.h> 67 #include <linux/lockdep.h>
68 #include <linux/kmemleak.h> 68 #include <linux/kmemleak.h>
69 #include <linux/padata.h> 69 #include <linux/padata.h>
70 #include <linux/pid_namespace.h> 70 #include <linux/pid_namespace.h>
71 #include <linux/device/driver.h> 71 #include <linux/device/driver.h>
72 #include <linux/kthread.h> 72 #include <linux/kthread.h>
73 #include <linux/sched.h> 73 #include <linux/sched.h>
74 #include <linux/sched/init.h> 74 #include <linux/sched/init.h>
75 #include <linux/signal.h> 75 #include <linux/signal.h>
76 #include <linux/idr.h> 76 #include <linux/idr.h>
77 #include <linux/kgdb.h> 77 #include <linux/kgdb.h>
78 #include <linux/ftrace.h> 78 #include <linux/ftrace.h>
79 #include <linux/async.h> 79 #include <linux/async.h>
80 #include <linux/shmem_fs.h> 80 #include <linux/shmem_fs.h>
81 #include <linux/slab.h> 81 #include <linux/slab.h>
82 #include <linux/perf_event.h> 82 #include <linux/perf_event.h>
83 #include <linux/ptrace.h> 83 #include <linux/ptrace.h>
84 #include <linux/pti.h> 84 #include <linux/pti.h>
85 #include <linux/blkdev.h> 85 #include <linux/blkdev.h>
86 #include <linux/sched/clock.h> 86 #include <linux/sched/clock.h>
87 #include <linux/sched/task.h> 87 #include <linux/sched/task.h>
88 #include <linux/sched/task_stack.h> 88 #include <linux/sched/task_stack.h>
89 #include <linux/context_tracking.h> 89 #include <linux/context_tracking.h>
90 #include <linux/random.h> 90 #include <linux/random.h>
91 #include <linux/moduleloader.h> <<
92 #include <linux/list.h> 91 #include <linux/list.h>
93 #include <linux/integrity.h> 92 #include <linux/integrity.h>
94 #include <linux/proc_ns.h> 93 #include <linux/proc_ns.h>
95 #include <linux/io.h> 94 #include <linux/io.h>
96 #include <linux/cache.h> 95 #include <linux/cache.h>
97 #include <linux/rodata_test.h> 96 #include <linux/rodata_test.h>
98 #include <linux/jump_label.h> 97 #include <linux/jump_label.h>
99 #include <linux/kcsan.h> 98 #include <linux/kcsan.h>
100 #include <linux/init_syscalls.h> 99 #include <linux/init_syscalls.h>
101 #include <linux/stackdepot.h> 100 #include <linux/stackdepot.h>
102 #include <linux/randomize_kstack.h> 101 #include <linux/randomize_kstack.h>
103 #include <linux/pidfs.h> <<
104 #include <linux/ptdump.h> <<
105 #include <net/net_namespace.h> 102 #include <net/net_namespace.h>
106 103
107 #include <asm/io.h> 104 #include <asm/io.h>
108 #include <asm/setup.h> 105 #include <asm/setup.h>
109 #include <asm/sections.h> 106 #include <asm/sections.h>
110 #include <asm/cacheflush.h> 107 #include <asm/cacheflush.h>
111 108
112 #define CREATE_TRACE_POINTS 109 #define CREATE_TRACE_POINTS
113 #include <trace/events/initcall.h> 110 #include <trace/events/initcall.h>
114 111
115 #include <kunit/test.h> 112 #include <kunit/test.h>
116 113
117 static int kernel_init(void *); 114 static int kernel_init(void *);
118 115
>> 116 extern void init_IRQ(void);
>> 117 extern void radix_tree_init(void);
>> 118 extern void maple_tree_init(void);
>> 119
119 /* 120 /*
120 * Debug helper: via this flag we know that we 121 * Debug helper: via this flag we know that we are in 'early bootup code'
121 * where only the boot processor is running wi 122 * where only the boot processor is running with IRQ disabled. This means
122 * two things - IRQ must not be enabled before 123 * two things - IRQ must not be enabled before the flag is cleared and some
123 * operations which are not allowed with IRQ d 124 * operations which are not allowed with IRQ disabled are allowed while the
124 * flag is set. 125 * flag is set.
125 */ 126 */
126 bool early_boot_irqs_disabled __read_mostly; 127 bool early_boot_irqs_disabled __read_mostly;
127 128
128 enum system_states system_state __read_mostly; 129 enum system_states system_state __read_mostly;
129 EXPORT_SYMBOL(system_state); 130 EXPORT_SYMBOL(system_state);
130 131
131 /* 132 /*
132 * Boot command-line arguments 133 * Boot command-line arguments
133 */ 134 */
134 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMI 135 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
135 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMI 136 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
136 137
>> 138 extern void time_init(void);
137 /* Default late time init is NULL. archs can o 139 /* Default late time init is NULL. archs can override this later. */
138 void (*__initdata late_time_init)(void); 140 void (*__initdata late_time_init)(void);
139 141
140 /* Untouched command line saved by arch-specif 142 /* Untouched command line saved by arch-specific code. */
141 char __initdata boot_command_line[COMMAND_LINE 143 char __initdata boot_command_line[COMMAND_LINE_SIZE];
142 /* Untouched saved command line (eg. for /proc 144 /* Untouched saved command line (eg. for /proc) */
143 char *saved_command_line __ro_after_init; 145 char *saved_command_line __ro_after_init;
144 unsigned int saved_command_line_len __ro_after 146 unsigned int saved_command_line_len __ro_after_init;
145 /* Command line for parameter parsing */ 147 /* Command line for parameter parsing */
146 static char *static_command_line; 148 static char *static_command_line;
147 /* Untouched extra command line */ 149 /* Untouched extra command line */
148 static char *extra_command_line; 150 static char *extra_command_line;
149 /* Extra init arguments */ 151 /* Extra init arguments */
150 static char *extra_init_args; 152 static char *extra_init_args;
151 153
152 #ifdef CONFIG_BOOT_CONFIG 154 #ifdef CONFIG_BOOT_CONFIG
153 /* Is bootconfig on command line? */ 155 /* Is bootconfig on command line? */
154 static bool bootconfig_found; 156 static bool bootconfig_found;
155 static size_t initargs_offs; 157 static size_t initargs_offs;
156 #else 158 #else
157 # define bootconfig_found false 159 # define bootconfig_found false
158 # define initargs_offs 0 160 # define initargs_offs 0
159 #endif 161 #endif
160 162
161 static char *execute_command; 163 static char *execute_command;
162 static char *ramdisk_execute_command = "/init" 164 static char *ramdisk_execute_command = "/init";
163 165
164 /* 166 /*
165 * Used to generate warnings if static_key man 167 * Used to generate warnings if static_key manipulation functions are used
166 * before jump_label_init is called. 168 * before jump_label_init is called.
167 */ 169 */
168 bool static_key_initialized __read_mostly; 170 bool static_key_initialized __read_mostly;
169 EXPORT_SYMBOL_GPL(static_key_initialized); 171 EXPORT_SYMBOL_GPL(static_key_initialized);
170 172
171 /* 173 /*
172 * If set, this is an indication to the driver 174 * If set, this is an indication to the drivers that reset the underlying
173 * device before going ahead with the initiali 175 * device before going ahead with the initialization otherwise driver might
174 * rely on the BIOS and skip the reset operati 176 * rely on the BIOS and skip the reset operation.
175 * 177 *
176 * This is useful if kernel is booting in an u 178 * This is useful if kernel is booting in an unreliable environment.
177 * For ex. kdump situation where previous kern 179 * For ex. kdump situation where previous kernel has crashed, BIOS has been
178 * skipped and devices will be in unknown stat 180 * skipped and devices will be in unknown state.
179 */ 181 */
180 unsigned int reset_devices; 182 unsigned int reset_devices;
181 EXPORT_SYMBOL(reset_devices); 183 EXPORT_SYMBOL(reset_devices);
182 184
183 static int __init set_reset_devices(char *str) 185 static int __init set_reset_devices(char *str)
184 { 186 {
185 reset_devices = 1; 187 reset_devices = 1;
186 return 1; 188 return 1;
187 } 189 }
188 190
189 __setup("reset_devices", set_reset_devices); 191 __setup("reset_devices", set_reset_devices);
190 192
191 static const char *argv_init[MAX_INIT_ARGS+2] 193 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
192 const char *envp_init[MAX_INIT_ENVS+2] = { "HO 194 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
193 static const char *panic_later, *panic_param; 195 static const char *panic_later, *panic_param;
194 196
>> 197 extern const struct obs_kernel_param __setup_start[], __setup_end[];
>> 198
195 static bool __init obsolete_checksetup(char *l 199 static bool __init obsolete_checksetup(char *line)
196 { 200 {
197 const struct obs_kernel_param *p; 201 const struct obs_kernel_param *p;
198 bool had_early_param = false; 202 bool had_early_param = false;
199 203
200 p = __setup_start; 204 p = __setup_start;
201 do { 205 do {
202 int n = strlen(p->str); 206 int n = strlen(p->str);
203 if (parameqn(line, p->str, n)) 207 if (parameqn(line, p->str, n)) {
204 if (p->early) { 208 if (p->early) {
205 /* Already don 209 /* Already done in parse_early_param?
206 * (Needs exac 210 * (Needs exact match on param part).
207 * Keep iterat 211 * Keep iterating, as we can have early
208 * params and 212 * params and __setups of same names 8( */
209 if (line[n] == 213 if (line[n] == '\0' || line[n] == '=')
210 had_ea 214 had_early_param = true;
211 } else if (!p->setup_f 215 } else if (!p->setup_func) {
212 pr_warn("Param 216 pr_warn("Parameter %s is obsolete, ignored\n",
213 p->str 217 p->str);
214 return true; 218 return true;
215 } else if (p->setup_fu 219 } else if (p->setup_func(line + n))
216 return true; 220 return true;
217 } 221 }
218 p++; 222 p++;
219 } while (p < __setup_end); 223 } while (p < __setup_end);
220 224
221 return had_early_param; 225 return had_early_param;
222 } 226 }
223 227
224 /* 228 /*
225 * This should be approx 2 Bo*oMips to start ( 229 * This should be approx 2 Bo*oMips to start (note initial shift), and will
226 * still work even if initially too large, it 230 * still work even if initially too large, it will just take slightly longer
227 */ 231 */
228 unsigned long loops_per_jiffy = (1<<12); 232 unsigned long loops_per_jiffy = (1<<12);
229 EXPORT_SYMBOL(loops_per_jiffy); 233 EXPORT_SYMBOL(loops_per_jiffy);
230 234
231 static int __init debug_kernel(char *str) 235 static int __init debug_kernel(char *str)
232 { 236 {
233 console_loglevel = CONSOLE_LOGLEVEL_DE 237 console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
234 return 0; 238 return 0;
235 } 239 }
236 240
237 static int __init quiet_kernel(char *str) 241 static int __init quiet_kernel(char *str)
238 { 242 {
239 console_loglevel = CONSOLE_LOGLEVEL_QU 243 console_loglevel = CONSOLE_LOGLEVEL_QUIET;
240 return 0; 244 return 0;
241 } 245 }
242 246
243 early_param("debug", debug_kernel); 247 early_param("debug", debug_kernel);
244 early_param("quiet", quiet_kernel); 248 early_param("quiet", quiet_kernel);
245 249
246 static int __init loglevel(char *str) 250 static int __init loglevel(char *str)
247 { 251 {
248 int newlevel; 252 int newlevel;
249 253
250 /* 254 /*
251 * Only update loglevel value when a c 255 * Only update loglevel value when a correct setting was passed,
252 * to prevent blind crashes (when logl 256 * to prevent blind crashes (when loglevel being set to 0) that
253 * are quite hard to debug 257 * are quite hard to debug
254 */ 258 */
255 if (get_option(&str, &newlevel)) { 259 if (get_option(&str, &newlevel)) {
256 console_loglevel = newlevel; 260 console_loglevel = newlevel;
257 return 0; 261 return 0;
258 } 262 }
259 263
260 return -EINVAL; 264 return -EINVAL;
261 } 265 }
262 266
263 early_param("loglevel", loglevel); 267 early_param("loglevel", loglevel);
264 268
265 #ifdef CONFIG_BLK_DEV_INITRD 269 #ifdef CONFIG_BLK_DEV_INITRD
266 static void * __init get_boot_config_from_init 270 static void * __init get_boot_config_from_initrd(size_t *_size)
267 { 271 {
268 u32 size, csum; 272 u32 size, csum;
269 char *data; 273 char *data;
270 u32 *hdr; 274 u32 *hdr;
271 int i; 275 int i;
272 276
273 if (!initrd_end) 277 if (!initrd_end)
274 return NULL; 278 return NULL;
275 279
276 data = (char *)initrd_end - BOOTCONFIG 280 data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
277 /* 281 /*
278 * Since Grub may align the size of in 282 * Since Grub may align the size of initrd to 4, we must
279 * check the preceding 3 bytes as well 283 * check the preceding 3 bytes as well.
280 */ 284 */
281 for (i = 0; i < 4; i++) { 285 for (i = 0; i < 4; i++) {
282 if (!memcmp(data, BOOTCONFIG_M 286 if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
283 goto found; 287 goto found;
284 data--; 288 data--;
285 } 289 }
286 return NULL; 290 return NULL;
287 291
288 found: 292 found:
289 hdr = (u32 *)(data - 8); 293 hdr = (u32 *)(data - 8);
290 size = le32_to_cpu(hdr[0]); 294 size = le32_to_cpu(hdr[0]);
291 csum = le32_to_cpu(hdr[1]); 295 csum = le32_to_cpu(hdr[1]);
292 296
293 data = ((void *)hdr) - size; 297 data = ((void *)hdr) - size;
294 if ((unsigned long)data < initrd_start 298 if ((unsigned long)data < initrd_start) {
295 pr_err("bootconfig size %d is 299 pr_err("bootconfig size %d is greater than initrd size %ld\n",
296 size, initrd_end - ini 300 size, initrd_end - initrd_start);
297 return NULL; 301 return NULL;
298 } 302 }
299 303
300 if (xbc_calc_checksum(data, size) != c 304 if (xbc_calc_checksum(data, size) != csum) {
301 pr_err("bootconfig checksum fa 305 pr_err("bootconfig checksum failed\n");
302 return NULL; 306 return NULL;
303 } 307 }
304 308
305 /* Remove bootconfig from initramfs/in 309 /* Remove bootconfig from initramfs/initrd */
306 initrd_end = (unsigned long)data; 310 initrd_end = (unsigned long)data;
307 if (_size) 311 if (_size)
308 *_size = size; 312 *_size = size;
309 313
310 return data; 314 return data;
311 } 315 }
312 #else 316 #else
313 static void * __init get_boot_config_from_init 317 static void * __init get_boot_config_from_initrd(size_t *_size)
314 { 318 {
315 return NULL; 319 return NULL;
316 } 320 }
317 #endif 321 #endif
318 322
319 #ifdef CONFIG_BOOT_CONFIG 323 #ifdef CONFIG_BOOT_CONFIG
320 324
321 static char xbc_namebuf[XBC_KEYLEN_MAX] __init 325 static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
322 326
323 #define rest(dst, end) ((end) > (dst) ? (end) 327 #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
324 328
325 static int __init xbc_snprint_cmdline(char *bu 329 static int __init xbc_snprint_cmdline(char *buf, size_t size,
326 struct x 330 struct xbc_node *root)
327 { 331 {
328 struct xbc_node *knode, *vnode; 332 struct xbc_node *knode, *vnode;
329 char *end = buf + size; 333 char *end = buf + size;
330 const char *val, *q; !! 334 const char *val;
331 int ret; 335 int ret;
332 336
333 xbc_node_for_each_key_value(root, knod 337 xbc_node_for_each_key_value(root, knode, val) {
334 ret = xbc_node_compose_key_aft 338 ret = xbc_node_compose_key_after(root, knode,
335 xbc_na 339 xbc_namebuf, XBC_KEYLEN_MAX);
336 if (ret < 0) 340 if (ret < 0)
337 return ret; 341 return ret;
338 342
339 vnode = xbc_node_get_child(kno 343 vnode = xbc_node_get_child(knode);
340 if (!vnode) { 344 if (!vnode) {
341 ret = snprintf(buf, re 345 ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
342 if (ret < 0) 346 if (ret < 0)
343 return ret; 347 return ret;
344 buf += ret; 348 buf += ret;
345 continue; 349 continue;
346 } 350 }
347 xbc_array_for_each_value(vnode 351 xbc_array_for_each_value(vnode, val) {
348 /* !! 352 ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ",
349 * For prettier and mo !! 353 xbc_namebuf, val);
350 * quote the value whe <<
351 * whitespace. <<
352 */ <<
353 q = strpbrk(val, " \t\ <<
354 ret = snprintf(buf, re <<
355 xbc_nam <<
356 if (ret < 0) 354 if (ret < 0)
357 return ret; 355 return ret;
358 buf += ret; 356 buf += ret;
359 } 357 }
360 } 358 }
361 359
362 return buf - (end - size); 360 return buf - (end - size);
363 } 361 }
364 #undef rest 362 #undef rest
365 363
366 /* Make an extra command line under given key 364 /* Make an extra command line under given key word */
367 static char * __init xbc_make_cmdline(const ch 365 static char * __init xbc_make_cmdline(const char *key)
368 { 366 {
369 struct xbc_node *root; 367 struct xbc_node *root;
370 char *new_cmdline; 368 char *new_cmdline;
371 int ret, len = 0; 369 int ret, len = 0;
372 370
373 root = xbc_find_node(key); 371 root = xbc_find_node(key);
374 if (!root) 372 if (!root)
375 return NULL; 373 return NULL;
376 374
377 /* Count required buffer size */ 375 /* Count required buffer size */
378 len = xbc_snprint_cmdline(NULL, 0, roo 376 len = xbc_snprint_cmdline(NULL, 0, root);
379 if (len <= 0) 377 if (len <= 0)
380 return NULL; 378 return NULL;
381 379
382 new_cmdline = memblock_alloc(len + 1, 380 new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
383 if (!new_cmdline) { 381 if (!new_cmdline) {
384 pr_err("Failed to allocate mem 382 pr_err("Failed to allocate memory for extra kernel cmdline.\n");
385 return NULL; 383 return NULL;
386 } 384 }
387 385
388 ret = xbc_snprint_cmdline(new_cmdline, 386 ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
389 if (ret < 0 || ret > len) { 387 if (ret < 0 || ret > len) {
390 pr_err("Failed to print extra 388 pr_err("Failed to print extra kernel cmdline.\n");
391 memblock_free(new_cmdline, len 389 memblock_free(new_cmdline, len + 1);
392 return NULL; 390 return NULL;
393 } 391 }
394 392
395 return new_cmdline; 393 return new_cmdline;
396 } 394 }
397 395
398 static int __init bootconfig_params(char *para 396 static int __init bootconfig_params(char *param, char *val,
399 const char 397 const char *unused, void *arg)
400 { 398 {
401 if (strcmp(param, "bootconfig") == 0) 399 if (strcmp(param, "bootconfig") == 0) {
402 bootconfig_found = true; 400 bootconfig_found = true;
403 } 401 }
404 return 0; 402 return 0;
405 } 403 }
406 404
407 static int __init warn_bootconfig(char *str) 405 static int __init warn_bootconfig(char *str)
408 { 406 {
409 /* The 'bootconfig' has been handled b 407 /* The 'bootconfig' has been handled by bootconfig_params(). */
410 return 0; 408 return 0;
411 } 409 }
412 410
413 static void __init setup_boot_config(void) 411 static void __init setup_boot_config(void)
414 { 412 {
415 static char tmp_cmdline[COMMAND_LINE_S 413 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
416 const char *msg, *data; 414 const char *msg, *data;
417 int pos, ret; 415 int pos, ret;
418 size_t size; 416 size_t size;
419 char *err; 417 char *err;
420 418
421 /* Cut out the bootconfig data even if 419 /* Cut out the bootconfig data even if we have no bootconfig option */
422 data = get_boot_config_from_initrd(&si 420 data = get_boot_config_from_initrd(&size);
423 /* If there is no bootconfig in initrd 421 /* If there is no bootconfig in initrd, try embedded one. */
424 if (!data) 422 if (!data)
425 data = xbc_get_embedded_bootco 423 data = xbc_get_embedded_bootconfig(&size);
426 424
427 strscpy(tmp_cmdline, boot_command_line 425 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
428 err = parse_args("bootconfig", tmp_cmd 426 err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
429 bootconfig_params); 427 bootconfig_params);
430 428
431 if (IS_ERR(err) || !(bootconfig_found 429 if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
432 return; 430 return;
433 431
434 /* parse_args() stops at the next para 432 /* parse_args() stops at the next param of '--' and returns an address */
435 if (err) 433 if (err)
436 initargs_offs = err - tmp_cmdl 434 initargs_offs = err - tmp_cmdline;
437 435
438 if (!data) { 436 if (!data) {
439 /* If user intended to use boo 437 /* If user intended to use bootconfig, show an error level message */
440 if (bootconfig_found) 438 if (bootconfig_found)
441 pr_err("'bootconfig' f 439 pr_err("'bootconfig' found on command line, but no bootconfig found\n");
442 else 440 else
443 pr_info("No bootconfig 441 pr_info("No bootconfig data provided, so skipping bootconfig");
444 return; 442 return;
445 } 443 }
446 444
447 if (size >= XBC_DATA_MAX) { 445 if (size >= XBC_DATA_MAX) {
448 pr_err("bootconfig size %ld gr 446 pr_err("bootconfig size %ld greater than max size %d\n",
449 (long)size, XBC_DATA_M 447 (long)size, XBC_DATA_MAX);
450 return; 448 return;
451 } 449 }
452 450
453 ret = xbc_init(data, size, &msg, &pos) 451 ret = xbc_init(data, size, &msg, &pos);
454 if (ret < 0) { 452 if (ret < 0) {
455 if (pos < 0) 453 if (pos < 0)
456 pr_err("Failed to init 454 pr_err("Failed to init bootconfig: %s.\n", msg);
457 else 455 else
458 pr_err("Failed to pars 456 pr_err("Failed to parse bootconfig: %s at %d.\n",
459 msg, pos); 457 msg, pos);
460 } else { 458 } else {
461 xbc_get_info(&ret, NULL); 459 xbc_get_info(&ret, NULL);
462 pr_info("Load bootconfig: %ld 460 pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
463 /* keys starting with "kernel. 461 /* keys starting with "kernel." are passed via cmdline */
464 extra_command_line = xbc_make_ 462 extra_command_line = xbc_make_cmdline("kernel");
465 /* Also, "init." keys are init 463 /* Also, "init." keys are init arguments */
466 extra_init_args = xbc_make_cmd 464 extra_init_args = xbc_make_cmdline("init");
467 } 465 }
468 return; 466 return;
469 } 467 }
470 468
471 static void __init exit_boot_config(void) 469 static void __init exit_boot_config(void)
472 { 470 {
473 xbc_exit(); 471 xbc_exit();
474 } 472 }
475 473
476 #else /* !CONFIG_BOOT_CONFIG */ 474 #else /* !CONFIG_BOOT_CONFIG */
477 475
478 static void __init setup_boot_config(void) 476 static void __init setup_boot_config(void)
479 { 477 {
480 /* Remove bootconfig data from initrd 478 /* Remove bootconfig data from initrd */
481 get_boot_config_from_initrd(NULL); 479 get_boot_config_from_initrd(NULL);
482 } 480 }
483 481
484 static int __init warn_bootconfig(char *str) 482 static int __init warn_bootconfig(char *str)
485 { 483 {
486 pr_warn("WARNING: 'bootconfig' found o 484 pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
487 return 0; 485 return 0;
488 } 486 }
489 487
490 #define exit_boot_config() do {} while (0 488 #define exit_boot_config() do {} while (0)
491 489
492 #endif /* CONFIG_BOOT_CONFIG */ 490 #endif /* CONFIG_BOOT_CONFIG */
493 491
494 early_param("bootconfig", warn_bootconfig); 492 early_param("bootconfig", warn_bootconfig);
495 493
496 bool __init cmdline_has_extra_options(void) <<
497 { <<
498 return extra_command_line || extra_ini <<
499 } <<
500 <<
501 /* Change NUL term back to "=", to make "param 494 /* Change NUL term back to "=", to make "param" the whole string. */
502 static void __init repair_env_string(char *par 495 static void __init repair_env_string(char *param, char *val)
503 { 496 {
504 if (val) { 497 if (val) {
505 /* param=val or param="val"? * 498 /* param=val or param="val"? */
506 if (val == param+strlen(param) 499 if (val == param+strlen(param)+1)
507 val[-1] = '='; 500 val[-1] = '=';
508 else if (val == param+strlen(p 501 else if (val == param+strlen(param)+2) {
509 val[-2] = '='; 502 val[-2] = '=';
510 memmove(val-1, val, st 503 memmove(val-1, val, strlen(val)+1);
511 } else 504 } else
512 BUG(); 505 BUG();
513 } 506 }
514 } 507 }
515 508
516 /* Anything after -- gets handed straight to i 509 /* Anything after -- gets handed straight to init. */
517 static int __init set_init_arg(char *param, ch 510 static int __init set_init_arg(char *param, char *val,
518 const char *unu 511 const char *unused, void *arg)
519 { 512 {
520 unsigned int i; 513 unsigned int i;
521 514
522 if (panic_later) 515 if (panic_later)
523 return 0; 516 return 0;
524 517
525 repair_env_string(param, val); 518 repair_env_string(param, val);
526 519
527 for (i = 0; argv_init[i]; i++) { 520 for (i = 0; argv_init[i]; i++) {
528 if (i == MAX_INIT_ARGS) { 521 if (i == MAX_INIT_ARGS) {
529 panic_later = "init"; 522 panic_later = "init";
530 panic_param = param; 523 panic_param = param;
531 return 0; 524 return 0;
532 } 525 }
533 } 526 }
534 argv_init[i] = param; 527 argv_init[i] = param;
535 return 0; 528 return 0;
536 } 529 }
537 530
538 /* 531 /*
539 * Unknown boot options get handed to init, un 532 * Unknown boot options get handed to init, unless they look like
540 * unused parameters (modprobe will find them 533 * unused parameters (modprobe will find them in /proc/cmdline).
541 */ 534 */
542 static int __init unknown_bootoption(char *par 535 static int __init unknown_bootoption(char *param, char *val,
543 const cha 536 const char *unused, void *arg)
544 { 537 {
545 size_t len = strlen(param); 538 size_t len = strlen(param);
546 539
547 /* Handle params aliased to sysctls */ <<
548 if (sysctl_is_alias(param)) <<
549 return 0; <<
550 <<
551 repair_env_string(param, val); 540 repair_env_string(param, val);
552 541
553 /* Handle obsolete-style parameters */ 542 /* Handle obsolete-style parameters */
554 if (obsolete_checksetup(param)) 543 if (obsolete_checksetup(param))
555 return 0; 544 return 0;
556 545
557 /* Unused module parameter. */ 546 /* Unused module parameter. */
558 if (strnchr(param, len, '.')) 547 if (strnchr(param, len, '.'))
559 return 0; 548 return 0;
560 549
561 if (panic_later) 550 if (panic_later)
562 return 0; 551 return 0;
563 552
564 if (val) { 553 if (val) {
565 /* Environment option */ 554 /* Environment option */
566 unsigned int i; 555 unsigned int i;
567 for (i = 0; envp_init[i]; i++) 556 for (i = 0; envp_init[i]; i++) {
568 if (i == MAX_INIT_ENVS 557 if (i == MAX_INIT_ENVS) {
569 panic_later = 558 panic_later = "env";
570 panic_param = 559 panic_param = param;
571 } 560 }
572 if (!strncmp(param, en 561 if (!strncmp(param, envp_init[i], len+1))
573 break; 562 break;
574 } 563 }
575 envp_init[i] = param; 564 envp_init[i] = param;
576 } else { 565 } else {
577 /* Command line option */ 566 /* Command line option */
578 unsigned int i; 567 unsigned int i;
579 for (i = 0; argv_init[i]; i++) 568 for (i = 0; argv_init[i]; i++) {
580 if (i == MAX_INIT_ARGS 569 if (i == MAX_INIT_ARGS) {
581 panic_later = 570 panic_later = "init";
582 panic_param = 571 panic_param = param;
583 } 572 }
584 } 573 }
585 argv_init[i] = param; 574 argv_init[i] = param;
586 } 575 }
587 return 0; 576 return 0;
588 } 577 }
589 578
590 static int __init init_setup(char *str) 579 static int __init init_setup(char *str)
591 { 580 {
592 unsigned int i; 581 unsigned int i;
593 582
594 execute_command = str; 583 execute_command = str;
595 /* 584 /*
596 * In case LILO is going to boot us wi 585 * In case LILO is going to boot us with default command line,
597 * it prepends "auto" before the whole 586 * it prepends "auto" before the whole cmdline which makes
598 * the shell think it should execute a 587 * the shell think it should execute a script with such name.
599 * So we ignore all arguments entered 588 * So we ignore all arguments entered _before_ init=... [MJ]
600 */ 589 */
601 for (i = 1; i < MAX_INIT_ARGS; i++) 590 for (i = 1; i < MAX_INIT_ARGS; i++)
602 argv_init[i] = NULL; 591 argv_init[i] = NULL;
603 return 1; 592 return 1;
604 } 593 }
605 __setup("init=", init_setup); 594 __setup("init=", init_setup);
606 595
607 static int __init rdinit_setup(char *str) 596 static int __init rdinit_setup(char *str)
608 { 597 {
609 unsigned int i; 598 unsigned int i;
610 599
611 ramdisk_execute_command = str; 600 ramdisk_execute_command = str;
612 /* See "auto" comment in init_setup */ 601 /* See "auto" comment in init_setup */
613 for (i = 1; i < MAX_INIT_ARGS; i++) 602 for (i = 1; i < MAX_INIT_ARGS; i++)
614 argv_init[i] = NULL; 603 argv_init[i] = NULL;
615 return 1; 604 return 1;
616 } 605 }
617 __setup("rdinit=", rdinit_setup); 606 __setup("rdinit=", rdinit_setup);
618 607
619 #ifndef CONFIG_SMP 608 #ifndef CONFIG_SMP
>> 609 static const unsigned int setup_max_cpus = NR_CPUS;
620 static inline void setup_nr_cpu_ids(void) { } 610 static inline void setup_nr_cpu_ids(void) { }
621 static inline void smp_prepare_cpus(unsigned i 611 static inline void smp_prepare_cpus(unsigned int maxcpus) { }
622 #endif 612 #endif
623 613
624 /* 614 /*
625 * We need to store the untouched command line 615 * We need to store the untouched command line for future reference.
626 * We also need to store the touched command l 616 * We also need to store the touched command line since the parameter
627 * parsing is performed in place, and we shoul 617 * parsing is performed in place, and we should allow a component to
628 * store reference of name/value for future re 618 * store reference of name/value for future reference.
629 */ 619 */
630 static void __init setup_command_line(char *co 620 static void __init setup_command_line(char *command_line)
631 { 621 {
632 size_t len, xlen = 0, ilen = 0; 622 size_t len, xlen = 0, ilen = 0;
633 623
634 if (extra_command_line) 624 if (extra_command_line)
635 xlen = strlen(extra_command_li 625 xlen = strlen(extra_command_line);
636 if (extra_init_args) { !! 626 if (extra_init_args)
637 extra_init_args = strim(extra_ <<
638 ilen = strlen(extra_init_args) 627 ilen = strlen(extra_init_args) + 4; /* for " -- " */
639 } <<
640 628
641 len = xlen + strlen(boot_command_line) !! 629 len = xlen + strlen(boot_command_line) + 1;
642 630
643 saved_command_line = memblock_alloc(le !! 631 saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES);
644 if (!saved_command_line) 632 if (!saved_command_line)
645 panic("%s: Failed to allocate !! 633 panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen);
646 <<
647 len = xlen + strlen(command_line) + 1; <<
648 634
649 static_command_line = memblock_alloc(l 635 static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
650 if (!static_command_line) 636 if (!static_command_line)
651 panic("%s: Failed to allocate 637 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
652 638
653 if (xlen) { 639 if (xlen) {
654 /* 640 /*
655 * We have to put extra_comman 641 * We have to put extra_command_line before boot command
656 * lines because there could b 642 * lines because there could be dashes (separator of init
657 * command line) in the comman 643 * command line) in the command lines.
658 */ 644 */
659 strcpy(saved_command_line, ext 645 strcpy(saved_command_line, extra_command_line);
660 strcpy(static_command_line, ex 646 strcpy(static_command_line, extra_command_line);
661 } 647 }
662 strcpy(saved_command_line + xlen, boot 648 strcpy(saved_command_line + xlen, boot_command_line);
663 strcpy(static_command_line + xlen, com 649 strcpy(static_command_line + xlen, command_line);
664 650
665 if (ilen) { 651 if (ilen) {
666 /* 652 /*
667 * Append supplemental init bo 653 * Append supplemental init boot args to saved_command_line
668 * so that user can check what 654 * so that user can check what command line options passed
669 * to init. 655 * to init.
670 * The order should always be 656 * The order should always be
671 * " -- "[bootconfig init-para 657 * " -- "[bootconfig init-param][cmdline init-param]
672 */ 658 */
673 if (initargs_offs) { 659 if (initargs_offs) {
674 len = xlen + initargs_ 660 len = xlen + initargs_offs;
675 strcpy(saved_command_l 661 strcpy(saved_command_line + len, extra_init_args);
676 len += ilen - 4; 662 len += ilen - 4; /* strlen(extra_init_args) */
677 strcpy(saved_command_l 663 strcpy(saved_command_line + len,
678 boot_command_l 664 boot_command_line + initargs_offs - 1);
679 } else { 665 } else {
680 len = strlen(saved_com 666 len = strlen(saved_command_line);
681 strcpy(saved_command_l 667 strcpy(saved_command_line + len, " -- ");
682 len += 4; 668 len += 4;
683 strcpy(saved_command_l 669 strcpy(saved_command_line + len, extra_init_args);
684 } 670 }
685 } 671 }
686 672
687 saved_command_line_len = strlen(saved_ 673 saved_command_line_len = strlen(saved_command_line);
688 } 674 }
689 675
690 /* 676 /*
691 * We need to finalize in a non-__init functio 677 * We need to finalize in a non-__init function or else race conditions
692 * between the root thread and the init thread 678 * between the root thread and the init thread may cause start_kernel to
693 * be reaped by free_initmem before the root t 679 * be reaped by free_initmem before the root thread has proceeded to
694 * cpu_idle. 680 * cpu_idle.
695 * 681 *
696 * gcc-3.4 accidentally inlines this function, 682 * gcc-3.4 accidentally inlines this function, so use noinline.
697 */ 683 */
698 684
699 static __initdata DECLARE_COMPLETION(kthreadd_ 685 static __initdata DECLARE_COMPLETION(kthreadd_done);
700 686
701 static noinline void __ref __noreturn rest_ini !! 687 noinline void __ref __noreturn rest_init(void)
702 { 688 {
703 struct task_struct *tsk; 689 struct task_struct *tsk;
704 int pid; 690 int pid;
705 691
706 rcu_scheduler_starting(); 692 rcu_scheduler_starting();
707 /* 693 /*
708 * We need to spawn init first so that 694 * We need to spawn init first so that it obtains pid 1, however
709 * the init task will end up wanting t 695 * the init task will end up wanting to create kthreads, which, if
710 * we schedule it before we create kth 696 * we schedule it before we create kthreadd, will OOPS.
711 */ 697 */
712 pid = user_mode_thread(kernel_init, NU 698 pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
713 /* 699 /*
714 * Pin init on the boot CPU. Task migr 700 * Pin init on the boot CPU. Task migration is not properly working
715 * until sched_init_smp() has been run 701 * until sched_init_smp() has been run. It will set the allowed
716 * CPUs for init to the non isolated C 702 * CPUs for init to the non isolated CPUs.
717 */ 703 */
718 rcu_read_lock(); 704 rcu_read_lock();
719 tsk = find_task_by_pid_ns(pid, &init_p 705 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
720 tsk->flags |= PF_NO_SETAFFINITY; 706 tsk->flags |= PF_NO_SETAFFINITY;
721 set_cpus_allowed_ptr(tsk, cpumask_of(s 707 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
722 rcu_read_unlock(); 708 rcu_read_unlock();
723 709
724 numa_default_policy(); 710 numa_default_policy();
725 pid = kernel_thread(kthreadd, NULL, NU 711 pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES);
726 rcu_read_lock(); 712 rcu_read_lock();
727 kthreadd_task = find_task_by_pid_ns(pi 713 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
728 rcu_read_unlock(); 714 rcu_read_unlock();
729 715
730 /* 716 /*
731 * Enable might_sleep() and smp_proces 717 * Enable might_sleep() and smp_processor_id() checks.
732 * They cannot be enabled earlier beca 718 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
733 * kernel_thread() would trigger might 719 * kernel_thread() would trigger might_sleep() splats. With
734 * CONFIG_PREEMPT_VOLUNTARY=y the init 720 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
735 * already, but it's stuck on the kthr 721 * already, but it's stuck on the kthreadd_done completion.
736 */ 722 */
737 system_state = SYSTEM_SCHEDULING; 723 system_state = SYSTEM_SCHEDULING;
738 724
739 complete(&kthreadd_done); 725 complete(&kthreadd_done);
740 726
741 /* 727 /*
742 * The boot idle thread must execute s 728 * The boot idle thread must execute schedule()
743 * at least once to get things moving: 729 * at least once to get things moving:
744 */ 730 */
745 schedule_preempt_disabled(); 731 schedule_preempt_disabled();
746 /* Call into cpu_idle with preempt dis 732 /* Call into cpu_idle with preempt disabled */
747 cpu_startup_entry(CPUHP_ONLINE); 733 cpu_startup_entry(CPUHP_ONLINE);
748 } 734 }
749 735
750 /* Check for early params. */ 736 /* Check for early params. */
751 static int __init do_early_param(char *param, 737 static int __init do_early_param(char *param, char *val,
752 const char *u 738 const char *unused, void *arg)
753 { 739 {
754 const struct obs_kernel_param *p; 740 const struct obs_kernel_param *p;
755 741
756 for (p = __setup_start; p < __setup_en 742 for (p = __setup_start; p < __setup_end; p++) {
757 if ((p->early && parameq(param 743 if ((p->early && parameq(param, p->str)) ||
758 (strcmp(param, "console") 744 (strcmp(param, "console") == 0 &&
759 strcmp(p->str, "earlycon" 745 strcmp(p->str, "earlycon") == 0)
760 ) { 746 ) {
761 if (p->setup_func(val) 747 if (p->setup_func(val) != 0)
762 pr_warn("Malfo 748 pr_warn("Malformed early option '%s'\n", param);
763 } 749 }
764 } 750 }
765 /* We accept everything at this stage. 751 /* We accept everything at this stage. */
766 return 0; 752 return 0;
767 } 753 }
768 754
769 void __init parse_early_options(char *cmdline) 755 void __init parse_early_options(char *cmdline)
770 { 756 {
771 parse_args("early options", cmdline, N 757 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
772 do_early_param); 758 do_early_param);
773 } 759 }
774 760
775 /* Arch code calls this early on, or if not, j 761 /* Arch code calls this early on, or if not, just before other parsing. */
776 void __init parse_early_param(void) 762 void __init parse_early_param(void)
777 { 763 {
778 static int done __initdata; 764 static int done __initdata;
779 static char tmp_cmdline[COMMAND_LINE_S 765 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
780 766
781 if (done) 767 if (done)
782 return; 768 return;
783 769
784 /* All fall through to do_early_param. 770 /* All fall through to do_early_param. */
785 strscpy(tmp_cmdline, boot_command_line 771 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
786 parse_early_options(tmp_cmdline); 772 parse_early_options(tmp_cmdline);
787 done = 1; 773 done = 1;
788 } 774 }
789 775
790 void __init __weak arch_post_acpi_subsys_init( 776 void __init __weak arch_post_acpi_subsys_init(void) { }
791 777
792 void __init __weak smp_setup_processor_id(void 778 void __init __weak smp_setup_processor_id(void)
793 { 779 {
794 } 780 }
795 781
796 void __init __weak smp_prepare_boot_cpu(void) <<
797 { <<
798 } <<
799 <<
800 # if THREAD_SIZE >= PAGE_SIZE 782 # if THREAD_SIZE >= PAGE_SIZE
801 void __init __weak thread_stack_cache_init(voi 783 void __init __weak thread_stack_cache_init(void)
802 { 784 {
803 } 785 }
804 #endif 786 #endif
805 787
806 void __init __weak poking_init(void) { } 788 void __init __weak poking_init(void) { }
807 789
808 void __init __weak pgtable_cache_init(void) { 790 void __init __weak pgtable_cache_init(void) { }
809 791
810 void __init __weak trap_init(void) { } 792 void __init __weak trap_init(void) { }
811 793
812 bool initcall_debug; 794 bool initcall_debug;
813 core_param(initcall_debug, initcall_debug, boo 795 core_param(initcall_debug, initcall_debug, bool, 0644);
814 796
815 #ifdef TRACEPOINTS_ENABLED 797 #ifdef TRACEPOINTS_ENABLED
816 static void __init initcall_debug_enable(void) 798 static void __init initcall_debug_enable(void);
817 #else 799 #else
818 static inline void initcall_debug_enable(void) 800 static inline void initcall_debug_enable(void)
819 { 801 {
820 } 802 }
821 #endif 803 #endif
822 804
823 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET 805 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
824 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KS 806 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
825 randomize_kstack_of 807 randomize_kstack_offset);
826 DEFINE_PER_CPU(u32, kstack_offset); 808 DEFINE_PER_CPU(u32, kstack_offset);
827 809
828 static int __init early_randomize_kstack_offse 810 static int __init early_randomize_kstack_offset(char *buf)
829 { 811 {
830 int ret; 812 int ret;
831 bool bool_result; 813 bool bool_result;
832 814
833 ret = kstrtobool(buf, &bool_result); 815 ret = kstrtobool(buf, &bool_result);
834 if (ret) 816 if (ret)
835 return ret; 817 return ret;
836 818
837 if (bool_result) 819 if (bool_result)
838 static_branch_enable(&randomiz 820 static_branch_enable(&randomize_kstack_offset);
839 else 821 else
840 static_branch_disable(&randomi 822 static_branch_disable(&randomize_kstack_offset);
841 return 0; 823 return 0;
842 } 824 }
843 early_param("randomize_kstack_offset", early_r 825 early_param("randomize_kstack_offset", early_randomize_kstack_offset);
844 #endif 826 #endif
845 827
>> 828 void __init __weak __noreturn arch_call_rest_init(void)
>> 829 {
>> 830 rest_init();
>> 831 }
>> 832
846 static void __init print_unknown_bootoptions(v 833 static void __init print_unknown_bootoptions(void)
847 { 834 {
848 char *unknown_options; 835 char *unknown_options;
849 char *end; 836 char *end;
850 const char *const *p; 837 const char *const *p;
851 size_t len; 838 size_t len;
852 839
853 if (panic_later || (!argv_init[1] && ! 840 if (panic_later || (!argv_init[1] && !envp_init[2]))
854 return; 841 return;
855 842
856 /* 843 /*
857 * Determine how many options we have 844 * Determine how many options we have to print out, plus a space
858 * before each 845 * before each
859 */ 846 */
860 len = 1; /* null terminator */ 847 len = 1; /* null terminator */
861 for (p = &argv_init[1]; *p; p++) { 848 for (p = &argv_init[1]; *p; p++) {
862 len++; 849 len++;
863 len += strlen(*p); 850 len += strlen(*p);
864 } 851 }
865 for (p = &envp_init[2]; *p; p++) { 852 for (p = &envp_init[2]; *p; p++) {
866 len++; 853 len++;
867 len += strlen(*p); 854 len += strlen(*p);
868 } 855 }
869 856
870 unknown_options = memblock_alloc(len, 857 unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
871 if (!unknown_options) { 858 if (!unknown_options) {
872 pr_err("%s: Failed to allocate 859 pr_err("%s: Failed to allocate %zu bytes\n",
873 __func__, len); 860 __func__, len);
874 return; 861 return;
875 } 862 }
876 end = unknown_options; 863 end = unknown_options;
877 864
878 for (p = &argv_init[1]; *p; p++) 865 for (p = &argv_init[1]; *p; p++)
879 end += sprintf(end, " %s", *p) 866 end += sprintf(end, " %s", *p);
880 for (p = &envp_init[2]; *p; p++) 867 for (p = &envp_init[2]; *p; p++)
881 end += sprintf(end, " %s", *p) 868 end += sprintf(end, " %s", *p);
882 869
883 /* Start at unknown_options[1] to skip 870 /* Start at unknown_options[1] to skip the initial space */
884 pr_notice("Unknown kernel command line 871 pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
885 &unknown_options[1]); 872 &unknown_options[1]);
886 memblock_free(unknown_options, len); 873 memblock_free(unknown_options, len);
887 } 874 }
888 875
889 static void __init early_numa_node_init(void) <<
890 { <<
891 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID <<
892 #ifndef cpu_to_node <<
893 int cpu; <<
894 <<
895 /* The early_cpu_to_node() should be r <<
896 for_each_possible_cpu(cpu) <<
897 set_cpu_numa_node(cpu, early_c <<
898 #endif <<
899 #endif <<
900 } <<
901 <<
902 asmlinkage __visible __init __no_sanitize_addr 876 asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector
903 void start_kernel(void) 877 void start_kernel(void)
904 { 878 {
905 char *command_line; 879 char *command_line;
906 char *after_dashes; 880 char *after_dashes;
907 881
908 set_task_stack_end_magic(&init_task); 882 set_task_stack_end_magic(&init_task);
909 smp_setup_processor_id(); 883 smp_setup_processor_id();
910 debug_objects_early_init(); 884 debug_objects_early_init();
911 init_vmlinux_build_id(); 885 init_vmlinux_build_id();
912 886
913 cgroup_init_early(); 887 cgroup_init_early();
914 888
915 local_irq_disable(); 889 local_irq_disable();
916 early_boot_irqs_disabled = true; 890 early_boot_irqs_disabled = true;
917 891
918 /* 892 /*
919 * Interrupts are still disabled. Do n 893 * Interrupts are still disabled. Do necessary setups, then
920 * enable them. 894 * enable them.
921 */ 895 */
922 boot_cpu_init(); 896 boot_cpu_init();
923 page_address_init(); 897 page_address_init();
924 pr_notice("%s", linux_banner); 898 pr_notice("%s", linux_banner);
925 setup_arch(&command_line); <<
926 /* Static keys and static calls are ne <<
927 jump_label_init(); <<
928 static_call_init(); <<
929 early_security_init(); 899 early_security_init();
>> 900 setup_arch(&command_line);
930 setup_boot_config(); 901 setup_boot_config();
931 setup_command_line(command_line); 902 setup_command_line(command_line);
932 setup_nr_cpu_ids(); 903 setup_nr_cpu_ids();
933 setup_per_cpu_areas(); 904 setup_per_cpu_areas();
934 smp_prepare_boot_cpu(); /* arch-specif 905 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
935 early_numa_node_init(); <<
936 boot_cpu_hotplug_init(); 906 boot_cpu_hotplug_init();
937 907
938 pr_notice("Kernel command line: %s\n", 908 pr_notice("Kernel command line: %s\n", saved_command_line);
939 /* parameters may set static keys */ 909 /* parameters may set static keys */
>> 910 jump_label_init();
940 parse_early_param(); 911 parse_early_param();
941 after_dashes = parse_args("Booting ker 912 after_dashes = parse_args("Booting kernel",
942 static_comma 913 static_command_line, __start___param,
943 __stop___par 914 __stop___param - __start___param,
944 -1, -1, NULL 915 -1, -1, NULL, &unknown_bootoption);
945 print_unknown_bootoptions(); 916 print_unknown_bootoptions();
946 if (!IS_ERR_OR_NULL(after_dashes)) 917 if (!IS_ERR_OR_NULL(after_dashes))
947 parse_args("Setting init args" 918 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
948 NULL, set_init_arg) 919 NULL, set_init_arg);
949 if (extra_init_args) 920 if (extra_init_args)
950 parse_args("Setting extra init 921 parse_args("Setting extra init args", extra_init_args,
951 NULL, 0, -1, -1, NU 922 NULL, 0, -1, -1, NULL, set_init_arg);
952 923
953 /* Architectural and non-timekeeping r 924 /* Architectural and non-timekeeping rng init, before allocator init */
954 random_init_early(command_line); 925 random_init_early(command_line);
955 926
956 /* 927 /*
957 * These use large bootmem allocations 928 * These use large bootmem allocations and must precede
958 * initalization of page allocator 929 * initalization of page allocator
959 */ 930 */
960 setup_log_buf(0); 931 setup_log_buf(0);
961 vfs_caches_init_early(); 932 vfs_caches_init_early();
962 sort_main_extable(); 933 sort_main_extable();
963 trap_init(); 934 trap_init();
964 mm_core_init(); 935 mm_core_init();
965 poking_init(); 936 poking_init();
966 ftrace_init(); 937 ftrace_init();
967 938
968 /* trace_printk can be enabled here */ 939 /* trace_printk can be enabled here */
969 early_trace_init(); 940 early_trace_init();
970 941
971 /* 942 /*
972 * Set up the scheduler prior starting 943 * Set up the scheduler prior starting any interrupts (such as the
973 * timer interrupt). Full topology set 944 * timer interrupt). Full topology setup happens at smp_init()
974 * time - but meanwhile we still have 945 * time - but meanwhile we still have a functioning scheduler.
975 */ 946 */
976 sched_init(); 947 sched_init();
977 948
978 if (WARN(!irqs_disabled(), 949 if (WARN(!irqs_disabled(),
979 "Interrupts were enabled *ver 950 "Interrupts were enabled *very* early, fixing it\n"))
980 local_irq_disable(); 951 local_irq_disable();
981 radix_tree_init(); 952 radix_tree_init();
982 maple_tree_init(); 953 maple_tree_init();
983 954
984 /* 955 /*
985 * Set up housekeeping before setting 956 * Set up housekeeping before setting up workqueues to allow the unbound
986 * workqueue to take non-housekeeping 957 * workqueue to take non-housekeeping into account.
987 */ 958 */
988 housekeeping_init(); 959 housekeeping_init();
989 960
990 /* 961 /*
991 * Allow workqueue creation and work i 962 * Allow workqueue creation and work item queueing/cancelling
992 * early. Work item execution depends 963 * early. Work item execution depends on kthreads and starts after
993 * workqueue_init(). 964 * workqueue_init().
994 */ 965 */
995 workqueue_init_early(); 966 workqueue_init_early();
996 967
997 rcu_init(); 968 rcu_init();
998 969
999 /* Trace events are available after th 970 /* Trace events are available after this */
1000 trace_init(); 971 trace_init();
1001 972
1002 if (initcall_debug) 973 if (initcall_debug)
1003 initcall_debug_enable(); 974 initcall_debug_enable();
1004 975
1005 context_tracking_init(); 976 context_tracking_init();
1006 /* init some links before init_ISA_ir 977 /* init some links before init_ISA_irqs() */
1007 early_irq_init(); 978 early_irq_init();
1008 init_IRQ(); 979 init_IRQ();
1009 tick_init(); 980 tick_init();
1010 rcu_init_nohz(); 981 rcu_init_nohz();
1011 init_timers(); 982 init_timers();
1012 srcu_init(); 983 srcu_init();
1013 hrtimers_init(); 984 hrtimers_init();
1014 softirq_init(); 985 softirq_init();
1015 timekeeping_init(); 986 timekeeping_init();
1016 time_init(); 987 time_init();
1017 988
1018 /* This must be after timekeeping is 989 /* This must be after timekeeping is initialized */
1019 random_init(); 990 random_init();
1020 991
1021 /* These make use of the fully initia 992 /* These make use of the fully initialized rng */
1022 kfence_init(); 993 kfence_init();
1023 boot_init_stack_canary(); 994 boot_init_stack_canary();
1024 995
1025 perf_event_init(); 996 perf_event_init();
1026 profile_init(); 997 profile_init();
1027 call_function_init(); 998 call_function_init();
1028 WARN(!irqs_disabled(), "Interrupts we 999 WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1029 1000
1030 early_boot_irqs_disabled = false; 1001 early_boot_irqs_disabled = false;
1031 local_irq_enable(); 1002 local_irq_enable();
1032 1003
1033 kmem_cache_init_late(); 1004 kmem_cache_init_late();
1034 1005
1035 /* 1006 /*
1036 * HACK ALERT! This is early. We're e 1007 * HACK ALERT! This is early. We're enabling the console before
1037 * we've done PCI setups etc, and con 1008 * we've done PCI setups etc, and console_init() must be aware of
1038 * this. But we do want output early, 1009 * this. But we do want output early, in case something goes wrong.
1039 */ 1010 */
1040 console_init(); 1011 console_init();
1041 if (panic_later) 1012 if (panic_later)
1042 panic("Too many boot %s vars 1013 panic("Too many boot %s vars at `%s'", panic_later,
1043 panic_param); 1014 panic_param);
1044 1015
1045 lockdep_init(); 1016 lockdep_init();
1046 1017
1047 /* 1018 /*
1048 * Need to run this when irqs are ena 1019 * Need to run this when irqs are enabled, because it wants
1049 * to self-test [hard/soft]-irqs on/o 1020 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1050 * too: 1021 * too:
1051 */ 1022 */
1052 locking_selftest(); 1023 locking_selftest();
1053 1024
1054 #ifdef CONFIG_BLK_DEV_INITRD 1025 #ifdef CONFIG_BLK_DEV_INITRD
1055 if (initrd_start && !initrd_below_sta 1026 if (initrd_start && !initrd_below_start_ok &&
1056 page_to_pfn(virt_to_page((void *) 1027 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1057 pr_crit("initrd overwritten ( 1028 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1058 page_to_pfn(virt_to_page( 1029 page_to_pfn(virt_to_page((void *)initrd_start)),
1059 min_low_pfn); 1030 min_low_pfn);
1060 initrd_start = 0; 1031 initrd_start = 0;
1061 } 1032 }
1062 #endif 1033 #endif
1063 setup_per_cpu_pageset(); 1034 setup_per_cpu_pageset();
1064 numa_policy_init(); 1035 numa_policy_init();
1065 acpi_early_init(); 1036 acpi_early_init();
1066 if (late_time_init) 1037 if (late_time_init)
1067 late_time_init(); 1038 late_time_init();
1068 sched_clock_init(); 1039 sched_clock_init();
1069 calibrate_delay(); 1040 calibrate_delay();
1070 1041
1071 arch_cpu_finalize_init(); 1042 arch_cpu_finalize_init();
1072 1043
1073 pid_idr_init(); 1044 pid_idr_init();
1074 anon_vma_init(); 1045 anon_vma_init();
1075 #ifdef CONFIG_X86 1046 #ifdef CONFIG_X86
1076 if (efi_enabled(EFI_RUNTIME_SERVICES) 1047 if (efi_enabled(EFI_RUNTIME_SERVICES))
1077 efi_enter_virtual_mode(); 1048 efi_enter_virtual_mode();
1078 #endif 1049 #endif
1079 thread_stack_cache_init(); 1050 thread_stack_cache_init();
1080 cred_init(); 1051 cred_init();
1081 fork_init(); 1052 fork_init();
1082 proc_caches_init(); 1053 proc_caches_init();
1083 uts_ns_init(); 1054 uts_ns_init();
1084 key_init(); 1055 key_init();
1085 security_init(); 1056 security_init();
1086 dbg_late_init(); 1057 dbg_late_init();
1087 net_ns_init(); 1058 net_ns_init();
1088 vfs_caches_init(); 1059 vfs_caches_init();
1089 pagecache_init(); 1060 pagecache_init();
1090 signals_init(); 1061 signals_init();
1091 seq_file_init(); 1062 seq_file_init();
1092 proc_root_init(); 1063 proc_root_init();
1093 nsfs_init(); 1064 nsfs_init();
1094 pidfs_init(); <<
1095 cpuset_init(); 1065 cpuset_init();
1096 cgroup_init(); 1066 cgroup_init();
1097 taskstats_init_early(); 1067 taskstats_init_early();
1098 delayacct_init(); 1068 delayacct_init();
1099 1069
1100 acpi_subsystem_init(); 1070 acpi_subsystem_init();
1101 arch_post_acpi_subsys_init(); 1071 arch_post_acpi_subsys_init();
1102 kcsan_init(); 1072 kcsan_init();
1103 1073
1104 /* Do the rest non-__init'ed, we're n 1074 /* Do the rest non-__init'ed, we're now alive */
1105 rest_init(); !! 1075 arch_call_rest_init();
1106 1076
1107 /* <<
1108 * Avoid stack canaries in callers of <<
1109 * and older. <<
1110 */ <<
1111 #if !__has_attribute(__no_stack_protector__) <<
1112 prevent_tail_call_optimization(); 1077 prevent_tail_call_optimization();
1113 #endif <<
1114 } 1078 }
1115 1079
1116 /* Call all constructor functions linked into 1080 /* Call all constructor functions linked into the kernel. */
1117 static void __init do_ctors(void) 1081 static void __init do_ctors(void)
1118 { 1082 {
1119 /* 1083 /*
1120 * For UML, the constructors have already bee 1084 * For UML, the constructors have already been called by the
1121 * normal setup code as it's just a normal EL 1085 * normal setup code as it's just a normal ELF binary, so we
1122 * cannot do it again - but we do need CONFIG 1086 * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1123 * even on UML for modules. 1087 * even on UML for modules.
1124 */ 1088 */
1125 #if defined(CONFIG_CONSTRUCTORS) && !defined( 1089 #if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1126 ctor_fn_t *fn = (ctor_fn_t *) __ctors 1090 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1127 1091
1128 for (; fn < (ctor_fn_t *) __ctors_end 1092 for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1129 (*fn)(); 1093 (*fn)();
1130 #endif 1094 #endif
1131 } 1095 }
1132 1096
1133 #ifdef CONFIG_KALLSYMS 1097 #ifdef CONFIG_KALLSYMS
1134 struct blacklist_entry { 1098 struct blacklist_entry {
1135 struct list_head next; 1099 struct list_head next;
1136 char *buf; 1100 char *buf;
1137 }; 1101 };
1138 1102
1139 static __initdata_or_module LIST_HEAD(blackli 1103 static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1140 1104
1141 static int __init initcall_blacklist(char *st 1105 static int __init initcall_blacklist(char *str)
1142 { 1106 {
1143 char *str_entry; 1107 char *str_entry;
1144 struct blacklist_entry *entry; 1108 struct blacklist_entry *entry;
1145 1109
1146 /* str argument is a comma-separated 1110 /* str argument is a comma-separated list of functions */
1147 do { 1111 do {
1148 str_entry = strsep(&str, ",") 1112 str_entry = strsep(&str, ",");
1149 if (str_entry) { 1113 if (str_entry) {
1150 pr_debug("blacklistin 1114 pr_debug("blacklisting initcall %s\n", str_entry);
1151 entry = memblock_allo 1115 entry = memblock_alloc(sizeof(*entry),
1152 1116 SMP_CACHE_BYTES);
1153 if (!entry) 1117 if (!entry)
1154 panic("%s: Fa 1118 panic("%s: Failed to allocate %zu bytes\n",
1155 __func_ 1119 __func__, sizeof(*entry));
1156 entry->buf = memblock 1120 entry->buf = memblock_alloc(strlen(str_entry) + 1,
1157 1121 SMP_CACHE_BYTES);
1158 if (!entry->buf) 1122 if (!entry->buf)
1159 panic("%s: Fa 1123 panic("%s: Failed to allocate %zu bytes\n",
1160 __func_ 1124 __func__, strlen(str_entry) + 1);
1161 strcpy(entry->buf, st 1125 strcpy(entry->buf, str_entry);
1162 list_add(&entry->next 1126 list_add(&entry->next, &blacklisted_initcalls);
1163 } 1127 }
1164 } while (str_entry); 1128 } while (str_entry);
1165 1129
1166 return 1; 1130 return 1;
1167 } 1131 }
1168 1132
1169 static bool __init_or_module initcall_blackli 1133 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1170 { 1134 {
1171 struct blacklist_entry *entry; 1135 struct blacklist_entry *entry;
1172 char fn_name[KSYM_SYMBOL_LEN]; 1136 char fn_name[KSYM_SYMBOL_LEN];
1173 unsigned long addr; 1137 unsigned long addr;
1174 1138
1175 if (list_empty(&blacklisted_initcalls 1139 if (list_empty(&blacklisted_initcalls))
1176 return false; 1140 return false;
1177 1141
1178 addr = (unsigned long) dereference_fu 1142 addr = (unsigned long) dereference_function_descriptor(fn);
1179 sprint_symbol_no_offset(fn_name, addr 1143 sprint_symbol_no_offset(fn_name, addr);
1180 1144
1181 /* 1145 /*
1182 * fn will be "function_name [module_ 1146 * fn will be "function_name [module_name]" where [module_name] is not
1183 * displayed for built-in init functi 1147 * displayed for built-in init functions. Strip off the [module_name].
1184 */ 1148 */
1185 strreplace(fn_name, ' ', '\0'); 1149 strreplace(fn_name, ' ', '\0');
1186 1150
1187 list_for_each_entry(entry, &blacklist 1151 list_for_each_entry(entry, &blacklisted_initcalls, next) {
1188 if (!strcmp(fn_name, entry->b 1152 if (!strcmp(fn_name, entry->buf)) {
1189 pr_debug("initcall %s 1153 pr_debug("initcall %s blacklisted\n", fn_name);
1190 return true; 1154 return true;
1191 } 1155 }
1192 } 1156 }
1193 1157
1194 return false; 1158 return false;
1195 } 1159 }
1196 #else 1160 #else
1197 static int __init initcall_blacklist(char *st 1161 static int __init initcall_blacklist(char *str)
1198 { 1162 {
1199 pr_warn("initcall_blacklist requires 1163 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1200 return 0; 1164 return 0;
1201 } 1165 }
1202 1166
1203 static bool __init_or_module initcall_blackli 1167 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1204 { 1168 {
1205 return false; 1169 return false;
1206 } 1170 }
1207 #endif 1171 #endif
1208 __setup("initcall_blacklist=", initcall_black 1172 __setup("initcall_blacklist=", initcall_blacklist);
1209 1173
1210 static __init_or_module void 1174 static __init_or_module void
1211 trace_initcall_start_cb(void *data, initcall_ 1175 trace_initcall_start_cb(void *data, initcall_t fn)
1212 { 1176 {
1213 ktime_t *calltime = data; 1177 ktime_t *calltime = data;
1214 1178
1215 printk(KERN_DEBUG "calling %pS @ %i\ 1179 printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current));
1216 *calltime = ktime_get(); 1180 *calltime = ktime_get();
1217 } 1181 }
1218 1182
1219 static __init_or_module void 1183 static __init_or_module void
1220 trace_initcall_finish_cb(void *data, initcall 1184 trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1221 { 1185 {
1222 ktime_t rettime, *calltime = data; 1186 ktime_t rettime, *calltime = data;
1223 1187
1224 rettime = ktime_get(); 1188 rettime = ktime_get();
1225 printk(KERN_DEBUG "initcall %pS retur 1189 printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1226 fn, ret, (unsigned long long 1190 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));
1227 } 1191 }
1228 1192
1229 static ktime_t initcall_calltime; 1193 static ktime_t initcall_calltime;
1230 1194
1231 #ifdef TRACEPOINTS_ENABLED 1195 #ifdef TRACEPOINTS_ENABLED
1232 static void __init initcall_debug_enable(void 1196 static void __init initcall_debug_enable(void)
1233 { 1197 {
1234 int ret; 1198 int ret;
1235 1199
1236 ret = register_trace_initcall_start(t 1200 ret = register_trace_initcall_start(trace_initcall_start_cb,
1237 & 1201 &initcall_calltime);
1238 ret |= register_trace_initcall_finish 1202 ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1239 1203 &initcall_calltime);
1240 WARN(ret, "Failed to register initcal 1204 WARN(ret, "Failed to register initcall tracepoints\n");
1241 } 1205 }
1242 # define do_trace_initcall_start trace 1206 # define do_trace_initcall_start trace_initcall_start
1243 # define do_trace_initcall_finish trace 1207 # define do_trace_initcall_finish trace_initcall_finish
1244 #else 1208 #else
1245 static inline void do_trace_initcall_start(in 1209 static inline void do_trace_initcall_start(initcall_t fn)
1246 { 1210 {
1247 if (!initcall_debug) 1211 if (!initcall_debug)
1248 return; 1212 return;
1249 trace_initcall_start_cb(&initcall_cal 1213 trace_initcall_start_cb(&initcall_calltime, fn);
1250 } 1214 }
1251 static inline void do_trace_initcall_finish(i 1215 static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1252 { 1216 {
1253 if (!initcall_debug) 1217 if (!initcall_debug)
1254 return; 1218 return;
1255 trace_initcall_finish_cb(&initcall_ca 1219 trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1256 } 1220 }
1257 #endif /* !TRACEPOINTS_ENABLED */ 1221 #endif /* !TRACEPOINTS_ENABLED */
1258 1222
1259 int __init_or_module do_one_initcall(initcall 1223 int __init_or_module do_one_initcall(initcall_t fn)
1260 { 1224 {
1261 int count = preempt_count(); 1225 int count = preempt_count();
1262 char msgbuf[64]; 1226 char msgbuf[64];
1263 int ret; 1227 int ret;
1264 1228
1265 if (initcall_blacklisted(fn)) 1229 if (initcall_blacklisted(fn))
1266 return -EPERM; 1230 return -EPERM;
1267 1231
1268 do_trace_initcall_start(fn); 1232 do_trace_initcall_start(fn);
1269 ret = fn(); 1233 ret = fn();
1270 do_trace_initcall_finish(fn, ret); 1234 do_trace_initcall_finish(fn, ret);
1271 1235
1272 msgbuf[0] = 0; 1236 msgbuf[0] = 0;
1273 1237
1274 if (preempt_count() != count) { 1238 if (preempt_count() != count) {
1275 sprintf(msgbuf, "preemption i 1239 sprintf(msgbuf, "preemption imbalance ");
1276 preempt_count_set(count); 1240 preempt_count_set(count);
1277 } 1241 }
1278 if (irqs_disabled()) { 1242 if (irqs_disabled()) {
1279 strlcat(msgbuf, "disabled int 1243 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1280 local_irq_enable(); 1244 local_irq_enable();
1281 } 1245 }
1282 WARN(msgbuf[0], "initcall %pS returne 1246 WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1283 1247
1284 add_latent_entropy(); 1248 add_latent_entropy();
1285 return ret; 1249 return ret;
1286 } 1250 }
1287 1251
1288 1252
>> 1253 extern initcall_entry_t __initcall_start[];
>> 1254 extern initcall_entry_t __initcall0_start[];
>> 1255 extern initcall_entry_t __initcall1_start[];
>> 1256 extern initcall_entry_t __initcall2_start[];
>> 1257 extern initcall_entry_t __initcall3_start[];
>> 1258 extern initcall_entry_t __initcall4_start[];
>> 1259 extern initcall_entry_t __initcall5_start[];
>> 1260 extern initcall_entry_t __initcall6_start[];
>> 1261 extern initcall_entry_t __initcall7_start[];
>> 1262 extern initcall_entry_t __initcall_end[];
>> 1263
1289 static initcall_entry_t *initcall_levels[] __ 1264 static initcall_entry_t *initcall_levels[] __initdata = {
1290 __initcall0_start, 1265 __initcall0_start,
1291 __initcall1_start, 1266 __initcall1_start,
1292 __initcall2_start, 1267 __initcall2_start,
1293 __initcall3_start, 1268 __initcall3_start,
1294 __initcall4_start, 1269 __initcall4_start,
1295 __initcall5_start, 1270 __initcall5_start,
1296 __initcall6_start, 1271 __initcall6_start,
1297 __initcall7_start, 1272 __initcall7_start,
1298 __initcall_end, 1273 __initcall_end,
1299 }; 1274 };
1300 1275
1301 /* Keep these in sync with initcalls in inclu 1276 /* Keep these in sync with initcalls in include/linux/init.h */
1302 static const char *initcall_level_names[] __i 1277 static const char *initcall_level_names[] __initdata = {
1303 "pure", 1278 "pure",
1304 "core", 1279 "core",
1305 "postcore", 1280 "postcore",
1306 "arch", 1281 "arch",
1307 "subsys", 1282 "subsys",
1308 "fs", 1283 "fs",
1309 "device", 1284 "device",
1310 "late", 1285 "late",
1311 }; 1286 };
1312 1287
1313 static int __init ignore_unknown_bootoption(c 1288 static int __init ignore_unknown_bootoption(char *param, char *val,
1314 const char *un 1289 const char *unused, void *arg)
1315 { 1290 {
1316 return 0; 1291 return 0;
1317 } 1292 }
1318 1293
1319 static void __init do_initcall_level(int leve 1294 static void __init do_initcall_level(int level, char *command_line)
1320 { 1295 {
1321 initcall_entry_t *fn; 1296 initcall_entry_t *fn;
1322 1297
1323 parse_args(initcall_level_names[level 1298 parse_args(initcall_level_names[level],
1324 command_line, __start___pa 1299 command_line, __start___param,
1325 __stop___param - __start__ 1300 __stop___param - __start___param,
1326 level, level, 1301 level, level,
1327 NULL, ignore_unknown_booto 1302 NULL, ignore_unknown_bootoption);
1328 1303
1329 trace_initcall_level(initcall_level_n 1304 trace_initcall_level(initcall_level_names[level]);
1330 for (fn = initcall_levels[level]; fn 1305 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1331 do_one_initcall(initcall_from 1306 do_one_initcall(initcall_from_entry(fn));
1332 } 1307 }
1333 1308
1334 static void __init do_initcalls(void) 1309 static void __init do_initcalls(void)
1335 { 1310 {
1336 int level; 1311 int level;
1337 size_t len = saved_command_line_len + 1312 size_t len = saved_command_line_len + 1;
1338 char *command_line; 1313 char *command_line;
1339 1314
1340 command_line = kzalloc(len, GFP_KERNE 1315 command_line = kzalloc(len, GFP_KERNEL);
1341 if (!command_line) 1316 if (!command_line)
1342 panic("%s: Failed to allocate 1317 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1343 1318
1344 for (level = 0; level < ARRAY_SIZE(in 1319 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1345 /* Parser modifies command_li 1320 /* Parser modifies command_line, restore it each time */
1346 strcpy(command_line, saved_co 1321 strcpy(command_line, saved_command_line);
1347 do_initcall_level(level, comm 1322 do_initcall_level(level, command_line);
1348 } 1323 }
1349 1324
1350 kfree(command_line); 1325 kfree(command_line);
1351 } 1326 }
1352 1327
1353 /* 1328 /*
1354 * Ok, the machine is now initialized. None o 1329 * Ok, the machine is now initialized. None of the devices
1355 * have been touched yet, but the CPU subsyst 1330 * have been touched yet, but the CPU subsystem is up and
1356 * running, and memory and process management 1331 * running, and memory and process management works.
1357 * 1332 *
1358 * Now we can finally start doing some real w 1333 * Now we can finally start doing some real work..
1359 */ 1334 */
1360 static void __init do_basic_setup(void) 1335 static void __init do_basic_setup(void)
1361 { 1336 {
1362 cpuset_init_smp(); 1337 cpuset_init_smp();
1363 driver_init(); 1338 driver_init();
1364 init_irq_proc(); 1339 init_irq_proc();
1365 do_ctors(); 1340 do_ctors();
1366 do_initcalls(); 1341 do_initcalls();
1367 } 1342 }
1368 1343
1369 static void __init do_pre_smp_initcalls(void) 1344 static void __init do_pre_smp_initcalls(void)
1370 { 1345 {
1371 initcall_entry_t *fn; 1346 initcall_entry_t *fn;
1372 1347
1373 trace_initcall_level("early"); 1348 trace_initcall_level("early");
1374 for (fn = __initcall_start; fn < __in 1349 for (fn = __initcall_start; fn < __initcall0_start; fn++)
1375 do_one_initcall(initcall_from 1350 do_one_initcall(initcall_from_entry(fn));
1376 } 1351 }
1377 1352
1378 static int run_init_process(const char *init_ 1353 static int run_init_process(const char *init_filename)
1379 { 1354 {
1380 const char *const *p; 1355 const char *const *p;
1381 1356
1382 argv_init[0] = init_filename; 1357 argv_init[0] = init_filename;
1383 pr_info("Run %s as init process\n", i 1358 pr_info("Run %s as init process\n", init_filename);
1384 pr_debug(" with arguments:\n"); 1359 pr_debug(" with arguments:\n");
1385 for (p = argv_init; *p; p++) 1360 for (p = argv_init; *p; p++)
1386 pr_debug(" %s\n", *p); 1361 pr_debug(" %s\n", *p);
1387 pr_debug(" with environment:\n"); 1362 pr_debug(" with environment:\n");
1388 for (p = envp_init; *p; p++) 1363 for (p = envp_init; *p; p++)
1389 pr_debug(" %s\n", *p); 1364 pr_debug(" %s\n", *p);
1390 return kernel_execve(init_filename, a 1365 return kernel_execve(init_filename, argv_init, envp_init);
1391 } 1366 }
1392 1367
1393 static int try_to_run_init_process(const char 1368 static int try_to_run_init_process(const char *init_filename)
1394 { 1369 {
1395 int ret; 1370 int ret;
1396 1371
1397 ret = run_init_process(init_filename) 1372 ret = run_init_process(init_filename);
1398 1373
1399 if (ret && ret != -ENOENT) { 1374 if (ret && ret != -ENOENT) {
1400 pr_err("Starting init: %s exi 1375 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1401 init_filename, ret); 1376 init_filename, ret);
1402 } 1377 }
1403 1378
1404 return ret; 1379 return ret;
1405 } 1380 }
1406 1381
1407 static noinline void __init kernel_init_freea 1382 static noinline void __init kernel_init_freeable(void);
1408 1383
1409 #if defined(CONFIG_STRICT_KERNEL_RWX) || defi 1384 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1410 bool rodata_enabled __ro_after_init = true; 1385 bool rodata_enabled __ro_after_init = true;
1411 1386
1412 #ifndef arch_parse_debug_rodata 1387 #ifndef arch_parse_debug_rodata
1413 static inline bool arch_parse_debug_rodata(ch 1388 static inline bool arch_parse_debug_rodata(char *str) { return false; }
1414 #endif 1389 #endif
1415 1390
1416 static int __init set_debug_rodata(char *str) 1391 static int __init set_debug_rodata(char *str)
1417 { 1392 {
1418 if (arch_parse_debug_rodata(str)) 1393 if (arch_parse_debug_rodata(str))
1419 return 0; 1394 return 0;
1420 1395
1421 if (str && !strcmp(str, "on")) 1396 if (str && !strcmp(str, "on"))
1422 rodata_enabled = true; 1397 rodata_enabled = true;
1423 else if (str && !strcmp(str, "off")) 1398 else if (str && !strcmp(str, "off"))
1424 rodata_enabled = false; 1399 rodata_enabled = false;
1425 else 1400 else
1426 pr_warn("Invalid option strin 1401 pr_warn("Invalid option string for rodata: '%s'\n", str);
1427 return 0; 1402 return 0;
1428 } 1403 }
1429 early_param("rodata", set_debug_rodata); 1404 early_param("rodata", set_debug_rodata);
1430 #endif 1405 #endif
1431 1406
>> 1407 #ifdef CONFIG_STRICT_KERNEL_RWX
1432 static void mark_readonly(void) 1408 static void mark_readonly(void)
1433 { 1409 {
1434 if (IS_ENABLED(CONFIG_STRICT_KERNEL_R !! 1410 if (rodata_enabled) {
1435 /* 1411 /*
1436 * load_module() results in W 1412 * load_module() results in W+X mappings, which are cleaned
1437 * up with init_free_wq. Let' !! 1413 * up with call_rcu(). Let's make sure that queued work is
1438 * flushed so that we don't h 1414 * flushed so that we don't hit false positives looking for
1439 * insecure pages which are W 1415 * insecure pages which are W+X.
1440 */ 1416 */
1441 flush_module_init_free_work() !! 1417 rcu_barrier();
1442 jump_label_init_ro(); <<
1443 mark_rodata_ro(); 1418 mark_rodata_ro();
1444 debug_checkwx(); <<
1445 rodata_test(); 1419 rodata_test();
1446 } else if (IS_ENABLED(CONFIG_STRICT_K !! 1420 } else
1447 pr_info("Kernel memory protec 1421 pr_info("Kernel memory protection disabled.\n");
1448 } else if (IS_ENABLED(CONFIG_ARCH_HAS <<
1449 pr_warn("Kernel memory protec <<
1450 } else { <<
1451 pr_warn("This architecture do <<
1452 } <<
1453 } 1422 }
>> 1423 #elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)
>> 1424 static inline void mark_readonly(void)
>> 1425 {
>> 1426 pr_warn("Kernel memory protection not selected by kernel config.\n");
>> 1427 }
>> 1428 #else
>> 1429 static inline void mark_readonly(void)
>> 1430 {
>> 1431 pr_warn("This architecture does not have kernel memory protection.\n");
>> 1432 }
>> 1433 #endif
1454 1434
1455 void __weak free_initmem(void) 1435 void __weak free_initmem(void)
1456 { 1436 {
1457 free_initmem_default(POISON_FREE_INIT 1437 free_initmem_default(POISON_FREE_INITMEM);
1458 } 1438 }
1459 1439
1460 static int __ref kernel_init(void *unused) 1440 static int __ref kernel_init(void *unused)
1461 { 1441 {
1462 int ret; 1442 int ret;
1463 1443
1464 /* 1444 /*
1465 * Wait until kthreadd is all set-up. 1445 * Wait until kthreadd is all set-up.
1466 */ 1446 */
1467 wait_for_completion(&kthreadd_done); 1447 wait_for_completion(&kthreadd_done);
1468 1448
1469 kernel_init_freeable(); 1449 kernel_init_freeable();
1470 /* need to finish all async __init co 1450 /* need to finish all async __init code before freeing the memory */
1471 async_synchronize_full(); 1451 async_synchronize_full();
1472 1452
1473 system_state = SYSTEM_FREEING_INITMEM 1453 system_state = SYSTEM_FREEING_INITMEM;
1474 kprobe_free_init_mem(); 1454 kprobe_free_init_mem();
1475 ftrace_free_init_mem(); 1455 ftrace_free_init_mem();
1476 kgdb_free_init_mem(); 1456 kgdb_free_init_mem();
1477 exit_boot_config(); 1457 exit_boot_config();
1478 free_initmem(); 1458 free_initmem();
1479 mark_readonly(); 1459 mark_readonly();
1480 1460
1481 /* 1461 /*
1482 * Kernel mappings are now finalized 1462 * Kernel mappings are now finalized - update the userspace page-table
1483 * to finalize PTI. 1463 * to finalize PTI.
1484 */ 1464 */
1485 pti_finalize(); 1465 pti_finalize();
1486 1466
1487 system_state = SYSTEM_RUNNING; 1467 system_state = SYSTEM_RUNNING;
1488 numa_default_policy(); 1468 numa_default_policy();
1489 1469
1490 rcu_end_inkernel_boot(); 1470 rcu_end_inkernel_boot();
1491 1471
1492 do_sysctl_args(); 1472 do_sysctl_args();
1493 1473
1494 if (ramdisk_execute_command) { 1474 if (ramdisk_execute_command) {
1495 ret = run_init_process(ramdis 1475 ret = run_init_process(ramdisk_execute_command);
1496 if (!ret) 1476 if (!ret)
1497 return 0; 1477 return 0;
1498 pr_err("Failed to execute %s 1478 pr_err("Failed to execute %s (error %d)\n",
1499 ramdisk_execute_comman 1479 ramdisk_execute_command, ret);
1500 } 1480 }
1501 1481
1502 /* 1482 /*
1503 * We try each of these until one suc 1483 * We try each of these until one succeeds.
1504 * 1484 *
1505 * The Bourne shell can be used inste 1485 * The Bourne shell can be used instead of init if we are
1506 * trying to recover a really broken 1486 * trying to recover a really broken machine.
1507 */ 1487 */
1508 if (execute_command) { 1488 if (execute_command) {
1509 ret = run_init_process(execut 1489 ret = run_init_process(execute_command);
1510 if (!ret) 1490 if (!ret)
1511 return 0; 1491 return 0;
1512 panic("Requested init %s fail 1492 panic("Requested init %s failed (error %d).",
1513 execute_command, ret); 1493 execute_command, ret);
1514 } 1494 }
1515 1495
1516 if (CONFIG_DEFAULT_INIT[0] != '\0') { 1496 if (CONFIG_DEFAULT_INIT[0] != '\0') {
1517 ret = run_init_process(CONFIG 1497 ret = run_init_process(CONFIG_DEFAULT_INIT);
1518 if (ret) 1498 if (ret)
1519 pr_err("Default init 1499 pr_err("Default init %s failed (error %d)\n",
1520 CONFIG_DEFAULT 1500 CONFIG_DEFAULT_INIT, ret);
1521 else 1501 else
1522 return 0; 1502 return 0;
1523 } 1503 }
1524 1504
1525 if (!try_to_run_init_process("/sbin/i 1505 if (!try_to_run_init_process("/sbin/init") ||
1526 !try_to_run_init_process("/etc/in 1506 !try_to_run_init_process("/etc/init") ||
1527 !try_to_run_init_process("/bin/in 1507 !try_to_run_init_process("/bin/init") ||
1528 !try_to_run_init_process("/bin/sh 1508 !try_to_run_init_process("/bin/sh"))
1529 return 0; 1509 return 0;
1530 1510
1531 panic("No working init found. Try pa 1511 panic("No working init found. Try passing init= option to kernel. "
1532 "See Linux Documentation/admin- 1512 "See Linux Documentation/admin-guide/init.rst for guidance.");
1533 } 1513 }
1534 1514
1535 /* Open /dev/console, for stdin/stdout/stderr 1515 /* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1536 void __init console_on_rootfs(void) 1516 void __init console_on_rootfs(void)
1537 { 1517 {
1538 struct file *file = filp_open("/dev/c 1518 struct file *file = filp_open("/dev/console", O_RDWR, 0);
1539 1519
1540 if (IS_ERR(file)) { 1520 if (IS_ERR(file)) {
1541 pr_err("Warning: unable to op 1521 pr_err("Warning: unable to open an initial console.\n");
1542 return; 1522 return;
1543 } 1523 }
1544 init_dup(file); 1524 init_dup(file);
1545 init_dup(file); 1525 init_dup(file);
1546 init_dup(file); 1526 init_dup(file);
1547 fput(file); 1527 fput(file);
1548 } 1528 }
1549 1529
1550 static noinline void __init kernel_init_freea 1530 static noinline void __init kernel_init_freeable(void)
1551 { 1531 {
1552 /* Now the scheduler is fully set up 1532 /* Now the scheduler is fully set up and can do blocking allocations */
1553 gfp_allowed_mask = __GFP_BITS_MASK; 1533 gfp_allowed_mask = __GFP_BITS_MASK;
1554 1534
1555 /* 1535 /*
1556 * init can allocate pages on any nod 1536 * init can allocate pages on any node
1557 */ 1537 */
1558 set_mems_allowed(node_states[N_MEMORY 1538 set_mems_allowed(node_states[N_MEMORY]);
1559 1539
1560 cad_pid = get_pid(task_pid(current)); 1540 cad_pid = get_pid(task_pid(current));
1561 1541
1562 smp_prepare_cpus(setup_max_cpus); 1542 smp_prepare_cpus(setup_max_cpus);
1563 1543
1564 workqueue_init(); 1544 workqueue_init();
1565 1545
1566 init_mm_internals(); 1546 init_mm_internals();
1567 1547
1568 rcu_init_tasks_generic(); 1548 rcu_init_tasks_generic();
1569 do_pre_smp_initcalls(); 1549 do_pre_smp_initcalls();
1570 lockup_detector_init(); 1550 lockup_detector_init();
1571 1551
1572 smp_init(); 1552 smp_init();
1573 sched_init_smp(); 1553 sched_init_smp();
1574 1554
1575 workqueue_init_topology(); <<
1576 async_init(); <<
1577 padata_init(); 1555 padata_init();
1578 page_alloc_init_late(); 1556 page_alloc_init_late();
1579 1557
1580 do_basic_setup(); 1558 do_basic_setup();
1581 1559
1582 kunit_run_all_tests(); 1560 kunit_run_all_tests();
1583 1561
1584 wait_for_initramfs(); 1562 wait_for_initramfs();
1585 console_on_rootfs(); 1563 console_on_rootfs();
1586 1564
1587 /* 1565 /*
1588 * check if there is an early userspa 1566 * check if there is an early userspace init. If yes, let it do all
1589 * the work 1567 * the work
1590 */ 1568 */
1591 if (init_eaccess(ramdisk_execute_comm 1569 if (init_eaccess(ramdisk_execute_command) != 0) {
1592 ramdisk_execute_command = NUL 1570 ramdisk_execute_command = NULL;
1593 prepare_namespace(); 1571 prepare_namespace();
1594 } 1572 }
1595 1573
1596 /* 1574 /*
1597 * Ok, we have completed the initial 1575 * Ok, we have completed the initial bootup, and
1598 * we're essentially up and running. 1576 * we're essentially up and running. Get rid of the
1599 * initmem segments and start the use 1577 * initmem segments and start the user-mode stuff..
1600 * 1578 *
1601 * rootfs is available now, try loadi 1579 * rootfs is available now, try loading the public keys
1602 * and default modules 1580 * and default modules
1603 */ 1581 */
1604 1582
1605 integrity_load_keys(); 1583 integrity_load_keys();
1606 } 1584 }
1607 1585
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.