1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * linux/init/main.c 3 * linux/init/main.c
4 * 4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 1991, 1992 Linus Torvalds
6 * 6 *
7 * GK 2/5/95 - Changed to support mounting 7 * GK 2/5/95 - Changed to support mounting root fs via NFS
8 * Added initrd & change_root: Werner Almesbe 8 * Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
9 * Moan early if gcc is old, avoiding bogus k 9 * Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
10 * Simplified starting of init: Michael A. G 10 * Simplified starting of init: Michael A. Griffith <grif@acm.org>
11 */ 11 */
12 12
13 #define DEBUG /* Enable initcall_deb 13 #define DEBUG /* Enable initcall_debug */
14 14
15 #include <linux/types.h> 15 #include <linux/types.h>
16 #include <linux/extable.h> 16 #include <linux/extable.h>
17 #include <linux/module.h> 17 #include <linux/module.h>
18 #include <linux/proc_fs.h> 18 #include <linux/proc_fs.h>
19 #include <linux/binfmts.h> 19 #include <linux/binfmts.h>
20 #include <linux/kernel.h> 20 #include <linux/kernel.h>
21 #include <linux/syscalls.h> 21 #include <linux/syscalls.h>
22 #include <linux/stackprotector.h> 22 #include <linux/stackprotector.h>
23 #include <linux/string.h> 23 #include <linux/string.h>
24 #include <linux/ctype.h> 24 #include <linux/ctype.h>
25 #include <linux/delay.h> 25 #include <linux/delay.h>
26 #include <linux/ioport.h> 26 #include <linux/ioport.h>
27 #include <linux/init.h> 27 #include <linux/init.h>
28 #include <linux/initrd.h> 28 #include <linux/initrd.h>
29 #include <linux/memblock.h> 29 #include <linux/memblock.h>
30 #include <linux/acpi.h> 30 #include <linux/acpi.h>
31 #include <linux/bootconfig.h> 31 #include <linux/bootconfig.h>
32 #include <linux/console.h> 32 #include <linux/console.h>
33 #include <linux/nmi.h> 33 #include <linux/nmi.h>
34 #include <linux/percpu.h> 34 #include <linux/percpu.h>
35 #include <linux/kmod.h> 35 #include <linux/kmod.h>
36 #include <linux/kprobes.h> 36 #include <linux/kprobes.h>
37 #include <linux/kmsan.h> 37 #include <linux/kmsan.h>
38 #include <linux/vmalloc.h> 38 #include <linux/vmalloc.h>
39 #include <linux/kernel_stat.h> 39 #include <linux/kernel_stat.h>
40 #include <linux/start_kernel.h> 40 #include <linux/start_kernel.h>
41 #include <linux/security.h> 41 #include <linux/security.h>
42 #include <linux/smp.h> 42 #include <linux/smp.h>
43 #include <linux/profile.h> 43 #include <linux/profile.h>
44 #include <linux/kfence.h> 44 #include <linux/kfence.h>
45 #include <linux/rcupdate.h> 45 #include <linux/rcupdate.h>
46 #include <linux/srcu.h> 46 #include <linux/srcu.h>
47 #include <linux/moduleparam.h> 47 #include <linux/moduleparam.h>
48 #include <linux/kallsyms.h> 48 #include <linux/kallsyms.h>
49 #include <linux/buildid.h> 49 #include <linux/buildid.h>
50 #include <linux/writeback.h> 50 #include <linux/writeback.h>
51 #include <linux/cpu.h> 51 #include <linux/cpu.h>
52 #include <linux/cpuset.h> 52 #include <linux/cpuset.h>
53 #include <linux/cgroup.h> 53 #include <linux/cgroup.h>
54 #include <linux/efi.h> 54 #include <linux/efi.h>
55 #include <linux/tick.h> 55 #include <linux/tick.h>
56 #include <linux/sched/isolation.h> 56 #include <linux/sched/isolation.h>
57 #include <linux/interrupt.h> 57 #include <linux/interrupt.h>
58 #include <linux/taskstats_kern.h> 58 #include <linux/taskstats_kern.h>
59 #include <linux/delayacct.h> 59 #include <linux/delayacct.h>
60 #include <linux/unistd.h> 60 #include <linux/unistd.h>
61 #include <linux/utsname.h> 61 #include <linux/utsname.h>
62 #include <linux/rmap.h> 62 #include <linux/rmap.h>
63 #include <linux/mempolicy.h> 63 #include <linux/mempolicy.h>
64 #include <linux/key.h> 64 #include <linux/key.h>
65 #include <linux/debug_locks.h> 65 #include <linux/debug_locks.h>
66 #include <linux/debugobjects.h> 66 #include <linux/debugobjects.h>
67 #include <linux/lockdep.h> 67 #include <linux/lockdep.h>
68 #include <linux/kmemleak.h> 68 #include <linux/kmemleak.h>
69 #include <linux/padata.h> 69 #include <linux/padata.h>
70 #include <linux/pid_namespace.h> 70 #include <linux/pid_namespace.h>
71 #include <linux/device/driver.h> 71 #include <linux/device/driver.h>
72 #include <linux/kthread.h> 72 #include <linux/kthread.h>
73 #include <linux/sched.h> 73 #include <linux/sched.h>
74 #include <linux/sched/init.h> 74 #include <linux/sched/init.h>
75 #include <linux/signal.h> 75 #include <linux/signal.h>
76 #include <linux/idr.h> 76 #include <linux/idr.h>
77 #include <linux/kgdb.h> 77 #include <linux/kgdb.h>
78 #include <linux/ftrace.h> 78 #include <linux/ftrace.h>
79 #include <linux/async.h> 79 #include <linux/async.h>
80 #include <linux/shmem_fs.h> 80 #include <linux/shmem_fs.h>
81 #include <linux/slab.h> 81 #include <linux/slab.h>
82 #include <linux/perf_event.h> 82 #include <linux/perf_event.h>
83 #include <linux/ptrace.h> 83 #include <linux/ptrace.h>
84 #include <linux/pti.h> 84 #include <linux/pti.h>
85 #include <linux/blkdev.h> 85 #include <linux/blkdev.h>
86 #include <linux/sched/clock.h> 86 #include <linux/sched/clock.h>
87 #include <linux/sched/task.h> 87 #include <linux/sched/task.h>
88 #include <linux/sched/task_stack.h> 88 #include <linux/sched/task_stack.h>
89 #include <linux/context_tracking.h> 89 #include <linux/context_tracking.h>
90 #include <linux/random.h> 90 #include <linux/random.h>
91 #include <linux/moduleloader.h> 91 #include <linux/moduleloader.h>
92 #include <linux/list.h> 92 #include <linux/list.h>
93 #include <linux/integrity.h> 93 #include <linux/integrity.h>
94 #include <linux/proc_ns.h> 94 #include <linux/proc_ns.h>
95 #include <linux/io.h> 95 #include <linux/io.h>
96 #include <linux/cache.h> 96 #include <linux/cache.h>
97 #include <linux/rodata_test.h> 97 #include <linux/rodata_test.h>
98 #include <linux/jump_label.h> 98 #include <linux/jump_label.h>
99 #include <linux/kcsan.h> 99 #include <linux/kcsan.h>
100 #include <linux/init_syscalls.h> 100 #include <linux/init_syscalls.h>
101 #include <linux/stackdepot.h> 101 #include <linux/stackdepot.h>
102 #include <linux/randomize_kstack.h> 102 #include <linux/randomize_kstack.h>
103 #include <linux/pidfs.h> <<
104 #include <linux/ptdump.h> <<
105 #include <net/net_namespace.h> 103 #include <net/net_namespace.h>
106 104
107 #include <asm/io.h> 105 #include <asm/io.h>
108 #include <asm/setup.h> 106 #include <asm/setup.h>
109 #include <asm/sections.h> 107 #include <asm/sections.h>
110 #include <asm/cacheflush.h> 108 #include <asm/cacheflush.h>
111 109
112 #define CREATE_TRACE_POINTS 110 #define CREATE_TRACE_POINTS
113 #include <trace/events/initcall.h> 111 #include <trace/events/initcall.h>
114 112
115 #include <kunit/test.h> 113 #include <kunit/test.h>
116 114
117 static int kernel_init(void *); 115 static int kernel_init(void *);
118 116
119 /* 117 /*
120 * Debug helper: via this flag we know that we 118 * Debug helper: via this flag we know that we are in 'early bootup code'
121 * where only the boot processor is running wi 119 * where only the boot processor is running with IRQ disabled. This means
122 * two things - IRQ must not be enabled before 120 * two things - IRQ must not be enabled before the flag is cleared and some
123 * operations which are not allowed with IRQ d 121 * operations which are not allowed with IRQ disabled are allowed while the
124 * flag is set. 122 * flag is set.
125 */ 123 */
126 bool early_boot_irqs_disabled __read_mostly; 124 bool early_boot_irqs_disabled __read_mostly;
127 125
128 enum system_states system_state __read_mostly; 126 enum system_states system_state __read_mostly;
129 EXPORT_SYMBOL(system_state); 127 EXPORT_SYMBOL(system_state);
130 128
131 /* 129 /*
132 * Boot command-line arguments 130 * Boot command-line arguments
133 */ 131 */
134 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMI 132 #define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
135 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMI 133 #define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT
136 134
137 /* Default late time init is NULL. archs can o 135 /* Default late time init is NULL. archs can override this later. */
138 void (*__initdata late_time_init)(void); 136 void (*__initdata late_time_init)(void);
139 137
140 /* Untouched command line saved by arch-specif 138 /* Untouched command line saved by arch-specific code. */
141 char __initdata boot_command_line[COMMAND_LINE 139 char __initdata boot_command_line[COMMAND_LINE_SIZE];
142 /* Untouched saved command line (eg. for /proc 140 /* Untouched saved command line (eg. for /proc) */
143 char *saved_command_line __ro_after_init; 141 char *saved_command_line __ro_after_init;
144 unsigned int saved_command_line_len __ro_after 142 unsigned int saved_command_line_len __ro_after_init;
145 /* Command line for parameter parsing */ 143 /* Command line for parameter parsing */
146 static char *static_command_line; 144 static char *static_command_line;
147 /* Untouched extra command line */ 145 /* Untouched extra command line */
148 static char *extra_command_line; 146 static char *extra_command_line;
149 /* Extra init arguments */ 147 /* Extra init arguments */
150 static char *extra_init_args; 148 static char *extra_init_args;
151 149
152 #ifdef CONFIG_BOOT_CONFIG 150 #ifdef CONFIG_BOOT_CONFIG
153 /* Is bootconfig on command line? */ 151 /* Is bootconfig on command line? */
154 static bool bootconfig_found; 152 static bool bootconfig_found;
155 static size_t initargs_offs; 153 static size_t initargs_offs;
156 #else 154 #else
157 # define bootconfig_found false 155 # define bootconfig_found false
158 # define initargs_offs 0 156 # define initargs_offs 0
159 #endif 157 #endif
160 158
161 static char *execute_command; 159 static char *execute_command;
162 static char *ramdisk_execute_command = "/init" 160 static char *ramdisk_execute_command = "/init";
163 161
164 /* 162 /*
165 * Used to generate warnings if static_key man 163 * Used to generate warnings if static_key manipulation functions are used
166 * before jump_label_init is called. 164 * before jump_label_init is called.
167 */ 165 */
168 bool static_key_initialized __read_mostly; 166 bool static_key_initialized __read_mostly;
169 EXPORT_SYMBOL_GPL(static_key_initialized); 167 EXPORT_SYMBOL_GPL(static_key_initialized);
170 168
171 /* 169 /*
172 * If set, this is an indication to the driver 170 * If set, this is an indication to the drivers that reset the underlying
173 * device before going ahead with the initiali 171 * device before going ahead with the initialization otherwise driver might
174 * rely on the BIOS and skip the reset operati 172 * rely on the BIOS and skip the reset operation.
175 * 173 *
176 * This is useful if kernel is booting in an u 174 * This is useful if kernel is booting in an unreliable environment.
177 * For ex. kdump situation where previous kern 175 * For ex. kdump situation where previous kernel has crashed, BIOS has been
178 * skipped and devices will be in unknown stat 176 * skipped and devices will be in unknown state.
179 */ 177 */
180 unsigned int reset_devices; 178 unsigned int reset_devices;
181 EXPORT_SYMBOL(reset_devices); 179 EXPORT_SYMBOL(reset_devices);
182 180
183 static int __init set_reset_devices(char *str) 181 static int __init set_reset_devices(char *str)
184 { 182 {
185 reset_devices = 1; 183 reset_devices = 1;
186 return 1; 184 return 1;
187 } 185 }
188 186
189 __setup("reset_devices", set_reset_devices); 187 __setup("reset_devices", set_reset_devices);
190 188
191 static const char *argv_init[MAX_INIT_ARGS+2] 189 static const char *argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
192 const char *envp_init[MAX_INIT_ENVS+2] = { "HO 190 const char *envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
193 static const char *panic_later, *panic_param; 191 static const char *panic_later, *panic_param;
194 192
195 static bool __init obsolete_checksetup(char *l 193 static bool __init obsolete_checksetup(char *line)
196 { 194 {
197 const struct obs_kernel_param *p; 195 const struct obs_kernel_param *p;
198 bool had_early_param = false; 196 bool had_early_param = false;
199 197
200 p = __setup_start; 198 p = __setup_start;
201 do { 199 do {
202 int n = strlen(p->str); 200 int n = strlen(p->str);
203 if (parameqn(line, p->str, n)) 201 if (parameqn(line, p->str, n)) {
204 if (p->early) { 202 if (p->early) {
205 /* Already don 203 /* Already done in parse_early_param?
206 * (Needs exac 204 * (Needs exact match on param part).
207 * Keep iterat 205 * Keep iterating, as we can have early
208 * params and 206 * params and __setups of same names 8( */
209 if (line[n] == 207 if (line[n] == '\0' || line[n] == '=')
210 had_ea 208 had_early_param = true;
211 } else if (!p->setup_f 209 } else if (!p->setup_func) {
212 pr_warn("Param 210 pr_warn("Parameter %s is obsolete, ignored\n",
213 p->str 211 p->str);
214 return true; 212 return true;
215 } else if (p->setup_fu 213 } else if (p->setup_func(line + n))
216 return true; 214 return true;
217 } 215 }
218 p++; 216 p++;
219 } while (p < __setup_end); 217 } while (p < __setup_end);
220 218
221 return had_early_param; 219 return had_early_param;
222 } 220 }
223 221
224 /* 222 /*
225 * This should be approx 2 Bo*oMips to start ( 223 * This should be approx 2 Bo*oMips to start (note initial shift), and will
226 * still work even if initially too large, it 224 * still work even if initially too large, it will just take slightly longer
227 */ 225 */
228 unsigned long loops_per_jiffy = (1<<12); 226 unsigned long loops_per_jiffy = (1<<12);
229 EXPORT_SYMBOL(loops_per_jiffy); 227 EXPORT_SYMBOL(loops_per_jiffy);
230 228
231 static int __init debug_kernel(char *str) 229 static int __init debug_kernel(char *str)
232 { 230 {
233 console_loglevel = CONSOLE_LOGLEVEL_DE 231 console_loglevel = CONSOLE_LOGLEVEL_DEBUG;
234 return 0; 232 return 0;
235 } 233 }
236 234
237 static int __init quiet_kernel(char *str) 235 static int __init quiet_kernel(char *str)
238 { 236 {
239 console_loglevel = CONSOLE_LOGLEVEL_QU 237 console_loglevel = CONSOLE_LOGLEVEL_QUIET;
240 return 0; 238 return 0;
241 } 239 }
242 240
243 early_param("debug", debug_kernel); 241 early_param("debug", debug_kernel);
244 early_param("quiet", quiet_kernel); 242 early_param("quiet", quiet_kernel);
245 243
246 static int __init loglevel(char *str) 244 static int __init loglevel(char *str)
247 { 245 {
248 int newlevel; 246 int newlevel;
249 247
250 /* 248 /*
251 * Only update loglevel value when a c 249 * Only update loglevel value when a correct setting was passed,
252 * to prevent blind crashes (when logl 250 * to prevent blind crashes (when loglevel being set to 0) that
253 * are quite hard to debug 251 * are quite hard to debug
254 */ 252 */
255 if (get_option(&str, &newlevel)) { 253 if (get_option(&str, &newlevel)) {
256 console_loglevel = newlevel; 254 console_loglevel = newlevel;
257 return 0; 255 return 0;
258 } 256 }
259 257
260 return -EINVAL; 258 return -EINVAL;
261 } 259 }
262 260
263 early_param("loglevel", loglevel); 261 early_param("loglevel", loglevel);
264 262
265 #ifdef CONFIG_BLK_DEV_INITRD 263 #ifdef CONFIG_BLK_DEV_INITRD
266 static void * __init get_boot_config_from_init 264 static void * __init get_boot_config_from_initrd(size_t *_size)
267 { 265 {
268 u32 size, csum; 266 u32 size, csum;
269 char *data; 267 char *data;
270 u32 *hdr; 268 u32 *hdr;
271 int i; 269 int i;
272 270
273 if (!initrd_end) 271 if (!initrd_end)
274 return NULL; 272 return NULL;
275 273
276 data = (char *)initrd_end - BOOTCONFIG 274 data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN;
277 /* 275 /*
278 * Since Grub may align the size of in 276 * Since Grub may align the size of initrd to 4, we must
279 * check the preceding 3 bytes as well 277 * check the preceding 3 bytes as well.
280 */ 278 */
281 for (i = 0; i < 4; i++) { 279 for (i = 0; i < 4; i++) {
282 if (!memcmp(data, BOOTCONFIG_M 280 if (!memcmp(data, BOOTCONFIG_MAGIC, BOOTCONFIG_MAGIC_LEN))
283 goto found; 281 goto found;
284 data--; 282 data--;
285 } 283 }
286 return NULL; 284 return NULL;
287 285
288 found: 286 found:
289 hdr = (u32 *)(data - 8); 287 hdr = (u32 *)(data - 8);
290 size = le32_to_cpu(hdr[0]); 288 size = le32_to_cpu(hdr[0]);
291 csum = le32_to_cpu(hdr[1]); 289 csum = le32_to_cpu(hdr[1]);
292 290
293 data = ((void *)hdr) - size; 291 data = ((void *)hdr) - size;
294 if ((unsigned long)data < initrd_start 292 if ((unsigned long)data < initrd_start) {
295 pr_err("bootconfig size %d is 293 pr_err("bootconfig size %d is greater than initrd size %ld\n",
296 size, initrd_end - ini 294 size, initrd_end - initrd_start);
297 return NULL; 295 return NULL;
298 } 296 }
299 297
300 if (xbc_calc_checksum(data, size) != c 298 if (xbc_calc_checksum(data, size) != csum) {
301 pr_err("bootconfig checksum fa 299 pr_err("bootconfig checksum failed\n");
302 return NULL; 300 return NULL;
303 } 301 }
304 302
305 /* Remove bootconfig from initramfs/in 303 /* Remove bootconfig from initramfs/initrd */
306 initrd_end = (unsigned long)data; 304 initrd_end = (unsigned long)data;
307 if (_size) 305 if (_size)
308 *_size = size; 306 *_size = size;
309 307
310 return data; 308 return data;
311 } 309 }
312 #else 310 #else
313 static void * __init get_boot_config_from_init 311 static void * __init get_boot_config_from_initrd(size_t *_size)
314 { 312 {
315 return NULL; 313 return NULL;
316 } 314 }
317 #endif 315 #endif
318 316
319 #ifdef CONFIG_BOOT_CONFIG 317 #ifdef CONFIG_BOOT_CONFIG
320 318
321 static char xbc_namebuf[XBC_KEYLEN_MAX] __init 319 static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
322 320
323 #define rest(dst, end) ((end) > (dst) ? (end) 321 #define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
324 322
325 static int __init xbc_snprint_cmdline(char *bu 323 static int __init xbc_snprint_cmdline(char *buf, size_t size,
326 struct x 324 struct xbc_node *root)
327 { 325 {
328 struct xbc_node *knode, *vnode; 326 struct xbc_node *knode, *vnode;
329 char *end = buf + size; 327 char *end = buf + size;
330 const char *val, *q; !! 328 const char *val;
331 int ret; 329 int ret;
332 330
333 xbc_node_for_each_key_value(root, knod 331 xbc_node_for_each_key_value(root, knode, val) {
334 ret = xbc_node_compose_key_aft 332 ret = xbc_node_compose_key_after(root, knode,
335 xbc_na 333 xbc_namebuf, XBC_KEYLEN_MAX);
336 if (ret < 0) 334 if (ret < 0)
337 return ret; 335 return ret;
338 336
339 vnode = xbc_node_get_child(kno 337 vnode = xbc_node_get_child(knode);
340 if (!vnode) { 338 if (!vnode) {
341 ret = snprintf(buf, re 339 ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
342 if (ret < 0) 340 if (ret < 0)
343 return ret; 341 return ret;
344 buf += ret; 342 buf += ret;
345 continue; 343 continue;
346 } 344 }
347 xbc_array_for_each_value(vnode 345 xbc_array_for_each_value(vnode, val) {
348 /* !! 346 ret = snprintf(buf, rest(buf, end), "%s=\"%s\" ",
349 * For prettier and mo !! 347 xbc_namebuf, val);
350 * quote the value whe <<
351 * whitespace. <<
352 */ <<
353 q = strpbrk(val, " \t\ <<
354 ret = snprintf(buf, re <<
355 xbc_nam <<
356 if (ret < 0) 348 if (ret < 0)
357 return ret; 349 return ret;
358 buf += ret; 350 buf += ret;
359 } 351 }
360 } 352 }
361 353
362 return buf - (end - size); 354 return buf - (end - size);
363 } 355 }
364 #undef rest 356 #undef rest
365 357
366 /* Make an extra command line under given key 358 /* Make an extra command line under given key word */
367 static char * __init xbc_make_cmdline(const ch 359 static char * __init xbc_make_cmdline(const char *key)
368 { 360 {
369 struct xbc_node *root; 361 struct xbc_node *root;
370 char *new_cmdline; 362 char *new_cmdline;
371 int ret, len = 0; 363 int ret, len = 0;
372 364
373 root = xbc_find_node(key); 365 root = xbc_find_node(key);
374 if (!root) 366 if (!root)
375 return NULL; 367 return NULL;
376 368
377 /* Count required buffer size */ 369 /* Count required buffer size */
378 len = xbc_snprint_cmdline(NULL, 0, roo 370 len = xbc_snprint_cmdline(NULL, 0, root);
379 if (len <= 0) 371 if (len <= 0)
380 return NULL; 372 return NULL;
381 373
382 new_cmdline = memblock_alloc(len + 1, 374 new_cmdline = memblock_alloc(len + 1, SMP_CACHE_BYTES);
383 if (!new_cmdline) { 375 if (!new_cmdline) {
384 pr_err("Failed to allocate mem 376 pr_err("Failed to allocate memory for extra kernel cmdline.\n");
385 return NULL; 377 return NULL;
386 } 378 }
387 379
388 ret = xbc_snprint_cmdline(new_cmdline, 380 ret = xbc_snprint_cmdline(new_cmdline, len + 1, root);
389 if (ret < 0 || ret > len) { 381 if (ret < 0 || ret > len) {
390 pr_err("Failed to print extra 382 pr_err("Failed to print extra kernel cmdline.\n");
391 memblock_free(new_cmdline, len 383 memblock_free(new_cmdline, len + 1);
392 return NULL; 384 return NULL;
393 } 385 }
394 386
395 return new_cmdline; 387 return new_cmdline;
396 } 388 }
397 389
398 static int __init bootconfig_params(char *para 390 static int __init bootconfig_params(char *param, char *val,
399 const char 391 const char *unused, void *arg)
400 { 392 {
401 if (strcmp(param, "bootconfig") == 0) 393 if (strcmp(param, "bootconfig") == 0) {
402 bootconfig_found = true; 394 bootconfig_found = true;
403 } 395 }
404 return 0; 396 return 0;
405 } 397 }
406 398
407 static int __init warn_bootconfig(char *str) 399 static int __init warn_bootconfig(char *str)
408 { 400 {
409 /* The 'bootconfig' has been handled b 401 /* The 'bootconfig' has been handled by bootconfig_params(). */
410 return 0; 402 return 0;
411 } 403 }
412 404
413 static void __init setup_boot_config(void) 405 static void __init setup_boot_config(void)
414 { 406 {
415 static char tmp_cmdline[COMMAND_LINE_S 407 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
416 const char *msg, *data; 408 const char *msg, *data;
417 int pos, ret; 409 int pos, ret;
418 size_t size; 410 size_t size;
419 char *err; 411 char *err;
420 412
421 /* Cut out the bootconfig data even if 413 /* Cut out the bootconfig data even if we have no bootconfig option */
422 data = get_boot_config_from_initrd(&si 414 data = get_boot_config_from_initrd(&size);
423 /* If there is no bootconfig in initrd 415 /* If there is no bootconfig in initrd, try embedded one. */
424 if (!data) 416 if (!data)
425 data = xbc_get_embedded_bootco 417 data = xbc_get_embedded_bootconfig(&size);
426 418
427 strscpy(tmp_cmdline, boot_command_line 419 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
428 err = parse_args("bootconfig", tmp_cmd 420 err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL,
429 bootconfig_params); 421 bootconfig_params);
430 422
431 if (IS_ERR(err) || !(bootconfig_found 423 if (IS_ERR(err) || !(bootconfig_found || IS_ENABLED(CONFIG_BOOT_CONFIG_FORCE)))
432 return; 424 return;
433 425
434 /* parse_args() stops at the next para 426 /* parse_args() stops at the next param of '--' and returns an address */
435 if (err) 427 if (err)
436 initargs_offs = err - tmp_cmdl 428 initargs_offs = err - tmp_cmdline;
437 429
438 if (!data) { 430 if (!data) {
439 /* If user intended to use boo 431 /* If user intended to use bootconfig, show an error level message */
440 if (bootconfig_found) 432 if (bootconfig_found)
441 pr_err("'bootconfig' f 433 pr_err("'bootconfig' found on command line, but no bootconfig found\n");
442 else 434 else
443 pr_info("No bootconfig 435 pr_info("No bootconfig data provided, so skipping bootconfig");
444 return; 436 return;
445 } 437 }
446 438
447 if (size >= XBC_DATA_MAX) { 439 if (size >= XBC_DATA_MAX) {
448 pr_err("bootconfig size %ld gr 440 pr_err("bootconfig size %ld greater than max size %d\n",
449 (long)size, XBC_DATA_M 441 (long)size, XBC_DATA_MAX);
450 return; 442 return;
451 } 443 }
452 444
453 ret = xbc_init(data, size, &msg, &pos) 445 ret = xbc_init(data, size, &msg, &pos);
454 if (ret < 0) { 446 if (ret < 0) {
455 if (pos < 0) 447 if (pos < 0)
456 pr_err("Failed to init 448 pr_err("Failed to init bootconfig: %s.\n", msg);
457 else 449 else
458 pr_err("Failed to pars 450 pr_err("Failed to parse bootconfig: %s at %d.\n",
459 msg, pos); 451 msg, pos);
460 } else { 452 } else {
461 xbc_get_info(&ret, NULL); 453 xbc_get_info(&ret, NULL);
462 pr_info("Load bootconfig: %ld 454 pr_info("Load bootconfig: %ld bytes %d nodes\n", (long)size, ret);
463 /* keys starting with "kernel. 455 /* keys starting with "kernel." are passed via cmdline */
464 extra_command_line = xbc_make_ 456 extra_command_line = xbc_make_cmdline("kernel");
465 /* Also, "init." keys are init 457 /* Also, "init." keys are init arguments */
466 extra_init_args = xbc_make_cmd 458 extra_init_args = xbc_make_cmdline("init");
467 } 459 }
468 return; 460 return;
469 } 461 }
470 462
471 static void __init exit_boot_config(void) 463 static void __init exit_boot_config(void)
472 { 464 {
473 xbc_exit(); 465 xbc_exit();
474 } 466 }
475 467
476 #else /* !CONFIG_BOOT_CONFIG */ 468 #else /* !CONFIG_BOOT_CONFIG */
477 469
478 static void __init setup_boot_config(void) 470 static void __init setup_boot_config(void)
479 { 471 {
480 /* Remove bootconfig data from initrd 472 /* Remove bootconfig data from initrd */
481 get_boot_config_from_initrd(NULL); 473 get_boot_config_from_initrd(NULL);
482 } 474 }
483 475
484 static int __init warn_bootconfig(char *str) 476 static int __init warn_bootconfig(char *str)
485 { 477 {
486 pr_warn("WARNING: 'bootconfig' found o 478 pr_warn("WARNING: 'bootconfig' found on the kernel command line but CONFIG_BOOT_CONFIG is not set.\n");
487 return 0; 479 return 0;
488 } 480 }
489 481
490 #define exit_boot_config() do {} while (0 482 #define exit_boot_config() do {} while (0)
491 483
492 #endif /* CONFIG_BOOT_CONFIG */ 484 #endif /* CONFIG_BOOT_CONFIG */
493 485
494 early_param("bootconfig", warn_bootconfig); 486 early_param("bootconfig", warn_bootconfig);
495 487
496 bool __init cmdline_has_extra_options(void) 488 bool __init cmdline_has_extra_options(void)
497 { 489 {
498 return extra_command_line || extra_ini 490 return extra_command_line || extra_init_args;
499 } 491 }
500 492
501 /* Change NUL term back to "=", to make "param 493 /* Change NUL term back to "=", to make "param" the whole string. */
502 static void __init repair_env_string(char *par 494 static void __init repair_env_string(char *param, char *val)
503 { 495 {
504 if (val) { 496 if (val) {
505 /* param=val or param="val"? * 497 /* param=val or param="val"? */
506 if (val == param+strlen(param) 498 if (val == param+strlen(param)+1)
507 val[-1] = '='; 499 val[-1] = '=';
508 else if (val == param+strlen(p 500 else if (val == param+strlen(param)+2) {
509 val[-2] = '='; 501 val[-2] = '=';
510 memmove(val-1, val, st 502 memmove(val-1, val, strlen(val)+1);
511 } else 503 } else
512 BUG(); 504 BUG();
513 } 505 }
514 } 506 }
515 507
516 /* Anything after -- gets handed straight to i 508 /* Anything after -- gets handed straight to init. */
517 static int __init set_init_arg(char *param, ch 509 static int __init set_init_arg(char *param, char *val,
518 const char *unu 510 const char *unused, void *arg)
519 { 511 {
520 unsigned int i; 512 unsigned int i;
521 513
522 if (panic_later) 514 if (panic_later)
523 return 0; 515 return 0;
524 516
525 repair_env_string(param, val); 517 repair_env_string(param, val);
526 518
527 for (i = 0; argv_init[i]; i++) { 519 for (i = 0; argv_init[i]; i++) {
528 if (i == MAX_INIT_ARGS) { 520 if (i == MAX_INIT_ARGS) {
529 panic_later = "init"; 521 panic_later = "init";
530 panic_param = param; 522 panic_param = param;
531 return 0; 523 return 0;
532 } 524 }
533 } 525 }
534 argv_init[i] = param; 526 argv_init[i] = param;
535 return 0; 527 return 0;
536 } 528 }
537 529
538 /* 530 /*
539 * Unknown boot options get handed to init, un 531 * Unknown boot options get handed to init, unless they look like
540 * unused parameters (modprobe will find them 532 * unused parameters (modprobe will find them in /proc/cmdline).
541 */ 533 */
542 static int __init unknown_bootoption(char *par 534 static int __init unknown_bootoption(char *param, char *val,
543 const cha 535 const char *unused, void *arg)
544 { 536 {
545 size_t len = strlen(param); 537 size_t len = strlen(param);
546 538
547 /* Handle params aliased to sysctls */ 539 /* Handle params aliased to sysctls */
548 if (sysctl_is_alias(param)) 540 if (sysctl_is_alias(param))
549 return 0; 541 return 0;
550 542
551 repair_env_string(param, val); 543 repair_env_string(param, val);
552 544
553 /* Handle obsolete-style parameters */ 545 /* Handle obsolete-style parameters */
554 if (obsolete_checksetup(param)) 546 if (obsolete_checksetup(param))
555 return 0; 547 return 0;
556 548
557 /* Unused module parameter. */ 549 /* Unused module parameter. */
558 if (strnchr(param, len, '.')) 550 if (strnchr(param, len, '.'))
559 return 0; 551 return 0;
560 552
561 if (panic_later) 553 if (panic_later)
562 return 0; 554 return 0;
563 555
564 if (val) { 556 if (val) {
565 /* Environment option */ 557 /* Environment option */
566 unsigned int i; 558 unsigned int i;
567 for (i = 0; envp_init[i]; i++) 559 for (i = 0; envp_init[i]; i++) {
568 if (i == MAX_INIT_ENVS 560 if (i == MAX_INIT_ENVS) {
569 panic_later = 561 panic_later = "env";
570 panic_param = 562 panic_param = param;
571 } 563 }
572 if (!strncmp(param, en 564 if (!strncmp(param, envp_init[i], len+1))
573 break; 565 break;
574 } 566 }
575 envp_init[i] = param; 567 envp_init[i] = param;
576 } else { 568 } else {
577 /* Command line option */ 569 /* Command line option */
578 unsigned int i; 570 unsigned int i;
579 for (i = 0; argv_init[i]; i++) 571 for (i = 0; argv_init[i]; i++) {
580 if (i == MAX_INIT_ARGS 572 if (i == MAX_INIT_ARGS) {
581 panic_later = 573 panic_later = "init";
582 panic_param = 574 panic_param = param;
583 } 575 }
584 } 576 }
585 argv_init[i] = param; 577 argv_init[i] = param;
586 } 578 }
587 return 0; 579 return 0;
588 } 580 }
589 581
590 static int __init init_setup(char *str) 582 static int __init init_setup(char *str)
591 { 583 {
592 unsigned int i; 584 unsigned int i;
593 585
594 execute_command = str; 586 execute_command = str;
595 /* 587 /*
596 * In case LILO is going to boot us wi 588 * In case LILO is going to boot us with default command line,
597 * it prepends "auto" before the whole 589 * it prepends "auto" before the whole cmdline which makes
598 * the shell think it should execute a 590 * the shell think it should execute a script with such name.
599 * So we ignore all arguments entered 591 * So we ignore all arguments entered _before_ init=... [MJ]
600 */ 592 */
601 for (i = 1; i < MAX_INIT_ARGS; i++) 593 for (i = 1; i < MAX_INIT_ARGS; i++)
602 argv_init[i] = NULL; 594 argv_init[i] = NULL;
603 return 1; 595 return 1;
604 } 596 }
605 __setup("init=", init_setup); 597 __setup("init=", init_setup);
606 598
607 static int __init rdinit_setup(char *str) 599 static int __init rdinit_setup(char *str)
608 { 600 {
609 unsigned int i; 601 unsigned int i;
610 602
611 ramdisk_execute_command = str; 603 ramdisk_execute_command = str;
612 /* See "auto" comment in init_setup */ 604 /* See "auto" comment in init_setup */
613 for (i = 1; i < MAX_INIT_ARGS; i++) 605 for (i = 1; i < MAX_INIT_ARGS; i++)
614 argv_init[i] = NULL; 606 argv_init[i] = NULL;
615 return 1; 607 return 1;
616 } 608 }
617 __setup("rdinit=", rdinit_setup); 609 __setup("rdinit=", rdinit_setup);
618 610
619 #ifndef CONFIG_SMP 611 #ifndef CONFIG_SMP
>> 612 static const unsigned int setup_max_cpus = NR_CPUS;
620 static inline void setup_nr_cpu_ids(void) { } 613 static inline void setup_nr_cpu_ids(void) { }
621 static inline void smp_prepare_cpus(unsigned i 614 static inline void smp_prepare_cpus(unsigned int maxcpus) { }
622 #endif 615 #endif
623 616
624 /* 617 /*
625 * We need to store the untouched command line 618 * We need to store the untouched command line for future reference.
626 * We also need to store the touched command l 619 * We also need to store the touched command line since the parameter
627 * parsing is performed in place, and we shoul 620 * parsing is performed in place, and we should allow a component to
628 * store reference of name/value for future re 621 * store reference of name/value for future reference.
629 */ 622 */
630 static void __init setup_command_line(char *co 623 static void __init setup_command_line(char *command_line)
631 { 624 {
632 size_t len, xlen = 0, ilen = 0; 625 size_t len, xlen = 0, ilen = 0;
633 626
634 if (extra_command_line) 627 if (extra_command_line)
635 xlen = strlen(extra_command_li 628 xlen = strlen(extra_command_line);
636 if (extra_init_args) { !! 629 if (extra_init_args)
637 extra_init_args = strim(extra_ <<
638 ilen = strlen(extra_init_args) 630 ilen = strlen(extra_init_args) + 4; /* for " -- " */
639 } <<
640 631
641 len = xlen + strlen(boot_command_line) !! 632 len = xlen + strlen(boot_command_line) + 1;
642 633
643 saved_command_line = memblock_alloc(le !! 634 saved_command_line = memblock_alloc(len + ilen, SMP_CACHE_BYTES);
644 if (!saved_command_line) 635 if (!saved_command_line)
645 panic("%s: Failed to allocate !! 636 panic("%s: Failed to allocate %zu bytes\n", __func__, len + ilen);
646 637
647 len = xlen + strlen(command_line) + 1; 638 len = xlen + strlen(command_line) + 1;
648 639
649 static_command_line = memblock_alloc(l 640 static_command_line = memblock_alloc(len, SMP_CACHE_BYTES);
650 if (!static_command_line) 641 if (!static_command_line)
651 panic("%s: Failed to allocate 642 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
652 643
653 if (xlen) { 644 if (xlen) {
654 /* 645 /*
655 * We have to put extra_comman 646 * We have to put extra_command_line before boot command
656 * lines because there could b 647 * lines because there could be dashes (separator of init
657 * command line) in the comman 648 * command line) in the command lines.
658 */ 649 */
659 strcpy(saved_command_line, ext 650 strcpy(saved_command_line, extra_command_line);
660 strcpy(static_command_line, ex 651 strcpy(static_command_line, extra_command_line);
661 } 652 }
662 strcpy(saved_command_line + xlen, boot 653 strcpy(saved_command_line + xlen, boot_command_line);
663 strcpy(static_command_line + xlen, com 654 strcpy(static_command_line + xlen, command_line);
664 655
665 if (ilen) { 656 if (ilen) {
666 /* 657 /*
667 * Append supplemental init bo 658 * Append supplemental init boot args to saved_command_line
668 * so that user can check what 659 * so that user can check what command line options passed
669 * to init. 660 * to init.
670 * The order should always be 661 * The order should always be
671 * " -- "[bootconfig init-para 662 * " -- "[bootconfig init-param][cmdline init-param]
672 */ 663 */
673 if (initargs_offs) { 664 if (initargs_offs) {
674 len = xlen + initargs_ 665 len = xlen + initargs_offs;
675 strcpy(saved_command_l 666 strcpy(saved_command_line + len, extra_init_args);
676 len += ilen - 4; 667 len += ilen - 4; /* strlen(extra_init_args) */
677 strcpy(saved_command_l 668 strcpy(saved_command_line + len,
678 boot_command_l 669 boot_command_line + initargs_offs - 1);
679 } else { 670 } else {
680 len = strlen(saved_com 671 len = strlen(saved_command_line);
681 strcpy(saved_command_l 672 strcpy(saved_command_line + len, " -- ");
682 len += 4; 673 len += 4;
683 strcpy(saved_command_l 674 strcpy(saved_command_line + len, extra_init_args);
684 } 675 }
685 } 676 }
686 677
687 saved_command_line_len = strlen(saved_ 678 saved_command_line_len = strlen(saved_command_line);
688 } 679 }
689 680
690 /* 681 /*
691 * We need to finalize in a non-__init functio 682 * We need to finalize in a non-__init function or else race conditions
692 * between the root thread and the init thread 683 * between the root thread and the init thread may cause start_kernel to
693 * be reaped by free_initmem before the root t 684 * be reaped by free_initmem before the root thread has proceeded to
694 * cpu_idle. 685 * cpu_idle.
695 * 686 *
696 * gcc-3.4 accidentally inlines this function, 687 * gcc-3.4 accidentally inlines this function, so use noinline.
697 */ 688 */
698 689
699 static __initdata DECLARE_COMPLETION(kthreadd_ 690 static __initdata DECLARE_COMPLETION(kthreadd_done);
700 691
701 static noinline void __ref __noreturn rest_ini !! 692 noinline void __ref __noreturn rest_init(void)
702 { 693 {
703 struct task_struct *tsk; 694 struct task_struct *tsk;
704 int pid; 695 int pid;
705 696
706 rcu_scheduler_starting(); 697 rcu_scheduler_starting();
707 /* 698 /*
708 * We need to spawn init first so that 699 * We need to spawn init first so that it obtains pid 1, however
709 * the init task will end up wanting t 700 * the init task will end up wanting to create kthreads, which, if
710 * we schedule it before we create kth 701 * we schedule it before we create kthreadd, will OOPS.
711 */ 702 */
712 pid = user_mode_thread(kernel_init, NU 703 pid = user_mode_thread(kernel_init, NULL, CLONE_FS);
713 /* 704 /*
714 * Pin init on the boot CPU. Task migr 705 * Pin init on the boot CPU. Task migration is not properly working
715 * until sched_init_smp() has been run 706 * until sched_init_smp() has been run. It will set the allowed
716 * CPUs for init to the non isolated C 707 * CPUs for init to the non isolated CPUs.
717 */ 708 */
718 rcu_read_lock(); 709 rcu_read_lock();
719 tsk = find_task_by_pid_ns(pid, &init_p 710 tsk = find_task_by_pid_ns(pid, &init_pid_ns);
720 tsk->flags |= PF_NO_SETAFFINITY; 711 tsk->flags |= PF_NO_SETAFFINITY;
721 set_cpus_allowed_ptr(tsk, cpumask_of(s 712 set_cpus_allowed_ptr(tsk, cpumask_of(smp_processor_id()));
722 rcu_read_unlock(); 713 rcu_read_unlock();
723 714
724 numa_default_policy(); 715 numa_default_policy();
725 pid = kernel_thread(kthreadd, NULL, NU 716 pid = kernel_thread(kthreadd, NULL, NULL, CLONE_FS | CLONE_FILES);
726 rcu_read_lock(); 717 rcu_read_lock();
727 kthreadd_task = find_task_by_pid_ns(pi 718 kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
728 rcu_read_unlock(); 719 rcu_read_unlock();
729 720
730 /* 721 /*
731 * Enable might_sleep() and smp_proces 722 * Enable might_sleep() and smp_processor_id() checks.
732 * They cannot be enabled earlier beca 723 * They cannot be enabled earlier because with CONFIG_PREEMPTION=y
733 * kernel_thread() would trigger might 724 * kernel_thread() would trigger might_sleep() splats. With
734 * CONFIG_PREEMPT_VOLUNTARY=y the init 725 * CONFIG_PREEMPT_VOLUNTARY=y the init task might have scheduled
735 * already, but it's stuck on the kthr 726 * already, but it's stuck on the kthreadd_done completion.
736 */ 727 */
737 system_state = SYSTEM_SCHEDULING; 728 system_state = SYSTEM_SCHEDULING;
738 729
739 complete(&kthreadd_done); 730 complete(&kthreadd_done);
740 731
741 /* 732 /*
742 * The boot idle thread must execute s 733 * The boot idle thread must execute schedule()
743 * at least once to get things moving: 734 * at least once to get things moving:
744 */ 735 */
745 schedule_preempt_disabled(); 736 schedule_preempt_disabled();
746 /* Call into cpu_idle with preempt dis 737 /* Call into cpu_idle with preempt disabled */
747 cpu_startup_entry(CPUHP_ONLINE); 738 cpu_startup_entry(CPUHP_ONLINE);
748 } 739 }
749 740
750 /* Check for early params. */ 741 /* Check for early params. */
751 static int __init do_early_param(char *param, 742 static int __init do_early_param(char *param, char *val,
752 const char *u 743 const char *unused, void *arg)
753 { 744 {
754 const struct obs_kernel_param *p; 745 const struct obs_kernel_param *p;
755 746
756 for (p = __setup_start; p < __setup_en 747 for (p = __setup_start; p < __setup_end; p++) {
757 if ((p->early && parameq(param 748 if ((p->early && parameq(param, p->str)) ||
758 (strcmp(param, "console") 749 (strcmp(param, "console") == 0 &&
759 strcmp(p->str, "earlycon" 750 strcmp(p->str, "earlycon") == 0)
760 ) { 751 ) {
761 if (p->setup_func(val) 752 if (p->setup_func(val) != 0)
762 pr_warn("Malfo 753 pr_warn("Malformed early option '%s'\n", param);
763 } 754 }
764 } 755 }
765 /* We accept everything at this stage. 756 /* We accept everything at this stage. */
766 return 0; 757 return 0;
767 } 758 }
768 759
769 void __init parse_early_options(char *cmdline) 760 void __init parse_early_options(char *cmdline)
770 { 761 {
771 parse_args("early options", cmdline, N 762 parse_args("early options", cmdline, NULL, 0, 0, 0, NULL,
772 do_early_param); 763 do_early_param);
773 } 764 }
774 765
775 /* Arch code calls this early on, or if not, j 766 /* Arch code calls this early on, or if not, just before other parsing. */
776 void __init parse_early_param(void) 767 void __init parse_early_param(void)
777 { 768 {
778 static int done __initdata; 769 static int done __initdata;
779 static char tmp_cmdline[COMMAND_LINE_S 770 static char tmp_cmdline[COMMAND_LINE_SIZE] __initdata;
780 771
781 if (done) 772 if (done)
782 return; 773 return;
783 774
784 /* All fall through to do_early_param. 775 /* All fall through to do_early_param. */
785 strscpy(tmp_cmdline, boot_command_line 776 strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
786 parse_early_options(tmp_cmdline); 777 parse_early_options(tmp_cmdline);
787 done = 1; 778 done = 1;
788 } 779 }
789 780
790 void __init __weak arch_post_acpi_subsys_init( 781 void __init __weak arch_post_acpi_subsys_init(void) { }
791 782
792 void __init __weak smp_setup_processor_id(void 783 void __init __weak smp_setup_processor_id(void)
793 { 784 {
794 } 785 }
795 786
796 void __init __weak smp_prepare_boot_cpu(void) <<
797 { <<
798 } <<
799 <<
800 # if THREAD_SIZE >= PAGE_SIZE 787 # if THREAD_SIZE >= PAGE_SIZE
801 void __init __weak thread_stack_cache_init(voi 788 void __init __weak thread_stack_cache_init(void)
802 { 789 {
803 } 790 }
804 #endif 791 #endif
805 792
806 void __init __weak poking_init(void) { } 793 void __init __weak poking_init(void) { }
807 794
808 void __init __weak pgtable_cache_init(void) { 795 void __init __weak pgtable_cache_init(void) { }
809 796
810 void __init __weak trap_init(void) { } 797 void __init __weak trap_init(void) { }
811 798
812 bool initcall_debug; 799 bool initcall_debug;
813 core_param(initcall_debug, initcall_debug, boo 800 core_param(initcall_debug, initcall_debug, bool, 0644);
814 801
815 #ifdef TRACEPOINTS_ENABLED 802 #ifdef TRACEPOINTS_ENABLED
816 static void __init initcall_debug_enable(void) 803 static void __init initcall_debug_enable(void);
817 #else 804 #else
818 static inline void initcall_debug_enable(void) 805 static inline void initcall_debug_enable(void)
819 { 806 {
820 } 807 }
821 #endif 808 #endif
822 809
823 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET 810 #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET
824 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KS 811 DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT,
825 randomize_kstack_of 812 randomize_kstack_offset);
826 DEFINE_PER_CPU(u32, kstack_offset); 813 DEFINE_PER_CPU(u32, kstack_offset);
827 814
828 static int __init early_randomize_kstack_offse 815 static int __init early_randomize_kstack_offset(char *buf)
829 { 816 {
830 int ret; 817 int ret;
831 bool bool_result; 818 bool bool_result;
832 819
833 ret = kstrtobool(buf, &bool_result); 820 ret = kstrtobool(buf, &bool_result);
834 if (ret) 821 if (ret)
835 return ret; 822 return ret;
836 823
837 if (bool_result) 824 if (bool_result)
838 static_branch_enable(&randomiz 825 static_branch_enable(&randomize_kstack_offset);
839 else 826 else
840 static_branch_disable(&randomi 827 static_branch_disable(&randomize_kstack_offset);
841 return 0; 828 return 0;
842 } 829 }
843 early_param("randomize_kstack_offset", early_r 830 early_param("randomize_kstack_offset", early_randomize_kstack_offset);
844 #endif 831 #endif
845 832
>> 833 void __init __weak __noreturn arch_call_rest_init(void)
>> 834 {
>> 835 rest_init();
>> 836 }
>> 837
846 static void __init print_unknown_bootoptions(v 838 static void __init print_unknown_bootoptions(void)
847 { 839 {
848 char *unknown_options; 840 char *unknown_options;
849 char *end; 841 char *end;
850 const char *const *p; 842 const char *const *p;
851 size_t len; 843 size_t len;
852 844
853 if (panic_later || (!argv_init[1] && ! 845 if (panic_later || (!argv_init[1] && !envp_init[2]))
854 return; 846 return;
855 847
856 /* 848 /*
857 * Determine how many options we have 849 * Determine how many options we have to print out, plus a space
858 * before each 850 * before each
859 */ 851 */
860 len = 1; /* null terminator */ 852 len = 1; /* null terminator */
861 for (p = &argv_init[1]; *p; p++) { 853 for (p = &argv_init[1]; *p; p++) {
862 len++; 854 len++;
863 len += strlen(*p); 855 len += strlen(*p);
864 } 856 }
865 for (p = &envp_init[2]; *p; p++) { 857 for (p = &envp_init[2]; *p; p++) {
866 len++; 858 len++;
867 len += strlen(*p); 859 len += strlen(*p);
868 } 860 }
869 861
870 unknown_options = memblock_alloc(len, 862 unknown_options = memblock_alloc(len, SMP_CACHE_BYTES);
871 if (!unknown_options) { 863 if (!unknown_options) {
872 pr_err("%s: Failed to allocate 864 pr_err("%s: Failed to allocate %zu bytes\n",
873 __func__, len); 865 __func__, len);
874 return; 866 return;
875 } 867 }
876 end = unknown_options; 868 end = unknown_options;
877 869
878 for (p = &argv_init[1]; *p; p++) 870 for (p = &argv_init[1]; *p; p++)
879 end += sprintf(end, " %s", *p) 871 end += sprintf(end, " %s", *p);
880 for (p = &envp_init[2]; *p; p++) 872 for (p = &envp_init[2]; *p; p++)
881 end += sprintf(end, " %s", *p) 873 end += sprintf(end, " %s", *p);
882 874
883 /* Start at unknown_options[1] to skip 875 /* Start at unknown_options[1] to skip the initial space */
884 pr_notice("Unknown kernel command line 876 pr_notice("Unknown kernel command line parameters \"%s\", will be passed to user space.\n",
885 &unknown_options[1]); 877 &unknown_options[1]);
886 memblock_free(unknown_options, len); 878 memblock_free(unknown_options, len);
887 } 879 }
888 880
889 static void __init early_numa_node_init(void) <<
890 { <<
891 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID <<
892 #ifndef cpu_to_node <<
893 int cpu; <<
894 <<
895 /* The early_cpu_to_node() should be r <<
896 for_each_possible_cpu(cpu) <<
897 set_cpu_numa_node(cpu, early_c <<
898 #endif <<
899 #endif <<
900 } <<
901 <<
902 asmlinkage __visible __init __no_sanitize_addr 881 asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector
903 void start_kernel(void) 882 void start_kernel(void)
904 { 883 {
905 char *command_line; 884 char *command_line;
906 char *after_dashes; 885 char *after_dashes;
907 886
908 set_task_stack_end_magic(&init_task); 887 set_task_stack_end_magic(&init_task);
909 smp_setup_processor_id(); 888 smp_setup_processor_id();
910 debug_objects_early_init(); 889 debug_objects_early_init();
911 init_vmlinux_build_id(); 890 init_vmlinux_build_id();
912 891
913 cgroup_init_early(); 892 cgroup_init_early();
914 893
915 local_irq_disable(); 894 local_irq_disable();
916 early_boot_irqs_disabled = true; 895 early_boot_irqs_disabled = true;
917 896
918 /* 897 /*
919 * Interrupts are still disabled. Do n 898 * Interrupts are still disabled. Do necessary setups, then
920 * enable them. 899 * enable them.
921 */ 900 */
922 boot_cpu_init(); 901 boot_cpu_init();
923 page_address_init(); 902 page_address_init();
924 pr_notice("%s", linux_banner); 903 pr_notice("%s", linux_banner);
925 setup_arch(&command_line); <<
926 /* Static keys and static calls are ne <<
927 jump_label_init(); <<
928 static_call_init(); <<
929 early_security_init(); 904 early_security_init();
>> 905 setup_arch(&command_line);
930 setup_boot_config(); 906 setup_boot_config();
931 setup_command_line(command_line); 907 setup_command_line(command_line);
932 setup_nr_cpu_ids(); 908 setup_nr_cpu_ids();
933 setup_per_cpu_areas(); 909 setup_per_cpu_areas();
934 smp_prepare_boot_cpu(); /* arch-specif 910 smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */
935 early_numa_node_init(); <<
936 boot_cpu_hotplug_init(); 911 boot_cpu_hotplug_init();
937 912
938 pr_notice("Kernel command line: %s\n", 913 pr_notice("Kernel command line: %s\n", saved_command_line);
939 /* parameters may set static keys */ 914 /* parameters may set static keys */
>> 915 jump_label_init();
940 parse_early_param(); 916 parse_early_param();
941 after_dashes = parse_args("Booting ker 917 after_dashes = parse_args("Booting kernel",
942 static_comma 918 static_command_line, __start___param,
943 __stop___par 919 __stop___param - __start___param,
944 -1, -1, NULL 920 -1, -1, NULL, &unknown_bootoption);
945 print_unknown_bootoptions(); 921 print_unknown_bootoptions();
946 if (!IS_ERR_OR_NULL(after_dashes)) 922 if (!IS_ERR_OR_NULL(after_dashes))
947 parse_args("Setting init args" 923 parse_args("Setting init args", after_dashes, NULL, 0, -1, -1,
948 NULL, set_init_arg) 924 NULL, set_init_arg);
949 if (extra_init_args) 925 if (extra_init_args)
950 parse_args("Setting extra init 926 parse_args("Setting extra init args", extra_init_args,
951 NULL, 0, -1, -1, NU 927 NULL, 0, -1, -1, NULL, set_init_arg);
952 928
953 /* Architectural and non-timekeeping r 929 /* Architectural and non-timekeeping rng init, before allocator init */
954 random_init_early(command_line); 930 random_init_early(command_line);
955 931
956 /* 932 /*
957 * These use large bootmem allocations 933 * These use large bootmem allocations and must precede
958 * initalization of page allocator 934 * initalization of page allocator
959 */ 935 */
960 setup_log_buf(0); 936 setup_log_buf(0);
961 vfs_caches_init_early(); 937 vfs_caches_init_early();
962 sort_main_extable(); 938 sort_main_extable();
963 trap_init(); 939 trap_init();
964 mm_core_init(); 940 mm_core_init();
965 poking_init(); 941 poking_init();
966 ftrace_init(); 942 ftrace_init();
967 943
968 /* trace_printk can be enabled here */ 944 /* trace_printk can be enabled here */
969 early_trace_init(); 945 early_trace_init();
970 946
971 /* 947 /*
972 * Set up the scheduler prior starting 948 * Set up the scheduler prior starting any interrupts (such as the
973 * timer interrupt). Full topology set 949 * timer interrupt). Full topology setup happens at smp_init()
974 * time - but meanwhile we still have 950 * time - but meanwhile we still have a functioning scheduler.
975 */ 951 */
976 sched_init(); 952 sched_init();
977 953
978 if (WARN(!irqs_disabled(), 954 if (WARN(!irqs_disabled(),
979 "Interrupts were enabled *ver 955 "Interrupts were enabled *very* early, fixing it\n"))
980 local_irq_disable(); 956 local_irq_disable();
981 radix_tree_init(); 957 radix_tree_init();
982 maple_tree_init(); 958 maple_tree_init();
983 959
984 /* 960 /*
985 * Set up housekeeping before setting 961 * Set up housekeeping before setting up workqueues to allow the unbound
986 * workqueue to take non-housekeeping 962 * workqueue to take non-housekeeping into account.
987 */ 963 */
988 housekeeping_init(); 964 housekeeping_init();
989 965
990 /* 966 /*
991 * Allow workqueue creation and work i 967 * Allow workqueue creation and work item queueing/cancelling
992 * early. Work item execution depends 968 * early. Work item execution depends on kthreads and starts after
993 * workqueue_init(). 969 * workqueue_init().
994 */ 970 */
995 workqueue_init_early(); 971 workqueue_init_early();
996 972
997 rcu_init(); 973 rcu_init();
998 974
999 /* Trace events are available after th 975 /* Trace events are available after this */
1000 trace_init(); 976 trace_init();
1001 977
1002 if (initcall_debug) 978 if (initcall_debug)
1003 initcall_debug_enable(); 979 initcall_debug_enable();
1004 980
1005 context_tracking_init(); 981 context_tracking_init();
1006 /* init some links before init_ISA_ir 982 /* init some links before init_ISA_irqs() */
1007 early_irq_init(); 983 early_irq_init();
1008 init_IRQ(); 984 init_IRQ();
1009 tick_init(); 985 tick_init();
1010 rcu_init_nohz(); 986 rcu_init_nohz();
1011 init_timers(); 987 init_timers();
1012 srcu_init(); 988 srcu_init();
1013 hrtimers_init(); 989 hrtimers_init();
1014 softirq_init(); 990 softirq_init();
1015 timekeeping_init(); 991 timekeeping_init();
1016 time_init(); 992 time_init();
1017 993
1018 /* This must be after timekeeping is 994 /* This must be after timekeeping is initialized */
1019 random_init(); 995 random_init();
1020 996
1021 /* These make use of the fully initia 997 /* These make use of the fully initialized rng */
1022 kfence_init(); 998 kfence_init();
1023 boot_init_stack_canary(); 999 boot_init_stack_canary();
1024 1000
1025 perf_event_init(); 1001 perf_event_init();
1026 profile_init(); 1002 profile_init();
1027 call_function_init(); 1003 call_function_init();
1028 WARN(!irqs_disabled(), "Interrupts we 1004 WARN(!irqs_disabled(), "Interrupts were enabled early\n");
1029 1005
1030 early_boot_irqs_disabled = false; 1006 early_boot_irqs_disabled = false;
1031 local_irq_enable(); 1007 local_irq_enable();
1032 1008
1033 kmem_cache_init_late(); 1009 kmem_cache_init_late();
1034 1010
1035 /* 1011 /*
1036 * HACK ALERT! This is early. We're e 1012 * HACK ALERT! This is early. We're enabling the console before
1037 * we've done PCI setups etc, and con 1013 * we've done PCI setups etc, and console_init() must be aware of
1038 * this. But we do want output early, 1014 * this. But we do want output early, in case something goes wrong.
1039 */ 1015 */
1040 console_init(); 1016 console_init();
1041 if (panic_later) 1017 if (panic_later)
1042 panic("Too many boot %s vars 1018 panic("Too many boot %s vars at `%s'", panic_later,
1043 panic_param); 1019 panic_param);
1044 1020
1045 lockdep_init(); 1021 lockdep_init();
1046 1022
1047 /* 1023 /*
1048 * Need to run this when irqs are ena 1024 * Need to run this when irqs are enabled, because it wants
1049 * to self-test [hard/soft]-irqs on/o 1025 * to self-test [hard/soft]-irqs on/off lock inversion bugs
1050 * too: 1026 * too:
1051 */ 1027 */
1052 locking_selftest(); 1028 locking_selftest();
1053 1029
1054 #ifdef CONFIG_BLK_DEV_INITRD 1030 #ifdef CONFIG_BLK_DEV_INITRD
1055 if (initrd_start && !initrd_below_sta 1031 if (initrd_start && !initrd_below_start_ok &&
1056 page_to_pfn(virt_to_page((void *) 1032 page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
1057 pr_crit("initrd overwritten ( 1033 pr_crit("initrd overwritten (0x%08lx < 0x%08lx) - disabling it.\n",
1058 page_to_pfn(virt_to_page( 1034 page_to_pfn(virt_to_page((void *)initrd_start)),
1059 min_low_pfn); 1035 min_low_pfn);
1060 initrd_start = 0; 1036 initrd_start = 0;
1061 } 1037 }
1062 #endif 1038 #endif
1063 setup_per_cpu_pageset(); 1039 setup_per_cpu_pageset();
1064 numa_policy_init(); 1040 numa_policy_init();
1065 acpi_early_init(); 1041 acpi_early_init();
1066 if (late_time_init) 1042 if (late_time_init)
1067 late_time_init(); 1043 late_time_init();
1068 sched_clock_init(); 1044 sched_clock_init();
1069 calibrate_delay(); 1045 calibrate_delay();
1070 1046
1071 arch_cpu_finalize_init(); 1047 arch_cpu_finalize_init();
1072 1048
1073 pid_idr_init(); 1049 pid_idr_init();
1074 anon_vma_init(); 1050 anon_vma_init();
1075 #ifdef CONFIG_X86 1051 #ifdef CONFIG_X86
1076 if (efi_enabled(EFI_RUNTIME_SERVICES) 1052 if (efi_enabled(EFI_RUNTIME_SERVICES))
1077 efi_enter_virtual_mode(); 1053 efi_enter_virtual_mode();
1078 #endif 1054 #endif
1079 thread_stack_cache_init(); 1055 thread_stack_cache_init();
1080 cred_init(); 1056 cred_init();
1081 fork_init(); 1057 fork_init();
1082 proc_caches_init(); 1058 proc_caches_init();
1083 uts_ns_init(); 1059 uts_ns_init();
1084 key_init(); 1060 key_init();
1085 security_init(); 1061 security_init();
1086 dbg_late_init(); 1062 dbg_late_init();
1087 net_ns_init(); 1063 net_ns_init();
1088 vfs_caches_init(); 1064 vfs_caches_init();
1089 pagecache_init(); 1065 pagecache_init();
1090 signals_init(); 1066 signals_init();
1091 seq_file_init(); 1067 seq_file_init();
1092 proc_root_init(); 1068 proc_root_init();
1093 nsfs_init(); 1069 nsfs_init();
1094 pidfs_init(); <<
1095 cpuset_init(); 1070 cpuset_init();
1096 cgroup_init(); 1071 cgroup_init();
1097 taskstats_init_early(); 1072 taskstats_init_early();
1098 delayacct_init(); 1073 delayacct_init();
1099 1074
1100 acpi_subsystem_init(); 1075 acpi_subsystem_init();
1101 arch_post_acpi_subsys_init(); 1076 arch_post_acpi_subsys_init();
1102 kcsan_init(); 1077 kcsan_init();
1103 1078
1104 /* Do the rest non-__init'ed, we're n 1079 /* Do the rest non-__init'ed, we're now alive */
1105 rest_init(); !! 1080 arch_call_rest_init();
1106 1081
1107 /* 1082 /*
1108 * Avoid stack canaries in callers of 1083 * Avoid stack canaries in callers of boot_init_stack_canary for gcc-10
1109 * and older. 1084 * and older.
1110 */ 1085 */
1111 #if !__has_attribute(__no_stack_protector__) 1086 #if !__has_attribute(__no_stack_protector__)
1112 prevent_tail_call_optimization(); 1087 prevent_tail_call_optimization();
1113 #endif 1088 #endif
1114 } 1089 }
1115 1090
1116 /* Call all constructor functions linked into 1091 /* Call all constructor functions linked into the kernel. */
1117 static void __init do_ctors(void) 1092 static void __init do_ctors(void)
1118 { 1093 {
1119 /* 1094 /*
1120 * For UML, the constructors have already bee 1095 * For UML, the constructors have already been called by the
1121 * normal setup code as it's just a normal EL 1096 * normal setup code as it's just a normal ELF binary, so we
1122 * cannot do it again - but we do need CONFIG 1097 * cannot do it again - but we do need CONFIG_CONSTRUCTORS
1123 * even on UML for modules. 1098 * even on UML for modules.
1124 */ 1099 */
1125 #if defined(CONFIG_CONSTRUCTORS) && !defined( 1100 #if defined(CONFIG_CONSTRUCTORS) && !defined(CONFIG_UML)
1126 ctor_fn_t *fn = (ctor_fn_t *) __ctors 1101 ctor_fn_t *fn = (ctor_fn_t *) __ctors_start;
1127 1102
1128 for (; fn < (ctor_fn_t *) __ctors_end 1103 for (; fn < (ctor_fn_t *) __ctors_end; fn++)
1129 (*fn)(); 1104 (*fn)();
1130 #endif 1105 #endif
1131 } 1106 }
1132 1107
1133 #ifdef CONFIG_KALLSYMS 1108 #ifdef CONFIG_KALLSYMS
1134 struct blacklist_entry { 1109 struct blacklist_entry {
1135 struct list_head next; 1110 struct list_head next;
1136 char *buf; 1111 char *buf;
1137 }; 1112 };
1138 1113
1139 static __initdata_or_module LIST_HEAD(blackli 1114 static __initdata_or_module LIST_HEAD(blacklisted_initcalls);
1140 1115
1141 static int __init initcall_blacklist(char *st 1116 static int __init initcall_blacklist(char *str)
1142 { 1117 {
1143 char *str_entry; 1118 char *str_entry;
1144 struct blacklist_entry *entry; 1119 struct blacklist_entry *entry;
1145 1120
1146 /* str argument is a comma-separated 1121 /* str argument is a comma-separated list of functions */
1147 do { 1122 do {
1148 str_entry = strsep(&str, ",") 1123 str_entry = strsep(&str, ",");
1149 if (str_entry) { 1124 if (str_entry) {
1150 pr_debug("blacklistin 1125 pr_debug("blacklisting initcall %s\n", str_entry);
1151 entry = memblock_allo 1126 entry = memblock_alloc(sizeof(*entry),
1152 1127 SMP_CACHE_BYTES);
1153 if (!entry) 1128 if (!entry)
1154 panic("%s: Fa 1129 panic("%s: Failed to allocate %zu bytes\n",
1155 __func_ 1130 __func__, sizeof(*entry));
1156 entry->buf = memblock 1131 entry->buf = memblock_alloc(strlen(str_entry) + 1,
1157 1132 SMP_CACHE_BYTES);
1158 if (!entry->buf) 1133 if (!entry->buf)
1159 panic("%s: Fa 1134 panic("%s: Failed to allocate %zu bytes\n",
1160 __func_ 1135 __func__, strlen(str_entry) + 1);
1161 strcpy(entry->buf, st 1136 strcpy(entry->buf, str_entry);
1162 list_add(&entry->next 1137 list_add(&entry->next, &blacklisted_initcalls);
1163 } 1138 }
1164 } while (str_entry); 1139 } while (str_entry);
1165 1140
1166 return 1; 1141 return 1;
1167 } 1142 }
1168 1143
1169 static bool __init_or_module initcall_blackli 1144 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1170 { 1145 {
1171 struct blacklist_entry *entry; 1146 struct blacklist_entry *entry;
1172 char fn_name[KSYM_SYMBOL_LEN]; 1147 char fn_name[KSYM_SYMBOL_LEN];
1173 unsigned long addr; 1148 unsigned long addr;
1174 1149
1175 if (list_empty(&blacklisted_initcalls 1150 if (list_empty(&blacklisted_initcalls))
1176 return false; 1151 return false;
1177 1152
1178 addr = (unsigned long) dereference_fu 1153 addr = (unsigned long) dereference_function_descriptor(fn);
1179 sprint_symbol_no_offset(fn_name, addr 1154 sprint_symbol_no_offset(fn_name, addr);
1180 1155
1181 /* 1156 /*
1182 * fn will be "function_name [module_ 1157 * fn will be "function_name [module_name]" where [module_name] is not
1183 * displayed for built-in init functi 1158 * displayed for built-in init functions. Strip off the [module_name].
1184 */ 1159 */
1185 strreplace(fn_name, ' ', '\0'); 1160 strreplace(fn_name, ' ', '\0');
1186 1161
1187 list_for_each_entry(entry, &blacklist 1162 list_for_each_entry(entry, &blacklisted_initcalls, next) {
1188 if (!strcmp(fn_name, entry->b 1163 if (!strcmp(fn_name, entry->buf)) {
1189 pr_debug("initcall %s 1164 pr_debug("initcall %s blacklisted\n", fn_name);
1190 return true; 1165 return true;
1191 } 1166 }
1192 } 1167 }
1193 1168
1194 return false; 1169 return false;
1195 } 1170 }
1196 #else 1171 #else
1197 static int __init initcall_blacklist(char *st 1172 static int __init initcall_blacklist(char *str)
1198 { 1173 {
1199 pr_warn("initcall_blacklist requires 1174 pr_warn("initcall_blacklist requires CONFIG_KALLSYMS\n");
1200 return 0; 1175 return 0;
1201 } 1176 }
1202 1177
1203 static bool __init_or_module initcall_blackli 1178 static bool __init_or_module initcall_blacklisted(initcall_t fn)
1204 { 1179 {
1205 return false; 1180 return false;
1206 } 1181 }
1207 #endif 1182 #endif
1208 __setup("initcall_blacklist=", initcall_black 1183 __setup("initcall_blacklist=", initcall_blacklist);
1209 1184
1210 static __init_or_module void 1185 static __init_or_module void
1211 trace_initcall_start_cb(void *data, initcall_ 1186 trace_initcall_start_cb(void *data, initcall_t fn)
1212 { 1187 {
1213 ktime_t *calltime = data; 1188 ktime_t *calltime = data;
1214 1189
1215 printk(KERN_DEBUG "calling %pS @ %i\ 1190 printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current));
1216 *calltime = ktime_get(); 1191 *calltime = ktime_get();
1217 } 1192 }
1218 1193
1219 static __init_or_module void 1194 static __init_or_module void
1220 trace_initcall_finish_cb(void *data, initcall 1195 trace_initcall_finish_cb(void *data, initcall_t fn, int ret)
1221 { 1196 {
1222 ktime_t rettime, *calltime = data; 1197 ktime_t rettime, *calltime = data;
1223 1198
1224 rettime = ktime_get(); 1199 rettime = ktime_get();
1225 printk(KERN_DEBUG "initcall %pS retur 1200 printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n",
1226 fn, ret, (unsigned long long 1201 fn, ret, (unsigned long long)ktime_us_delta(rettime, *calltime));
1227 } 1202 }
1228 1203
1229 static ktime_t initcall_calltime; 1204 static ktime_t initcall_calltime;
1230 1205
1231 #ifdef TRACEPOINTS_ENABLED 1206 #ifdef TRACEPOINTS_ENABLED
1232 static void __init initcall_debug_enable(void 1207 static void __init initcall_debug_enable(void)
1233 { 1208 {
1234 int ret; 1209 int ret;
1235 1210
1236 ret = register_trace_initcall_start(t 1211 ret = register_trace_initcall_start(trace_initcall_start_cb,
1237 & 1212 &initcall_calltime);
1238 ret |= register_trace_initcall_finish 1213 ret |= register_trace_initcall_finish(trace_initcall_finish_cb,
1239 1214 &initcall_calltime);
1240 WARN(ret, "Failed to register initcal 1215 WARN(ret, "Failed to register initcall tracepoints\n");
1241 } 1216 }
1242 # define do_trace_initcall_start trace 1217 # define do_trace_initcall_start trace_initcall_start
1243 # define do_trace_initcall_finish trace 1218 # define do_trace_initcall_finish trace_initcall_finish
1244 #else 1219 #else
1245 static inline void do_trace_initcall_start(in 1220 static inline void do_trace_initcall_start(initcall_t fn)
1246 { 1221 {
1247 if (!initcall_debug) 1222 if (!initcall_debug)
1248 return; 1223 return;
1249 trace_initcall_start_cb(&initcall_cal 1224 trace_initcall_start_cb(&initcall_calltime, fn);
1250 } 1225 }
1251 static inline void do_trace_initcall_finish(i 1226 static inline void do_trace_initcall_finish(initcall_t fn, int ret)
1252 { 1227 {
1253 if (!initcall_debug) 1228 if (!initcall_debug)
1254 return; 1229 return;
1255 trace_initcall_finish_cb(&initcall_ca 1230 trace_initcall_finish_cb(&initcall_calltime, fn, ret);
1256 } 1231 }
1257 #endif /* !TRACEPOINTS_ENABLED */ 1232 #endif /* !TRACEPOINTS_ENABLED */
1258 1233
1259 int __init_or_module do_one_initcall(initcall 1234 int __init_or_module do_one_initcall(initcall_t fn)
1260 { 1235 {
1261 int count = preempt_count(); 1236 int count = preempt_count();
1262 char msgbuf[64]; 1237 char msgbuf[64];
1263 int ret; 1238 int ret;
1264 1239
1265 if (initcall_blacklisted(fn)) 1240 if (initcall_blacklisted(fn))
1266 return -EPERM; 1241 return -EPERM;
1267 1242
1268 do_trace_initcall_start(fn); 1243 do_trace_initcall_start(fn);
1269 ret = fn(); 1244 ret = fn();
1270 do_trace_initcall_finish(fn, ret); 1245 do_trace_initcall_finish(fn, ret);
1271 1246
1272 msgbuf[0] = 0; 1247 msgbuf[0] = 0;
1273 1248
1274 if (preempt_count() != count) { 1249 if (preempt_count() != count) {
1275 sprintf(msgbuf, "preemption i 1250 sprintf(msgbuf, "preemption imbalance ");
1276 preempt_count_set(count); 1251 preempt_count_set(count);
1277 } 1252 }
1278 if (irqs_disabled()) { 1253 if (irqs_disabled()) {
1279 strlcat(msgbuf, "disabled int 1254 strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
1280 local_irq_enable(); 1255 local_irq_enable();
1281 } 1256 }
1282 WARN(msgbuf[0], "initcall %pS returne 1257 WARN(msgbuf[0], "initcall %pS returned with %s\n", fn, msgbuf);
1283 1258
1284 add_latent_entropy(); 1259 add_latent_entropy();
1285 return ret; 1260 return ret;
1286 } 1261 }
1287 1262
1288 1263
1289 static initcall_entry_t *initcall_levels[] __ 1264 static initcall_entry_t *initcall_levels[] __initdata = {
1290 __initcall0_start, 1265 __initcall0_start,
1291 __initcall1_start, 1266 __initcall1_start,
1292 __initcall2_start, 1267 __initcall2_start,
1293 __initcall3_start, 1268 __initcall3_start,
1294 __initcall4_start, 1269 __initcall4_start,
1295 __initcall5_start, 1270 __initcall5_start,
1296 __initcall6_start, 1271 __initcall6_start,
1297 __initcall7_start, 1272 __initcall7_start,
1298 __initcall_end, 1273 __initcall_end,
1299 }; 1274 };
1300 1275
1301 /* Keep these in sync with initcalls in inclu 1276 /* Keep these in sync with initcalls in include/linux/init.h */
1302 static const char *initcall_level_names[] __i 1277 static const char *initcall_level_names[] __initdata = {
1303 "pure", 1278 "pure",
1304 "core", 1279 "core",
1305 "postcore", 1280 "postcore",
1306 "arch", 1281 "arch",
1307 "subsys", 1282 "subsys",
1308 "fs", 1283 "fs",
1309 "device", 1284 "device",
1310 "late", 1285 "late",
1311 }; 1286 };
1312 1287
1313 static int __init ignore_unknown_bootoption(c 1288 static int __init ignore_unknown_bootoption(char *param, char *val,
1314 const char *un 1289 const char *unused, void *arg)
1315 { 1290 {
1316 return 0; 1291 return 0;
1317 } 1292 }
1318 1293
1319 static void __init do_initcall_level(int leve 1294 static void __init do_initcall_level(int level, char *command_line)
1320 { 1295 {
1321 initcall_entry_t *fn; 1296 initcall_entry_t *fn;
1322 1297
1323 parse_args(initcall_level_names[level 1298 parse_args(initcall_level_names[level],
1324 command_line, __start___pa 1299 command_line, __start___param,
1325 __stop___param - __start__ 1300 __stop___param - __start___param,
1326 level, level, 1301 level, level,
1327 NULL, ignore_unknown_booto 1302 NULL, ignore_unknown_bootoption);
1328 1303
1329 trace_initcall_level(initcall_level_n 1304 trace_initcall_level(initcall_level_names[level]);
1330 for (fn = initcall_levels[level]; fn 1305 for (fn = initcall_levels[level]; fn < initcall_levels[level+1]; fn++)
1331 do_one_initcall(initcall_from 1306 do_one_initcall(initcall_from_entry(fn));
1332 } 1307 }
1333 1308
1334 static void __init do_initcalls(void) 1309 static void __init do_initcalls(void)
1335 { 1310 {
1336 int level; 1311 int level;
1337 size_t len = saved_command_line_len + 1312 size_t len = saved_command_line_len + 1;
1338 char *command_line; 1313 char *command_line;
1339 1314
1340 command_line = kzalloc(len, GFP_KERNE 1315 command_line = kzalloc(len, GFP_KERNEL);
1341 if (!command_line) 1316 if (!command_line)
1342 panic("%s: Failed to allocate 1317 panic("%s: Failed to allocate %zu bytes\n", __func__, len);
1343 1318
1344 for (level = 0; level < ARRAY_SIZE(in 1319 for (level = 0; level < ARRAY_SIZE(initcall_levels) - 1; level++) {
1345 /* Parser modifies command_li 1320 /* Parser modifies command_line, restore it each time */
1346 strcpy(command_line, saved_co 1321 strcpy(command_line, saved_command_line);
1347 do_initcall_level(level, comm 1322 do_initcall_level(level, command_line);
1348 } 1323 }
1349 1324
1350 kfree(command_line); 1325 kfree(command_line);
1351 } 1326 }
1352 1327
1353 /* 1328 /*
1354 * Ok, the machine is now initialized. None o 1329 * Ok, the machine is now initialized. None of the devices
1355 * have been touched yet, but the CPU subsyst 1330 * have been touched yet, but the CPU subsystem is up and
1356 * running, and memory and process management 1331 * running, and memory and process management works.
1357 * 1332 *
1358 * Now we can finally start doing some real w 1333 * Now we can finally start doing some real work..
1359 */ 1334 */
1360 static void __init do_basic_setup(void) 1335 static void __init do_basic_setup(void)
1361 { 1336 {
1362 cpuset_init_smp(); 1337 cpuset_init_smp();
1363 driver_init(); 1338 driver_init();
1364 init_irq_proc(); 1339 init_irq_proc();
1365 do_ctors(); 1340 do_ctors();
1366 do_initcalls(); 1341 do_initcalls();
1367 } 1342 }
1368 1343
1369 static void __init do_pre_smp_initcalls(void) 1344 static void __init do_pre_smp_initcalls(void)
1370 { 1345 {
1371 initcall_entry_t *fn; 1346 initcall_entry_t *fn;
1372 1347
1373 trace_initcall_level("early"); 1348 trace_initcall_level("early");
1374 for (fn = __initcall_start; fn < __in 1349 for (fn = __initcall_start; fn < __initcall0_start; fn++)
1375 do_one_initcall(initcall_from 1350 do_one_initcall(initcall_from_entry(fn));
1376 } 1351 }
1377 1352
1378 static int run_init_process(const char *init_ 1353 static int run_init_process(const char *init_filename)
1379 { 1354 {
1380 const char *const *p; 1355 const char *const *p;
1381 1356
1382 argv_init[0] = init_filename; 1357 argv_init[0] = init_filename;
1383 pr_info("Run %s as init process\n", i 1358 pr_info("Run %s as init process\n", init_filename);
1384 pr_debug(" with arguments:\n"); 1359 pr_debug(" with arguments:\n");
1385 for (p = argv_init; *p; p++) 1360 for (p = argv_init; *p; p++)
1386 pr_debug(" %s\n", *p); 1361 pr_debug(" %s\n", *p);
1387 pr_debug(" with environment:\n"); 1362 pr_debug(" with environment:\n");
1388 for (p = envp_init; *p; p++) 1363 for (p = envp_init; *p; p++)
1389 pr_debug(" %s\n", *p); 1364 pr_debug(" %s\n", *p);
1390 return kernel_execve(init_filename, a 1365 return kernel_execve(init_filename, argv_init, envp_init);
1391 } 1366 }
1392 1367
1393 static int try_to_run_init_process(const char 1368 static int try_to_run_init_process(const char *init_filename)
1394 { 1369 {
1395 int ret; 1370 int ret;
1396 1371
1397 ret = run_init_process(init_filename) 1372 ret = run_init_process(init_filename);
1398 1373
1399 if (ret && ret != -ENOENT) { 1374 if (ret && ret != -ENOENT) {
1400 pr_err("Starting init: %s exi 1375 pr_err("Starting init: %s exists but couldn't execute it (error %d)\n",
1401 init_filename, ret); 1376 init_filename, ret);
1402 } 1377 }
1403 1378
1404 return ret; 1379 return ret;
1405 } 1380 }
1406 1381
1407 static noinline void __init kernel_init_freea 1382 static noinline void __init kernel_init_freeable(void);
1408 1383
1409 #if defined(CONFIG_STRICT_KERNEL_RWX) || defi 1384 #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
1410 bool rodata_enabled __ro_after_init = true; 1385 bool rodata_enabled __ro_after_init = true;
1411 1386
1412 #ifndef arch_parse_debug_rodata 1387 #ifndef arch_parse_debug_rodata
1413 static inline bool arch_parse_debug_rodata(ch 1388 static inline bool arch_parse_debug_rodata(char *str) { return false; }
1414 #endif 1389 #endif
1415 1390
1416 static int __init set_debug_rodata(char *str) 1391 static int __init set_debug_rodata(char *str)
1417 { 1392 {
1418 if (arch_parse_debug_rodata(str)) 1393 if (arch_parse_debug_rodata(str))
1419 return 0; 1394 return 0;
1420 1395
1421 if (str && !strcmp(str, "on")) 1396 if (str && !strcmp(str, "on"))
1422 rodata_enabled = true; 1397 rodata_enabled = true;
1423 else if (str && !strcmp(str, "off")) 1398 else if (str && !strcmp(str, "off"))
1424 rodata_enabled = false; 1399 rodata_enabled = false;
1425 else 1400 else
1426 pr_warn("Invalid option strin 1401 pr_warn("Invalid option string for rodata: '%s'\n", str);
1427 return 0; 1402 return 0;
1428 } 1403 }
1429 early_param("rodata", set_debug_rodata); 1404 early_param("rodata", set_debug_rodata);
1430 #endif 1405 #endif
1431 1406
>> 1407 #ifdef CONFIG_STRICT_KERNEL_RWX
1432 static void mark_readonly(void) 1408 static void mark_readonly(void)
1433 { 1409 {
1434 if (IS_ENABLED(CONFIG_STRICT_KERNEL_R !! 1410 if (rodata_enabled) {
1435 /* 1411 /*
1436 * load_module() results in W 1412 * load_module() results in W+X mappings, which are cleaned
1437 * up with init_free_wq. Let' 1413 * up with init_free_wq. Let's make sure that queued work is
1438 * flushed so that we don't h 1414 * flushed so that we don't hit false positives looking for
1439 * insecure pages which are W 1415 * insecure pages which are W+X.
1440 */ 1416 */
1441 flush_module_init_free_work() 1417 flush_module_init_free_work();
1442 jump_label_init_ro(); <<
1443 mark_rodata_ro(); 1418 mark_rodata_ro();
1444 debug_checkwx(); <<
1445 rodata_test(); 1419 rodata_test();
1446 } else if (IS_ENABLED(CONFIG_STRICT_K !! 1420 } else
1447 pr_info("Kernel memory protec 1421 pr_info("Kernel memory protection disabled.\n");
1448 } else if (IS_ENABLED(CONFIG_ARCH_HAS <<
1449 pr_warn("Kernel memory protec <<
1450 } else { <<
1451 pr_warn("This architecture do <<
1452 } <<
1453 } 1422 }
>> 1423 #elif defined(CONFIG_ARCH_HAS_STRICT_KERNEL_RWX)
>> 1424 static inline void mark_readonly(void)
>> 1425 {
>> 1426 pr_warn("Kernel memory protection not selected by kernel config.\n");
>> 1427 }
>> 1428 #else
>> 1429 static inline void mark_readonly(void)
>> 1430 {
>> 1431 pr_warn("This architecture does not have kernel memory protection.\n");
>> 1432 }
>> 1433 #endif
1454 1434
1455 void __weak free_initmem(void) 1435 void __weak free_initmem(void)
1456 { 1436 {
1457 free_initmem_default(POISON_FREE_INIT 1437 free_initmem_default(POISON_FREE_INITMEM);
1458 } 1438 }
1459 1439
1460 static int __ref kernel_init(void *unused) 1440 static int __ref kernel_init(void *unused)
1461 { 1441 {
1462 int ret; 1442 int ret;
1463 1443
1464 /* 1444 /*
1465 * Wait until kthreadd is all set-up. 1445 * Wait until kthreadd is all set-up.
1466 */ 1446 */
1467 wait_for_completion(&kthreadd_done); 1447 wait_for_completion(&kthreadd_done);
1468 1448
1469 kernel_init_freeable(); 1449 kernel_init_freeable();
1470 /* need to finish all async __init co 1450 /* need to finish all async __init code before freeing the memory */
1471 async_synchronize_full(); 1451 async_synchronize_full();
1472 1452
1473 system_state = SYSTEM_FREEING_INITMEM 1453 system_state = SYSTEM_FREEING_INITMEM;
1474 kprobe_free_init_mem(); 1454 kprobe_free_init_mem();
1475 ftrace_free_init_mem(); 1455 ftrace_free_init_mem();
1476 kgdb_free_init_mem(); 1456 kgdb_free_init_mem();
1477 exit_boot_config(); 1457 exit_boot_config();
1478 free_initmem(); 1458 free_initmem();
1479 mark_readonly(); 1459 mark_readonly();
1480 1460
1481 /* 1461 /*
1482 * Kernel mappings are now finalized 1462 * Kernel mappings are now finalized - update the userspace page-table
1483 * to finalize PTI. 1463 * to finalize PTI.
1484 */ 1464 */
1485 pti_finalize(); 1465 pti_finalize();
1486 1466
1487 system_state = SYSTEM_RUNNING; 1467 system_state = SYSTEM_RUNNING;
1488 numa_default_policy(); 1468 numa_default_policy();
1489 1469
1490 rcu_end_inkernel_boot(); 1470 rcu_end_inkernel_boot();
1491 1471
1492 do_sysctl_args(); 1472 do_sysctl_args();
1493 1473
1494 if (ramdisk_execute_command) { 1474 if (ramdisk_execute_command) {
1495 ret = run_init_process(ramdis 1475 ret = run_init_process(ramdisk_execute_command);
1496 if (!ret) 1476 if (!ret)
1497 return 0; 1477 return 0;
1498 pr_err("Failed to execute %s 1478 pr_err("Failed to execute %s (error %d)\n",
1499 ramdisk_execute_comman 1479 ramdisk_execute_command, ret);
1500 } 1480 }
1501 1481
1502 /* 1482 /*
1503 * We try each of these until one suc 1483 * We try each of these until one succeeds.
1504 * 1484 *
1505 * The Bourne shell can be used inste 1485 * The Bourne shell can be used instead of init if we are
1506 * trying to recover a really broken 1486 * trying to recover a really broken machine.
1507 */ 1487 */
1508 if (execute_command) { 1488 if (execute_command) {
1509 ret = run_init_process(execut 1489 ret = run_init_process(execute_command);
1510 if (!ret) 1490 if (!ret)
1511 return 0; 1491 return 0;
1512 panic("Requested init %s fail 1492 panic("Requested init %s failed (error %d).",
1513 execute_command, ret); 1493 execute_command, ret);
1514 } 1494 }
1515 1495
1516 if (CONFIG_DEFAULT_INIT[0] != '\0') { 1496 if (CONFIG_DEFAULT_INIT[0] != '\0') {
1517 ret = run_init_process(CONFIG 1497 ret = run_init_process(CONFIG_DEFAULT_INIT);
1518 if (ret) 1498 if (ret)
1519 pr_err("Default init 1499 pr_err("Default init %s failed (error %d)\n",
1520 CONFIG_DEFAULT 1500 CONFIG_DEFAULT_INIT, ret);
1521 else 1501 else
1522 return 0; 1502 return 0;
1523 } 1503 }
1524 1504
1525 if (!try_to_run_init_process("/sbin/i 1505 if (!try_to_run_init_process("/sbin/init") ||
1526 !try_to_run_init_process("/etc/in 1506 !try_to_run_init_process("/etc/init") ||
1527 !try_to_run_init_process("/bin/in 1507 !try_to_run_init_process("/bin/init") ||
1528 !try_to_run_init_process("/bin/sh 1508 !try_to_run_init_process("/bin/sh"))
1529 return 0; 1509 return 0;
1530 1510
1531 panic("No working init found. Try pa 1511 panic("No working init found. Try passing init= option to kernel. "
1532 "See Linux Documentation/admin- 1512 "See Linux Documentation/admin-guide/init.rst for guidance.");
1533 } 1513 }
1534 1514
1535 /* Open /dev/console, for stdin/stdout/stderr 1515 /* Open /dev/console, for stdin/stdout/stderr, this should never fail */
1536 void __init console_on_rootfs(void) 1516 void __init console_on_rootfs(void)
1537 { 1517 {
1538 struct file *file = filp_open("/dev/c 1518 struct file *file = filp_open("/dev/console", O_RDWR, 0);
1539 1519
1540 if (IS_ERR(file)) { 1520 if (IS_ERR(file)) {
1541 pr_err("Warning: unable to op 1521 pr_err("Warning: unable to open an initial console.\n");
1542 return; 1522 return;
1543 } 1523 }
1544 init_dup(file); 1524 init_dup(file);
1545 init_dup(file); 1525 init_dup(file);
1546 init_dup(file); 1526 init_dup(file);
1547 fput(file); 1527 fput(file);
1548 } 1528 }
1549 1529
1550 static noinline void __init kernel_init_freea 1530 static noinline void __init kernel_init_freeable(void)
1551 { 1531 {
1552 /* Now the scheduler is fully set up 1532 /* Now the scheduler is fully set up and can do blocking allocations */
1553 gfp_allowed_mask = __GFP_BITS_MASK; 1533 gfp_allowed_mask = __GFP_BITS_MASK;
1554 1534
1555 /* 1535 /*
1556 * init can allocate pages on any nod 1536 * init can allocate pages on any node
1557 */ 1537 */
1558 set_mems_allowed(node_states[N_MEMORY 1538 set_mems_allowed(node_states[N_MEMORY]);
1559 1539
1560 cad_pid = get_pid(task_pid(current)); 1540 cad_pid = get_pid(task_pid(current));
1561 1541
1562 smp_prepare_cpus(setup_max_cpus); 1542 smp_prepare_cpus(setup_max_cpus);
1563 1543
1564 workqueue_init(); 1544 workqueue_init();
1565 1545
1566 init_mm_internals(); 1546 init_mm_internals();
1567 1547
1568 rcu_init_tasks_generic(); 1548 rcu_init_tasks_generic();
1569 do_pre_smp_initcalls(); 1549 do_pre_smp_initcalls();
1570 lockup_detector_init(); 1550 lockup_detector_init();
1571 1551
1572 smp_init(); 1552 smp_init();
1573 sched_init_smp(); 1553 sched_init_smp();
1574 1554
1575 workqueue_init_topology(); 1555 workqueue_init_topology();
1576 async_init(); <<
1577 padata_init(); 1556 padata_init();
1578 page_alloc_init_late(); 1557 page_alloc_init_late();
1579 1558
1580 do_basic_setup(); 1559 do_basic_setup();
1581 1560
1582 kunit_run_all_tests(); 1561 kunit_run_all_tests();
1583 1562
1584 wait_for_initramfs(); 1563 wait_for_initramfs();
1585 console_on_rootfs(); 1564 console_on_rootfs();
1586 1565
1587 /* 1566 /*
1588 * check if there is an early userspa 1567 * check if there is an early userspace init. If yes, let it do all
1589 * the work 1568 * the work
1590 */ 1569 */
1591 if (init_eaccess(ramdisk_execute_comm 1570 if (init_eaccess(ramdisk_execute_command) != 0) {
1592 ramdisk_execute_command = NUL 1571 ramdisk_execute_command = NULL;
1593 prepare_namespace(); 1572 prepare_namespace();
1594 } 1573 }
1595 1574
1596 /* 1575 /*
1597 * Ok, we have completed the initial 1576 * Ok, we have completed the initial bootup, and
1598 * we're essentially up and running. 1577 * we're essentially up and running. Get rid of the
1599 * initmem segments and start the use 1578 * initmem segments and start the user-mode stuff..
1600 * 1579 *
1601 * rootfs is available now, try loadi 1580 * rootfs is available now, try loading the public keys
1602 * and default modules 1581 * and default modules
1603 */ 1582 */
1604 1583
1605 integrity_load_keys(); 1584 integrity_load_keys();
1606 } 1585 }
1607 1586
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.