1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * linux/boot/head.S
4 *
5 * Copyright (C) 1991, 1992, 1993 Linus Torv
6 */
7
8 /*
9 * head.S contains the 32-bit startup code.
10 *
11 * NOTE!!! Startup happens at absolute address
12 * the page directory will exist. The startup
13 * the page directory. [According to comments
14 * kernel it will end up at 0x1000 + 1Mb I hop
15 *
16 * Page 0 is deliberately kept safe, since Sys
17 * laptops may need to access the BIOS data st
18 * useful for future device drivers that eithe
19 * mode.
20 */
21
22 /*
23 * High loaded stuff by Hans Lermen & Werner A
24 */
25 .code32
26 .text
27
28 #include <linux/init.h>
29 #include <linux/linkage.h>
30 #include <asm/segment.h>
31 #include <asm/boot.h>
32 #include <asm/msr.h>
33 #include <asm/processor-flags.h>
34 #include <asm/asm-offsets.h>
35 #include <asm/bootparam.h>
36 #include <asm/desc_defs.h>
37 #include <asm/trapnr.h>
38 #include "pgtable.h"
39
40 /*
41 * Fix alignment at 16 bytes. Following CONFIG
42 * in assembly errors due to trying to move .o
43 * alignment.
44 */
45 #undef __ALIGN
46 #define __ALIGN .balign 16, 0x90
47
48 /*
49 * Locally defined symbols should be marked hi
50 */
51 .hidden _bss
52 .hidden _ebss
53 .hidden _end
54
55 __HEAD
56
57 /*
58 * This macro gives the relative virtual addre
59 * from startup_32. This is the same as the li
60 * since startup_32 is at 0, but defining it t
61 * assembler/linker that we do not want the ac
62 * prevents the linker from trying to create u
63 * entries for the reference when the compress
64 *
65 * A reference X(%reg) will result in the link
66 * the instruction, and a run-time R_X86_64_RE
67 * adds the 64-bit base address where the kern
68 *
69 * Replacing it with (X-startup_32)(%reg) resu
70 * and no run-time relocation.
71 *
72 * The macro should be used as a displacement
73 * the run-time address of startup_32 [i.e. rv
74 * [$ rva(X)].
75 *
76 * This macro can only be used from within the
77 * expression requires startup_32 to be in the
78 * assembled.
79 */
80 #define rva(X) ((X) - startup_32)
81
82 .code32
83 SYM_FUNC_START(startup_32)
84 /*
85 * 32bit entry is 0 and it is ABI so i
86 * If we come here directly from a boo
87 * kernel(text+data+bss+brk) ramdisk,
88 * all need to be under the 4G limit.
89 */
90 cld
91 cli
92
93 /*
94 * Calculate the delta between where we were c
95 * at and where we were actually loaded at. T
96 * with a short local call on x86. Nothing e
97 * address we are running at. The reserved ch
98 * data at 0x1e4 (defined as a scratch field)
99 * for this calculation. Only 4 bytes are need
100 */
101 leal (BP_scratch+4)(%esi), %esp
102 call 1f
103 1: popl %ebp
104 subl $ rva(1b), %ebp
105
106 /* Load new GDT with the 64bit segment
107 leal rva(gdt)(%ebp), %eax
108 movl %eax, 2(%eax)
109 lgdt (%eax)
110
111 /* Load segment registers with our des
112 movl $__BOOT_DS, %eax
113 movl %eax, %ds
114 movl %eax, %es
115 movl %eax, %fs
116 movl %eax, %gs
117 movl %eax, %ss
118
119 /* Setup a stack and load CS from curr
120 leal rva(boot_stack_end)(%ebp), %es
121
122 pushl $__KERNEL32_CS
123 leal rva(1f)(%ebp), %eax
124 pushl %eax
125 lretl
126 1:
127
128 /* Setup Exception handling for SEV-ES
129 #ifdef CONFIG_AMD_MEM_ENCRYPT
130 call startup32_load_idt
131 #endif
132
133 /* Make sure cpu supports long mode. *
134 call verify_cpu
135 testl %eax, %eax
136 jnz .Lno_longmode
137
138 /*
139 * Compute the delta between where we were com
140 * and where the code will actually run at.
141 *
142 * %ebp contains the address we are loaded at
143 * contains the address where we should move t
144 * for safe in-place decompression.
145 */
146
147 #ifdef CONFIG_RELOCATABLE
148 movl %ebp, %ebx
149 movl BP_kernel_alignment(%esi), %ea
150 decl %eax
151 addl %eax, %ebx
152 notl %eax
153 andl %eax, %ebx
154 cmpl $LOAD_PHYSICAL_ADDR, %ebx
155 jae 1f
156 #endif
157 movl $LOAD_PHYSICAL_ADDR, %ebx
158 1:
159
160 /* Target address to relocate to for d
161 addl BP_init_size(%esi), %ebx
162 subl $ rva(_end), %ebx
163
164 /*
165 * Prepare for entering 64 bit mode
166 */
167
168 /* Enable PAE mode */
169 movl %cr4, %eax
170 orl $X86_CR4_PAE, %eax
171 movl %eax, %cr4
172
173 /*
174 * Build early 4G boot pagetable
175 */
176 /*
177 * If SEV is active then set the encry
178 * This will ensure that when the kern
179 * it will be done so encrypted.
180 */
181 xorl %edx, %edx
182 #ifdef CONFIG_AMD_MEM_ENCRYPT
183 call get_sev_encryption_bit
184 xorl %edx, %edx
185 testl %eax, %eax
186 jz 1f
187 subl $32, %eax /* Encryption
188 bts %eax, %edx /* Set encrypt
189 /*
190 * Set MSR_AMD64_SEV_ENABLED_BIT in se
191 * startup32_check_sev_cbit() will do
192 * initialize sev_status with all the
193 * MSR_AMD_SEV_STATUS later, but only
194 * needs to be set for now.
195 */
196 movl $1, rva(sev_status)(%ebp)
197 1:
198 #endif
199
200 /* Initialize Page tables to 0 */
201 leal rva(pgtable)(%ebx), %edi
202 xorl %eax, %eax
203 movl $(BOOT_INIT_PGT_SIZE/4), %ecx
204 rep stosl
205
206 /* Build Level 4 */
207 leal rva(pgtable + 0)(%ebx), %edi
208 leal 0x1007 (%edi), %eax
209 movl %eax, 0(%edi)
210 addl %edx, 4(%edi)
211
212 /* Build Level 3 */
213 leal rva(pgtable + 0x1000)(%ebx), %
214 leal 0x1007(%edi), %eax
215 movl $4, %ecx
216 1: movl %eax, 0x00(%edi)
217 addl %edx, 0x04(%edi)
218 addl $0x00001000, %eax
219 addl $8, %edi
220 decl %ecx
221 jnz 1b
222
223 /* Build Level 2 */
224 leal rva(pgtable + 0x2000)(%ebx), %
225 movl $0x00000183, %eax
226 movl $2048, %ecx
227 1: movl %eax, 0(%edi)
228 addl %edx, 4(%edi)
229 addl $0x00200000, %eax
230 addl $8, %edi
231 decl %ecx
232 jnz 1b
233
234 /* Enable the boot page tables */
235 leal rva(pgtable)(%ebx), %eax
236 movl %eax, %cr3
237
238 /* Enable Long mode in EFER (Extended
239 movl $MSR_EFER, %ecx
240 rdmsr
241 btsl $_EFER_LME, %eax
242 wrmsr
243
244 /* After gdt is loaded */
245 xorl %eax, %eax
246 lldt %ax
247 movl $__BOOT_TSS, %eax
248 ltr %ax
249
250 #ifdef CONFIG_AMD_MEM_ENCRYPT
251 /* Check if the C-bit position is corr
252 call startup32_check_sev_cbit
253 #endif
254
255 /*
256 * Setup for the jump to 64bit mode
257 *
258 * When the jump is performed we will
259 * in 32bit compatibility mode with EF
260 * (and in turn EFER.LMA = 1). To jum
261 * the new gdt/idt that has __KERNEL_C
262 * We place all of the values on our m
263 * used to perform that far jump.
264 */
265 leal rva(startup_64)(%ebp), %eax
266 #ifdef CONFIG_EFI_MIXED
267 cmpb $1, rva(efi_is64)(%ebp)
268 je 1f
269 leal rva(startup_64_mixed_mode)(%eb
270 1:
271 #endif
272
273 pushl $__KERNEL_CS
274 pushl %eax
275
276 /* Enter paged protected Mode, activat
277 movl $CR0_STATE, %eax
278 movl %eax, %cr0
279
280 /* Jump from 32bit compatibility mode
281 lret
282 SYM_FUNC_END(startup_32)
283
284 .code64
285 .org 0x200
286 SYM_CODE_START(startup_64)
287 /*
288 * 64bit entry is 0x200 and it is ABI
289 * We come here either from startup_32
290 * 64bit bootloader.
291 * If we come here from a bootloader,
292 * ramdisk, zero_page, command line co
293 * We depend on an identity mapped pag
294 * that maps our entire kernel(text+da
295 * and command line.
296 */
297
298 cld
299 cli
300
301 /* Setup data segments. */
302 xorl %eax, %eax
303 movl %eax, %ds
304 movl %eax, %es
305 movl %eax, %ss
306 movl %eax, %fs
307 movl %eax, %gs
308
309 /*
310 * Compute the decompressed kernel sta
311 * we were loaded at aligned to a 2M b
312 * decompressed kernel start address.
313 *
314 * If it is a relocatable kernel then
315 * from load address aligned to 2MB ad
316 * run the kernel from LOAD_PHYSICAL_A
317 *
318 * We cannot rely on the calculation d
319 * may have been invoked via the 64-bi
320 */
321
322 /* Start with the delta to where the k
323 #ifdef CONFIG_RELOCATABLE
324 leaq startup_32(%rip) /* - $startup
325 movl BP_kernel_alignment(%rsi), %ea
326 decl %eax
327 addq %rax, %rbp
328 notq %rax
329 andq %rax, %rbp
330 cmpq $LOAD_PHYSICAL_ADDR, %rbp
331 jae 1f
332 #endif
333 movq $LOAD_PHYSICAL_ADDR, %rbp
334 1:
335
336 /* Target address to relocate to for d
337 movl BP_init_size(%rsi), %ebx
338 subl $ rva(_end), %ebx
339 addq %rbp, %rbx
340
341 /* Set up the stack */
342 leaq rva(boot_stack_end)(%rbx), %rs
343
344 /*
345 * At this point we are in long mode w
346 * but we might want to enable 5-level
347 *
348 * The problem is that we cannot do it
349 * CR4.LA57 in long mode would trigger
350 * long mode and paging first.
351 *
352 * We also need a trampoline in lower
353 * 4- to 5-level paging for cases when
354 * above 4G, but didn't enable 5-level
355 *
356 * The same trampoline can be used to
357 * mode, like when starting 4-level pa
358 * original kernel worked in 5-level p
359 *
360 * For the trampoline, we need the top
361 * memory as we don't have a way to lo
362 * 32-bit mode.
363 */
364
365 /* Make sure we have GDT with 32-bit c
366 leaq gdt64(%rip), %rax
367 addq %rax, 2(%rax)
368 lgdt (%rax)
369
370 /* Reload CS so IRET returns to a CS a
371 pushq $__KERNEL_CS
372 leaq .Lon_kernel_cs(%rip), %rax
373 pushq %rax
374 lretq
375
376 .Lon_kernel_cs:
377 /*
378 * RSI holds a pointer to a boot_param
379 * loader, and this needs to be preser
380 * move it into a callee saved registe
381 */
382 movq %rsi, %r15
383
384 call load_stage1_idt
385
386 #ifdef CONFIG_AMD_MEM_ENCRYPT
387 /*
388 * Now that the stage1 interrupt handl
389 * CPUID instructions can be properly
390 *
391 * For SEV-SNP, the CPUID table also n
392 * CPUID instructions being issued, so
393 * sev_enable(), which will also handl
394 * detection/setup to ensure that has
395 * code. Pass the boot_params pointer
396 */
397 movq %r15, %rdi
398 call sev_enable
399 #endif
400
401 /* Preserve only the CR4 bits that mus
402 movq %cr4, %rax
403 andl $(X86_CR4_PAE | X86_CR4_MCE |
404 movq %rax, %cr4
405
406 /*
407 * configure_5level_paging() updates t
408 * a trampoline in 32-bit addressable
409 * not match the desired number.
410 *
411 * Pass the boot_params pointer as the
412 * argument is the relocated address o
413 * of the page table in trampoline mem
414 */
415 movq %r15, %rdi
416 leaq rva(top_pgtable)(%rbx), %rsi
417 call configure_5level_paging
418
419 /* Zero EFLAGS */
420 pushq $0
421 popfq
422
423 /*
424 * Copy the compressed kernel to the end of ou
425 * where decompression in place becomes safe.
426 */
427 leaq (_bss-8)(%rip), %rsi
428 leaq rva(_bss-8)(%rbx), %rdi
429 movl $(_bss - startup_32), %ecx
430 shrl $3, %ecx
431 std
432 rep movsq
433 cld
434
435 /*
436 * The GDT may get overwritten either
437 * during extract_kernel below. To avo
438 * to the new copy of the GDT.
439 */
440 leaq rva(gdt64)(%rbx), %rax
441 leaq rva(gdt)(%rbx), %rdx
442 movq %rdx, 2(%rax)
443 lgdt (%rax)
444
445 /*
446 * Jump to the relocated address.
447 */
448 leaq rva(.Lrelocated)(%rbx), %rax
449 jmp *%rax
450 SYM_CODE_END(startup_64)
451
452 .text
453 SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated)
454
455 /*
456 * Clear BSS (stack is currently empty)
457 */
458 xorl %eax, %eax
459 leaq _bss(%rip), %rdi
460 leaq _ebss(%rip), %rcx
461 subq %rdi, %rcx
462 shrq $3, %rcx
463 rep stosq
464
465 call load_stage2_idt
466
467 /* Pass boot_params to initialize_iden
468 movq %r15, %rdi
469 call initialize_identity_maps
470
471 /*
472 * Do the extraction, and jump to the new kern
473 */
474 /* pass struct boot_params pointer and
475 movq %r15, %rdi
476 movq %rbp, %rsi
477 call extract_kernel /* ret
478
479 /*
480 * Jump to the decompressed kernel.
481 */
482 movq %r15, %rsi
483 jmp *%rax
484 SYM_FUNC_END(.Lrelocated)
485
486 /*
487 * This is the 32-bit trampoline that will be
488 * will be called using the ordinary 64-bit ca
489 * running in 64-bit mode.
490 *
491 * Return address is at the top of the stack (
492 * The first argument (EDI) contains the addre
493 * page table in 32-bit addressable memory whi
494 * register CR3.
495 */
496 .section ".rodata", "a", @progbits
497 SYM_CODE_START(trampoline_32bit_src)
498 /*
499 * Preserve callee save 64-bit registe
500 * necessary because the architecture
501 * retain their full 64-bit values acr
502 */
503 pushq %r15
504 pushq %r14
505 pushq %r13
506 pushq %r12
507 pushq %rbp
508 pushq %rbx
509
510 /* Preserve top half of RSP in a legac
511 movq %rsp, %rbx
512 shrq $32, %rbx
513
514 /* Switch to compatibility mode (CS.L
515 pushq $__KERNEL32_CS
516 leaq 0f(%rip), %rax
517 pushq %rax
518 lretq
519
520 /*
521 * The 32-bit code below will do a far
522 * up here after reconfiguring the num
523 * stack pointer needs to be restored
524 * the callee save register contents c
525 */
526 .Lret:
527 shlq $32, %rbx
528 orq %rbx, %rsp
529
530 /* Restore the preserved 64-bit regist
531 popq %rbx
532 popq %rbp
533 popq %r12
534 popq %r13
535 popq %r14
536 popq %r15
537 retq
538
539 .code32
540 0:
541 /* Disable paging */
542 movl %cr0, %eax
543 btrl $X86_CR0_PG_BIT, %eax
544 movl %eax, %cr0
545
546 /* Point CR3 to the trampoline's new t
547 movl %edi, %cr3
548
549 /* Set EFER.LME=1 as a precaution in c
550 movl $MSR_EFER, %ecx
551 rdmsr
552 btsl $_EFER_LME, %eax
553 /* Avoid writing EFER if no change was
554 jc 1f
555 wrmsr
556 1:
557 /* Toggle CR4.LA57 */
558 movl %cr4, %eax
559 btcl $X86_CR4_LA57_BIT, %eax
560 movl %eax, %cr4
561
562 /* Enable paging again. */
563 movl %cr0, %eax
564 btsl $X86_CR0_PG_BIT, %eax
565 movl %eax, %cr0
566
567 /*
568 * Return to the 64-bit calling code u
569 * avoid the need for a 32-bit address
570 * address will be adjusted after the
571 * 32-bit addressable buffer.
572 */
573 .Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampo
574 SYM_CODE_END(trampoline_32bit_src)
575
576 /*
577 * This symbol is placed right after trampolin
578 * be used to infer the size of the trampoline
579 */
580 SYM_DATA(trampoline_ljmp_imm_offset, .word .L
581
582 /*
583 * The trampoline code has a size limi
584 * Make sure we fail to compile if the
585 * beyond TRAMPOLINE_32BIT_CODE_SIZE b
586 */
587 .org trampoline_32bit_src + TRAMPOL
588
589 .text
590 SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmode)
591 /* This isn't an x86-64 CPU, so hang i
592 1:
593 hlt
594 jmp 1b
595 SYM_FUNC_END(.Lno_longmode)
596
597 .globl verify_cpu
598 #include "../../kernel/verify_cpu.S"
599
600 .data
601 SYM_DATA_START_LOCAL(gdt64)
602 .word gdt_end - gdt - 1
603 .quad gdt - gdt64
604 SYM_DATA_END(gdt64)
605 .balign 8
606 SYM_DATA_START_LOCAL(gdt)
607 .word gdt_end - gdt - 1
608 .long 0
609 .word 0
610 .quad 0x00cf9a000000ffff /* __K
611 .quad 0x00af9a000000ffff /* __K
612 .quad 0x00cf92000000ffff /* __K
613 .quad 0x0080890000000000 /* TS
614 .quad 0x0000000000000000 /* TS
615 SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end)
616
617 SYM_DATA_START(boot_idt_desc)
618 .word boot_idt_end - boot_idt - 1
619 .quad 0
620 SYM_DATA_END(boot_idt_desc)
621 .balign 8
622 SYM_DATA_START(boot_idt)
623 .rept BOOT_IDT_ENTRIES
624 .quad 0
625 .quad 0
626 .endr
627 SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBAL, boo
628
629 /*
630 * Stack and heap for uncompression
631 */
632 .bss
633 .balign 4
634 SYM_DATA_START_LOCAL(boot_stack)
635 .fill BOOT_STACK_SIZE, 1, 0
636 .balign 16
637 SYM_DATA_END_LABEL(boot_stack, SYM_L_LOCAL, bo
638
639 /*
640 * Space for page tables (not in .bss so not z
641 */
642 .section ".pgtable","aw",@nobits
643 .balign 4096
644 SYM_DATA_LOCAL(pgtable, .fill BOOT_PGT
645
646 /*
647 * The page table is going to be used instead
648 * memory.
649 */
650 SYM_DATA_LOCAL(top_pgtable, .fill PAGE_SIZ
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.