1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * 4 * Trampoline.S Derived from Setup.S by Linus Torvalds 5 * 6 * 4 Jan 1997 Michael Chastain: changed to gnu as. 7 * 15 Sept 2005 Eric Biederman: 64bit PIC support 8 * 9 * Entry: CS:IP point to the start of our code, we are 10 * in real mode with no stack, but the rest of the 11 * trampoline page to make our stack and everything else 12 * is a mystery. 13 * 14 * On entry to trampoline_start, the processor is in real mode 15 * with 16-bit addressing and 16-bit data. CS has some value 16 * and IP is zero. Thus, data addresses need to be absolute 17 * (no relocation) and are taken with regard to r_base. 18 * 19 * With the addition of trampoline_level4_pgt this code can 20 * now enter a 64bit kernel that lives at arbitrary 64bit 21 * physical addresses. 22 * 23 * If you work on this file, check the object module with objdump 24 * --full-contents --reloc to make sure there are no relocation 25 * entries. 26 */ 27 28 #include <linux/linkage.h> 29 #include <asm/pgtable_types.h> 30 #include <asm/page_types.h> 31 #include <asm/msr.h> 32 #include <asm/segment.h> 33 #include <asm/processor-flags.h> 34 #include <asm/realmode.h> 35 #include "realmode.h" 36 37 .text 38 .code16 39 40 .macro LOCK_AND_LOAD_REALMODE_ESP lock_pa=0 lock_rip=0 41 /* 42 * Make sure only one CPU fiddles with the realmode stack 43 */ 44 .Llock_rm\@: 45 .if \lock_pa 46 lock btsl $0, pa_tr_lock 47 .elseif \lock_rip 48 lock btsl $0, tr_lock(%rip) 49 .else 50 lock btsl $0, tr_lock 51 .endif 52 jnc 2f 53 pause 54 jmp .Llock_rm\@ 55 2: 56 # Setup stack 57 movl $rm_stack_end, %esp 58 .endm 59 60 .balign PAGE_SIZE 61 SYM_CODE_START(trampoline_start) 62 cli # We should be safe anyway 63 wbinvd 64 65 LJMPW_RM(1f) 66 1: 67 mov %cs, %ax # Code and data in the same place 68 mov %ax, %ds 69 mov %ax, %es 70 mov %ax, %ss 71 72 LOCK_AND_LOAD_REALMODE_ESP 73 74 call verify_cpu # Verify the cpu supports long mode 75 testl %eax, %eax # Check for return code 76 jnz no_longmode 77 78 .Lswitch_to_protected: 79 /* 80 * GDT tables in non default location kernel can be beyond 16MB and 81 * lgdt will not be able to load the address as in real mode default 82 * operand size is 16bit. Use lgdtl instead to force operand size 83 * to 32 bit. 84 */ 85 86 lidtl tr_idt # load idt with 0, 0 87 lgdtl tr_gdt # load gdt with whatever is appropriate 88 89 movw $__KERNEL_DS, %dx # Data segment descriptor 90 91 # Enable protected mode 92 movl $(CR0_STATE & ~X86_CR0_PG), %eax 93 movl %eax, %cr0 # into protected mode 94 95 # flush prefetch and jump to startup_32 96 ljmpl $__KERNEL32_CS, $pa_startup_32 97 98 no_longmode: 99 hlt 100 jmp no_longmode 101 SYM_CODE_END(trampoline_start) 102 103 #ifdef CONFIG_AMD_MEM_ENCRYPT 104 /* SEV-ES supports non-zero IP for entry points - no alignment needed */ 105 SYM_CODE_START(sev_es_trampoline_start) 106 cli # We should be safe anyway 107 108 LJMPW_RM(1f) 109 1: 110 mov %cs, %ax # Code and data in the same place 111 mov %ax, %ds 112 mov %ax, %es 113 mov %ax, %ss 114 115 LOCK_AND_LOAD_REALMODE_ESP 116 117 jmp .Lswitch_to_protected 118 SYM_CODE_END(sev_es_trampoline_start) 119 #endif /* CONFIG_AMD_MEM_ENCRYPT */ 120 121 #include "../kernel/verify_cpu.S" 122 123 .section ".text32","ax" 124 .code32 125 .balign 4 126 SYM_CODE_START(startup_32) 127 movl %edx, %ss 128 addl $pa_real_mode_base, %esp 129 movl %edx, %ds 130 movl %edx, %es 131 movl %edx, %fs 132 movl %edx, %gs 133 134 /* 135 * Check for memory encryption support. This is a safety net in 136 * case BIOS hasn't done the necessary step of setting the bit in 137 * the MSR for this AP. If SME is active and we've gotten this far 138 * then it is safe for us to set the MSR bit and continue. If we 139 * don't we'll eventually crash trying to execute encrypted 140 * instructions. 141 */ 142 btl $TH_FLAGS_SME_ACTIVE_BIT, pa_tr_flags 143 jnc .Ldone 144 movl $MSR_AMD64_SYSCFG, %ecx 145 rdmsr 146 bts $MSR_AMD64_SYSCFG_MEM_ENCRYPT_BIT, %eax 147 jc .Ldone 148 149 /* 150 * Memory encryption is enabled but the SME enable bit for this 151 * CPU has has not been set. It is safe to set it, so do so. 152 */ 153 wrmsr 154 .Ldone: 155 156 movl pa_tr_cr4, %eax 157 movl %eax, %cr4 # Enable PAE mode 158 159 # Setup trampoline 4 level pagetables 160 movl $pa_trampoline_pgd, %eax 161 movl %eax, %cr3 162 163 # Set up EFER 164 movl $MSR_EFER, %ecx 165 rdmsr 166 /* 167 * Skip writing to EFER if the register already has desired 168 * value (to avoid #VE for the TDX guest). 169 */ 170 cmp pa_tr_efer, %eax 171 jne .Lwrite_efer 172 cmp pa_tr_efer + 4, %edx 173 je .Ldone_efer 174 .Lwrite_efer: 175 movl pa_tr_efer, %eax 176 movl pa_tr_efer + 4, %edx 177 wrmsr 178 179 .Ldone_efer: 180 # Enable paging and in turn activate Long Mode. 181 movl $CR0_STATE, %eax 182 movl %eax, %cr0 183 184 /* 185 * At this point we're in long mode but in 32bit compatibility mode 186 * with EFER.LME = 1, CS.L = 0, CS.D = 1 (and in turn 187 * EFER.LMA = 1). Now we want to jump in 64bit mode, to do that we use 188 * the new gdt/idt that has __KERNEL_CS with CS.L = 1. 189 */ 190 ljmpl $__KERNEL_CS, $pa_startup_64 191 SYM_CODE_END(startup_32) 192 193 SYM_CODE_START(pa_trampoline_compat) 194 /* 195 * In compatibility mode. Prep ESP and DX for startup_32, then disable 196 * paging and complete the switch to legacy 32-bit mode. 197 */ 198 LOCK_AND_LOAD_REALMODE_ESP lock_pa=1 199 movw $__KERNEL_DS, %dx 200 201 movl $(CR0_STATE & ~X86_CR0_PG), %eax 202 movl %eax, %cr0 203 ljmpl $__KERNEL32_CS, $pa_startup_32 204 SYM_CODE_END(pa_trampoline_compat) 205 206 .section ".text64","ax" 207 .code64 208 .balign 4 209 SYM_CODE_START(startup_64) 210 # Now jump into the kernel using virtual addresses 211 jmpq *tr_start(%rip) 212 SYM_CODE_END(startup_64) 213 214 SYM_CODE_START(trampoline_start64) 215 /* 216 * APs start here on a direct transfer from 64-bit BIOS with identity 217 * mapped page tables. Load the kernel's GDT in order to gear down to 218 * 32-bit mode (to handle 4-level vs. 5-level paging), and to (re)load 219 * segment registers. Load the zero IDT so any fault triggers a 220 * shutdown instead of jumping back into BIOS. 221 */ 222 lidt tr_idt(%rip) 223 lgdt tr_gdt64(%rip) 224 225 /* Check if paging mode has to be changed */ 226 movq %cr4, %rax 227 xorl tr_cr4(%rip), %eax 228 testl $X86_CR4_LA57, %eax 229 jnz .L_switch_paging 230 231 /* Paging mode is correct proceed in 64-bit mode */ 232 233 LOCK_AND_LOAD_REALMODE_ESP lock_rip=1 234 235 movw $__KERNEL_DS, %dx 236 movl %edx, %ss 237 addl $pa_real_mode_base, %esp 238 movl %edx, %ds 239 movl %edx, %es 240 movl %edx, %fs 241 movl %edx, %gs 242 243 movl $pa_trampoline_pgd, %eax 244 movq %rax, %cr3 245 246 pushq $__KERNEL_CS 247 pushq tr_start(%rip) 248 lretq 249 .L_switch_paging: 250 /* 251 * To switch between 4- and 5-level paging modes, it is necessary 252 * to disable paging. This must be done in the compatibility mode. 253 */ 254 ljmpl *tr_compat(%rip) 255 SYM_CODE_END(trampoline_start64) 256 257 .section ".rodata","a" 258 # Duplicate the global descriptor table 259 # so the kernel can live anywhere 260 .balign 16 261 SYM_DATA_START(tr_gdt) 262 .short tr_gdt_end - tr_gdt - 1 # gdt limit 263 .long pa_tr_gdt 264 .short 0 265 .quad 0x00cf9b000000ffff # __KERNEL32_CS 266 .quad 0x00af9b000000ffff # __KERNEL_CS 267 .quad 0x00cf93000000ffff # __KERNEL_DS 268 SYM_DATA_END_LABEL(tr_gdt, SYM_L_LOCAL, tr_gdt_end) 269 270 SYM_DATA_START(tr_gdt64) 271 .short tr_gdt_end - tr_gdt - 1 # gdt limit 272 .long pa_tr_gdt 273 .long 0 274 SYM_DATA_END(tr_gdt64) 275 276 SYM_DATA_START(tr_compat) 277 .long pa_trampoline_compat 278 .short __KERNEL32_CS 279 SYM_DATA_END(tr_compat) 280 281 .bss 282 .balign PAGE_SIZE 283 SYM_DATA(trampoline_pgd, .space PAGE_SIZE) 284 285 .balign 8 286 SYM_DATA_START(trampoline_header) 287 SYM_DATA_LOCAL(tr_start, .space 8) 288 SYM_DATA(tr_efer, .space 8) 289 SYM_DATA(tr_cr4, .space 4) 290 SYM_DATA(tr_flags, .space 4) 291 SYM_DATA(tr_lock, .space 4) 292 SYM_DATA_END(trampoline_header) 293 294 #include "trampoline_common.S"
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.