1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/mm/proc.S 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 * Author: Catalin Marinas <catalin.marinas@arm.com> 8 */ 9 10 #include <linux/init.h> 11 #include <linux/linkage.h> 12 #include <linux/pgtable.h> 13 #include <linux/cfi_types.h> 14 #include <asm/assembler.h> 15 #include <asm/asm-offsets.h> 16 #include <asm/asm_pointer_auth.h> 17 #include <asm/hwcap.h> 18 #include <asm/kernel-pgtable.h> 19 #include <asm/pgtable-hwdef.h> 20 #include <asm/cpufeature.h> 21 #include <asm/alternative.h> 22 #include <asm/smp.h> 23 #include <asm/sysreg.h> 24 25 #ifdef CONFIG_ARM64_64K_PAGES 26 #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 27 #elif defined(CONFIG_ARM64_16K_PAGES) 28 #define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K 29 #else /* CONFIG_ARM64_4K_PAGES */ 30 #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 31 #endif 32 33 #ifdef CONFIG_RANDOMIZE_BASE 34 #define TCR_KASLR_FLAGS TCR_NFD1 35 #else 36 #define TCR_KASLR_FLAGS 0 37 #endif 38 39 #define TCR_SMP_FLAGS TCR_SHARED 40 41 /* PTWs cacheable, inner/outer WBWA */ 42 #define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA 43 44 #ifdef CONFIG_KASAN_SW_TAGS 45 #define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1 46 #else 47 #define TCR_KASAN_SW_FLAGS 0 48 #endif 49 50 #ifdef CONFIG_KASAN_HW_TAGS 51 #define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1 52 #elif defined(CONFIG_ARM64_MTE) 53 /* 54 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on 55 * TBI being enabled at EL1. 56 */ 57 #define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 58 #else 59 #define TCR_MTE_FLAGS 0 60 #endif 61 62 /* 63 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and 64 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. 65 */ 66 #define MAIR_EL1_SET \ 67 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ 68 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ 69 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ 70 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ 71 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) 72 73 #ifdef CONFIG_CPU_PM 74 /** 75 * cpu_do_suspend - save CPU registers context 76 * 77 * x0: virtual address of context pointer 78 * 79 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>. 80 */ 81 SYM_FUNC_START(cpu_do_suspend) 82 mrs x2, tpidr_el0 83 mrs x3, tpidrro_el0 84 mrs x4, contextidr_el1 85 mrs x5, osdlr_el1 86 mrs x6, cpacr_el1 87 mrs x7, tcr_el1 88 mrs x8, vbar_el1 89 mrs x9, mdscr_el1 90 mrs x10, oslsr_el1 91 mrs x11, sctlr_el1 92 get_this_cpu_offset x12 93 mrs x13, sp_el0 94 stp x2, x3, [x0] 95 stp x4, x5, [x0, #16] 96 stp x6, x7, [x0, #32] 97 stp x8, x9, [x0, #48] 98 stp x10, x11, [x0, #64] 99 stp x12, x13, [x0, #80] 100 /* 101 * Save x18 as it may be used as a platform register, e.g. by shadow 102 * call stack. 103 */ 104 str x18, [x0, #96] 105 ret 106 SYM_FUNC_END(cpu_do_suspend) 107 108 /** 109 * cpu_do_resume - restore CPU register context 110 * 111 * x0: Address of context pointer 112 */ 113 SYM_FUNC_START(cpu_do_resume) 114 ldp x2, x3, [x0] 115 ldp x4, x5, [x0, #16] 116 ldp x6, x8, [x0, #32] 117 ldp x9, x10, [x0, #48] 118 ldp x11, x12, [x0, #64] 119 ldp x13, x14, [x0, #80] 120 /* 121 * Restore x18, as it may be used as a platform register, and clear 122 * the buffer to minimize the risk of exposure when used for shadow 123 * call stack. 124 */ 125 ldr x18, [x0, #96] 126 str xzr, [x0, #96] 127 msr tpidr_el0, x2 128 msr tpidrro_el0, x3 129 msr contextidr_el1, x4 130 msr cpacr_el1, x6 131 132 /* Don't change t0sz here, mask those bits when restoring */ 133 mrs x7, tcr_el1 134 bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH 135 136 msr tcr_el1, x8 137 msr vbar_el1, x9 138 msr mdscr_el1, x10 139 140 msr sctlr_el1, x12 141 set_this_cpu_offset x13 142 msr sp_el0, x14 143 /* 144 * Restore oslsr_el1 by writing oslar_el1 145 */ 146 msr osdlr_el1, x5 147 ubfx x11, x11, #1, #1 148 msr oslar_el1, x11 149 reset_pmuserenr_el0 x0 // Disable PMU access from EL0 150 reset_amuserenr_el0 x0 // Disable AMU access from EL0 151 152 alternative_if ARM64_HAS_RAS_EXTN 153 msr_s SYS_DISR_EL1, xzr 154 alternative_else_nop_endif 155 156 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 157 isb 158 ret 159 SYM_FUNC_END(cpu_do_resume) 160 #endif 161 162 .pushsection ".idmap.text", "a" 163 164 .macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2 165 adrp \tmp1, reserved_pg_dir 166 phys_to_ttbr \tmp2, \tmp1 167 offset_ttbr1 \tmp2, \tmp1 168 msr ttbr1_el1, \tmp2 169 isb 170 tlbi vmalle1 171 dsb nsh 172 isb 173 .endm 174 175 /* 176 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1) 177 * 178 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be 179 * called by anything else. It can only be executed from a TTBR0 mapping. 180 */ 181 SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) 182 __idmap_cpu_set_reserved_ttbr1 x1, x3 183 184 offset_ttbr1 x0, x3 185 msr ttbr1_el1, x0 186 isb 187 188 ret 189 SYM_FUNC_END(idmap_cpu_replace_ttbr1) 190 SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1) 191 .popsection 192 193 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 194 195 #define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \ 196 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE) 197 198 .pushsection ".idmap.text", "a" 199 200 .macro pte_to_phys, phys, pte 201 and \phys, \pte, #PTE_ADDR_LOW 202 #ifdef CONFIG_ARM64_PA_BITS_52 203 and \pte, \pte, #PTE_ADDR_HIGH 204 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT 205 #endif 206 .endm 207 208 .macro kpti_mk_tbl_ng, type, num_entries 209 add end_\type\()p, cur_\type\()p, #\num_entries * 8 210 .Ldo_\type: 211 ldr \type, [cur_\type\()p], #8 // Load the entry and advance 212 tbz \type, #0, .Lnext_\type // Skip invalid and 213 tbnz \type, #11, .Lnext_\type // non-global entries 214 orr \type, \type, #PTE_NG // Same bit for blocks and pages 215 str \type, [cur_\type\()p, #-8] // Update the entry 216 .ifnc \type, pte 217 tbnz \type, #1, .Lderef_\type 218 .endif 219 .Lnext_\type: 220 cmp cur_\type\()p, end_\type\()p 221 b.ne .Ldo_\type 222 .endm 223 224 /* 225 * Dereference the current table entry and map it into the temporary 226 * fixmap slot associated with the current level. 227 */ 228 .macro kpti_map_pgtbl, type, level 229 str xzr, [temp_pte, #8 * (\level + 2)] // break before make 230 dsb nshst 231 add pte, temp_pte, #PAGE_SIZE * (\level + 2) 232 lsr pte, pte, #12 233 tlbi vaae1, pte 234 dsb nsh 235 isb 236 237 phys_to_pte pte, cur_\type\()p 238 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2) 239 orr pte, pte, pte_flags 240 str pte, [temp_pte, #8 * (\level + 2)] 241 dsb nshst 242 .endm 243 244 /* 245 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd, 246 * unsigned long temp_pte_va) 247 * 248 * Called exactly once from stop_machine context by each CPU found during boot. 249 */ 250 .pushsection ".data", "aw", %progbits 251 SYM_DATA(__idmap_kpti_flag, .long 1) 252 .popsection 253 254 SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) 255 cpu .req w0 256 temp_pte .req x0 257 num_cpus .req w1 258 pte_flags .req x1 259 temp_pgd_phys .req x2 260 swapper_ttb .req x3 261 flag_ptr .req x4 262 cur_pgdp .req x5 263 end_pgdp .req x6 264 pgd .req x7 265 cur_pudp .req x8 266 end_pudp .req x9 267 cur_pmdp .req x11 268 end_pmdp .req x12 269 cur_ptep .req x14 270 end_ptep .req x15 271 pte .req x16 272 valid .req x17 273 cur_p4dp .req x19 274 end_p4dp .req x20 275 276 mov x5, x3 // preserve temp_pte arg 277 mrs swapper_ttb, ttbr1_el1 278 adr_l flag_ptr, __idmap_kpti_flag 279 280 cbnz cpu, __idmap_kpti_secondary 281 282 #if CONFIG_PGTABLE_LEVELS > 4 283 stp x29, x30, [sp, #-32]! 284 mov x29, sp 285 stp x19, x20, [sp, #16] 286 #endif 287 288 /* We're the boot CPU. Wait for the others to catch up */ 289 sevl 290 1: wfe 291 ldaxr w17, [flag_ptr] 292 eor w17, w17, num_cpus 293 cbnz w17, 1b 294 295 /* Switch to the temporary page tables on this CPU only */ 296 __idmap_cpu_set_reserved_ttbr1 x8, x9 297 offset_ttbr1 temp_pgd_phys, x8 298 msr ttbr1_el1, temp_pgd_phys 299 isb 300 301 mov temp_pte, x5 302 mov_q pte_flags, KPTI_NG_PTE_FLAGS 303 304 /* Everybody is enjoying the idmap, so we can rewrite swapper. */ 305 306 #ifdef CONFIG_ARM64_LPA2 307 /* 308 * If LPA2 support is configured, but 52-bit virtual addressing is not 309 * enabled at runtime, we will fall back to one level of paging less, 310 * and so we have to walk swapper_pg_dir as if we dereferenced its 311 * address from a PGD level entry, and terminate the PGD level loop 312 * right after. 313 */ 314 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level 315 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop 316 alternative_if_not ARM64_HAS_VA52 317 b .Lderef_pgd // skip to the next level 318 alternative_else_nop_endif 319 /* 320 * LPA2 based 52-bit virtual addressing requires 52-bit physical 321 * addressing to be enabled as well. In this case, the shareability 322 * bits are repurposed as physical address bits, and should not be 323 * set in pte_flags. 324 */ 325 bic pte_flags, pte_flags, #PTE_SHARED 326 #endif 327 328 /* PGD */ 329 adrp cur_pgdp, swapper_pg_dir 330 kpti_map_pgtbl pgd, -1 331 kpti_mk_tbl_ng pgd, PTRS_PER_PGD 332 333 /* Ensure all the updated entries are visible to secondary CPUs */ 334 dsb ishst 335 336 /* We're done: fire up swapper_pg_dir again */ 337 __idmap_cpu_set_reserved_ttbr1 x8, x9 338 msr ttbr1_el1, swapper_ttb 339 isb 340 341 /* Set the flag to zero to indicate that we're all done */ 342 str wzr, [flag_ptr] 343 #if CONFIG_PGTABLE_LEVELS > 4 344 ldp x19, x20, [sp, #16] 345 ldp x29, x30, [sp], #32 346 #endif 347 ret 348 349 .Lderef_pgd: 350 /* P4D */ 351 .if CONFIG_PGTABLE_LEVELS > 4 352 p4d .req x30 353 pte_to_phys cur_p4dp, pgd 354 kpti_map_pgtbl p4d, 0 355 kpti_mk_tbl_ng p4d, PTRS_PER_P4D 356 b .Lnext_pgd 357 .else /* CONFIG_PGTABLE_LEVELS <= 4 */ 358 p4d .req pgd 359 .set .Lnext_p4d, .Lnext_pgd 360 .endif 361 362 .Lderef_p4d: 363 /* PUD */ 364 .if CONFIG_PGTABLE_LEVELS > 3 365 pud .req x10 366 pte_to_phys cur_pudp, p4d 367 kpti_map_pgtbl pud, 1 368 kpti_mk_tbl_ng pud, PTRS_PER_PUD 369 b .Lnext_p4d 370 .else /* CONFIG_PGTABLE_LEVELS <= 3 */ 371 pud .req pgd 372 .set .Lnext_pud, .Lnext_pgd 373 .endif 374 375 .Lderef_pud: 376 /* PMD */ 377 .if CONFIG_PGTABLE_LEVELS > 2 378 pmd .req x13 379 pte_to_phys cur_pmdp, pud 380 kpti_map_pgtbl pmd, 2 381 kpti_mk_tbl_ng pmd, PTRS_PER_PMD 382 b .Lnext_pud 383 .else /* CONFIG_PGTABLE_LEVELS <= 2 */ 384 pmd .req pgd 385 .set .Lnext_pmd, .Lnext_pgd 386 .endif 387 388 .Lderef_pmd: 389 /* PTE */ 390 pte_to_phys cur_ptep, pmd 391 kpti_map_pgtbl pte, 3 392 kpti_mk_tbl_ng pte, PTRS_PER_PTE 393 b .Lnext_pmd 394 395 .unreq cpu 396 .unreq temp_pte 397 .unreq num_cpus 398 .unreq pte_flags 399 .unreq temp_pgd_phys 400 .unreq cur_pgdp 401 .unreq end_pgdp 402 .unreq pgd 403 .unreq cur_pudp 404 .unreq end_pudp 405 .unreq pud 406 .unreq cur_pmdp 407 .unreq end_pmdp 408 .unreq pmd 409 .unreq cur_ptep 410 .unreq end_ptep 411 .unreq pte 412 .unreq valid 413 .unreq cur_p4dp 414 .unreq end_p4dp 415 .unreq p4d 416 417 /* Secondary CPUs end up here */ 418 __idmap_kpti_secondary: 419 /* Uninstall swapper before surgery begins */ 420 __idmap_cpu_set_reserved_ttbr1 x16, x17 421 422 /* Increment the flag to let the boot CPU we're ready */ 423 1: ldxr w16, [flag_ptr] 424 add w16, w16, #1 425 stxr w17, w16, [flag_ptr] 426 cbnz w17, 1b 427 428 /* Wait for the boot CPU to finish messing around with swapper */ 429 sevl 430 1: wfe 431 ldxr w16, [flag_ptr] 432 cbnz w16, 1b 433 434 /* All done, act like nothing happened */ 435 msr ttbr1_el1, swapper_ttb 436 isb 437 ret 438 439 .unreq swapper_ttb 440 .unreq flag_ptr 441 SYM_FUNC_END(idmap_kpti_install_ng_mappings) 442 .popsection 443 #endif 444 445 /* 446 * __cpu_setup 447 * 448 * Initialise the processor for turning the MMU on. 449 * 450 * Output: 451 * Return in x0 the value of the SCTLR_EL1 register. 452 */ 453 .pushsection ".idmap.text", "a" 454 SYM_FUNC_START(__cpu_setup) 455 tlbi vmalle1 // Invalidate local TLB 456 dsb nsh 457 458 msr cpacr_el1, xzr // Reset cpacr_el1 459 mov x1, #1 << 12 // Reset mdscr_el1 and disable 460 msr mdscr_el1, x1 // access to the DCC from EL0 461 reset_pmuserenr_el0 x1 // Disable PMU access from EL0 462 reset_amuserenr_el0 x1 // Disable AMU access from EL0 463 464 /* 465 * Default values for VMSA control registers. These will be adjusted 466 * below depending on detected CPU features. 467 */ 468 mair .req x17 469 tcr .req x16 470 mov_q mair, MAIR_EL1_SET 471 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \ 472 TCR_SMP_FLAGS | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ 473 TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS 474 475 tcr_clear_errata_bits tcr, x9, x5 476 477 #ifdef CONFIG_ARM64_VA_BITS_52 478 mov x9, #64 - VA_BITS 479 alternative_if ARM64_HAS_VA52 480 tcr_set_t1sz tcr, x9 481 #ifdef CONFIG_ARM64_LPA2 482 orr tcr, tcr, #TCR_DS 483 #endif 484 alternative_else_nop_endif 485 #endif 486 487 /* 488 * Set the IPS bits in TCR_EL1. 489 */ 490 tcr_compute_pa_size tcr, #TCR_IPS_SHIFT, x5, x6 491 #ifdef CONFIG_ARM64_HW_AFDBM 492 /* 493 * Enable hardware update of the Access Flags bit. 494 * Hardware dirty bit management is enabled later, 495 * via capabilities. 496 */ 497 mrs x9, ID_AA64MMFR1_EL1 498 and x9, x9, ID_AA64MMFR1_EL1_HAFDBS_MASK 499 cbz x9, 1f 500 orr tcr, tcr, #TCR_HA // hardware Access flag update 501 1: 502 #endif /* CONFIG_ARM64_HW_AFDBM */ 503 msr mair_el1, mair 504 msr tcr_el1, tcr 505 506 mrs_s x1, SYS_ID_AA64MMFR3_EL1 507 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4 508 cbz x1, .Lskip_indirection 509 510 /* 511 * The PROT_* macros describing the various memory types may resolve to 512 * C expressions if they include the PTE_MAYBE_* macros, and so they 513 * can only be used from C code. The PIE_E* constants below are also 514 * defined in terms of those macros, but will mask out those 515 * PTE_MAYBE_* constants, whether they are set or not. So #define them 516 * as 0x0 here so we can evaluate the PIE_E* constants in asm context. 517 */ 518 519 #define PTE_MAYBE_NG 0 520 #define PTE_MAYBE_SHARED 0 521 522 mov_q x0, PIE_E0 523 msr REG_PIRE0_EL1, x0 524 mov_q x0, PIE_E1 525 msr REG_PIR_EL1, x0 526 527 #undef PTE_MAYBE_NG 528 #undef PTE_MAYBE_SHARED 529 530 mov x0, TCR2_EL1x_PIE 531 msr REG_TCR2_EL1, x0 532 533 .Lskip_indirection: 534 535 /* 536 * Prepare SCTLR 537 */ 538 mov_q x0, INIT_SCTLR_EL1_MMU_ON 539 ret // return to head.S 540 541 .unreq mair 542 .unreq tcr 543 SYM_FUNC_END(__cpu_setup)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.