1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* !! 2 /* head.S: Initial boot code for the Sparc64 port of Linux.
3 * linux/arch/x86/kernel/head_64.S -- start i <<
4 * 3 *
5 * Copyright (C) 2000 Andrea Arcangeli <andrea !! 4 * Copyright (C) 1996, 1997, 2007 David S. Miller (davem@davemloft.net)
6 * Copyright (C) 2000 Pavel Machek <pavel@suse !! 5 * Copyright (C) 1996 David Sitsky (David.Sitsky@anu.edu.au)
7 * Copyright (C) 2000 Karsten Keil <kkeil@suse !! 6 * Copyright (C) 1997, 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 * Copyright (C) 2001,2002 Andi Kleen <ak@suse !! 7 * Copyright (C) 1997 Miguel de Icaza (miguel@nuclecu.unam.mx)
9 * Copyright (C) 2005 Eric Biederman <ebiederm <<
10 */ 8 */
11 9
>> 10 #include <linux/version.h>
>> 11 #include <linux/errno.h>
12 #include <linux/export.h> 12 #include <linux/export.h>
13 #include <linux/linkage.h> <<
14 #include <linux/threads.h> 13 #include <linux/threads.h>
15 #include <linux/init.h> 14 #include <linux/init.h>
>> 15 #include <linux/linkage.h>
16 #include <linux/pgtable.h> 16 #include <linux/pgtable.h>
17 #include <asm/segment.h> <<
18 #include <asm/page.h> <<
19 #include <asm/msr.h> <<
20 #include <asm/cache.h> <<
21 #include <asm/processor-flags.h> <<
22 #include <asm/percpu.h> <<
23 #include <asm/nops.h> <<
24 #include "../entry/calling.h" <<
25 #include <asm/nospec-branch.h> <<
26 #include <asm/apicdef.h> <<
27 #include <asm/fixmap.h> <<
28 #include <asm/smp.h> <<
29 #include <asm/thread_info.h> 17 #include <asm/thread_info.h>
>> 18 #include <asm/asi.h>
>> 19 #include <asm/pstate.h>
>> 20 #include <asm/ptrace.h>
>> 21 #include <asm/spitfire.h>
>> 22 #include <asm/page.h>
>> 23 #include <asm/errno.h>
>> 24 #include <asm/signal.h>
>> 25 #include <asm/processor.h>
>> 26 #include <asm/lsu.h>
>> 27 #include <asm/dcr.h>
>> 28 #include <asm/dcu.h>
>> 29 #include <asm/head.h>
>> 30 #include <asm/ttable.h>
>> 31 #include <asm/mmu.h>
>> 32 #include <asm/cpudata.h>
>> 33 #include <asm/pil.h>
>> 34 #include <asm/estate.h>
>> 35 #include <asm/sfafsr.h>
>> 36 #include <asm/unistd.h>
30 37
31 /* !! 38 /* This section from from _start to sparc64_boot_end should fit into
32 * We are not able to switch in one step to th !! 39 * 0x0000000000404000 to 0x0000000000408000.
33 * because we need identity-mapped pages. <<
34 */ 40 */
35 <<
36 __HEAD <<
37 .code64 <<
38 SYM_CODE_START_NOALIGN(startup_64) <<
39 UNWIND_HINT_END_OF_STACK <<
40 /* <<
41 * At this point the CPU runs in 64bit <<
42 * and someone has loaded an identity <<
43 * for us. These identity mapped page <<
44 * kernel pages and possibly all of me <<
45 * <<
46 * %RSI holds the physical address of <<
47 * provided by the bootloader. Preserv <<
48 * will not clobber it. <<
49 * <<
50 * We come here either directly from a <<
51 * arch/x86/boot/compressed/head_64.S. <<
52 * <<
53 * We only come here initially at boot <<
54 * <<
55 * Since we may be loaded at an addres <<
56 * compiled to run at we first fixup t <<
57 * tables and then reload them. <<
58 */ <<
59 mov %rsi, %r15 <<
60 <<
61 /* Set up the stack for verify_cpu() * <<
62 leaq __top_init_kernel_stack(%rip), <<
63 <<
64 /* Setup GSBASE to allow stack canary <<
65 movl $MSR_GS_BASE, %ecx <<
66 leaq INIT_PER_CPU_VAR(fixed_percpu_ <<
67 movl %edx, %eax <<
68 shrq $32, %rdx <<
69 wrmsr <<
70 <<
71 call startup_64_setup_gdt_idt <<
72 <<
73 /* Now switch to __KERNEL_CS so IRET w <<
74 pushq $__KERNEL_CS <<
75 leaq .Lon_kernel_cs(%rip), %rax <<
76 pushq %rax <<
77 lretq <<
78 <<
79 .Lon_kernel_cs: <<
80 UNWIND_HINT_END_OF_STACK <<
81 <<
82 #ifdef CONFIG_AMD_MEM_ENCRYPT <<
83 /* <<
84 * Activate SEV/SME memory encryption <<
85 * be done now, since this also includ <<
86 * which needs to be done before any C <<
87 * subsequent code. Pass the boot_para <<
88 */ <<
89 movq %r15, %rdi <<
90 call sme_enable <<
91 #endif <<
92 <<
93 /* Sanitize CPU configuration */ <<
94 call verify_cpu <<
95 <<
96 /* <<
97 * Perform pagetable fixups. Additiona <<
98 * the kernel and retrieve the modifie <<
99 * is active) to be added to the initi <<
100 * programmed into CR3. <<
101 */ <<
102 leaq _text(%rip), %rdi <<
103 movq %r15, %rsi <<
104 call __startup_64 <<
105 <<
106 /* Form the CR3 value being sure to in <<
107 leaq early_top_pgt(%rip), %rcx <<
108 addq %rcx, %rax <<
109 <<
110 #ifdef CONFIG_AMD_MEM_ENCRYPT <<
111 mov %rax, %rdi <<
112 <<
113 /* <<
114 * For SEV guests: Verify that the C-b <<
115 * hypervisor could lie about the C-bi <<
116 * attack on the guest by writing to t <<
117 * the next RET instruction. <<
118 */ <<
119 call sev_verify_cbit <<
120 #endif <<
121 <<
122 /* <<
123 * Switch to early_top_pgt which still <<
124 * present. <<
125 */ <<
126 movq %rax, %cr3 <<
127 <<
128 /* Branch to the common startup code a <<
129 ANNOTATE_RETPOLINE_SAFE <<
130 jmp *0f(%rip) <<
131 SYM_CODE_END(startup_64) <<
132 <<
133 __INITRODATA <<
134 0: .quad common_startup_64 <<
135 <<
136 .text 41 .text
137 SYM_CODE_START(secondary_startup_64) !! 42 .globl start, _start, stext, _stext
138 UNWIND_HINT_END_OF_STACK !! 43 _start:
139 ANNOTATE_NOENDBR !! 44 start:
140 /* !! 45 _stext:
141 * At this point the CPU runs in 64bit !! 46 stext:
142 * and someone has loaded a mapped pag !! 47 ! 0x0000000000404000
143 * !! 48 b sparc64_boot
144 * We come here either from startup_64 !! 49 flushw /* Flush register file. */
145 * or from trampoline.S (using virtual !! 50
146 * !! 51 /* This stuff has to be in sync with SILO and other potential boot loaders
147 * Using virtual addresses from trampo !! 52 * Fields should be kept upward compatible and whenever any change is made,
148 * to have any identity mapped pages i !! 53 * HdrS version should be incremented.
149 * after the boot processor executes t !! 54 */
150 */ !! 55 .global root_flags, ram_flags, root_dev
151 !! 56 .global sparc_ramdisk_image, sparc_ramdisk_size
152 /* Sanitize CPU configuration */ !! 57 .global sparc_ramdisk_image64
153 call verify_cpu !! 58
154 !! 59 .ascii "HdrS"
155 /* !! 60 .word LINUX_VERSION_CODE
156 * The secondary_startup_64_no_verify !! 61
157 * SEV-ES guests. In those guests the !! 62 /* History:
158 * #VC exceptions which can not be han !! 63 *
159 * CPU bringup. !! 64 * 0x0300 : Supports being located at other than 0x4000
160 * !! 65 * 0x0202 : Supports kernel params string
161 * All non SEV-ES systems, especially !! 66 * 0x0201 : Supports reboot_command
162 * verify_cpu() above to make sure NX !! 67 */
163 */ !! 68 .half 0x0301 /* HdrS version */
164 SYM_INNER_LABEL(secondary_startup_64_no_verify !! 69
165 UNWIND_HINT_END_OF_STACK !! 70 root_flags:
166 ANNOTATE_NOENDBR !! 71 .half 1
167 !! 72 root_dev:
168 /* Clear %R15 which holds the boot_par !! 73 .half 0
169 xorl %r15d, %r15d !! 74 ram_flags:
170 !! 75 .half 0
171 /* Derive the runtime physical address !! 76 sparc_ramdisk_image:
172 movq phys_base(%rip), %rax !! 77 .word 0
173 addq $(init_top_pgt - __START_KERNE !! 78 sparc_ramdisk_size:
174 !! 79 .word 0
175 /* !! 80 .xword reboot_command
176 * Retrieve the modifier (SME encrypti !! 81 .xword bootstr_info
177 * added to the initial pgdir entry th !! 82 sparc_ramdisk_image64:
178 */ !! 83 .xword 0
179 #ifdef CONFIG_AMD_MEM_ENCRYPT !! 84 .word _end
180 addq sme_me_mask(%rip), %rax !! 85
181 #endif !! 86 /* PROM cif handler code address is in %o4. */
182 /* !! 87 sparc64_boot:
183 * Switch to the init_top_pgt here, aw !! 88 mov %o4, %l7
184 * unmap the identity mapped ranges. !! 89
185 */ !! 90 /* We need to remap the kernel. Use position independent
186 movq %rax, %cr3 !! 91 * code to remap us to KERNBASE.
187 !! 92 *
188 SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL !! 93 * SILO can invoke us with 32-bit address masking enabled,
189 UNWIND_HINT_END_OF_STACK !! 94 * so make sure that's clear.
190 ANNOTATE_NOENDBR !! 95 */
191 !! 96 rdpr %pstate, %g1
192 /* !! 97 andn %g1, PSTATE_AM, %g1
193 * Create a mask of CR4 bits to preser !! 98 wrpr %g1, 0x0, %pstate
194 * global 1:1 translations from the TL !! 99 ba,a,pt %xcc, 1f
195 * !! 100 nop
196 * From the SDM: !! 101
197 * "If CR4.PGE is changing from 0 to 1 !! 102 .globl prom_finddev_name, prom_chosen_path, prom_root_node
198 * entries before the execution; if C !! 103 .globl prom_getprop_name, prom_mmu_name, prom_peer_name
199 * there will be no global TLB entrie !! 104 .globl prom_callmethod_name, prom_translate_name, prom_root_compatible
200 */ !! 105 .globl prom_map_name, prom_unmap_name, prom_mmu_ihandle_cache
201 movl $(X86_CR4_PAE | X86_CR4_LA57), !! 106 .globl prom_boot_mapped_pc, prom_boot_mapping_mode
202 #ifdef CONFIG_X86_MCE !! 107 .globl prom_boot_mapping_phys_high, prom_boot_mapping_phys_low
203 /* !! 108 .globl prom_compatible_name, prom_cpu_path, prom_cpu_compatible
204 * Preserve CR4.MCE if the kernel will !! 109 .globl is_sun4v, sun4v_chip_type, prom_set_trap_table_name
205 * Clearing MCE may fault in some envi !! 110 prom_peer_name:
206 * support). Any machine check that oc !! 111 .asciz "peer"
207 * configured will crash the system re !! 112 prom_compatible_name:
208 * here. !! 113 .asciz "compatible"
209 */ !! 114 prom_finddev_name:
210 orl $X86_CR4_MCE, %edx !! 115 .asciz "finddevice"
211 #endif !! 116 prom_chosen_path:
212 movq %cr4, %rcx !! 117 .asciz "/chosen"
213 andl %edx, %ecx !! 118 prom_cpu_path:
214 !! 119 .asciz "/cpu"
215 /* Even if ignored in long mode, set P !! 120 prom_getprop_name:
216 btsl $X86_CR4_PSE_BIT, %ecx !! 121 .asciz "getprop"
217 movq %rcx, %cr4 !! 122 prom_mmu_name:
218 !! 123 .asciz "mmu"
219 /* !! 124 prom_callmethod_name:
220 * Set CR4.PGE to re-enable global tra !! 125 .asciz "call-method"
221 */ !! 126 prom_translate_name:
222 btsl $X86_CR4_PGE_BIT, %ecx !! 127 .asciz "translate"
223 movq %rcx, %cr4 !! 128 prom_map_name:
224 !! 129 .asciz "map"
225 #ifdef CONFIG_SMP !! 130 prom_unmap_name:
226 /* !! 131 .asciz "unmap"
227 * For parallel boot, the APIC ID is r !! 132 prom_set_trap_table_name:
228 * used to look up the CPU number. Fo !! 133 .asciz "SUNW,set-trap-table"
229 * CPU number is encoded in smpboot_co !! 134 prom_sun4v_name:
230 * !! 135 .asciz "sun4v"
231 * Bit 31 STARTUP_READ_APICID (R !! 136 prom_niagara_prefix:
232 * Bit 0-23 CPU# if STARTUP_xx fla !! 137 .asciz "SUNW,UltraSPARC-T"
233 */ !! 138 prom_sparc_prefix:
234 movl smpboot_control(%rip), %ecx !! 139 .asciz "SPARC-"
235 testl $STARTUP_READ_APICID, %ecx !! 140 prom_sparc64x_prefix:
236 jnz .Lread_apicid !! 141 .asciz "SPARC64-X"
237 /* !! 142 .align 4
238 * No control bit set, single CPU brin !! 143 prom_root_compatible:
239 * in bit 0-23. This is also the boot !! 144 .skip 64
240 */ !! 145 prom_cpu_compatible:
241 andl $(~STARTUP_PARALLEL_MASK), %ec !! 146 .skip 64
242 jmp .Lsetup_cpu !! 147 prom_root_node:
243 !! 148 .word 0
244 .Lread_apicid: !! 149 EXPORT_SYMBOL(prom_root_node)
245 /* Check whether X2APIC mode is alread !! 150 prom_mmu_ihandle_cache:
246 mov $MSR_IA32_APICBASE, %ecx !! 151 .word 0
247 rdmsr !! 152 prom_boot_mapped_pc:
248 testl $X2APIC_ENABLE, %eax !! 153 .word 0
249 jnz .Lread_apicid_msr !! 154 prom_boot_mapping_mode:
250 !! 155 .word 0
251 #ifdef CONFIG_X86_X2APIC !! 156 .align 8
252 /* !! 157 prom_boot_mapping_phys_high:
253 * If system is in X2APIC mode then MM !! 158 .xword 0
254 * mapped causing the MMIO read below !! 159 prom_boot_mapping_phys_low:
255 * be handled at that point. !! 160 .xword 0
256 */ !! 161 is_sun4v:
257 cmpl $0, x2apic_mode(%rip) !! 162 .word 0
258 jz .Lread_apicid_mmio !! 163 sun4v_chip_type:
259 !! 164 .word SUN4V_CHIP_INVALID
260 /* Force the AP into X2APIC mode. */ !! 165 EXPORT_SYMBOL(sun4v_chip_type)
261 orl $X2APIC_ENABLE, %eax <<
262 wrmsr <<
263 jmp .Lread_apicid_msr <<
264 #endif <<
265 <<
266 .Lread_apicid_mmio: <<
267 /* Read the APIC ID from the fix-mappe <<
268 movq apic_mmio_base(%rip), %rcx <<
269 addq $APIC_ID, %rcx <<
270 movl (%rcx), %eax <<
271 shr $24, %eax <<
272 jmp .Llookup_AP <<
273 <<
274 .Lread_apicid_msr: <<
275 mov $APIC_X2APIC_ID_MSR, %ecx <<
276 rdmsr <<
277 <<
278 .Llookup_AP: <<
279 /* EAX contains the APIC ID of the cur <<
280 xorl %ecx, %ecx <<
281 leaq cpuid_to_apicid(%rip), %rbx <<
282 <<
283 .Lfind_cpunr: <<
284 cmpl (%rbx,%rcx,4), %eax <<
285 jz .Lsetup_cpu <<
286 inc %ecx <<
287 #ifdef CONFIG_FORCE_NR_CPUS <<
288 cmpl $NR_CPUS, %ecx <<
289 #else <<
290 cmpl nr_cpu_ids(%rip), %ecx <<
291 #endif <<
292 jb .Lfind_cpunr <<
293 <<
294 /* APIC ID not found in the table. Dr <<
295 movq trampoline_lock(%rip), %rax <<
296 movl $0, (%rax) <<
297 <<
298 1: cli <<
299 hlt <<
300 jmp 1b <<
301 <<
302 .Lsetup_cpu: <<
303 /* Get the per cpu offset for the give <<
304 movq __per_cpu_offset(,%rcx,8), %rd <<
305 #else <<
306 xorl %edx, %edx /* zero-extended to <<
307 #endif /* CONFIG_SMP */ <<
308 <<
309 /* <<
310 * Setup a boot time stack - Any secon <<
311 * by now because the cr3-switch above <<
312 * <<
313 * RDX contains the per-cpu offset <<
314 */ <<
315 movq pcpu_hot + X86_current_task(%r <<
316 movq TASK_threadsp(%rax), %rsp <<
317 <<
318 /* <<
319 * Now that this CPU is running on its <<
320 * protection. For the boot CPU the po <<
321 */ <<
322 movq trampoline_lock(%rip), %rax <<
323 testq %rax, %rax <<
324 jz .Lsetup_gdt <<
325 movl $0, (%rax) <<
326 <<
327 .Lsetup_gdt: <<
328 /* <<
329 * We must switch to a new descriptor <<
330 * because soon the kernel won't have <<
331 * addresses where we're currently run <<
332 * because in 32bit we couldn't load a <<
333 */ <<
334 subq $16, %rsp <<
335 movw $(GDT_SIZE-1), (%rsp) <<
336 leaq gdt_page(%rdx), %rax <<
337 movq %rax, 2(%rsp) <<
338 lgdt (%rsp) <<
339 addq $16, %rsp <<
340 <<
341 /* set up data segments */ <<
342 xorl %eax,%eax <<
343 movl %eax,%ds <<
344 movl %eax,%ss <<
345 movl %eax,%es <<
346 <<
347 /* <<
348 * We don't really need to load %fs or <<
349 * to kill any stale realmode selector <<
350 * under VT hardware. <<
351 */ <<
352 movl %eax,%fs <<
353 movl %eax,%gs <<
354 <<
355 /* Set up %gs. <<
356 * <<
357 * The base of %gs always points to fi <<
358 * stack protector canary is enabled, <<
359 * Note that, on SMP, the boot cpu use <<
360 * the per cpu areas are set up. <<
361 */ <<
362 movl $MSR_GS_BASE,%ecx <<
363 #ifndef CONFIG_SMP <<
364 leaq INIT_PER_CPU_VAR(fixed_percpu_ <<
365 #endif <<
366 movl %edx, %eax <<
367 shrq $32, %rdx <<
368 wrmsr <<
369 <<
370 /* Setup and Load IDT */ <<
371 call early_setup_idt <<
372 <<
373 /* Check if nx is implemented */ <<
374 movl $0x80000001, %eax <<
375 cpuid <<
376 movl %edx,%edi <<
377 <<
378 /* Setup EFER (Extended Feature Enable <<
379 movl $MSR_EFER, %ecx <<
380 rdmsr <<
381 /* <<
382 * Preserve current value of EFER for <<
383 * EFER writes if no change was made ( <<
384 */ <<
385 movl %eax, %edx <<
386 btsl $_EFER_SCE, %eax /* Ena <<
387 btl $20,%edi /* No <<
388 jnc 1f <<
389 btsl $_EFER_NX, %eax <<
390 btsq $_PAGE_BIT_NX,early_pmd_flags( <<
391 <<
392 /* Avoid writing EFER if no change was <<
393 1: cmpl %edx, %eax <<
394 je 1f <<
395 xor %edx, %edx <<
396 wrmsr /* Mak <<
397 1: 166 1:
398 /* Setup cr0 */ !! 167 rd %pc, %l0
399 movl $CR0_STATE, %eax <<
400 /* Make changes effective */ <<
401 movq %rax, %cr0 <<
402 <<
403 /* zero EFLAGS after setting rsp */ <<
404 pushq $0 <<
405 popfq <<
406 <<
407 /* Pass the boot_params pointer as fir <<
408 movq %r15, %rdi <<
409 <<
410 .Ljump_to_C_code: <<
411 xorl %ebp, %ebp # clear frame <<
412 ANNOTATE_RETPOLINE_SAFE <<
413 callq *initial_code(%rip) <<
414 ud2 <<
415 SYM_CODE_END(secondary_startup_64) <<
416 168
417 #include "verify_cpu.S" !! 169 mov (1b - prom_peer_name), %l1
418 #include "sev_verify_cbit.S" !! 170 sub %l0, %l1, %l1
>> 171 mov 0, %l2
>> 172
>> 173 /* prom_root_node = prom_peer(0) */
>> 174 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "peer"
>> 175 mov 1, %l3
>> 176 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1
>> 177 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 178 stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, 0
>> 179 stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1
>> 180 call %l7
>> 181 add %sp, (2047 + 128), %o0 ! argument array
>> 182
>> 183 ldx [%sp + 2047 + 128 + 0x20], %l4 ! prom root node
>> 184 mov (1b - prom_root_node), %l1
>> 185 sub %l0, %l1, %l1
>> 186 stw %l4, [%l1]
>> 187
>> 188 mov (1b - prom_getprop_name), %l1
>> 189 mov (1b - prom_compatible_name), %l2
>> 190 mov (1b - prom_root_compatible), %l5
>> 191 sub %l0, %l1, %l1
>> 192 sub %l0, %l2, %l2
>> 193 sub %l0, %l5, %l5
>> 194
>> 195 /* prom_getproperty(prom_root_node, "compatible",
>> 196 * &prom_root_compatible, 64)
>> 197 */
>> 198 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop"
>> 199 mov 4, %l3
>> 200 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4
>> 201 mov 1, %l3
>> 202 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 203 stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, prom_root_node
>> 204 stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "compatible"
>> 205 stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_root_compatible
>> 206 mov 64, %l3
>> 207 stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, size
>> 208 stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1
>> 209 call %l7
>> 210 add %sp, (2047 + 128), %o0 ! argument array
>> 211
>> 212 mov (1b - prom_finddev_name), %l1
>> 213 mov (1b - prom_chosen_path), %l2
>> 214 mov (1b - prom_boot_mapped_pc), %l3
>> 215 sub %l0, %l1, %l1
>> 216 sub %l0, %l2, %l2
>> 217 sub %l0, %l3, %l3
>> 218 stw %l0, [%l3]
>> 219 sub %sp, (192 + 128), %sp
>> 220
>> 221 /* chosen_node = prom_finddevice("/chosen") */
>> 222 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "finddevice"
>> 223 mov 1, %l3
>> 224 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1
>> 225 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 226 stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, "/chosen"
>> 227 stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1
>> 228 call %l7
>> 229 add %sp, (2047 + 128), %o0 ! argument array
>> 230
>> 231 ldx [%sp + 2047 + 128 + 0x20], %l4 ! chosen device node
>> 232
>> 233 mov (1b - prom_getprop_name), %l1
>> 234 mov (1b - prom_mmu_name), %l2
>> 235 mov (1b - prom_mmu_ihandle_cache), %l5
>> 236 sub %l0, %l1, %l1
>> 237 sub %l0, %l2, %l2
>> 238 sub %l0, %l5, %l5
>> 239
>> 240 /* prom_mmu_ihandle_cache = prom_getint(chosen_node, "mmu") */
>> 241 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop"
>> 242 mov 4, %l3
>> 243 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4
>> 244 mov 1, %l3
>> 245 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 246 stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, chosen_node
>> 247 stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "mmu"
>> 248 stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_mmu_ihandle_cache
>> 249 mov 4, %l3
>> 250 stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, sizeof(arg3)
>> 251 stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1
>> 252 call %l7
>> 253 add %sp, (2047 + 128), %o0 ! argument array
>> 254
>> 255 mov (1b - prom_callmethod_name), %l1
>> 256 mov (1b - prom_translate_name), %l2
>> 257 sub %l0, %l1, %l1
>> 258 sub %l0, %l2, %l2
>> 259 lduw [%l5], %l5 ! prom_mmu_ihandle_cache
>> 260
>> 261 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "call-method"
>> 262 mov 3, %l3
>> 263 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 3
>> 264 mov 5, %l3
>> 265 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 5
>> 266 stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1: "translate"
>> 267 stx %l5, [%sp + 2047 + 128 + 0x20] ! arg2: prom_mmu_ihandle_cache
>> 268 /* PAGE align */
>> 269 srlx %l0, 13, %l3
>> 270 sllx %l3, 13, %l3
>> 271 stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: vaddr, our PC
>> 272 stx %g0, [%sp + 2047 + 128 + 0x30] ! res1
>> 273 stx %g0, [%sp + 2047 + 128 + 0x38] ! res2
>> 274 stx %g0, [%sp + 2047 + 128 + 0x40] ! res3
>> 275 stx %g0, [%sp + 2047 + 128 + 0x48] ! res4
>> 276 stx %g0, [%sp + 2047 + 128 + 0x50] ! res5
>> 277 call %l7
>> 278 add %sp, (2047 + 128), %o0 ! argument array
>> 279
>> 280 ldx [%sp + 2047 + 128 + 0x40], %l1 ! translation mode
>> 281 mov (1b - prom_boot_mapping_mode), %l4
>> 282 sub %l0, %l4, %l4
>> 283 stw %l1, [%l4]
>> 284 mov (1b - prom_boot_mapping_phys_high), %l4
>> 285 sub %l0, %l4, %l4
>> 286 ldx [%sp + 2047 + 128 + 0x48], %l2 ! physaddr high
>> 287 stx %l2, [%l4 + 0x0]
>> 288 ldx [%sp + 2047 + 128 + 0x50], %l3 ! physaddr low
>> 289 /* 4MB align */
>> 290 srlx %l3, ILOG2_4MB, %l3
>> 291 sllx %l3, ILOG2_4MB, %l3
>> 292 stx %l3, [%l4 + 0x8]
>> 293
>> 294 /* Leave service as-is, "call-method" */
>> 295 mov 7, %l3
>> 296 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 7
>> 297 mov 1, %l3
>> 298 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 299 mov (1b - prom_map_name), %l3
>> 300 sub %l0, %l3, %l3
>> 301 stx %l3, [%sp + 2047 + 128 + 0x18] ! arg1: "map"
>> 302 /* Leave arg2 as-is, prom_mmu_ihandle_cache */
>> 303 mov -1, %l3
>> 304 stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: mode (-1 default)
>> 305 /* 4MB align the kernel image size. */
>> 306 set (_end - KERNBASE), %l3
>> 307 set ((4 * 1024 * 1024) - 1), %l4
>> 308 add %l3, %l4, %l3
>> 309 andn %l3, %l4, %l3
>> 310 stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4: roundup(ksize, 4MB)
>> 311 sethi %hi(KERNBASE), %l3
>> 312 stx %l3, [%sp + 2047 + 128 + 0x38] ! arg5: vaddr (KERNBASE)
>> 313 stx %g0, [%sp + 2047 + 128 + 0x40] ! arg6: empty
>> 314 mov (1b - prom_boot_mapping_phys_low), %l3
>> 315 sub %l0, %l3, %l3
>> 316 ldx [%l3], %l3
>> 317 stx %l3, [%sp + 2047 + 128 + 0x48] ! arg7: phys addr
>> 318 call %l7
>> 319 add %sp, (2047 + 128), %o0 ! argument array
>> 320
>> 321 add %sp, (192 + 128), %sp
>> 322
>> 323 sethi %hi(prom_root_compatible), %g1
>> 324 or %g1, %lo(prom_root_compatible), %g1
>> 325 sethi %hi(prom_sun4v_name), %g7
>> 326 or %g7, %lo(prom_sun4v_name), %g7
>> 327 mov 5, %g3
>> 328 90: ldub [%g7], %g2
>> 329 ldub [%g1], %g4
>> 330 cmp %g2, %g4
>> 331 bne,pn %icc, 80f
>> 332 add %g7, 1, %g7
>> 333 subcc %g3, 1, %g3
>> 334 bne,pt %xcc, 90b
>> 335 add %g1, 1, %g1
>> 336
>> 337 sethi %hi(is_sun4v), %g1
>> 338 or %g1, %lo(is_sun4v), %g1
>> 339 mov 1, %g7
>> 340 stw %g7, [%g1]
>> 341
>> 342 /* cpu_node = prom_finddevice("/cpu") */
>> 343 mov (1b - prom_finddev_name), %l1
>> 344 mov (1b - prom_cpu_path), %l2
>> 345 sub %l0, %l1, %l1
>> 346 sub %l0, %l2, %l2
>> 347 sub %sp, (192 + 128), %sp
>> 348
>> 349 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "finddevice"
>> 350 mov 1, %l3
>> 351 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1
>> 352 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 353 stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, "/cpu"
>> 354 stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1
>> 355 call %l7
>> 356 add %sp, (2047 + 128), %o0 ! argument array
>> 357
>> 358 ldx [%sp + 2047 + 128 + 0x20], %l4 ! cpu device node
>> 359
>> 360 mov (1b - prom_getprop_name), %l1
>> 361 mov (1b - prom_compatible_name), %l2
>> 362 mov (1b - prom_cpu_compatible), %l5
>> 363 sub %l0, %l1, %l1
>> 364 sub %l0, %l2, %l2
>> 365 sub %l0, %l5, %l5
>> 366
>> 367 /* prom_getproperty(cpu_node, "compatible",
>> 368 * &prom_cpu_compatible, 64)
>> 369 */
>> 370 stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop"
>> 371 mov 4, %l3
>> 372 stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4
>> 373 mov 1, %l3
>> 374 stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
>> 375 stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, cpu_node
>> 376 stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "compatible"
>> 377 stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_cpu_compatible
>> 378 mov 64, %l3
>> 379 stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, size
>> 380 stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1
>> 381 call %l7
>> 382 add %sp, (2047 + 128), %o0 ! argument array
>> 383
>> 384 add %sp, (192 + 128), %sp
>> 385
>> 386 sethi %hi(prom_cpu_compatible), %g1
>> 387 or %g1, %lo(prom_cpu_compatible), %g1
>> 388 sethi %hi(prom_niagara_prefix), %g7
>> 389 or %g7, %lo(prom_niagara_prefix), %g7
>> 390 mov 17, %g3
>> 391 90: ldub [%g7], %g2
>> 392 ldub [%g1], %g4
>> 393 cmp %g2, %g4
>> 394 bne,pn %icc, 89f
>> 395 add %g7, 1, %g7
>> 396 subcc %g3, 1, %g3
>> 397 bne,pt %xcc, 90b
>> 398 add %g1, 1, %g1
>> 399 ba,pt %xcc, 91f
>> 400 nop
>> 401
>> 402 89: sethi %hi(prom_cpu_compatible), %g1
>> 403 or %g1, %lo(prom_cpu_compatible), %g1
>> 404 sethi %hi(prom_sparc_prefix), %g7
>> 405 or %g7, %lo(prom_sparc_prefix), %g7
>> 406 mov 6, %g3
>> 407 90: ldub [%g7], %g2
>> 408 ldub [%g1], %g4
>> 409 cmp %g2, %g4
>> 410 bne,pn %icc, 4f
>> 411 add %g7, 1, %g7
>> 412 subcc %g3, 1, %g3
>> 413 bne,pt %xcc, 90b
>> 414 add %g1, 1, %g1
>> 415
>> 416 sethi %hi(prom_cpu_compatible), %g1
>> 417 or %g1, %lo(prom_cpu_compatible), %g1
>> 418 ldub [%g1 + 6], %g2
>> 419 cmp %g2, 'T'
>> 420 be,pt %xcc, 70f
>> 421 cmp %g2, 'M'
>> 422 be,pt %xcc, 70f
>> 423 cmp %g2, 'S'
>> 424 bne,pn %xcc, 49f
>> 425 nop
>> 426
>> 427 70: ldub [%g1 + 7], %g2
>> 428 cmp %g2, CPU_ID_NIAGARA3
>> 429 be,pt %xcc, 5f
>> 430 mov SUN4V_CHIP_NIAGARA3, %g4
>> 431 cmp %g2, CPU_ID_NIAGARA4
>> 432 be,pt %xcc, 5f
>> 433 mov SUN4V_CHIP_NIAGARA4, %g4
>> 434 cmp %g2, CPU_ID_NIAGARA5
>> 435 be,pt %xcc, 5f
>> 436 mov SUN4V_CHIP_NIAGARA5, %g4
>> 437 cmp %g2, CPU_ID_M6
>> 438 be,pt %xcc, 5f
>> 439 mov SUN4V_CHIP_SPARC_M6, %g4
>> 440 cmp %g2, CPU_ID_M7
>> 441 be,pt %xcc, 5f
>> 442 mov SUN4V_CHIP_SPARC_M7, %g4
>> 443 cmp %g2, CPU_ID_M8
>> 444 be,pt %xcc, 5f
>> 445 mov SUN4V_CHIP_SPARC_M8, %g4
>> 446 cmp %g2, CPU_ID_SONOMA1
>> 447 be,pt %xcc, 5f
>> 448 mov SUN4V_CHIP_SPARC_SN, %g4
>> 449 ba,pt %xcc, 49f
>> 450 nop
>> 451
>> 452 91: sethi %hi(prom_cpu_compatible), %g1
>> 453 or %g1, %lo(prom_cpu_compatible), %g1
>> 454 ldub [%g1 + 17], %g2
>> 455 cmp %g2, CPU_ID_NIAGARA1
>> 456 be,pt %xcc, 5f
>> 457 mov SUN4V_CHIP_NIAGARA1, %g4
>> 458 cmp %g2, CPU_ID_NIAGARA2
>> 459 be,pt %xcc, 5f
>> 460 mov SUN4V_CHIP_NIAGARA2, %g4
>> 461
>> 462 4:
>> 463 /* Athena */
>> 464 sethi %hi(prom_cpu_compatible), %g1
>> 465 or %g1, %lo(prom_cpu_compatible), %g1
>> 466 sethi %hi(prom_sparc64x_prefix), %g7
>> 467 or %g7, %lo(prom_sparc64x_prefix), %g7
>> 468 mov 9, %g3
>> 469 41: ldub [%g7], %g2
>> 470 ldub [%g1], %g4
>> 471 cmp %g2, %g4
>> 472 bne,pn %icc, 49f
>> 473 add %g7, 1, %g7
>> 474 subcc %g3, 1, %g3
>> 475 bne,pt %xcc, 41b
>> 476 add %g1, 1, %g1
>> 477 ba,pt %xcc, 5f
>> 478 mov SUN4V_CHIP_SPARC64X, %g4
>> 479
>> 480 49:
>> 481 mov SUN4V_CHIP_UNKNOWN, %g4
>> 482 5: sethi %hi(sun4v_chip_type), %g2
>> 483 or %g2, %lo(sun4v_chip_type), %g2
>> 484 stw %g4, [%g2]
>> 485
>> 486 80:
>> 487 BRANCH_IF_SUN4V(g1, jump_to_sun4u_init)
>> 488 BRANCH_IF_CHEETAH_BASE(g1,g7,cheetah_boot)
>> 489 BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1,g7,cheetah_plus_boot)
>> 490 ba,pt %xcc, spitfire_boot
>> 491 nop
>> 492
>> 493 cheetah_plus_boot:
>> 494 /* Preserve OBP chosen DCU and DCR register settings. */
>> 495 ba,pt %xcc, cheetah_generic_boot
>> 496 nop
>> 497
>> 498 cheetah_boot:
>> 499 mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1
>> 500 wr %g1, %asr18
>> 501
>> 502 sethi %uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g7
>> 503 or %g7, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g7
>> 504 sllx %g7, 32, %g7
>> 505 or %g7, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g7
>> 506 stxa %g7, [%g0] ASI_DCU_CONTROL_REG
>> 507 membar #Sync
>> 508
>> 509 cheetah_generic_boot:
>> 510 mov TSB_EXTENSION_P, %g3
>> 511 stxa %g0, [%g3] ASI_DMMU
>> 512 stxa %g0, [%g3] ASI_IMMU
>> 513 membar #Sync
>> 514
>> 515 mov TSB_EXTENSION_S, %g3
>> 516 stxa %g0, [%g3] ASI_DMMU
>> 517 membar #Sync
>> 518
>> 519 mov TSB_EXTENSION_N, %g3
>> 520 stxa %g0, [%g3] ASI_DMMU
>> 521 stxa %g0, [%g3] ASI_IMMU
>> 522 membar #Sync
>> 523
>> 524 ba,a,pt %xcc, jump_to_sun4u_init
>> 525
>> 526 spitfire_boot:
>> 527 /* Typically PROM has already enabled both MMU's and both on-chip
>> 528 * caches, but we do it here anyway just to be paranoid.
>> 529 */
>> 530 mov (LSU_CONTROL_IC|LSU_CONTROL_DC|LSU_CONTROL_IM|LSU_CONTROL_DM), %g1
>> 531 stxa %g1, [%g0] ASI_LSU_CONTROL
>> 532 membar #Sync
>> 533
>> 534 jump_to_sun4u_init:
>> 535 /*
>> 536 * Make sure we are in privileged mode, have address masking,
>> 537 * using the ordinary globals and have enabled floating
>> 538 * point.
>> 539 *
>> 540 * Again, typically PROM has left %pil at 13 or similar, and
>> 541 * (PSTATE_PRIV | PSTATE_PEF | PSTATE_IE) in %pstate.
>> 542 */
>> 543 wrpr %g0, (PSTATE_PRIV|PSTATE_PEF|PSTATE_IE), %pstate
>> 544 wr %g0, 0, %fprs
>> 545
>> 546 set sun4u_init, %g2
>> 547 jmpl %g2 + %g0, %g0
>> 548 nop
>> 549
>> 550 __REF
>> 551 sun4u_init:
>> 552 BRANCH_IF_SUN4V(g1, sun4v_init)
>> 553
>> 554 /* Set ctx 0 */
>> 555 mov PRIMARY_CONTEXT, %g7
>> 556 stxa %g0, [%g7] ASI_DMMU
>> 557 membar #Sync
>> 558
>> 559 mov SECONDARY_CONTEXT, %g7
>> 560 stxa %g0, [%g7] ASI_DMMU
>> 561 membar #Sync
>> 562
>> 563 ba,a,pt %xcc, sun4u_continue
>> 564
>> 565 sun4v_init:
>> 566 /* Set ctx 0 */
>> 567 mov PRIMARY_CONTEXT, %g7
>> 568 stxa %g0, [%g7] ASI_MMU
>> 569 membar #Sync
>> 570
>> 571 mov SECONDARY_CONTEXT, %g7
>> 572 stxa %g0, [%g7] ASI_MMU
>> 573 membar #Sync
>> 574 ba,a,pt %xcc, niagara_tlb_fixup
>> 575
>> 576 sun4u_continue:
>> 577 BRANCH_IF_ANY_CHEETAH(g1, g7, cheetah_tlb_fixup)
>> 578
>> 579 ba,a,pt %xcc, spitfire_tlb_fixup
>> 580
>> 581 niagara_tlb_fixup:
>> 582 mov 3, %g2 /* Set TLB type to hypervisor. */
>> 583 sethi %hi(tlb_type), %g1
>> 584 stw %g2, [%g1 + %lo(tlb_type)]
>> 585
>> 586 /* Patch copy/clear ops. */
>> 587 sethi %hi(sun4v_chip_type), %g1
>> 588 lduw [%g1 + %lo(sun4v_chip_type)], %g1
>> 589 cmp %g1, SUN4V_CHIP_NIAGARA1
>> 590 be,pt %xcc, niagara_patch
>> 591 cmp %g1, SUN4V_CHIP_NIAGARA2
>> 592 be,pt %xcc, niagara2_patch
>> 593 nop
>> 594 cmp %g1, SUN4V_CHIP_NIAGARA3
>> 595 be,pt %xcc, niagara2_patch
>> 596 nop
>> 597 cmp %g1, SUN4V_CHIP_NIAGARA4
>> 598 be,pt %xcc, niagara4_patch
>> 599 nop
>> 600 cmp %g1, SUN4V_CHIP_NIAGARA5
>> 601 be,pt %xcc, niagara4_patch
>> 602 nop
>> 603 cmp %g1, SUN4V_CHIP_SPARC_M6
>> 604 be,pt %xcc, niagara4_patch
>> 605 nop
>> 606 cmp %g1, SUN4V_CHIP_SPARC_M7
>> 607 be,pt %xcc, sparc_m7_patch
>> 608 nop
>> 609 cmp %g1, SUN4V_CHIP_SPARC_M8
>> 610 be,pt %xcc, sparc_m7_patch
>> 611 nop
>> 612 cmp %g1, SUN4V_CHIP_SPARC_SN
>> 613 be,pt %xcc, niagara4_patch
>> 614 nop
>> 615
>> 616 call generic_patch_copyops
>> 617 nop
>> 618 call generic_patch_bzero
>> 619 nop
>> 620 call generic_patch_pageops
>> 621 nop
>> 622
>> 623 ba,a,pt %xcc, 80f
>> 624 nop
>> 625
>> 626 sparc_m7_patch:
>> 627 call m7_patch_copyops
>> 628 nop
>> 629 call m7_patch_bzero
>> 630 nop
>> 631 call m7_patch_pageops
>> 632 nop
>> 633
>> 634 ba,a,pt %xcc, 80f
>> 635 nop
>> 636
>> 637 niagara4_patch:
>> 638 call niagara4_patch_copyops
>> 639 nop
>> 640 call niagara4_patch_bzero
>> 641 nop
>> 642 call niagara4_patch_pageops
>> 643 nop
>> 644 call niagara4_patch_fls
>> 645 nop
>> 646
>> 647 ba,a,pt %xcc, 80f
>> 648 nop
>> 649
>> 650 niagara2_patch:
>> 651 call niagara2_patch_copyops
>> 652 nop
>> 653 call niagara_patch_bzero
>> 654 nop
>> 655 call niagara_patch_pageops
>> 656 nop
>> 657
>> 658 ba,a,pt %xcc, 80f
>> 659 nop
>> 660
>> 661 niagara_patch:
>> 662 call niagara_patch_copyops
>> 663 nop
>> 664 call niagara_patch_bzero
>> 665 nop
>> 666 call niagara_patch_pageops
>> 667 nop
>> 668
>> 669 80:
>> 670 /* Patch TLB/cache ops. */
>> 671 call hypervisor_patch_cachetlbops
>> 672 nop
>> 673
>> 674 ba,a,pt %xcc, tlb_fixup_done
>> 675
>> 676 cheetah_tlb_fixup:
>> 677 mov 2, %g2 /* Set TLB type to cheetah+. */
>> 678 BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1,g7,1f)
>> 679
>> 680 mov 1, %g2 /* Set TLB type to cheetah. */
>> 681
>> 682 1: sethi %hi(tlb_type), %g1
>> 683 stw %g2, [%g1 + %lo(tlb_type)]
>> 684
>> 685 /* Patch copy/page operations to cheetah optimized versions. */
>> 686 call cheetah_patch_copyops
>> 687 nop
>> 688 call cheetah_patch_copy_page
>> 689 nop
>> 690 call cheetah_patch_cachetlbops
>> 691 nop
>> 692
>> 693 ba,a,pt %xcc, tlb_fixup_done
>> 694
>> 695 spitfire_tlb_fixup:
>> 696 /* Set TLB type to spitfire. */
>> 697 mov 0, %g2
>> 698 sethi %hi(tlb_type), %g1
>> 699 stw %g2, [%g1 + %lo(tlb_type)]
>> 700
>> 701 tlb_fixup_done:
>> 702 sethi %hi(init_thread_union), %g6
>> 703 or %g6, %lo(init_thread_union), %g6
>> 704 ldx [%g6 + TI_TASK], %g4
>> 705
>> 706 wr %g0, ASI_P, %asi
>> 707 mov 1, %g1
>> 708 sllx %g1, THREAD_SHIFT, %g1
>> 709 sub %g1, (STACKFRAME_SZ + STACK_BIAS + TRACEREG_SZ), %g1
>> 710 add %g6, %g1, %sp
>> 711
>> 712 /* Set per-cpu pointer initially to zero, this makes
>> 713 * the boot-cpu use the in-kernel-image per-cpu areas
>> 714 * before setup_per_cpu_area() is invoked.
>> 715 */
>> 716 clr %g5
>> 717
>> 718 wrpr %g0, 0, %wstate
>> 719 wrpr %g0, 0x0, %tl
>> 720
>> 721 /* Clear the bss */
>> 722 sethi %hi(__bss_start), %o0
>> 723 or %o0, %lo(__bss_start), %o0
>> 724 sethi %hi(_end), %o1
>> 725 or %o1, %lo(_end), %o1
>> 726 call __bzero
>> 727 sub %o1, %o0, %o1
>> 728
>> 729 call prom_init
>> 730 mov %l7, %o0 ! OpenPROM cif handler
>> 731
>> 732 /* To create a one-register-window buffer between the kernel's
>> 733 * initial stack and the last stack frame we use from the firmware,
>> 734 * do the rest of the boot from a C helper function.
>> 735 */
>> 736 call start_early_boot
>> 737 nop
>> 738 /* Not reached... */
>> 739
>> 740 .previous
>> 741
>> 742 /* This is meant to allow the sharing of this code between
>> 743 * boot processor invocation (via setup_tba() below) and
>> 744 * secondary processor startup (via trampoline.S). The
>> 745 * former does use this code, the latter does not yet due
>> 746 * to some complexities. That should be fixed up at some
>> 747 * point.
>> 748 *
>> 749 * There used to be enormous complexity wrt. transferring
>> 750 * over from the firmware's trap table to the Linux kernel's.
>> 751 * For example, there was a chicken & egg problem wrt. building
>> 752 * the OBP page tables, yet needing to be on the Linux kernel
>> 753 * trap table (to translate PAGE_OFFSET addresses) in order to
>> 754 * do that.
>> 755 *
>> 756 * We now handle OBP tlb misses differently, via linear lookups
>> 757 * into the prom_trans[] array. So that specific problem no
>> 758 * longer exists. Yet, unfortunately there are still some issues
>> 759 * preventing trampoline.S from using this code... ho hum.
>> 760 */
>> 761 .globl setup_trap_table
>> 762 setup_trap_table:
>> 763 save %sp, -192, %sp
>> 764
>> 765 /* Force interrupts to be disabled. */
>> 766 rdpr %pstate, %l0
>> 767 andn %l0, PSTATE_IE, %o1
>> 768 wrpr %o1, 0x0, %pstate
>> 769 rdpr %pil, %l1
>> 770 wrpr %g0, PIL_NORMAL_MAX, %pil
>> 771
>> 772 /* Make the firmware call to jump over to the Linux trap table. */
>> 773 sethi %hi(is_sun4v), %o0
>> 774 lduw [%o0 + %lo(is_sun4v)], %o0
>> 775 brz,pt %o0, 1f
>> 776 nop
>> 777
>> 778 TRAP_LOAD_TRAP_BLOCK(%g2, %g3)
>> 779 add %g2, TRAP_PER_CPU_FAULT_INFO, %g2
>> 780 stxa %g2, [%g0] ASI_SCRATCHPAD
>> 781
>> 782 /* Compute physical address:
>> 783 *
>> 784 * paddr = kern_base + (mmfsa_vaddr - KERNBASE)
>> 785 */
>> 786 sethi %hi(KERNBASE), %g3
>> 787 sub %g2, %g3, %g2
>> 788 sethi %hi(kern_base), %g3
>> 789 ldx [%g3 + %lo(kern_base)], %g3
>> 790 add %g2, %g3, %o1
>> 791 sethi %hi(sparc64_ttable_tl0), %o0
>> 792
>> 793 set prom_set_trap_table_name, %g2
>> 794 stx %g2, [%sp + 2047 + 128 + 0x00]
>> 795 mov 2, %g2
>> 796 stx %g2, [%sp + 2047 + 128 + 0x08]
>> 797 mov 0, %g2
>> 798 stx %g2, [%sp + 2047 + 128 + 0x10]
>> 799 stx %o0, [%sp + 2047 + 128 + 0x18]
>> 800 stx %o1, [%sp + 2047 + 128 + 0x20]
>> 801 sethi %hi(p1275buf), %g2
>> 802 or %g2, %lo(p1275buf), %g2
>> 803 ldx [%g2 + 0x08], %o1
>> 804 call %o1
>> 805 add %sp, (2047 + 128), %o0
>> 806
>> 807 ba,a,pt %xcc, 2f
>> 808
>> 809 1: sethi %hi(sparc64_ttable_tl0), %o0
>> 810 set prom_set_trap_table_name, %g2
>> 811 stx %g2, [%sp + 2047 + 128 + 0x00]
>> 812 mov 1, %g2
>> 813 stx %g2, [%sp + 2047 + 128 + 0x08]
>> 814 mov 0, %g2
>> 815 stx %g2, [%sp + 2047 + 128 + 0x10]
>> 816 stx %o0, [%sp + 2047 + 128 + 0x18]
>> 817 sethi %hi(p1275buf), %g2
>> 818 or %g2, %lo(p1275buf), %g2
>> 819 ldx [%g2 + 0x08], %o1
>> 820 call %o1
>> 821 add %sp, (2047 + 128), %o0
>> 822
>> 823 /* Start using proper page size encodings in ctx register. */
>> 824 2: sethi %hi(sparc64_kern_pri_context), %g3
>> 825 ldx [%g3 + %lo(sparc64_kern_pri_context)], %g2
>> 826
>> 827 mov PRIMARY_CONTEXT, %g1
>> 828
>> 829 661: stxa %g2, [%g1] ASI_DMMU
>> 830 .section .sun4v_1insn_patch, "ax"
>> 831 .word 661b
>> 832 stxa %g2, [%g1] ASI_MMU
>> 833 .previous
>> 834
>> 835 membar #Sync
>> 836
>> 837 BRANCH_IF_SUN4V(o2, 1f)
>> 838
>> 839 /* Kill PROM timer */
>> 840 sethi %hi(0x80000000), %o2
>> 841 sllx %o2, 32, %o2
>> 842 wr %o2, 0, %tick_cmpr
419 843
420 #if defined(CONFIG_HOTPLUG_CPU) && defined(CON !! 844 BRANCH_IF_ANY_CHEETAH(o2, o3, 1f)
421 /* <<
422 * Entry point for soft restart of a CPU. Invo <<
423 * restarting the boot CPU or for restarting S <<
424 * unplug. Everything is set up already except <<
425 */ <<
426 SYM_CODE_START(soft_restart_cpu) <<
427 ANNOTATE_NOENDBR <<
428 UNWIND_HINT_END_OF_STACK <<
429 <<
430 /* Find the idle task stack */ <<
431 movq PER_CPU_VAR(pcpu_hot + X86_cur <<
432 movq TASK_threadsp(%rcx), %rsp <<
433 <<
434 jmp .Ljump_to_C_code <<
435 SYM_CODE_END(soft_restart_cpu) <<
436 #endif <<
437 845
438 #ifdef CONFIG_AMD_MEM_ENCRYPT !! 846 ba,a,pt %xcc, 2f
439 /* <<
440 * VC Exception handler used during early boot <<
441 * addresses, but before the switch to the idt <<
442 * The early_idt_handler_array can't be used h <<
443 * of __init code and this handler is also use <<
444 * Therefore this handler ends up in the .text <<
445 * when .init.text is freed. <<
446 */ <<
447 SYM_CODE_START_NOALIGN(vc_boot_ghcb) <<
448 UNWIND_HINT_IRET_REGS offset=8 <<
449 ENDBR <<
450 <<
451 /* Build pt_regs */ <<
452 PUSH_AND_CLEAR_REGS <<
453 <<
454 /* Call C handler */ <<
455 movq %rsp, %rdi <<
456 movq ORIG_RAX(%rsp), %rsi <<
457 movq initial_vc_handler(%rip), %rax <<
458 ANNOTATE_RETPOLINE_SAFE <<
459 call *%rax <<
460 <<
461 /* Unwind pt_regs */ <<
462 POP_REGS <<
463 <<
464 /* Remove Error Code */ <<
465 addq $8, %rsp <<
466 <<
467 iretq <<
468 SYM_CODE_END(vc_boot_ghcb) <<
469 #endif <<
470 <<
471 /* Both SMP bootup and ACPI suspend ch <<
472 __REFDATA <<
473 .balign 8 <<
474 SYM_DATA(initial_code, .quad x86_64_start_ker <<
475 #ifdef CONFIG_AMD_MEM_ENCRYPT <<
476 SYM_DATA(initial_vc_handler, .quad handle_v <<
477 #endif <<
478 <<
479 SYM_DATA(trampoline_lock, .quad 0); <<
480 __FINITDATA <<
481 <<
482 __INIT <<
483 SYM_CODE_START(early_idt_handler_array) <<
484 i = 0 <<
485 .rept NUM_EXCEPTION_VECTORS <<
486 .if ((EXCEPTION_ERRCODE_MASK >> i) & 1 <<
487 UNWIND_HINT_IRET_REGS <<
488 ENDBR <<
489 pushq $0 # Dummy error <<
490 .else <<
491 UNWIND_HINT_IRET_REGS offset=8 <<
492 ENDBR <<
493 .endif <<
494 pushq $i # 72(%rsp) Vec <<
495 jmp early_idt_handler_common <<
496 UNWIND_HINT_IRET_REGS <<
497 i = i + 1 <<
498 .fill early_idt_handler_array + i*EARL <<
499 .endr <<
500 SYM_CODE_END(early_idt_handler_array) <<
501 ANNOTATE_NOENDBR // early_idt_handler_ <<
502 847
503 SYM_CODE_START_LOCAL(early_idt_handler_common) !! 848 /* Disable STICK_INT interrupts. */
504 UNWIND_HINT_IRET_REGS offset=16 !! 849 1:
505 /* !! 850 sethi %hi(0x80000000), %o2
506 * The stack is the hardware frame, an !! 851 sllx %o2, 32, %o2
507 * vector number. !! 852 wr %o2, %asr25
508 */ !! 853
509 cld !! 854 2:
510 !! 855 wrpr %g0, %g0, %wstate
511 incl early_recursion_flag(%rip) !! 856
512 !! 857 call init_irqwork_curcpu
513 /* The vector number is currently in t !! 858 nop
514 pushq %rsi !! 859
515 movq 8(%rsp), %rsi !! 860 /* Now we can restore interrupt state. */
516 movq %rdi, 8(%rsp) !! 861 wrpr %l0, 0, %pstate
517 pushq %rdx !! 862 wrpr %l1, 0x0, %pil
518 pushq %rcx !! 863
519 pushq %rax !! 864 ret
520 pushq %r8 !! 865 restore
521 pushq %r9 !! 866
522 pushq %r10 !! 867 .globl setup_tba
523 pushq %r11 !! 868 setup_tba:
524 pushq %rbx !! 869 save %sp, -192, %sp
525 pushq %rbp !! 870
526 pushq %r12 !! 871 /* The boot processor is the only cpu which invokes this
527 pushq %r13 !! 872 * routine, the other cpus set things up via trampoline.S.
528 pushq %r14 !! 873 * So save the OBP trap table address here.
529 pushq %r15 !! 874 */
530 UNWIND_HINT_REGS !! 875 rdpr %tba, %g7
531 !! 876 sethi %hi(prom_tba), %o1
532 movq %rsp,%rdi /* RDI = pt_re !! 877 or %o1, %lo(prom_tba), %o1
533 call do_early_exception !! 878 stx %g7, [%o1]
534 !! 879
535 decl early_recursion_flag(%rip) !! 880 call setup_trap_table
536 jmp restore_regs_and_return_to_kernel !! 881 nop
537 SYM_CODE_END(early_idt_handler_common) !! 882
>> 883 ret
>> 884 restore
>> 885 sparc64_boot_end:
>> 886
>> 887 #include "etrap_64.S"
>> 888 #include "rtrap_64.S"
>> 889 #include "winfixup.S"
>> 890 #include "fpu_traps.S"
>> 891 #include "ivec.S"
>> 892 #include "getsetcc.S"
>> 893 #include "utrap.S"
>> 894 #include "spiterrs.S"
>> 895 #include "cherrs.S"
>> 896 #include "misctrap.S"
>> 897 #include "syscalls.S"
>> 898 #include "helpers.S"
>> 899 #include "sun4v_tlb_miss.S"
>> 900 #include "sun4v_mcd.S"
>> 901 #include "sun4v_ivec.S"
>> 902 #include "ktlb.S"
>> 903 #include "tsb.S"
538 904
539 #ifdef CONFIG_AMD_MEM_ENCRYPT <<
540 /* 905 /*
541 * VC Exception handler used during very early !! 906 * The following skip makes sure the trap table in ttable.S is aligned
542 * early_idt_handler_array can't be used becau !! 907 * on a 32K boundary as required by the v9 specs for TBA register.
543 * paravirtualized INTERRUPT_RETURN and pv-ops <<
544 * 908 *
545 * XXX it does, fix this. !! 909 * We align to a 32K boundary, then we have the 32K kernel TSB,
546 * !! 910 * the 64K kernel 4MB TSB, and then the 32K aligned trap table.
547 * This handler will end up in the .init.text <<
548 * available to boot secondary CPUs. <<
549 */ 911 */
550 SYM_CODE_START_NOALIGN(vc_no_ghcb) !! 912 1:
551 UNWIND_HINT_IRET_REGS offset=8 !! 913 .skip 0x4000 + _start - 1b
552 ENDBR <<
553 <<
554 /* Build pt_regs */ <<
555 PUSH_AND_CLEAR_REGS <<
556 <<
557 /* Call C handler */ <<
558 movq %rsp, %rdi <<
559 movq ORIG_RAX(%rsp), %rsi <<
560 call do_vc_no_ghcb <<
561 <<
562 /* Unwind pt_regs */ <<
563 POP_REGS <<
564 <<
565 /* Remove Error Code */ <<
566 addq $8, %rsp <<
567 <<
568 /* Pure iret required here - don't use <<
569 iretq <<
570 SYM_CODE_END(vc_no_ghcb) <<
571 #endif <<
572 914
573 #ifdef CONFIG_MITIGATION_PAGE_TABLE_ISOLATION !! 915 ! 0x0000000000408000
574 /* <<
575 * Each PGD needs to be 8k long and 8k aligned <<
576 * ever go out to userspace with these, so we <<
577 * strictly *need* the second page, but this a <<
578 * have a single set_pgd() implementation that <<
579 * need to worry about whether it has 4k or 8k <<
580 * with. <<
581 * <<
582 * This ensures PGDs are 8k long: <<
583 */ <<
584 #define PTI_USER_PGD_FILL 512 <<
585 /* This ensures they are 8k-aligned: */ <<
586 #define SYM_DATA_START_PTI_ALIGNED(name) \ <<
587 SYM_START(name, SYM_L_GLOBAL, .balign <<
588 #else <<
589 #define SYM_DATA_START_PTI_ALIGNED(name) \ <<
590 SYM_DATA_START_PAGE_ALIGNED(name) <<
591 #define PTI_USER_PGD_FILL 0 <<
592 #endif <<
593 <<
594 __INITDATA <<
595 .balign 4 <<
596 <<
597 SYM_DATA_START_PTI_ALIGNED(early_top_pgt) <<
598 .fill 511,8,0 <<
599 .quad level3_kernel_pgt - __START_KE <<
600 .fill PTI_USER_PGD_FILL,8,0 <<
601 SYM_DATA_END(early_top_pgt) <<
602 <<
603 SYM_DATA_START_PAGE_ALIGNED(early_dynamic_pgts <<
604 .fill 512*EARLY_DYNAMIC_PAGE_TABLES, <<
605 SYM_DATA_END(early_dynamic_pgts) <<
606 916
607 SYM_DATA(early_recursion_flag, .long 0) !! 917 .globl swapper_tsb
>> 918 swapper_tsb:
>> 919 .skip (32 * 1024)
608 920
609 .data !! 921 .globl swapper_4m_tsb
>> 922 swapper_4m_tsb:
>> 923 .skip (64 * 1024)
610 924
611 #if defined(CONFIG_XEN_PV) || defined(CONFIG_P !! 925 ! 0x0000000000420000
612 SYM_DATA_START_PTI_ALIGNED(init_top_pgt) <<
613 .quad level3_ident_pgt - __START_KER <<
614 .org init_top_pgt + L4_PAGE_OFFSET* <<
615 .quad level3_ident_pgt - __START_KER <<
616 .org init_top_pgt + L4_START_KERNEL <<
617 /* (2^48-(2*1024*1024*1024))/(2^39) = <<
618 .quad level3_kernel_pgt - __START_KE <<
619 .fill PTI_USER_PGD_FILL,8,0 <<
620 SYM_DATA_END(init_top_pgt) <<
621 <<
622 SYM_DATA_START_PAGE_ALIGNED(level3_ident_pgt) <<
623 .quad level2_ident_pgt - __START_KER <<
624 .fill 511, 8, 0 <<
625 SYM_DATA_END(level3_ident_pgt) <<
626 SYM_DATA_START_PAGE_ALIGNED(level2_ident_pgt) <<
627 /* <<
628 * Since I easily can, map the first 1 <<
629 * Don't set NX because code runs from <<
630 * <<
631 * Note: This sets _PAGE_GLOBAL despit <<
632 * the CPU supports it or it is enable <<
633 * the CPU should ignore the bit. <<
634 */ <<
635 PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC <<
636 SYM_DATA_END(level2_ident_pgt) <<
637 #else <<
638 SYM_DATA_START_PTI_ALIGNED(init_top_pgt) <<
639 .fill 512,8,0 <<
640 .fill PTI_USER_PGD_FILL,8,0 <<
641 SYM_DATA_END(init_top_pgt) <<
642 #endif <<
643 <<
644 #ifdef CONFIG_X86_5LEVEL <<
645 SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt) <<
646 .fill 511,8,0 <<
647 .quad level3_kernel_pgt - __START_KE <<
648 SYM_DATA_END(level4_kernel_pgt) <<
649 #endif <<
650 <<
651 SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt) <<
652 .fill L3_START_KERNEL,8,0 <<
653 /* (2^48-(2*1024*1024*1024)-((2^39)*51 <<
654 .quad level2_kernel_pgt - __START_KE <<
655 .quad level2_fixmap_pgt - __START_KE <<
656 SYM_DATA_END(level3_kernel_pgt) <<
657 926
658 SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt) !! 927 /* Some care needs to be exercised if you try to move the
659 /* !! 928 * location of the trap table relative to other things. For
660 * Kernel high mapping. !! 929 * one thing there are br* instructions in some of the
>> 930 * trap table entires which branch back to code in ktlb.S
>> 931 * Those instructions can only handle a signed 16-bit
>> 932 * displacement.
661 * 933 *
662 * The kernel code+data+bss must be lo !! 934 * There is a binutils bug (bugzilla #4558) which causes
663 * virtual address space, which is 1 G !! 935 * the relocation overflow checks for such instructions to
664 * 512 MiB otherwise. !! 936 * not be done correctly. So bintuils will not notice the
665 * !! 937 * error and will instead write junk into the relocation and
666 * (NOTE: after that starts the module !! 938 * you'll have an unbootable kernel.
667 * !! 939 */
668 * This table is eventually used by th !! 940 #include "ttable_64.S"
669 * Care must be taken to clear out und <<
670 * or _PAGE_GLOBAL in some cases. <<
671 */ <<
672 PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERN <<
673 SYM_DATA_END(level2_kernel_pgt) <<
674 <<
675 SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt) <<
676 .fill (512 - 4 - FIXMAP_PMD_NUM),8,0 <<
677 pgtno = 0 <<
678 .rept (FIXMAP_PMD_NUM) <<
679 .quad level1_fixmap_pgt + (pgtno << PA <<
680 + _PAGE_TABLE_NOENC; <<
681 pgtno = pgtno + 1 <<
682 .endr <<
683 /* 6 MB reserved space + a 2MB hole */ <<
684 .fill 4,8,0 <<
685 SYM_DATA_END(level2_fixmap_pgt) <<
686 <<
687 SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt) <<
688 .rept (FIXMAP_PMD_NUM) <<
689 .fill 512,8,0 <<
690 .endr <<
691 SYM_DATA_END(level1_fixmap_pgt) <<
692 <<
693 .data <<
694 .align 16 <<
695 941
696 SYM_DATA(smpboot_control, .long !! 942 ! 0x0000000000428000
697 943
698 .align 16 !! 944 #include "hvcalls.S"
699 /* This must match the first entry in level2_k !! 945 #include "systbls_64.S"
700 SYM_DATA(phys_base, .quad 0x0) <<
701 EXPORT_SYMBOL(phys_base) <<
702 <<
703 #include "../xen/xen-head.S" <<
704 <<
705 __PAGE_ALIGNED_BSS <<
706 SYM_DATA_START_PAGE_ALIGNED(empty_zero_page) <<
707 .skip PAGE_SIZE <<
708 SYM_DATA_END(empty_zero_page) <<
709 EXPORT_SYMBOL(empty_zero_page) <<
710 946
>> 947 .data
>> 948 .align 8
>> 949 .globl prom_tba, tlb_type
>> 950 prom_tba: .xword 0
>> 951 tlb_type: .word 0 /* Must NOT end up in BSS */
>> 952 EXPORT_SYMBOL(tlb_type)
>> 953 .section ".fixup",#alloc,#execinstr
>> 954
>> 955 ENTRY(__retl_efault)
>> 956 retl
>> 957 mov -EFAULT, %o0
>> 958 ENDPROC(__retl_efault)
>> 959
>> 960 ENTRY(__retl_o1)
>> 961 retl
>> 962 mov %o1, %o0
>> 963 ENDPROC(__retl_o1)
>> 964
>> 965 ENTRY(__retl_o1_asi)
>> 966 wr %o5, 0x0, %asi
>> 967 retl
>> 968 mov %o1, %o0
>> 969 ENDPROC(__retl_o1_asi)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.