1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * 4 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 5 * 6 * Derived from book3s_rmhandlers.S and other files, which are: 7 * 8 * Copyright SUSE Linux Products GmbH 2009 9 * 10 * Authors: Alexander Graf <agraf@suse.de> 11 */ 12 13 #include <linux/export.h> 14 #include <linux/linkage.h> 15 #include <linux/objtool.h> 16 #include <asm/ppc_asm.h> 17 #include <asm/code-patching-asm.h> 18 #include <asm/kvm_asm.h> 19 #include <asm/reg.h> 20 #include <asm/mmu.h> 21 #include <asm/page.h> 22 #include <asm/ptrace.h> 23 #include <asm/hvcall.h> 24 #include <asm/asm-offsets.h> 25 #include <asm/exception-64s.h> 26 #include <asm/kvm_book3s_asm.h> 27 #include <asm/book3s/64/mmu-hash.h> 28 #include <asm/tm.h> 29 #include <asm/opal.h> 30 #include <asm/thread_info.h> 31 #include <asm/asm-compat.h> 32 #include <asm/feature-fixups.h> 33 #include <asm/cpuidle.h> 34 35 /* Values in HSTATE_NAPPING(r13) */ 36 #define NAPPING_CEDE 1 37 #define NAPPING_NOVCPU 2 38 #define NAPPING_UNSPLIT 3 39 40 /* Stack frame offsets for kvmppc_hv_entry */ 41 #define SFS 160 42 #define STACK_SLOT_TRAP (SFS-4) 43 #define STACK_SLOT_TID (SFS-16) 44 #define STACK_SLOT_PSSCR (SFS-24) 45 #define STACK_SLOT_PID (SFS-32) 46 #define STACK_SLOT_IAMR (SFS-40) 47 #define STACK_SLOT_CIABR (SFS-48) 48 #define STACK_SLOT_DAWR0 (SFS-56) 49 #define STACK_SLOT_DAWRX0 (SFS-64) 50 #define STACK_SLOT_HFSCR (SFS-72) 51 #define STACK_SLOT_AMR (SFS-80) 52 #define STACK_SLOT_UAMOR (SFS-88) 53 #define STACK_SLOT_FSCR (SFS-96) 54 55 /* 56 * Use the last LPID (all implemented LPID bits = 1) for partition switching. 57 * This is reserved in the LPID allocator. POWER7 only implements 0x3ff, but 58 * we write 0xfff into the LPID SPR anyway, which seems to work and just 59 * ignores the top bits. 60 */ 61 #define LPID_RSVD 0xfff 62 63 /* 64 * Call kvmppc_hv_entry in real mode. 65 * Must be called with interrupts hard-disabled. 66 * 67 * Input Registers: 68 * 69 * LR = return address to continue at after eventually re-enabling MMU 70 */ 71 _GLOBAL_TOC(kvmppc_hv_entry_trampoline) 72 mflr r0 73 std r0, PPC_LR_STKOFF(r1) 74 stdu r1, -112(r1) 75 mfmsr r10 76 std r10, HSTATE_HOST_MSR(r13) 77 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry) 78 li r0,MSR_RI 79 andc r0,r10,r0 80 li r6,MSR_IR | MSR_DR 81 andc r6,r10,r6 82 mtmsrd r0,1 /* clear RI in MSR */ 83 mtsrr0 r5 84 mtsrr1 r6 85 RFI_TO_KERNEL 86 87 kvmppc_call_hv_entry: 88 ld r4, HSTATE_KVM_VCPU(r13) 89 bl kvmppc_hv_entry 90 91 /* Back from guest - restore host state and return to caller */ 92 93 BEGIN_FTR_SECTION 94 /* Restore host DABR and DABRX */ 95 ld r5,HSTATE_DABR(r13) 96 li r6,7 97 mtspr SPRN_DABR,r5 98 mtspr SPRN_DABRX,r6 99 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 100 101 /* Restore SPRG3 */ 102 ld r3,PACA_SPRG_VDSO(r13) 103 mtspr SPRN_SPRG_VDSO_WRITE,r3 104 105 /* Reload the host's PMU registers */ 106 bl kvmhv_load_host_pmu 107 108 /* 109 * Reload DEC. HDEC interrupts were disabled when 110 * we reloaded the host's LPCR value. 111 */ 112 ld r3, HSTATE_DECEXP(r13) 113 mftb r4 114 subf r4, r4, r3 115 mtspr SPRN_DEC, r4 116 117 /* hwthread_req may have got set by cede or no vcpu, so clear it */ 118 li r0, 0 119 stb r0, HSTATE_HWTHREAD_REQ(r13) 120 121 /* 122 * For external interrupts we need to call the Linux 123 * handler to process the interrupt. We do that by jumping 124 * to absolute address 0x500 for external interrupts. 125 * The [h]rfid at the end of the handler will return to 126 * the book3s_hv_interrupts.S code. For other interrupts 127 * we do the rfid to get back to the book3s_hv_interrupts.S 128 * code here. 129 */ 130 ld r8, 112+PPC_LR_STKOFF(r1) 131 addi r1, r1, 112 132 ld r7, HSTATE_HOST_MSR(r13) 133 134 /* Return the trap number on this thread as the return value */ 135 mr r3, r12 136 137 /* RFI into the highmem handler */ 138 mfmsr r6 139 li r0, MSR_RI 140 andc r6, r6, r0 141 mtmsrd r6, 1 /* Clear RI in MSR */ 142 mtsrr0 r8 143 mtsrr1 r7 144 RFI_TO_KERNEL 145 146 kvmppc_primary_no_guest: 147 /* We handle this much like a ceded vcpu */ 148 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ 149 /* HDEC may be larger than DEC for arch >= v3.00, but since the */ 150 /* HDEC value came from DEC in the first place, it will fit */ 151 mfspr r3, SPRN_HDEC 152 mtspr SPRN_DEC, r3 153 /* 154 * Make sure the primary has finished the MMU switch. 155 * We should never get here on a secondary thread, but 156 * check it for robustness' sake. 157 */ 158 ld r5, HSTATE_KVM_VCORE(r13) 159 65: lbz r0, VCORE_IN_GUEST(r5) 160 cmpwi r0, 0 161 beq 65b 162 /* Set LPCR. */ 163 ld r8,VCORE_LPCR(r5) 164 mtspr SPRN_LPCR,r8 165 isync 166 /* set our bit in napping_threads */ 167 ld r5, HSTATE_KVM_VCORE(r13) 168 lbz r7, HSTATE_PTID(r13) 169 li r0, 1 170 sld r0, r0, r7 171 addi r6, r5, VCORE_NAPPING_THREADS 172 1: lwarx r3, 0, r6 173 or r3, r3, r0 174 stwcx. r3, 0, r6 175 bne 1b 176 /* order napping_threads update vs testing entry_exit_map */ 177 isync 178 li r12, 0 179 lwz r7, VCORE_ENTRY_EXIT(r5) 180 cmpwi r7, 0x100 181 bge kvm_novcpu_exit /* another thread already exiting */ 182 li r3, NAPPING_NOVCPU 183 stb r3, HSTATE_NAPPING(r13) 184 185 li r3, 0 /* Don't wake on privileged (OS) doorbell */ 186 b kvm_do_nap 187 188 /* 189 * kvm_novcpu_wakeup 190 * Entered from kvm_start_guest if kvm_hstate.napping is set 191 * to NAPPING_NOVCPU 192 * r2 = kernel TOC 193 * r13 = paca 194 */ 195 kvm_novcpu_wakeup: 196 ld r1, HSTATE_HOST_R1(r13) 197 ld r5, HSTATE_KVM_VCORE(r13) 198 li r0, 0 199 stb r0, HSTATE_NAPPING(r13) 200 201 /* check the wake reason */ 202 bl kvmppc_check_wake_reason 203 204 /* 205 * Restore volatile registers since we could have called 206 * a C routine in kvmppc_check_wake_reason. 207 * r5 = VCORE 208 */ 209 ld r5, HSTATE_KVM_VCORE(r13) 210 211 /* see if any other thread is already exiting */ 212 lwz r0, VCORE_ENTRY_EXIT(r5) 213 cmpwi r0, 0x100 214 bge kvm_novcpu_exit 215 216 /* clear our bit in napping_threads */ 217 lbz r7, HSTATE_PTID(r13) 218 li r0, 1 219 sld r0, r0, r7 220 addi r6, r5, VCORE_NAPPING_THREADS 221 4: lwarx r7, 0, r6 222 andc r7, r7, r0 223 stwcx. r7, 0, r6 224 bne 4b 225 226 /* See if the wake reason means we need to exit */ 227 cmpdi r3, 0 228 bge kvm_novcpu_exit 229 230 /* See if our timeslice has expired (HDEC is negative) */ 231 mfspr r0, SPRN_HDEC 232 extsw r0, r0 233 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 234 cmpdi r0, 0 235 blt kvm_novcpu_exit 236 237 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */ 238 ld r4, HSTATE_KVM_VCPU(r13) 239 cmpdi r4, 0 240 beq kvmppc_primary_no_guest 241 242 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 243 addi r3, r4, VCPU_TB_RMENTRY 244 bl kvmhv_start_timing 245 #endif 246 b kvmppc_got_guest 247 248 kvm_novcpu_exit: 249 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 250 ld r4, HSTATE_KVM_VCPU(r13) 251 cmpdi r4, 0 252 beq 13f 253 addi r3, r4, VCPU_TB_RMEXIT 254 bl kvmhv_accumulate_time 255 #endif 256 13: mr r3, r12 257 stw r12, STACK_SLOT_TRAP(r1) 258 bl kvmhv_commence_exit 259 nop 260 b kvmhv_switch_to_host 261 262 /* 263 * We come in here when wakened from Linux offline idle code. 264 * Relocation is off 265 * r3 contains the SRR1 wakeup value, SRR1 is trashed. 266 */ 267 _GLOBAL(idle_kvm_start_guest) 268 mfcr r5 269 mflr r0 270 std r5, 8(r1) // Save CR in caller's frame 271 std r0, 16(r1) // Save LR in caller's frame 272 // Create frame on emergency stack 273 ld r4, PACAEMERGSP(r13) 274 stdu r1, -SWITCH_FRAME_SIZE(r4) 275 // Switch to new frame on emergency stack 276 mr r1, r4 277 std r3, 32(r1) // Save SRR1 wakeup value 278 SAVE_NVGPRS(r1) 279 280 /* 281 * Could avoid this and pass it through in r3. For now, 282 * code expects it to be in SRR1. 283 */ 284 mtspr SPRN_SRR1,r3 285 286 li r0,0 287 stb r0,PACA_FTRACE_ENABLED(r13) 288 289 li r0,KVM_HWTHREAD_IN_KVM 290 stb r0,HSTATE_HWTHREAD_STATE(r13) 291 292 /* kvm cede / napping does not come through here */ 293 lbz r0,HSTATE_NAPPING(r13) 294 twnei r0,0 295 296 b 1f 297 298 kvm_unsplit_wakeup: 299 li r0, 0 300 stb r0, HSTATE_NAPPING(r13) 301 302 1: 303 304 /* 305 * We weren't napping due to cede, so this must be a secondary 306 * thread being woken up to run a guest, or being woken up due 307 * to a stray IPI. (Or due to some machine check or hypervisor 308 * maintenance interrupt while the core is in KVM.) 309 */ 310 311 /* Check the wake reason in SRR1 to see why we got here */ 312 bl kvmppc_check_wake_reason 313 /* 314 * kvmppc_check_wake_reason could invoke a C routine, but we 315 * have no volatile registers to restore when we return. 316 */ 317 318 cmpdi r3, 0 319 bge kvm_no_guest 320 321 /* get vcore pointer, NULL if we have nothing to run */ 322 ld r5,HSTATE_KVM_VCORE(r13) 323 cmpdi r5,0 324 /* if we have no vcore to run, go back to sleep */ 325 beq kvm_no_guest 326 327 kvm_secondary_got_guest: 328 329 // About to go to guest, clear saved SRR1 330 li r0, 0 331 std r0, 32(r1) 332 333 /* Set HSTATE_DSCR(r13) to something sensible */ 334 ld r6, PACA_DSCR_DEFAULT(r13) 335 std r6, HSTATE_DSCR(r13) 336 337 /* On thread 0 of a subcore, set HDEC to max */ 338 lbz r4, HSTATE_PTID(r13) 339 cmpwi r4, 0 340 bne 63f 341 lis r6,0x7fff /* MAX_INT@h */ 342 mtspr SPRN_HDEC, r6 343 /* and set per-LPAR registers, if doing dynamic micro-threading */ 344 ld r6, HSTATE_SPLIT_MODE(r13) 345 cmpdi r6, 0 346 beq 63f 347 ld r0, KVM_SPLIT_RPR(r6) 348 mtspr SPRN_RPR, r0 349 ld r0, KVM_SPLIT_PMMAR(r6) 350 mtspr SPRN_PMMAR, r0 351 ld r0, KVM_SPLIT_LDBAR(r6) 352 mtspr SPRN_LDBAR, r0 353 isync 354 63: 355 /* Order load of vcpu after load of vcore */ 356 lwsync 357 ld r4, HSTATE_KVM_VCPU(r13) 358 bl kvmppc_hv_entry 359 360 /* Back from the guest, go back to nap */ 361 /* Clear our vcpu and vcore pointers so we don't come back in early */ 362 li r0, 0 363 std r0, HSTATE_KVM_VCPU(r13) 364 /* 365 * Once we clear HSTATE_KVM_VCORE(r13), the code in 366 * kvmppc_run_core() is going to assume that all our vcpu 367 * state is visible in memory. This lwsync makes sure 368 * that that is true. 369 */ 370 lwsync 371 std r0, HSTATE_KVM_VCORE(r13) 372 373 /* 374 * All secondaries exiting guest will fall through this path. 375 * Before proceeding, just check for HMI interrupt and 376 * invoke opal hmi handler. By now we are sure that the 377 * primary thread on this core/subcore has already made partition 378 * switch/TB resync and we are good to call opal hmi handler. 379 */ 380 cmpwi r12, BOOK3S_INTERRUPT_HMI 381 bne kvm_no_guest 382 383 li r3,0 /* NULL argument */ 384 bl CFUNC(hmi_exception_realmode) 385 /* 386 * At this point we have finished executing in the guest. 387 * We need to wait for hwthread_req to become zero, since 388 * we may not turn on the MMU while hwthread_req is non-zero. 389 * While waiting we also need to check if we get given a vcpu to run. 390 */ 391 kvm_no_guest: 392 lbz r3, HSTATE_HWTHREAD_REQ(r13) 393 cmpwi r3, 0 394 bne 53f 395 HMT_MEDIUM 396 li r0, KVM_HWTHREAD_IN_KERNEL 397 stb r0, HSTATE_HWTHREAD_STATE(r13) 398 /* need to recheck hwthread_req after a barrier, to avoid race */ 399 sync 400 lbz r3, HSTATE_HWTHREAD_REQ(r13) 401 cmpwi r3, 0 402 bne 54f 403 404 /* 405 * Jump to idle_return_gpr_loss, which returns to the 406 * idle_kvm_start_guest caller. 407 */ 408 li r3, LPCR_PECE0 409 mfspr r4, SPRN_LPCR 410 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 411 mtspr SPRN_LPCR, r4 412 // Return SRR1 wakeup value, or 0 if we went into the guest 413 ld r3, 32(r1) 414 REST_NVGPRS(r1) 415 ld r1, 0(r1) // Switch back to caller stack 416 ld r0, 16(r1) // Reload LR 417 ld r5, 8(r1) // Reload CR 418 mtlr r0 419 mtcr r5 420 blr 421 422 53: 423 HMT_LOW 424 ld r5, HSTATE_KVM_VCORE(r13) 425 cmpdi r5, 0 426 bne 60f 427 ld r3, HSTATE_SPLIT_MODE(r13) 428 cmpdi r3, 0 429 beq kvm_no_guest 430 lbz r0, KVM_SPLIT_DO_NAP(r3) 431 cmpwi r0, 0 432 beq kvm_no_guest 433 HMT_MEDIUM 434 b kvm_unsplit_nap 435 60: HMT_MEDIUM 436 b kvm_secondary_got_guest 437 438 54: li r0, KVM_HWTHREAD_IN_KVM 439 stb r0, HSTATE_HWTHREAD_STATE(r13) 440 b kvm_no_guest 441 442 /* 443 * Here the primary thread is trying to return the core to 444 * whole-core mode, so we need to nap. 445 */ 446 kvm_unsplit_nap: 447 /* 448 * When secondaries are napping in kvm_unsplit_nap() with 449 * hwthread_req = 1, HMI goes ignored even though subcores are 450 * already exited the guest. Hence HMI keeps waking up secondaries 451 * from nap in a loop and secondaries always go back to nap since 452 * no vcore is assigned to them. This makes impossible for primary 453 * thread to get hold of secondary threads resulting into a soft 454 * lockup in KVM path. 455 * 456 * Let us check if HMI is pending and handle it before we go to nap. 457 */ 458 cmpwi r12, BOOK3S_INTERRUPT_HMI 459 bne 55f 460 li r3, 0 /* NULL argument */ 461 bl CFUNC(hmi_exception_realmode) 462 55: 463 /* 464 * Ensure that secondary doesn't nap when it has 465 * its vcore pointer set. 466 */ 467 sync /* matches smp_mb() before setting split_info.do_nap */ 468 ld r0, HSTATE_KVM_VCORE(r13) 469 cmpdi r0, 0 470 bne kvm_no_guest 471 /* clear any pending message */ 472 BEGIN_FTR_SECTION 473 lis r6, (PPC_DBELL_SERVER << (63-36))@h 474 PPC_MSGCLR(6) 475 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 476 /* Set kvm_split_mode.napped[tid] = 1 */ 477 ld r3, HSTATE_SPLIT_MODE(r13) 478 li r0, 1 479 lhz r4, PACAPACAINDEX(r13) 480 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ 481 addi r4, r4, KVM_SPLIT_NAPPED 482 stbx r0, r3, r4 483 /* Check the do_nap flag again after setting napped[] */ 484 sync 485 lbz r0, KVM_SPLIT_DO_NAP(r3) 486 cmpwi r0, 0 487 beq 57f 488 li r3, NAPPING_UNSPLIT 489 stb r3, HSTATE_NAPPING(r13) 490 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 491 mfspr r5, SPRN_LPCR 492 rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) 493 b kvm_nap_sequence 494 495 57: li r0, 0 496 stbx r0, r3, r4 497 b kvm_no_guest 498 499 /****************************************************************************** 500 * * 501 * Entry code * 502 * * 503 *****************************************************************************/ 504 505 SYM_CODE_START_LOCAL(kvmppc_hv_entry) 506 507 /* Required state: 508 * 509 * R4 = vcpu pointer (or NULL) 510 * MSR = ~IR|DR 511 * R13 = PACA 512 * R1 = host R1 513 * R2 = TOC 514 * all other volatile GPRS = free 515 * Does not preserve non-volatile GPRs or CR fields 516 */ 517 mflr r0 518 std r0, PPC_LR_STKOFF(r1) 519 stdu r1, -SFS(r1) 520 521 /* Save R1 in the PACA */ 522 std r1, HSTATE_HOST_R1(r13) 523 524 li r6, KVM_GUEST_MODE_HOST_HV 525 stb r6, HSTATE_IN_GUEST(r13) 526 527 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 528 /* Store initial timestamp */ 529 cmpdi r4, 0 530 beq 1f 531 addi r3, r4, VCPU_TB_RMENTRY 532 bl kvmhv_start_timing 533 1: 534 #endif 535 536 ld r5, HSTATE_KVM_VCORE(r13) 537 ld r9, VCORE_KVM(r5) /* pointer to struct kvm */ 538 539 /* 540 * POWER7/POWER8 host -> guest partition switch code. 541 * We don't have to lock against concurrent tlbies, 542 * but we do have to coordinate across hardware threads. 543 */ 544 /* Set bit in entry map iff exit map is zero. */ 545 li r7, 1 546 lbz r6, HSTATE_PTID(r13) 547 sld r7, r7, r6 548 addi r8, r5, VCORE_ENTRY_EXIT 549 21: lwarx r3, 0, r8 550 cmpwi r3, 0x100 /* any threads starting to exit? */ 551 bge secondary_too_late /* if so we're too late to the party */ 552 or r3, r3, r7 553 stwcx. r3, 0, r8 554 bne 21b 555 556 /* Primary thread switches to guest partition. */ 557 cmpwi r6,0 558 bne 10f 559 560 lwz r7,KVM_LPID(r9) 561 ld r6,KVM_SDR1(r9) 562 li r0,LPID_RSVD /* switch to reserved LPID */ 563 mtspr SPRN_LPID,r0 564 ptesync 565 mtspr SPRN_SDR1,r6 /* switch to partition page table */ 566 mtspr SPRN_LPID,r7 567 isync 568 569 /* See if we need to flush the TLB. */ 570 mr r3, r9 /* kvm pointer */ 571 lhz r4, PACAPACAINDEX(r13) /* physical cpu number */ 572 li r5, 0 /* nested vcpu pointer */ 573 bl kvmppc_check_need_tlb_flush 574 nop 575 ld r5, HSTATE_KVM_VCORE(r13) 576 577 /* Add timebase offset onto timebase */ 578 22: ld r8,VCORE_TB_OFFSET(r5) 579 cmpdi r8,0 580 beq 37f 581 std r8, VCORE_TB_OFFSET_APPL(r5) 582 mftb r6 /* current host timebase */ 583 add r8,r8,r6 584 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 585 mftb r7 /* check if lower 24 bits overflowed */ 586 clrldi r6,r6,40 587 clrldi r7,r7,40 588 cmpld r7,r6 589 bge 37f 590 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 591 mtspr SPRN_TBU40,r8 592 593 /* Load guest PCR value to select appropriate compat mode */ 594 37: ld r7, VCORE_PCR(r5) 595 LOAD_REG_IMMEDIATE(r6, PCR_MASK) 596 cmpld r7, r6 597 beq 38f 598 or r7, r7, r6 599 mtspr SPRN_PCR, r7 600 38: 601 602 BEGIN_FTR_SECTION 603 /* DPDES and VTB are shared between threads */ 604 ld r8, VCORE_DPDES(r5) 605 ld r7, VCORE_VTB(r5) 606 mtspr SPRN_DPDES, r8 607 mtspr SPRN_VTB, r7 608 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 609 610 /* Mark the subcore state as inside guest */ 611 bl kvmppc_subcore_enter_guest 612 nop 613 ld r5, HSTATE_KVM_VCORE(r13) 614 ld r4, HSTATE_KVM_VCPU(r13) 615 li r0,1 616 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */ 617 618 /* Do we have a guest vcpu to run? */ 619 10: cmpdi r4, 0 620 beq kvmppc_primary_no_guest 621 kvmppc_got_guest: 622 /* Increment yield count if they have a VPA */ 623 ld r3, VCPU_VPA(r4) 624 cmpdi r3, 0 625 beq 25f 626 li r6, LPPACA_YIELDCOUNT 627 LWZX_BE r5, r3, r6 628 addi r5, r5, 1 629 STWX_BE r5, r3, r6 630 li r6, 1 631 stb r6, VCPU_VPA_DIRTY(r4) 632 25: 633 634 /* Save purr/spurr */ 635 mfspr r5,SPRN_PURR 636 mfspr r6,SPRN_SPURR 637 std r5,HSTATE_PURR(r13) 638 std r6,HSTATE_SPURR(r13) 639 ld r7,VCPU_PURR(r4) 640 ld r8,VCPU_SPURR(r4) 641 mtspr SPRN_PURR,r7 642 mtspr SPRN_SPURR,r8 643 644 /* Save host values of some registers */ 645 BEGIN_FTR_SECTION 646 mfspr r5, SPRN_CIABR 647 mfspr r6, SPRN_DAWR0 648 mfspr r7, SPRN_DAWRX0 649 mfspr r8, SPRN_IAMR 650 std r5, STACK_SLOT_CIABR(r1) 651 std r6, STACK_SLOT_DAWR0(r1) 652 std r7, STACK_SLOT_DAWRX0(r1) 653 std r8, STACK_SLOT_IAMR(r1) 654 mfspr r5, SPRN_FSCR 655 std r5, STACK_SLOT_FSCR(r1) 656 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 657 658 mfspr r5, SPRN_AMR 659 std r5, STACK_SLOT_AMR(r1) 660 mfspr r6, SPRN_UAMOR 661 std r6, STACK_SLOT_UAMOR(r1) 662 663 BEGIN_FTR_SECTION 664 /* Set partition DABR */ 665 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */ 666 lwz r5,VCPU_DABRX(r4) 667 ld r6,VCPU_DABR(r4) 668 mtspr SPRN_DABRX,r5 669 mtspr SPRN_DABR,r6 670 isync 671 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 672 673 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 674 BEGIN_FTR_SECTION 675 b 91f 676 END_FTR_SECTION_IFCLR(CPU_FTR_TM) 677 /* 678 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 679 */ 680 mr r3, r4 681 ld r4, VCPU_MSR(r3) 682 li r5, 0 /* don't preserve non-vol regs */ 683 bl kvmppc_restore_tm_hv 684 nop 685 ld r4, HSTATE_KVM_VCPU(r13) 686 91: 687 #endif 688 689 /* Load guest PMU registers; r4 = vcpu pointer here */ 690 mr r3, r4 691 bl kvmhv_load_guest_pmu 692 693 /* Load up FP, VMX and VSX registers */ 694 ld r4, HSTATE_KVM_VCPU(r13) 695 bl kvmppc_load_fp 696 697 ld r14, VCPU_GPR(R14)(r4) 698 ld r15, VCPU_GPR(R15)(r4) 699 ld r16, VCPU_GPR(R16)(r4) 700 ld r17, VCPU_GPR(R17)(r4) 701 ld r18, VCPU_GPR(R18)(r4) 702 ld r19, VCPU_GPR(R19)(r4) 703 ld r20, VCPU_GPR(R20)(r4) 704 ld r21, VCPU_GPR(R21)(r4) 705 ld r22, VCPU_GPR(R22)(r4) 706 ld r23, VCPU_GPR(R23)(r4) 707 ld r24, VCPU_GPR(R24)(r4) 708 ld r25, VCPU_GPR(R25)(r4) 709 ld r26, VCPU_GPR(R26)(r4) 710 ld r27, VCPU_GPR(R27)(r4) 711 ld r28, VCPU_GPR(R28)(r4) 712 ld r29, VCPU_GPR(R29)(r4) 713 ld r30, VCPU_GPR(R30)(r4) 714 ld r31, VCPU_GPR(R31)(r4) 715 716 /* Switch DSCR to guest value */ 717 ld r5, VCPU_DSCR(r4) 718 mtspr SPRN_DSCR, r5 719 720 BEGIN_FTR_SECTION 721 /* Skip next section on POWER7 */ 722 b 8f 723 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 724 /* Load up POWER8-specific registers */ 725 ld r5, VCPU_IAMR(r4) 726 lwz r6, VCPU_PSPB(r4) 727 ld r7, VCPU_FSCR(r4) 728 mtspr SPRN_IAMR, r5 729 mtspr SPRN_PSPB, r6 730 mtspr SPRN_FSCR, r7 731 /* 732 * Handle broken DAWR case by not writing it. This means we 733 * can still store the DAWR register for migration. 734 */ 735 LOAD_REG_ADDR(r5, dawr_force_enable) 736 lbz r5, 0(r5) 737 cmpdi r5, 0 738 beq 1f 739 ld r5, VCPU_DAWR0(r4) 740 ld r6, VCPU_DAWRX0(r4) 741 mtspr SPRN_DAWR0, r5 742 mtspr SPRN_DAWRX0, r6 743 1: 744 ld r7, VCPU_CIABR(r4) 745 ld r8, VCPU_TAR(r4) 746 mtspr SPRN_CIABR, r7 747 mtspr SPRN_TAR, r8 748 ld r5, VCPU_IC(r4) 749 ld r8, VCPU_EBBHR(r4) 750 mtspr SPRN_IC, r5 751 mtspr SPRN_EBBHR, r8 752 ld r5, VCPU_EBBRR(r4) 753 ld r6, VCPU_BESCR(r4) 754 lwz r7, VCPU_GUEST_PID(r4) 755 ld r8, VCPU_WORT(r4) 756 mtspr SPRN_EBBRR, r5 757 mtspr SPRN_BESCR, r6 758 mtspr SPRN_PID, r7 759 mtspr SPRN_WORT, r8 760 /* POWER8-only registers */ 761 ld r5, VCPU_TCSCR(r4) 762 ld r6, VCPU_ACOP(r4) 763 ld r7, VCPU_CSIGR(r4) 764 ld r8, VCPU_TACR(r4) 765 mtspr SPRN_TCSCR, r5 766 mtspr SPRN_ACOP, r6 767 mtspr SPRN_CSIGR, r7 768 mtspr SPRN_TACR, r8 769 nop 770 8: 771 772 ld r5, VCPU_SPRG0(r4) 773 ld r6, VCPU_SPRG1(r4) 774 ld r7, VCPU_SPRG2(r4) 775 ld r8, VCPU_SPRG3(r4) 776 mtspr SPRN_SPRG0, r5 777 mtspr SPRN_SPRG1, r6 778 mtspr SPRN_SPRG2, r7 779 mtspr SPRN_SPRG3, r8 780 781 /* Load up DAR and DSISR */ 782 ld r5, VCPU_DAR(r4) 783 lwz r6, VCPU_DSISR(r4) 784 mtspr SPRN_DAR, r5 785 mtspr SPRN_DSISR, r6 786 787 /* Restore AMR and UAMOR, set AMOR to all 1s */ 788 ld r5,VCPU_AMR(r4) 789 ld r6,VCPU_UAMOR(r4) 790 mtspr SPRN_AMR,r5 791 mtspr SPRN_UAMOR,r6 792 793 /* Restore state of CTRL run bit; the host currently has it set to 1 */ 794 lwz r5,VCPU_CTRL(r4) 795 andi. r5,r5,1 796 bne 4f 797 li r6,0 798 mtspr SPRN_CTRLT,r6 799 4: 800 /* Secondary threads wait for primary to have done partition switch */ 801 ld r5, HSTATE_KVM_VCORE(r13) 802 lbz r6, HSTATE_PTID(r13) 803 cmpwi r6, 0 804 beq 21f 805 lbz r0, VCORE_IN_GUEST(r5) 806 cmpwi r0, 0 807 bne 21f 808 HMT_LOW 809 20: lwz r3, VCORE_ENTRY_EXIT(r5) 810 cmpwi r3, 0x100 811 bge no_switch_exit 812 lbz r0, VCORE_IN_GUEST(r5) 813 cmpwi r0, 0 814 beq 20b 815 HMT_MEDIUM 816 21: 817 /* Set LPCR. */ 818 ld r8,VCORE_LPCR(r5) 819 mtspr SPRN_LPCR,r8 820 isync 821 822 /* 823 * Set the decrementer to the guest decrementer. 824 */ 825 ld r8,VCPU_DEC_EXPIRES(r4) 826 mftb r7 827 subf r3,r7,r8 828 mtspr SPRN_DEC,r3 829 830 /* Check if HDEC expires soon */ 831 mfspr r3, SPRN_HDEC 832 extsw r3, r3 833 cmpdi r3, 512 /* 1 microsecond */ 834 blt hdec_soon 835 836 /* Clear out and reload the SLB */ 837 li r6, 0 838 slbmte r6, r6 839 PPC_SLBIA(6) 840 ptesync 841 842 /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ 843 lwz r5,VCPU_SLB_MAX(r4) 844 cmpwi r5,0 845 beq 9f 846 mtctr r5 847 addi r6,r4,VCPU_SLB 848 1: ld r8,VCPU_SLB_E(r6) 849 ld r9,VCPU_SLB_V(r6) 850 slbmte r9,r8 851 addi r6,r6,VCPU_SLB_SIZE 852 bdnz 1b 853 9: 854 855 deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ 856 /* Check if we can deliver an external or decrementer interrupt now */ 857 ld r0, VCPU_PENDING_EXC(r4) 858 cmpdi r0, 0 859 beq 71f 860 mr r3, r4 861 bl CFUNC(kvmppc_guest_entry_inject_int) 862 ld r4, HSTATE_KVM_VCPU(r13) 863 71: 864 ld r6, VCPU_SRR0(r4) 865 ld r7, VCPU_SRR1(r4) 866 mtspr SPRN_SRR0, r6 867 mtspr SPRN_SRR1, r7 868 869 ld r10, VCPU_PC(r4) 870 ld r11, VCPU_MSR(r4) 871 /* r11 = vcpu->arch.msr & ~MSR_HV */ 872 rldicl r11, r11, 63 - MSR_HV_LG, 1 873 rotldi r11, r11, 1 + MSR_HV_LG 874 ori r11, r11, MSR_ME 875 876 ld r6, VCPU_CTR(r4) 877 ld r7, VCPU_XER(r4) 878 mtctr r6 879 mtxer r7 880 881 /* 882 * Required state: 883 * R4 = vcpu 884 * R10: value for HSRR0 885 * R11: value for HSRR1 886 * R13 = PACA 887 */ 888 fast_guest_return: 889 li r0,0 890 stb r0,VCPU_CEDED(r4) /* cancel cede */ 891 mtspr SPRN_HSRR0,r10 892 mtspr SPRN_HSRR1,r11 893 894 /* Activate guest mode, so faults get handled by KVM */ 895 li r9, KVM_GUEST_MODE_GUEST_HV 896 stb r9, HSTATE_IN_GUEST(r13) 897 898 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 899 /* Accumulate timing */ 900 addi r3, r4, VCPU_TB_GUEST 901 bl kvmhv_accumulate_time 902 #endif 903 904 /* Enter guest */ 905 906 BEGIN_FTR_SECTION 907 ld r5, VCPU_CFAR(r4) 908 mtspr SPRN_CFAR, r5 909 END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 910 BEGIN_FTR_SECTION 911 ld r0, VCPU_PPR(r4) 912 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 913 914 ld r5, VCPU_LR(r4) 915 mtlr r5 916 917 ld r1, VCPU_GPR(R1)(r4) 918 ld r5, VCPU_GPR(R5)(r4) 919 ld r8, VCPU_GPR(R8)(r4) 920 ld r9, VCPU_GPR(R9)(r4) 921 ld r10, VCPU_GPR(R10)(r4) 922 ld r11, VCPU_GPR(R11)(r4) 923 ld r12, VCPU_GPR(R12)(r4) 924 ld r13, VCPU_GPR(R13)(r4) 925 926 BEGIN_FTR_SECTION 927 mtspr SPRN_PPR, r0 928 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 929 930 ld r6, VCPU_GPR(R6)(r4) 931 ld r7, VCPU_GPR(R7)(r4) 932 933 ld r0, VCPU_CR(r4) 934 mtcr r0 935 936 ld r0, VCPU_GPR(R0)(r4) 937 ld r2, VCPU_GPR(R2)(r4) 938 ld r3, VCPU_GPR(R3)(r4) 939 ld r4, VCPU_GPR(R4)(r4) 940 HRFI_TO_GUEST 941 b . 942 SYM_CODE_END(kvmppc_hv_entry) 943 944 secondary_too_late: 945 li r12, 0 946 stw r12, STACK_SLOT_TRAP(r1) 947 cmpdi r4, 0 948 beq 11f 949 stw r12, VCPU_TRAP(r4) 950 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 951 addi r3, r4, VCPU_TB_RMEXIT 952 bl kvmhv_accumulate_time 953 #endif 954 11: b kvmhv_switch_to_host 955 956 no_switch_exit: 957 HMT_MEDIUM 958 li r12, 0 959 b 12f 960 hdec_soon: 961 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER 962 12: stw r12, VCPU_TRAP(r4) 963 mr r9, r4 964 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 965 addi r3, r4, VCPU_TB_RMEXIT 966 bl kvmhv_accumulate_time 967 #endif 968 b guest_bypass 969 970 /****************************************************************************** 971 * * 972 * Exit code * 973 * * 974 *****************************************************************************/ 975 976 /* 977 * We come here from the first-level interrupt handlers. 978 */ 979 .globl kvmppc_interrupt_hv 980 kvmppc_interrupt_hv: 981 /* 982 * Register contents: 983 * R9 = HSTATE_IN_GUEST 984 * R12 = (guest CR << 32) | interrupt vector 985 * R13 = PACA 986 * guest R12 saved in shadow VCPU SCRATCH0 987 * guest R13 saved in SPRN_SCRATCH0 988 * guest R9 saved in HSTATE_SCRATCH2 989 */ 990 /* We're now back in the host but in guest MMU context */ 991 cmpwi r9,KVM_GUEST_MODE_HOST_HV 992 beq kvmppc_bad_host_intr 993 li r9, KVM_GUEST_MODE_HOST_HV 994 stb r9, HSTATE_IN_GUEST(r13) 995 996 ld r9, HSTATE_KVM_VCPU(r13) 997 998 /* Save registers */ 999 1000 std r0, VCPU_GPR(R0)(r9) 1001 std r1, VCPU_GPR(R1)(r9) 1002 std r2, VCPU_GPR(R2)(r9) 1003 std r3, VCPU_GPR(R3)(r9) 1004 std r4, VCPU_GPR(R4)(r9) 1005 std r5, VCPU_GPR(R5)(r9) 1006 std r6, VCPU_GPR(R6)(r9) 1007 std r7, VCPU_GPR(R7)(r9) 1008 std r8, VCPU_GPR(R8)(r9) 1009 ld r0, HSTATE_SCRATCH2(r13) 1010 std r0, VCPU_GPR(R9)(r9) 1011 std r10, VCPU_GPR(R10)(r9) 1012 std r11, VCPU_GPR(R11)(r9) 1013 ld r3, HSTATE_SCRATCH0(r13) 1014 std r3, VCPU_GPR(R12)(r9) 1015 /* CR is in the high half of r12 */ 1016 srdi r4, r12, 32 1017 std r4, VCPU_CR(r9) 1018 BEGIN_FTR_SECTION 1019 ld r3, HSTATE_CFAR(r13) 1020 std r3, VCPU_CFAR(r9) 1021 END_FTR_SECTION_IFSET(CPU_FTR_CFAR) 1022 BEGIN_FTR_SECTION 1023 ld r4, HSTATE_PPR(r13) 1024 std r4, VCPU_PPR(r9) 1025 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1026 1027 /* Restore R1/R2 so we can handle faults */ 1028 ld r1, HSTATE_HOST_R1(r13) 1029 LOAD_PACA_TOC() 1030 1031 mfspr r10, SPRN_SRR0 1032 mfspr r11, SPRN_SRR1 1033 std r10, VCPU_SRR0(r9) 1034 std r11, VCPU_SRR1(r9) 1035 /* trap is in the low half of r12, clear CR from the high half */ 1036 clrldi r12, r12, 32 1037 andi. r0, r12, 2 /* need to read HSRR0/1? */ 1038 beq 1f 1039 mfspr r10, SPRN_HSRR0 1040 mfspr r11, SPRN_HSRR1 1041 clrrdi r12, r12, 2 1042 1: std r10, VCPU_PC(r9) 1043 std r11, VCPU_MSR(r9) 1044 1045 GET_SCRATCH0(r3) 1046 mflr r4 1047 std r3, VCPU_GPR(R13)(r9) 1048 std r4, VCPU_LR(r9) 1049 1050 stw r12,VCPU_TRAP(r9) 1051 1052 /* 1053 * Now that we have saved away SRR0/1 and HSRR0/1, 1054 * interrupts are recoverable in principle, so set MSR_RI. 1055 * This becomes important for relocation-on interrupts from 1056 * the guest, which we can get in radix mode on POWER9. 1057 */ 1058 li r0, MSR_RI 1059 mtmsrd r0, 1 1060 1061 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 1062 addi r3, r9, VCPU_TB_RMINTR 1063 mr r4, r9 1064 bl kvmhv_accumulate_time 1065 ld r5, VCPU_GPR(R5)(r9) 1066 ld r6, VCPU_GPR(R6)(r9) 1067 ld r7, VCPU_GPR(R7)(r9) 1068 ld r8, VCPU_GPR(R8)(r9) 1069 #endif 1070 1071 /* Save HEIR (HV emulation assist reg) in emul_inst 1072 if this is an HEI (HV emulation interrupt, e40) */ 1073 li r3,KVM_INST_FETCH_FAILED 1074 std r3,VCPU_LAST_INST(r9) 1075 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST 1076 bne 11f 1077 mfspr r3,SPRN_HEIR 1078 11: std r3,VCPU_HEIR(r9) 1079 1080 /* these are volatile across C function calls */ 1081 mfctr r3 1082 mfxer r4 1083 std r3, VCPU_CTR(r9) 1084 std r4, VCPU_XER(r9) 1085 1086 /* Save more register state */ 1087 mfdar r3 1088 mfdsisr r4 1089 std r3, VCPU_DAR(r9) 1090 stw r4, VCPU_DSISR(r9) 1091 1092 /* If this is a page table miss then see if it's theirs or ours */ 1093 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1094 beq kvmppc_hdsi 1095 std r3, VCPU_FAULT_DAR(r9) 1096 stw r4, VCPU_FAULT_DSISR(r9) 1097 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1098 beq kvmppc_hisi 1099 1100 /* See if this is a leftover HDEC interrupt */ 1101 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER 1102 bne 2f 1103 mfspr r3,SPRN_HDEC 1104 extsw r3, r3 1105 cmpdi r3,0 1106 mr r4,r9 1107 bge fast_guest_return 1108 2: 1109 /* See if this is an hcall we can handle in real mode */ 1110 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL 1111 beq hcall_try_real_mode 1112 1113 /* Hypervisor doorbell - exit only if host IPI flag set */ 1114 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL 1115 bne 3f 1116 lbz r0, HSTATE_HOST_IPI(r13) 1117 cmpwi r0, 0 1118 beq maybe_reenter_guest 1119 b guest_exit_cont 1120 3: 1121 /* If it's a hypervisor facility unavailable interrupt, save HFSCR */ 1122 cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL 1123 bne 14f 1124 mfspr r3, SPRN_HFSCR 1125 std r3, VCPU_HFSCR(r9) 1126 b guest_exit_cont 1127 14: 1128 /* External interrupt ? */ 1129 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL 1130 beq kvmppc_guest_external 1131 /* See if it is a machine check */ 1132 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK 1133 beq machine_check_realmode 1134 /* Or a hypervisor maintenance interrupt */ 1135 cmpwi r12, BOOK3S_INTERRUPT_HMI 1136 beq hmi_realmode 1137 1138 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ 1139 1140 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 1141 addi r3, r9, VCPU_TB_RMEXIT 1142 mr r4, r9 1143 bl kvmhv_accumulate_time 1144 #endif 1145 1146 /* 1147 * Possibly flush the link stack here, before we do a blr in 1148 * kvmhv_switch_to_host. 1149 */ 1150 1: nop 1151 patch_site 1b patch__call_kvm_flush_link_stack 1152 1153 /* For hash guest, read the guest SLB and save it away */ 1154 li r5, 0 1155 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ 1156 mtctr r0 1157 li r6,0 1158 addi r7,r9,VCPU_SLB 1159 1: slbmfee r8,r6 1160 andis. r0,r8,SLB_ESID_V@h 1161 beq 2f 1162 add r8,r8,r6 /* put index in */ 1163 slbmfev r3,r6 1164 std r8,VCPU_SLB_E(r7) 1165 std r3,VCPU_SLB_V(r7) 1166 addi r7,r7,VCPU_SLB_SIZE 1167 addi r5,r5,1 1168 2: addi r6,r6,1 1169 bdnz 1b 1170 /* Finally clear out the SLB */ 1171 li r0,0 1172 slbmte r0,r0 1173 PPC_SLBIA(6) 1174 ptesync 1175 stw r5,VCPU_SLB_MAX(r9) 1176 1177 /* load host SLB entries */ 1178 ld r8,PACA_SLBSHADOWPTR(r13) 1179 1180 .rept SLB_NUM_BOLTED 1181 li r3, SLBSHADOW_SAVEAREA 1182 LDX_BE r5, r8, r3 1183 addi r3, r3, 8 1184 LDX_BE r6, r8, r3 1185 andis. r7,r5,SLB_ESID_V@h 1186 beq 1f 1187 slbmte r6,r5 1188 1: addi r8,r8,16 1189 .endr 1190 1191 guest_bypass: 1192 stw r12, STACK_SLOT_TRAP(r1) 1193 1194 /* Save DEC */ 1195 /* Do this before kvmhv_commence_exit so we know TB is guest TB */ 1196 ld r3, HSTATE_KVM_VCORE(r13) 1197 mfspr r5,SPRN_DEC 1198 mftb r6 1199 extsw r5,r5 1200 16: add r5,r5,r6 1201 std r5,VCPU_DEC_EXPIRES(r9) 1202 1203 /* Increment exit count, poke other threads to exit */ 1204 mr r3, r12 1205 bl kvmhv_commence_exit 1206 nop 1207 ld r9, HSTATE_KVM_VCPU(r13) 1208 1209 /* Stop others sending VCPU interrupts to this physical CPU */ 1210 li r0, -1 1211 stw r0, VCPU_CPU(r9) 1212 stw r0, VCPU_THREAD_CPU(r9) 1213 1214 /* Save guest CTRL register, set runlatch to 1 if it was clear */ 1215 mfspr r6,SPRN_CTRLF 1216 stw r6,VCPU_CTRL(r9) 1217 andi. r0,r6,1 1218 bne 4f 1219 li r6,1 1220 mtspr SPRN_CTRLT,r6 1221 4: 1222 /* 1223 * Save the guest PURR/SPURR 1224 */ 1225 mfspr r5,SPRN_PURR 1226 mfspr r6,SPRN_SPURR 1227 ld r7,VCPU_PURR(r9) 1228 ld r8,VCPU_SPURR(r9) 1229 std r5,VCPU_PURR(r9) 1230 std r6,VCPU_SPURR(r9) 1231 subf r5,r7,r5 1232 subf r6,r8,r6 1233 1234 /* 1235 * Restore host PURR/SPURR and add guest times 1236 * so that the time in the guest gets accounted. 1237 */ 1238 ld r3,HSTATE_PURR(r13) 1239 ld r4,HSTATE_SPURR(r13) 1240 add r3,r3,r5 1241 add r4,r4,r6 1242 mtspr SPRN_PURR,r3 1243 mtspr SPRN_SPURR,r4 1244 1245 BEGIN_FTR_SECTION 1246 b 8f 1247 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) 1248 /* Save POWER8-specific registers */ 1249 mfspr r5, SPRN_IAMR 1250 mfspr r6, SPRN_PSPB 1251 mfspr r7, SPRN_FSCR 1252 std r5, VCPU_IAMR(r9) 1253 stw r6, VCPU_PSPB(r9) 1254 std r7, VCPU_FSCR(r9) 1255 mfspr r5, SPRN_IC 1256 mfspr r7, SPRN_TAR 1257 std r5, VCPU_IC(r9) 1258 std r7, VCPU_TAR(r9) 1259 mfspr r8, SPRN_EBBHR 1260 std r8, VCPU_EBBHR(r9) 1261 mfspr r5, SPRN_EBBRR 1262 mfspr r6, SPRN_BESCR 1263 mfspr r7, SPRN_PID 1264 mfspr r8, SPRN_WORT 1265 std r5, VCPU_EBBRR(r9) 1266 std r6, VCPU_BESCR(r9) 1267 stw r7, VCPU_GUEST_PID(r9) 1268 std r8, VCPU_WORT(r9) 1269 mfspr r5, SPRN_TCSCR 1270 mfspr r6, SPRN_ACOP 1271 mfspr r7, SPRN_CSIGR 1272 mfspr r8, SPRN_TACR 1273 std r5, VCPU_TCSCR(r9) 1274 std r6, VCPU_ACOP(r9) 1275 std r7, VCPU_CSIGR(r9) 1276 std r8, VCPU_TACR(r9) 1277 BEGIN_FTR_SECTION 1278 ld r5, STACK_SLOT_FSCR(r1) 1279 mtspr SPRN_FSCR, r5 1280 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1281 /* 1282 * Restore various registers to 0, where non-zero values 1283 * set by the guest could disrupt the host. 1284 */ 1285 li r0, 0 1286 mtspr SPRN_PSPB, r0 1287 mtspr SPRN_WORT, r0 1288 mtspr SPRN_TCSCR, r0 1289 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ 1290 li r0, 1 1291 sldi r0, r0, 31 1292 mtspr SPRN_MMCRS, r0 1293 1294 /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ 1295 ld r8, STACK_SLOT_IAMR(r1) 1296 mtspr SPRN_IAMR, r8 1297 1298 8: /* Power7 jumps back in here */ 1299 mfspr r5,SPRN_AMR 1300 mfspr r6,SPRN_UAMOR 1301 std r5,VCPU_AMR(r9) 1302 std r6,VCPU_UAMOR(r9) 1303 ld r5,STACK_SLOT_AMR(r1) 1304 ld r6,STACK_SLOT_UAMOR(r1) 1305 mtspr SPRN_AMR, r5 1306 mtspr SPRN_UAMOR, r6 1307 1308 /* Switch DSCR back to host value */ 1309 mfspr r8, SPRN_DSCR 1310 ld r7, HSTATE_DSCR(r13) 1311 std r8, VCPU_DSCR(r9) 1312 mtspr SPRN_DSCR, r7 1313 1314 /* Save non-volatile GPRs */ 1315 std r14, VCPU_GPR(R14)(r9) 1316 std r15, VCPU_GPR(R15)(r9) 1317 std r16, VCPU_GPR(R16)(r9) 1318 std r17, VCPU_GPR(R17)(r9) 1319 std r18, VCPU_GPR(R18)(r9) 1320 std r19, VCPU_GPR(R19)(r9) 1321 std r20, VCPU_GPR(R20)(r9) 1322 std r21, VCPU_GPR(R21)(r9) 1323 std r22, VCPU_GPR(R22)(r9) 1324 std r23, VCPU_GPR(R23)(r9) 1325 std r24, VCPU_GPR(R24)(r9) 1326 std r25, VCPU_GPR(R25)(r9) 1327 std r26, VCPU_GPR(R26)(r9) 1328 std r27, VCPU_GPR(R27)(r9) 1329 std r28, VCPU_GPR(R28)(r9) 1330 std r29, VCPU_GPR(R29)(r9) 1331 std r30, VCPU_GPR(R30)(r9) 1332 std r31, VCPU_GPR(R31)(r9) 1333 1334 /* Save SPRGs */ 1335 mfspr r3, SPRN_SPRG0 1336 mfspr r4, SPRN_SPRG1 1337 mfspr r5, SPRN_SPRG2 1338 mfspr r6, SPRN_SPRG3 1339 std r3, VCPU_SPRG0(r9) 1340 std r4, VCPU_SPRG1(r9) 1341 std r5, VCPU_SPRG2(r9) 1342 std r6, VCPU_SPRG3(r9) 1343 1344 /* save FP state */ 1345 mr r3, r9 1346 bl kvmppc_save_fp 1347 1348 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1349 BEGIN_FTR_SECTION 1350 b 91f 1351 END_FTR_SECTION_IFCLR(CPU_FTR_TM) 1352 /* 1353 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 1354 */ 1355 mr r3, r9 1356 ld r4, VCPU_MSR(r3) 1357 li r5, 0 /* don't preserve non-vol regs */ 1358 bl kvmppc_save_tm_hv 1359 nop 1360 ld r9, HSTATE_KVM_VCPU(r13) 1361 91: 1362 #endif 1363 1364 /* Increment yield count if they have a VPA */ 1365 ld r8, VCPU_VPA(r9) /* do they have a VPA? */ 1366 cmpdi r8, 0 1367 beq 25f 1368 li r4, LPPACA_YIELDCOUNT 1369 LWZX_BE r3, r8, r4 1370 addi r3, r3, 1 1371 STWX_BE r3, r8, r4 1372 li r3, 1 1373 stb r3, VCPU_VPA_DIRTY(r9) 1374 25: 1375 /* Save PMU registers if requested */ 1376 /* r8 and cr0.eq are live here */ 1377 mr r3, r9 1378 li r4, 1 1379 beq 21f /* if no VPA, save PMU stuff anyway */ 1380 lbz r4, LPPACA_PMCINUSE(r8) 1381 21: bl kvmhv_save_guest_pmu 1382 ld r9, HSTATE_KVM_VCPU(r13) 1383 1384 /* Restore host values of some registers */ 1385 BEGIN_FTR_SECTION 1386 ld r5, STACK_SLOT_CIABR(r1) 1387 ld r6, STACK_SLOT_DAWR0(r1) 1388 ld r7, STACK_SLOT_DAWRX0(r1) 1389 mtspr SPRN_CIABR, r5 1390 /* 1391 * If the DAWR doesn't work, it's ok to write these here as 1392 * this value should always be zero 1393 */ 1394 mtspr SPRN_DAWR0, r6 1395 mtspr SPRN_DAWRX0, r7 1396 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1397 1398 /* 1399 * POWER7/POWER8 guest -> host partition switch code. 1400 * We don't have to lock against tlbies but we do 1401 * have to coordinate the hardware threads. 1402 * Here STACK_SLOT_TRAP(r1) contains the trap number. 1403 */ 1404 kvmhv_switch_to_host: 1405 /* Secondary threads wait for primary to do partition switch */ 1406 ld r5,HSTATE_KVM_VCORE(r13) 1407 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1408 lbz r3,HSTATE_PTID(r13) 1409 cmpwi r3,0 1410 beq 15f 1411 HMT_LOW 1412 13: lbz r3,VCORE_IN_GUEST(r5) 1413 cmpwi r3,0 1414 bne 13b 1415 HMT_MEDIUM 1416 b 16f 1417 1418 /* Primary thread waits for all the secondaries to exit guest */ 1419 15: lwz r3,VCORE_ENTRY_EXIT(r5) 1420 rlwinm r0,r3,32-8,0xff 1421 clrldi r3,r3,56 1422 cmpw r3,r0 1423 bne 15b 1424 isync 1425 1426 /* Did we actually switch to the guest at all? */ 1427 lbz r6, VCORE_IN_GUEST(r5) 1428 cmpwi r6, 0 1429 beq 19f 1430 1431 /* Primary thread switches back to host partition */ 1432 lwz r7,KVM_HOST_LPID(r4) 1433 ld r6,KVM_HOST_SDR1(r4) 1434 li r8,LPID_RSVD /* switch to reserved LPID */ 1435 mtspr SPRN_LPID,r8 1436 ptesync 1437 mtspr SPRN_SDR1,r6 /* switch to host page table */ 1438 mtspr SPRN_LPID,r7 1439 isync 1440 1441 BEGIN_FTR_SECTION 1442 /* DPDES and VTB are shared between threads */ 1443 mfspr r7, SPRN_DPDES 1444 mfspr r8, SPRN_VTB 1445 std r7, VCORE_DPDES(r5) 1446 std r8, VCORE_VTB(r5) 1447 /* clear DPDES so we don't get guest doorbells in the host */ 1448 li r8, 0 1449 mtspr SPRN_DPDES, r8 1450 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 1451 1452 /* Subtract timebase offset from timebase */ 1453 ld r8, VCORE_TB_OFFSET_APPL(r5) 1454 cmpdi r8,0 1455 beq 17f 1456 li r0, 0 1457 std r0, VCORE_TB_OFFSET_APPL(r5) 1458 mftb r6 /* current guest timebase */ 1459 subf r8,r8,r6 1460 mtspr SPRN_TBU40,r8 /* update upper 40 bits */ 1461 mftb r7 /* check if lower 24 bits overflowed */ 1462 clrldi r6,r6,40 1463 clrldi r7,r7,40 1464 cmpld r7,r6 1465 bge 17f 1466 addis r8,r8,0x100 /* if so, increment upper 40 bits */ 1467 mtspr SPRN_TBU40,r8 1468 1469 17: 1470 /* 1471 * If this is an HMI, we called kvmppc_realmode_hmi_handler 1472 * above, which may or may not have already called 1473 * kvmppc_subcore_exit_guest. Fortunately, all that 1474 * kvmppc_subcore_exit_guest does is clear a flag, so calling 1475 * it again here is benign even if kvmppc_realmode_hmi_handler 1476 * has already called it. 1477 */ 1478 bl kvmppc_subcore_exit_guest 1479 nop 1480 30: ld r5,HSTATE_KVM_VCORE(r13) 1481 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */ 1482 1483 /* Reset PCR */ 1484 ld r0, VCORE_PCR(r5) 1485 LOAD_REG_IMMEDIATE(r6, PCR_MASK) 1486 cmpld r0, r6 1487 beq 18f 1488 mtspr SPRN_PCR, r6 1489 18: 1490 /* Signal secondary CPUs to continue */ 1491 li r0, 0 1492 stb r0,VCORE_IN_GUEST(r5) 1493 19: lis r8,0x7fff /* MAX_INT@h */ 1494 mtspr SPRN_HDEC,r8 1495 1496 16: ld r8,KVM_HOST_LPCR(r4) 1497 mtspr SPRN_LPCR,r8 1498 isync 1499 1500 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 1501 /* Finish timing, if we have a vcpu */ 1502 ld r4, HSTATE_KVM_VCPU(r13) 1503 cmpdi r4, 0 1504 li r3, 0 1505 beq 2f 1506 bl kvmhv_accumulate_time 1507 2: 1508 #endif 1509 /* Unset guest mode */ 1510 li r0, KVM_GUEST_MODE_NONE 1511 stb r0, HSTATE_IN_GUEST(r13) 1512 1513 lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */ 1514 ld r0, SFS+PPC_LR_STKOFF(r1) 1515 addi r1, r1, SFS 1516 mtlr r0 1517 blr 1518 1519 .balign 32 1520 .global kvm_flush_link_stack 1521 kvm_flush_link_stack: 1522 /* Save LR into r0 */ 1523 mflr r0 1524 1525 /* Flush the link stack. On Power8 it's up to 32 entries in size. */ 1526 .rept 32 1527 ANNOTATE_INTRA_FUNCTION_CALL 1528 bl .+4 1529 .endr 1530 1531 /* And on Power9 it's up to 64. */ 1532 BEGIN_FTR_SECTION 1533 .rept 32 1534 ANNOTATE_INTRA_FUNCTION_CALL 1535 bl .+4 1536 .endr 1537 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) 1538 1539 /* Restore LR */ 1540 mtlr r0 1541 blr 1542 1543 kvmppc_guest_external: 1544 /* External interrupt, first check for host_ipi. If this is 1545 * set, we know the host wants us out so let's do it now 1546 */ 1547 bl CFUNC(kvmppc_read_intr) 1548 1549 /* 1550 * Restore the active volatile registers after returning from 1551 * a C function. 1552 */ 1553 ld r9, HSTATE_KVM_VCPU(r13) 1554 li r12, BOOK3S_INTERRUPT_EXTERNAL 1555 1556 /* 1557 * kvmppc_read_intr return codes: 1558 * 1559 * Exit to host (r3 > 0) 1560 * 1 An interrupt is pending that needs to be handled by the host 1561 * Exit guest and return to host by branching to guest_exit_cont 1562 * 1563 * 2 Passthrough that needs completion in the host 1564 * Exit guest and return to host by branching to guest_exit_cont 1565 * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD 1566 * to indicate to the host to complete handling the interrupt 1567 * 1568 * Before returning to guest, we check if any CPU is heading out 1569 * to the host and if so, we head out also. If no CPUs are heading 1570 * check return values <= 0. 1571 * 1572 * Return to guest (r3 <= 0) 1573 * 0 No external interrupt is pending 1574 * -1 A guest wakeup IPI (which has now been cleared) 1575 * In either case, we return to guest to deliver any pending 1576 * guest interrupts. 1577 * 1578 * -2 A PCI passthrough external interrupt was handled 1579 * (interrupt was delivered directly to guest) 1580 * Return to guest to deliver any pending guest interrupts. 1581 */ 1582 1583 cmpdi r3, 1 1584 ble 1f 1585 1586 /* Return code = 2 */ 1587 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 1588 stw r12, VCPU_TRAP(r9) 1589 b guest_exit_cont 1590 1591 1: /* Return code <= 1 */ 1592 cmpdi r3, 0 1593 bgt guest_exit_cont 1594 1595 /* Return code <= 0 */ 1596 maybe_reenter_guest: 1597 ld r5, HSTATE_KVM_VCORE(r13) 1598 lwz r0, VCORE_ENTRY_EXIT(r5) 1599 cmpwi r0, 0x100 1600 mr r4, r9 1601 blt deliver_guest_interrupt 1602 b guest_exit_cont 1603 1604 /* 1605 * Check whether an HDSI is an HPTE not found fault or something else. 1606 * If it is an HPTE not found fault that is due to the guest accessing 1607 * a page that they have mapped but which we have paged out, then 1608 * we continue on with the guest exit path. In all other cases, 1609 * reflect the HDSI to the guest as a DSI. 1610 */ 1611 kvmppc_hdsi: 1612 mfspr r4, SPRN_HDAR 1613 mfspr r6, SPRN_HDSISR 1614 /* HPTE not found fault or protection fault? */ 1615 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h 1616 beq 1f /* if not, send it to the guest */ 1617 andi. r0, r11, MSR_DR /* data relocation enabled? */ 1618 beq 3f 1619 clrrdi r0, r4, 28 1620 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1621 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT 1622 bne 7f /* if no SLB entry found */ 1623 4: std r4, VCPU_FAULT_DAR(r9) 1624 stw r6, VCPU_FAULT_DSISR(r9) 1625 1626 /* Search the hash table. */ 1627 mr r3, r9 /* vcpu pointer */ 1628 li r7, 1 /* data fault */ 1629 bl CFUNC(kvmppc_hpte_hv_fault) 1630 ld r9, HSTATE_KVM_VCPU(r13) 1631 ld r10, VCPU_PC(r9) 1632 ld r11, VCPU_MSR(r9) 1633 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE 1634 cmpdi r3, 0 /* retry the instruction */ 1635 beq 6f 1636 cmpdi r3, -1 /* handle in kernel mode */ 1637 beq guest_exit_cont 1638 cmpdi r3, -2 /* MMIO emulation; need instr word */ 1639 beq 2f 1640 1641 /* Synthesize a DSI (or DSegI) for the guest */ 1642 ld r4, VCPU_FAULT_DAR(r9) 1643 mr r6, r3 1644 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE 1645 mtspr SPRN_DSISR, r6 1646 7: mtspr SPRN_DAR, r4 1647 mtspr SPRN_SRR0, r10 1648 mtspr SPRN_SRR1, r11 1649 mr r10, r0 1650 bl kvmppc_msr_interrupt 1651 fast_interrupt_c_return: 1652 6: ld r7, VCPU_CTR(r9) 1653 ld r8, VCPU_XER(r9) 1654 mtctr r7 1655 mtxer r8 1656 mr r4, r9 1657 b fast_guest_return 1658 1659 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */ 1660 ld r5, KVM_VRMA_SLB_V(r5) 1661 b 4b 1662 1663 /* If this is for emulated MMIO, load the instruction word */ 1664 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */ 1665 1666 /* Set guest mode to 'jump over instruction' so if lwz faults 1667 * we'll just continue at the next IP. */ 1668 li r0, KVM_GUEST_MODE_SKIP 1669 stb r0, HSTATE_IN_GUEST(r13) 1670 1671 /* Do the access with MSR:DR enabled */ 1672 mfmsr r3 1673 ori r4, r3, MSR_DR /* Enable paging for data */ 1674 mtmsrd r4 1675 lwz r8, 0(r10) 1676 mtmsrd r3 1677 1678 /* Store the result */ 1679 std r8, VCPU_LAST_INST(r9) 1680 1681 /* Unset guest mode. */ 1682 li r0, KVM_GUEST_MODE_HOST_HV 1683 stb r0, HSTATE_IN_GUEST(r13) 1684 b guest_exit_cont 1685 1686 /* 1687 * Similarly for an HISI, reflect it to the guest as an ISI unless 1688 * it is an HPTE not found fault for a page that we have paged out. 1689 */ 1690 kvmppc_hisi: 1691 andis. r0, r11, SRR1_ISI_NOPT@h 1692 beq 1f 1693 andi. r0, r11, MSR_IR /* instruction relocation enabled? */ 1694 beq 3f 1695 clrrdi r0, r10, 28 1696 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ 1697 li r0, BOOK3S_INTERRUPT_INST_SEGMENT 1698 bne 7f /* if no SLB entry found */ 1699 4: 1700 /* Search the hash table. */ 1701 mr r3, r9 /* vcpu pointer */ 1702 mr r4, r10 1703 mr r6, r11 1704 li r7, 0 /* instruction fault */ 1705 bl CFUNC(kvmppc_hpte_hv_fault) 1706 ld r9, HSTATE_KVM_VCPU(r13) 1707 ld r10, VCPU_PC(r9) 1708 ld r11, VCPU_MSR(r9) 1709 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE 1710 cmpdi r3, 0 /* retry the instruction */ 1711 beq fast_interrupt_c_return 1712 cmpdi r3, -1 /* handle in kernel mode */ 1713 beq guest_exit_cont 1714 1715 /* Synthesize an ISI (or ISegI) for the guest */ 1716 mr r11, r3 1717 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE 1718 7: mtspr SPRN_SRR0, r10 1719 mtspr SPRN_SRR1, r11 1720 mr r10, r0 1721 bl kvmppc_msr_interrupt 1722 b fast_interrupt_c_return 1723 1724 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */ 1725 ld r5, KVM_VRMA_SLB_V(r6) 1726 b 4b 1727 1728 /* 1729 * Try to handle an hcall in real mode. 1730 * Returns to the guest if we handle it, or continues on up to 1731 * the kernel if we can't (i.e. if we don't have a handler for 1732 * it, or if the handler returns H_TOO_HARD). 1733 * 1734 * r5 - r8 contain hcall args, 1735 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca 1736 */ 1737 hcall_try_real_mode: 1738 ld r3,VCPU_GPR(R3)(r9) 1739 andi. r0,r11,MSR_PR 1740 /* sc 1 from userspace - reflect to guest syscall */ 1741 bne sc_1_fast_return 1742 clrrdi r3,r3,2 1743 cmpldi r3,hcall_real_table_end - hcall_real_table 1744 bge guest_exit_cont 1745 /* See if this hcall is enabled for in-kernel handling */ 1746 ld r4, VCPU_KVM(r9) 1747 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */ 1748 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */ 1749 add r4, r4, r0 1750 ld r0, KVM_ENABLED_HCALLS(r4) 1751 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */ 1752 srd r0, r0, r4 1753 andi. r0, r0, 1 1754 beq guest_exit_cont 1755 /* Get pointer to handler, if any, and call it */ 1756 LOAD_REG_ADDR(r4, hcall_real_table) 1757 lwax r3,r3,r4 1758 cmpwi r3,0 1759 beq guest_exit_cont 1760 add r12,r3,r4 1761 mtctr r12 1762 mr r3,r9 /* get vcpu pointer */ 1763 ld r4,VCPU_GPR(R4)(r9) 1764 bctrl 1765 cmpdi r3,H_TOO_HARD 1766 beq hcall_real_fallback 1767 ld r4,HSTATE_KVM_VCPU(r13) 1768 std r3,VCPU_GPR(R3)(r4) 1769 ld r10,VCPU_PC(r4) 1770 ld r11,VCPU_MSR(r4) 1771 b fast_guest_return 1772 1773 sc_1_fast_return: 1774 mtspr SPRN_SRR0,r10 1775 mtspr SPRN_SRR1,r11 1776 li r10, BOOK3S_INTERRUPT_SYSCALL 1777 bl kvmppc_msr_interrupt 1778 mr r4,r9 1779 b fast_guest_return 1780 1781 /* We've attempted a real mode hcall, but it's punted it back 1782 * to userspace. We need to restore some clobbered volatiles 1783 * before resuming the pass-it-to-qemu path */ 1784 hcall_real_fallback: 1785 li r12,BOOK3S_INTERRUPT_SYSCALL 1786 ld r9, HSTATE_KVM_VCPU(r13) 1787 1788 b guest_exit_cont 1789 1790 .globl hcall_real_table 1791 hcall_real_table: 1792 .long 0 /* 0 - unused */ 1793 .long DOTSYM(kvmppc_h_remove) - hcall_real_table 1794 .long DOTSYM(kvmppc_h_enter) - hcall_real_table 1795 .long DOTSYM(kvmppc_h_read) - hcall_real_table 1796 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table 1797 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table 1798 .long DOTSYM(kvmppc_h_protect) - hcall_real_table 1799 .long 0 /* 0x1c */ 1800 .long 0 /* 0x20 */ 1801 .long 0 /* 0x24 - H_SET_SPRG0 */ 1802 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table 1803 .long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table 1804 .long 0 /* 0x30 */ 1805 .long 0 /* 0x34 */ 1806 .long 0 /* 0x38 */ 1807 .long 0 /* 0x3c */ 1808 .long 0 /* 0x40 */ 1809 .long 0 /* 0x44 */ 1810 .long 0 /* 0x48 */ 1811 .long 0 /* 0x4c */ 1812 .long 0 /* 0x50 */ 1813 .long 0 /* 0x54 */ 1814 .long 0 /* 0x58 */ 1815 .long 0 /* 0x5c */ 1816 .long 0 /* 0x60 */ 1817 #ifdef CONFIG_KVM_XICS 1818 .long DOTSYM(xics_rm_h_eoi) - hcall_real_table 1819 .long DOTSYM(xics_rm_h_cppr) - hcall_real_table 1820 .long DOTSYM(xics_rm_h_ipi) - hcall_real_table 1821 .long 0 /* 0x70 - H_IPOLL */ 1822 .long DOTSYM(xics_rm_h_xirr) - hcall_real_table 1823 #else 1824 .long 0 /* 0x64 - H_EOI */ 1825 .long 0 /* 0x68 - H_CPPR */ 1826 .long 0 /* 0x6c - H_IPI */ 1827 .long 0 /* 0x70 - H_IPOLL */ 1828 .long 0 /* 0x74 - H_XIRR */ 1829 #endif 1830 .long 0 /* 0x78 */ 1831 .long 0 /* 0x7c */ 1832 .long 0 /* 0x80 */ 1833 .long 0 /* 0x84 */ 1834 .long 0 /* 0x88 */ 1835 .long 0 /* 0x8c */ 1836 .long 0 /* 0x90 */ 1837 .long 0 /* 0x94 */ 1838 .long 0 /* 0x98 */ 1839 .long 0 /* 0x9c */ 1840 .long 0 /* 0xa0 */ 1841 .long 0 /* 0xa4 */ 1842 .long 0 /* 0xa8 */ 1843 .long 0 /* 0xac */ 1844 .long 0 /* 0xb0 */ 1845 .long 0 /* 0xb4 */ 1846 .long 0 /* 0xb8 */ 1847 .long 0 /* 0xbc */ 1848 .long 0 /* 0xc0 */ 1849 .long 0 /* 0xc4 */ 1850 .long 0 /* 0xc8 */ 1851 .long 0 /* 0xcc */ 1852 .long 0 /* 0xd0 */ 1853 .long 0 /* 0xd4 */ 1854 .long 0 /* 0xd8 */ 1855 .long 0 /* 0xdc */ 1856 .long DOTSYM(kvmppc_h_cede) - hcall_real_table 1857 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table 1858 .long 0 /* 0xe8 */ 1859 .long 0 /* 0xec */ 1860 .long 0 /* 0xf0 */ 1861 .long 0 /* 0xf4 */ 1862 .long 0 /* 0xf8 */ 1863 .long 0 /* 0xfc */ 1864 .long 0 /* 0x100 */ 1865 .long 0 /* 0x104 */ 1866 .long 0 /* 0x108 */ 1867 .long 0 /* 0x10c */ 1868 .long 0 /* 0x110 */ 1869 .long 0 /* 0x114 */ 1870 .long 0 /* 0x118 */ 1871 .long 0 /* 0x11c */ 1872 .long 0 /* 0x120 */ 1873 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table 1874 .long 0 /* 0x128 */ 1875 .long 0 /* 0x12c */ 1876 .long 0 /* 0x130 */ 1877 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table 1878 .long 0 /* 0x138 */ 1879 .long 0 /* 0x13c */ 1880 .long 0 /* 0x140 */ 1881 .long 0 /* 0x144 */ 1882 .long 0 /* 0x148 */ 1883 .long 0 /* 0x14c */ 1884 .long 0 /* 0x150 */ 1885 .long 0 /* 0x154 */ 1886 .long 0 /* 0x158 */ 1887 .long 0 /* 0x15c */ 1888 .long 0 /* 0x160 */ 1889 .long 0 /* 0x164 */ 1890 .long 0 /* 0x168 */ 1891 .long 0 /* 0x16c */ 1892 .long 0 /* 0x170 */ 1893 .long 0 /* 0x174 */ 1894 .long 0 /* 0x178 */ 1895 .long 0 /* 0x17c */ 1896 .long 0 /* 0x180 */ 1897 .long 0 /* 0x184 */ 1898 .long 0 /* 0x188 */ 1899 .long 0 /* 0x18c */ 1900 .long 0 /* 0x190 */ 1901 .long 0 /* 0x194 */ 1902 .long 0 /* 0x198 */ 1903 .long 0 /* 0x19c */ 1904 .long 0 /* 0x1a0 */ 1905 .long 0 /* 0x1a4 */ 1906 .long 0 /* 0x1a8 */ 1907 .long 0 /* 0x1ac */ 1908 .long 0 /* 0x1b0 */ 1909 .long 0 /* 0x1b4 */ 1910 .long 0 /* 0x1b8 */ 1911 .long 0 /* 0x1bc */ 1912 .long 0 /* 0x1c0 */ 1913 .long 0 /* 0x1c4 */ 1914 .long 0 /* 0x1c8 */ 1915 .long 0 /* 0x1cc */ 1916 .long 0 /* 0x1d0 */ 1917 .long 0 /* 0x1d4 */ 1918 .long 0 /* 0x1d8 */ 1919 .long 0 /* 0x1dc */ 1920 .long 0 /* 0x1e0 */ 1921 .long 0 /* 0x1e4 */ 1922 .long 0 /* 0x1e8 */ 1923 .long 0 /* 0x1ec */ 1924 .long 0 /* 0x1f0 */ 1925 .long 0 /* 0x1f4 */ 1926 .long 0 /* 0x1f8 */ 1927 .long 0 /* 0x1fc */ 1928 .long 0 /* 0x200 */ 1929 .long 0 /* 0x204 */ 1930 .long 0 /* 0x208 */ 1931 .long 0 /* 0x20c */ 1932 .long 0 /* 0x210 */ 1933 .long 0 /* 0x214 */ 1934 .long 0 /* 0x218 */ 1935 .long 0 /* 0x21c */ 1936 .long 0 /* 0x220 */ 1937 .long 0 /* 0x224 */ 1938 .long 0 /* 0x228 */ 1939 .long 0 /* 0x22c */ 1940 .long 0 /* 0x230 */ 1941 .long 0 /* 0x234 */ 1942 .long 0 /* 0x238 */ 1943 .long 0 /* 0x23c */ 1944 .long 0 /* 0x240 */ 1945 .long 0 /* 0x244 */ 1946 .long 0 /* 0x248 */ 1947 .long 0 /* 0x24c */ 1948 .long 0 /* 0x250 */ 1949 .long 0 /* 0x254 */ 1950 .long 0 /* 0x258 */ 1951 .long 0 /* 0x25c */ 1952 .long 0 /* 0x260 */ 1953 .long 0 /* 0x264 */ 1954 .long 0 /* 0x268 */ 1955 .long 0 /* 0x26c */ 1956 .long 0 /* 0x270 */ 1957 .long 0 /* 0x274 */ 1958 .long 0 /* 0x278 */ 1959 .long 0 /* 0x27c */ 1960 .long 0 /* 0x280 */ 1961 .long 0 /* 0x284 */ 1962 .long 0 /* 0x288 */ 1963 .long 0 /* 0x28c */ 1964 .long 0 /* 0x290 */ 1965 .long 0 /* 0x294 */ 1966 .long 0 /* 0x298 */ 1967 .long 0 /* 0x29c */ 1968 .long 0 /* 0x2a0 */ 1969 .long 0 /* 0x2a4 */ 1970 .long 0 /* 0x2a8 */ 1971 .long 0 /* 0x2ac */ 1972 .long 0 /* 0x2b0 */ 1973 .long 0 /* 0x2b4 */ 1974 .long 0 /* 0x2b8 */ 1975 .long 0 /* 0x2bc */ 1976 .long 0 /* 0x2c0 */ 1977 .long 0 /* 0x2c4 */ 1978 .long 0 /* 0x2c8 */ 1979 .long 0 /* 0x2cc */ 1980 .long 0 /* 0x2d0 */ 1981 .long 0 /* 0x2d4 */ 1982 .long 0 /* 0x2d8 */ 1983 .long 0 /* 0x2dc */ 1984 .long 0 /* 0x2e0 */ 1985 .long 0 /* 0x2e4 */ 1986 .long 0 /* 0x2e8 */ 1987 .long 0 /* 0x2ec */ 1988 .long 0 /* 0x2f0 */ 1989 .long 0 /* 0x2f4 */ 1990 .long 0 /* 0x2f8 */ 1991 #ifdef CONFIG_KVM_XICS 1992 .long DOTSYM(xics_rm_h_xirr_x) - hcall_real_table 1993 #else 1994 .long 0 /* 0x2fc - H_XIRR_X*/ 1995 #endif 1996 .long DOTSYM(kvmppc_rm_h_random) - hcall_real_table 1997 .globl hcall_real_table_end 1998 hcall_real_table_end: 1999 2000 _GLOBAL_TOC(kvmppc_h_set_xdabr) 2001 EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr) 2002 andi. r0, r5, DABRX_USER | DABRX_KERNEL 2003 beq 6f 2004 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI 2005 andc. r0, r5, r0 2006 beq 3f 2007 6: li r3, H_PARAMETER 2008 blr 2009 2010 _GLOBAL_TOC(kvmppc_h_set_dabr) 2011 EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr) 2012 li r5, DABRX_USER | DABRX_KERNEL 2013 3: 2014 BEGIN_FTR_SECTION 2015 b 2f 2016 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2017 std r4,VCPU_DABR(r3) 2018 stw r5, VCPU_DABRX(r3) 2019 mtspr SPRN_DABRX, r5 2020 /* Work around P7 bug where DABR can get corrupted on mtspr */ 2021 1: mtspr SPRN_DABR,r4 2022 mfspr r5, SPRN_DABR 2023 cmpd r4, r5 2024 bne 1b 2025 isync 2026 li r3,0 2027 blr 2028 2029 2: 2030 LOAD_REG_ADDR(r11, dawr_force_enable) 2031 lbz r11, 0(r11) 2032 cmpdi r11, 0 2033 bne 3f 2034 li r3, H_HARDWARE 2035 blr 2036 3: 2037 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */ 2038 rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW 2039 rlwimi r5, r4, 2, DAWRX_WT 2040 clrrdi r4, r4, 3 2041 std r4, VCPU_DAWR0(r3) 2042 std r5, VCPU_DAWRX0(r3) 2043 /* 2044 * If came in through the real mode hcall handler then it is necessary 2045 * to write the registers since the return path won't. Otherwise it is 2046 * sufficient to store then in the vcpu struct as they will be loaded 2047 * next time the vcpu is run. 2048 */ 2049 mfmsr r6 2050 andi. r6, r6, MSR_DR /* in real mode? */ 2051 bne 4f 2052 mtspr SPRN_DAWR0, r4 2053 mtspr SPRN_DAWRX0, r5 2054 4: li r3, 0 2055 blr 2056 2057 _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ 2058 ori r11,r11,MSR_EE 2059 std r11,VCPU_MSR(r3) 2060 li r0,1 2061 stb r0,VCPU_CEDED(r3) 2062 sync /* order setting ceded vs. testing prodded */ 2063 lbz r5,VCPU_PRODDED(r3) 2064 cmpwi r5,0 2065 bne kvm_cede_prodded 2066 li r12,0 /* set trap to 0 to say hcall is handled */ 2067 stw r12,VCPU_TRAP(r3) 2068 li r0,H_SUCCESS 2069 std r0,VCPU_GPR(R3)(r3) 2070 2071 /* 2072 * Set our bit in the bitmask of napping threads unless all the 2073 * other threads are already napping, in which case we send this 2074 * up to the host. 2075 */ 2076 ld r5,HSTATE_KVM_VCORE(r13) 2077 lbz r6,HSTATE_PTID(r13) 2078 lwz r8,VCORE_ENTRY_EXIT(r5) 2079 clrldi r8,r8,56 2080 li r0,1 2081 sld r0,r0,r6 2082 addi r6,r5,VCORE_NAPPING_THREADS 2083 31: lwarx r4,0,r6 2084 or r4,r4,r0 2085 cmpw r4,r8 2086 beq kvm_cede_exit 2087 stwcx. r4,0,r6 2088 bne 31b 2089 /* order napping_threads update vs testing entry_exit_map */ 2090 isync 2091 li r0,NAPPING_CEDE 2092 stb r0,HSTATE_NAPPING(r13) 2093 lwz r7,VCORE_ENTRY_EXIT(r5) 2094 cmpwi r7,0x100 2095 bge 33f /* another thread already exiting */ 2096 2097 /* 2098 * Although not specifically required by the architecture, POWER7 2099 * preserves the following registers in nap mode, even if an SMT mode 2100 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3, 2101 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR. 2102 */ 2103 /* Save non-volatile GPRs */ 2104 std r14, VCPU_GPR(R14)(r3) 2105 std r15, VCPU_GPR(R15)(r3) 2106 std r16, VCPU_GPR(R16)(r3) 2107 std r17, VCPU_GPR(R17)(r3) 2108 std r18, VCPU_GPR(R18)(r3) 2109 std r19, VCPU_GPR(R19)(r3) 2110 std r20, VCPU_GPR(R20)(r3) 2111 std r21, VCPU_GPR(R21)(r3) 2112 std r22, VCPU_GPR(R22)(r3) 2113 std r23, VCPU_GPR(R23)(r3) 2114 std r24, VCPU_GPR(R24)(r3) 2115 std r25, VCPU_GPR(R25)(r3) 2116 std r26, VCPU_GPR(R26)(r3) 2117 std r27, VCPU_GPR(R27)(r3) 2118 std r28, VCPU_GPR(R28)(r3) 2119 std r29, VCPU_GPR(R29)(r3) 2120 std r30, VCPU_GPR(R30)(r3) 2121 std r31, VCPU_GPR(R31)(r3) 2122 2123 /* save FP state */ 2124 bl kvmppc_save_fp 2125 2126 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2127 BEGIN_FTR_SECTION 2128 b 91f 2129 END_FTR_SECTION_IFCLR(CPU_FTR_TM) 2130 /* 2131 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2132 */ 2133 ld r3, HSTATE_KVM_VCPU(r13) 2134 ld r4, VCPU_MSR(r3) 2135 li r5, 0 /* don't preserve non-vol regs */ 2136 bl kvmppc_save_tm_hv 2137 nop 2138 91: 2139 #endif 2140 2141 /* 2142 * Set DEC to the smaller of DEC and HDEC, so that we wake 2143 * no later than the end of our timeslice (HDEC interrupts 2144 * don't wake us from nap). 2145 */ 2146 mfspr r3, SPRN_DEC 2147 mfspr r4, SPRN_HDEC 2148 mftb r5 2149 extsw r3, r3 2150 extsw r4, r4 2151 cmpd r3, r4 2152 ble 67f 2153 mtspr SPRN_DEC, r4 2154 67: 2155 /* save expiry time of guest decrementer */ 2156 add r3, r3, r5 2157 ld r4, HSTATE_KVM_VCPU(r13) 2158 std r3, VCPU_DEC_EXPIRES(r4) 2159 2160 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 2161 ld r4, HSTATE_KVM_VCPU(r13) 2162 addi r3, r4, VCPU_TB_CEDE 2163 bl kvmhv_accumulate_time 2164 #endif 2165 2166 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */ 2167 2168 /* Go back to host stack */ 2169 ld r1, HSTATE_HOST_R1(r13) 2170 2171 /* 2172 * Take a nap until a decrementer or external or doobell interrupt 2173 * occurs, with PECE1 and PECE0 set in LPCR. 2174 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP. 2175 * Also clear the runlatch bit before napping. 2176 */ 2177 kvm_do_nap: 2178 li r0,0 2179 mtspr SPRN_CTRLT, r0 2180 2181 li r0,1 2182 stb r0,HSTATE_HWTHREAD_REQ(r13) 2183 mfspr r5,SPRN_LPCR 2184 ori r5,r5,LPCR_PECE0 | LPCR_PECE1 2185 BEGIN_FTR_SECTION 2186 ori r5, r5, LPCR_PECEDH 2187 rlwimi r5, r3, 0, LPCR_PECEDP 2188 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2189 2190 kvm_nap_sequence: /* desired LPCR value in r5 */ 2191 li r3, PNV_THREAD_NAP 2192 mtspr SPRN_LPCR,r5 2193 isync 2194 2195 bl isa206_idle_insn_mayloss 2196 2197 li r0,1 2198 mtspr SPRN_CTRLT, r0 2199 2200 mtspr SPRN_SRR1, r3 2201 2202 li r0, 0 2203 stb r0, PACA_FTRACE_ENABLED(r13) 2204 2205 li r0, KVM_HWTHREAD_IN_KVM 2206 stb r0, HSTATE_HWTHREAD_STATE(r13) 2207 2208 lbz r0, HSTATE_NAPPING(r13) 2209 cmpwi r0, NAPPING_CEDE 2210 beq kvm_end_cede 2211 cmpwi r0, NAPPING_NOVCPU 2212 beq kvm_novcpu_wakeup 2213 cmpwi r0, NAPPING_UNSPLIT 2214 beq kvm_unsplit_wakeup 2215 twi 31,0,0 /* Nap state must not be zero */ 2216 2217 33: mr r4, r3 2218 li r3, 0 2219 li r12, 0 2220 b 34f 2221 2222 kvm_end_cede: 2223 /* Woken by external or decrementer interrupt */ 2224 2225 /* get vcpu pointer */ 2226 ld r4, HSTATE_KVM_VCPU(r13) 2227 2228 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 2229 addi r3, r4, VCPU_TB_RMINTR 2230 bl kvmhv_accumulate_time 2231 #endif 2232 2233 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2234 BEGIN_FTR_SECTION 2235 b 91f 2236 END_FTR_SECTION_IFCLR(CPU_FTR_TM) 2237 /* 2238 * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) 2239 */ 2240 mr r3, r4 2241 ld r4, VCPU_MSR(r3) 2242 li r5, 0 /* don't preserve non-vol regs */ 2243 bl kvmppc_restore_tm_hv 2244 nop 2245 ld r4, HSTATE_KVM_VCPU(r13) 2246 91: 2247 #endif 2248 2249 /* load up FP state */ 2250 bl kvmppc_load_fp 2251 2252 /* Restore guest decrementer */ 2253 ld r3, VCPU_DEC_EXPIRES(r4) 2254 mftb r7 2255 subf r3, r7, r3 2256 mtspr SPRN_DEC, r3 2257 2258 /* Load NV GPRS */ 2259 ld r14, VCPU_GPR(R14)(r4) 2260 ld r15, VCPU_GPR(R15)(r4) 2261 ld r16, VCPU_GPR(R16)(r4) 2262 ld r17, VCPU_GPR(R17)(r4) 2263 ld r18, VCPU_GPR(R18)(r4) 2264 ld r19, VCPU_GPR(R19)(r4) 2265 ld r20, VCPU_GPR(R20)(r4) 2266 ld r21, VCPU_GPR(R21)(r4) 2267 ld r22, VCPU_GPR(R22)(r4) 2268 ld r23, VCPU_GPR(R23)(r4) 2269 ld r24, VCPU_GPR(R24)(r4) 2270 ld r25, VCPU_GPR(R25)(r4) 2271 ld r26, VCPU_GPR(R26)(r4) 2272 ld r27, VCPU_GPR(R27)(r4) 2273 ld r28, VCPU_GPR(R28)(r4) 2274 ld r29, VCPU_GPR(R29)(r4) 2275 ld r30, VCPU_GPR(R30)(r4) 2276 ld r31, VCPU_GPR(R31)(r4) 2277 2278 /* Check the wake reason in SRR1 to see why we got here */ 2279 bl kvmppc_check_wake_reason 2280 2281 /* 2282 * Restore volatile registers since we could have called a 2283 * C routine in kvmppc_check_wake_reason 2284 * r4 = VCPU 2285 * r3 tells us whether we need to return to host or not 2286 * WARNING: it gets checked further down: 2287 * should not modify r3 until this check is done. 2288 */ 2289 ld r4, HSTATE_KVM_VCPU(r13) 2290 2291 /* clear our bit in vcore->napping_threads */ 2292 34: ld r5,HSTATE_KVM_VCORE(r13) 2293 lbz r7,HSTATE_PTID(r13) 2294 li r0,1 2295 sld r0,r0,r7 2296 addi r6,r5,VCORE_NAPPING_THREADS 2297 32: lwarx r7,0,r6 2298 andc r7,r7,r0 2299 stwcx. r7,0,r6 2300 bne 32b 2301 li r0,0 2302 stb r0,HSTATE_NAPPING(r13) 2303 2304 /* See if the wake reason saved in r3 means we need to exit */ 2305 stw r12, VCPU_TRAP(r4) 2306 mr r9, r4 2307 cmpdi r3, 0 2308 bgt guest_exit_cont 2309 b maybe_reenter_guest 2310 2311 /* cede when already previously prodded case */ 2312 kvm_cede_prodded: 2313 li r0,0 2314 stb r0,VCPU_PRODDED(r3) 2315 sync /* order testing prodded vs. clearing ceded */ 2316 stb r0,VCPU_CEDED(r3) 2317 li r3,H_SUCCESS 2318 blr 2319 2320 /* we've ceded but we want to give control to the host */ 2321 kvm_cede_exit: 2322 ld r9, HSTATE_KVM_VCPU(r13) 2323 b guest_exit_cont 2324 2325 /* Try to do machine check recovery in real mode */ 2326 machine_check_realmode: 2327 mr r3, r9 /* get vcpu pointer */ 2328 bl kvmppc_realmode_machine_check 2329 nop 2330 /* all machine checks go to virtual mode for further handling */ 2331 ld r9, HSTATE_KVM_VCPU(r13) 2332 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK 2333 b guest_exit_cont 2334 2335 /* 2336 * Call C code to handle a HMI in real mode. 2337 * Only the primary thread does the call, secondary threads are handled 2338 * by calling hmi_exception_realmode() after kvmppc_hv_entry returns. 2339 * r9 points to the vcpu on entry 2340 */ 2341 hmi_realmode: 2342 lbz r0, HSTATE_PTID(r13) 2343 cmpwi r0, 0 2344 bne guest_exit_cont 2345 bl CFUNC(kvmppc_realmode_hmi_handler) 2346 ld r9, HSTATE_KVM_VCPU(r13) 2347 li r12, BOOK3S_INTERRUPT_HMI 2348 b guest_exit_cont 2349 2350 /* 2351 * Check the reason we woke from nap, and take appropriate action. 2352 * Returns (in r3): 2353 * 0 if nothing needs to be done 2354 * 1 if something happened that needs to be handled by the host 2355 * -1 if there was a guest wakeup (IPI or msgsnd) 2356 * -2 if we handled a PCI passthrough interrupt (returned by 2357 * kvmppc_read_intr only) 2358 * 2359 * Also sets r12 to the interrupt vector for any interrupt that needs 2360 * to be handled now by the host (0x500 for external interrupt), or zero. 2361 * Modifies all volatile registers (since it may call a C function). 2362 * This routine calls kvmppc_read_intr, a C function, if an external 2363 * interrupt is pending. 2364 */ 2365 SYM_FUNC_START_LOCAL(kvmppc_check_wake_reason) 2366 mfspr r6, SPRN_SRR1 2367 BEGIN_FTR_SECTION 2368 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */ 2369 FTR_SECTION_ELSE 2370 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */ 2371 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S) 2372 cmpwi r6, 8 /* was it an external interrupt? */ 2373 beq 7f /* if so, see what it was */ 2374 li r3, 0 2375 li r12, 0 2376 cmpwi r6, 6 /* was it the decrementer? */ 2377 beq 0f 2378 BEGIN_FTR_SECTION 2379 cmpwi r6, 5 /* privileged doorbell? */ 2380 beq 0f 2381 cmpwi r6, 3 /* hypervisor doorbell? */ 2382 beq 3f 2383 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2384 cmpwi r6, 0xa /* Hypervisor maintenance ? */ 2385 beq 4f 2386 li r3, 1 /* anything else, return 1 */ 2387 0: blr 2388 2389 /* hypervisor doorbell */ 2390 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL 2391 2392 /* 2393 * Clear the doorbell as we will invoke the handler 2394 * explicitly in the guest exit path. 2395 */ 2396 lis r6, (PPC_DBELL_SERVER << (63-36))@h 2397 PPC_MSGCLR(6) 2398 /* see if it's a host IPI */ 2399 li r3, 1 2400 lbz r0, HSTATE_HOST_IPI(r13) 2401 cmpwi r0, 0 2402 bnelr 2403 /* if not, return -1 */ 2404 li r3, -1 2405 blr 2406 2407 /* Woken up due to Hypervisor maintenance interrupt */ 2408 4: li r12, BOOK3S_INTERRUPT_HMI 2409 li r3, 1 2410 blr 2411 2412 /* external interrupt - create a stack frame so we can call C */ 2413 7: mflr r0 2414 std r0, PPC_LR_STKOFF(r1) 2415 stdu r1, -PPC_MIN_STKFRM(r1) 2416 bl CFUNC(kvmppc_read_intr) 2417 nop 2418 li r12, BOOK3S_INTERRUPT_EXTERNAL 2419 cmpdi r3, 1 2420 ble 1f 2421 2422 /* 2423 * Return code of 2 means PCI passthrough interrupt, but 2424 * we need to return back to host to complete handling the 2425 * interrupt. Trap reason is expected in r12 by guest 2426 * exit code. 2427 */ 2428 li r12, BOOK3S_INTERRUPT_HV_RM_HARD 2429 1: 2430 ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1) 2431 addi r1, r1, PPC_MIN_STKFRM 2432 mtlr r0 2433 blr 2434 SYM_FUNC_END(kvmppc_check_wake_reason) 2435 2436 /* 2437 * Save away FP, VMX and VSX registers. 2438 * r3 = vcpu pointer 2439 * N.B. r30 and r31 are volatile across this function, 2440 * thus it is not callable from C. 2441 */ 2442 SYM_FUNC_START_LOCAL(kvmppc_save_fp) 2443 mflr r30 2444 mr r31,r3 2445 mfmsr r5 2446 ori r8,r5,MSR_FP 2447 #ifdef CONFIG_ALTIVEC 2448 BEGIN_FTR_SECTION 2449 oris r8,r8,MSR_VEC@h 2450 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2451 #endif 2452 #ifdef CONFIG_VSX 2453 BEGIN_FTR_SECTION 2454 oris r8,r8,MSR_VSX@h 2455 END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2456 #endif 2457 mtmsrd r8 2458 addi r3,r3,VCPU_FPRS 2459 bl store_fp_state 2460 #ifdef CONFIG_ALTIVEC 2461 BEGIN_FTR_SECTION 2462 addi r3,r31,VCPU_VRS 2463 bl store_vr_state 2464 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2465 #endif 2466 mfspr r6,SPRN_VRSAVE 2467 stw r6,VCPU_VRSAVE(r31) 2468 mtlr r30 2469 blr 2470 SYM_FUNC_END(kvmppc_save_fp) 2471 2472 /* 2473 * Load up FP, VMX and VSX registers 2474 * r4 = vcpu pointer 2475 * N.B. r30 and r31 are volatile across this function, 2476 * thus it is not callable from C. 2477 */ 2478 SYM_FUNC_START_LOCAL(kvmppc_load_fp) 2479 mflr r30 2480 mr r31,r4 2481 mfmsr r9 2482 ori r8,r9,MSR_FP 2483 #ifdef CONFIG_ALTIVEC 2484 BEGIN_FTR_SECTION 2485 oris r8,r8,MSR_VEC@h 2486 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2487 #endif 2488 #ifdef CONFIG_VSX 2489 BEGIN_FTR_SECTION 2490 oris r8,r8,MSR_VSX@h 2491 END_FTR_SECTION_IFSET(CPU_FTR_VSX) 2492 #endif 2493 mtmsrd r8 2494 addi r3,r4,VCPU_FPRS 2495 bl load_fp_state 2496 #ifdef CONFIG_ALTIVEC 2497 BEGIN_FTR_SECTION 2498 addi r3,r31,VCPU_VRS 2499 bl load_vr_state 2500 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) 2501 #endif 2502 lwz r7,VCPU_VRSAVE(r31) 2503 mtspr SPRN_VRSAVE,r7 2504 mtlr r30 2505 mr r4,r31 2506 blr 2507 SYM_FUNC_END(kvmppc_load_fp) 2508 2509 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2510 /* 2511 * Save transactional state and TM-related registers. 2512 * Called with r3 pointing to the vcpu struct and r4 containing 2513 * the guest MSR value. 2514 * r5 is non-zero iff non-volatile register state needs to be maintained. 2515 * If r5 == 0, this can modify all checkpointed registers, but 2516 * restores r1 and r2 before exit. 2517 */ 2518 _GLOBAL_TOC(kvmppc_save_tm_hv) 2519 EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv) 2520 /* See if we need to handle fake suspend mode */ 2521 BEGIN_FTR_SECTION 2522 b __kvmppc_save_tm 2523 END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 2524 2525 lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */ 2526 cmpwi r0, 0 2527 beq __kvmppc_save_tm 2528 2529 /* The following code handles the fake_suspend = 1 case */ 2530 mflr r0 2531 std r0, PPC_LR_STKOFF(r1) 2532 stdu r1, -TM_FRAME_SIZE(r1) 2533 2534 /* Turn on TM. */ 2535 mfmsr r8 2536 li r0, 1 2537 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG 2538 mtmsrd r8 2539 2540 rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */ 2541 beq 4f 2542 BEGIN_FTR_SECTION 2543 bl pnv_power9_force_smt4_catch 2544 END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 2545 nop 2546 2547 /* 2548 * It's possible that treclaim. may modify registers, if we have lost 2549 * track of fake-suspend state in the guest due to it using rfscv. 2550 * Save and restore registers in case this occurs. 2551 */ 2552 mfspr r3, SPRN_DSCR 2553 mfspr r4, SPRN_XER 2554 mfspr r5, SPRN_AMR 2555 /* SPRN_TAR would need to be saved here if the kernel ever used it */ 2556 mfcr r12 2557 SAVE_NVGPRS(r1) 2558 SAVE_GPR(2, r1) 2559 SAVE_GPR(3, r1) 2560 SAVE_GPR(4, r1) 2561 SAVE_GPR(5, r1) 2562 stw r12, 8(r1) 2563 std r1, HSTATE_HOST_R1(r13) 2564 2565 /* We have to treclaim here because that's the only way to do S->N */ 2566 li r3, TM_CAUSE_KVM_RESCHED 2567 TRECLAIM(R3) 2568 2569 GET_PACA(r13) 2570 ld r1, HSTATE_HOST_R1(r13) 2571 REST_GPR(2, r1) 2572 REST_GPR(3, r1) 2573 REST_GPR(4, r1) 2574 REST_GPR(5, r1) 2575 lwz r12, 8(r1) 2576 REST_NVGPRS(r1) 2577 mtspr SPRN_DSCR, r3 2578 mtspr SPRN_XER, r4 2579 mtspr SPRN_AMR, r5 2580 mtcr r12 2581 HMT_MEDIUM 2582 2583 /* 2584 * We were in fake suspend, so we are not going to save the 2585 * register state as the guest checkpointed state (since 2586 * we already have it), therefore we can now use any volatile GPR. 2587 * In fact treclaim in fake suspend state doesn't modify 2588 * any registers. 2589 */ 2590 2591 BEGIN_FTR_SECTION 2592 bl pnv_power9_force_smt4_release 2593 END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) 2594 nop 2595 2596 4: 2597 mfspr r3, SPRN_PSSCR 2598 /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */ 2599 li r0, PSSCR_FAKE_SUSPEND 2600 andc r3, r3, r0 2601 mtspr SPRN_PSSCR, r3 2602 2603 /* Don't save TEXASR, use value from last exit in real suspend state */ 2604 ld r9, HSTATE_KVM_VCPU(r13) 2605 mfspr r5, SPRN_TFHAR 2606 mfspr r6, SPRN_TFIAR 2607 std r5, VCPU_TFHAR(r9) 2608 std r6, VCPU_TFIAR(r9) 2609 2610 addi r1, r1, TM_FRAME_SIZE 2611 ld r0, PPC_LR_STKOFF(r1) 2612 mtlr r0 2613 blr 2614 2615 /* 2616 * Restore transactional state and TM-related registers. 2617 * Called with r3 pointing to the vcpu struct 2618 * and r4 containing the guest MSR value. 2619 * r5 is non-zero iff non-volatile register state needs to be maintained. 2620 * This potentially modifies all checkpointed registers. 2621 * It restores r1 and r2 from the PACA. 2622 */ 2623 _GLOBAL_TOC(kvmppc_restore_tm_hv) 2624 EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv) 2625 /* 2626 * If we are doing TM emulation for the guest on a POWER9 DD2, 2627 * then we don't actually do a trechkpt -- we either set up 2628 * fake-suspend mode, or emulate a TM rollback. 2629 */ 2630 BEGIN_FTR_SECTION 2631 b __kvmppc_restore_tm 2632 END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST) 2633 mflr r0 2634 std r0, PPC_LR_STKOFF(r1) 2635 2636 li r0, 0 2637 stb r0, HSTATE_FAKE_SUSPEND(r13) 2638 2639 /* Turn on TM so we can restore TM SPRs */ 2640 mfmsr r5 2641 li r0, 1 2642 rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG 2643 mtmsrd r5 2644 2645 /* 2646 * The user may change these outside of a transaction, so they must 2647 * always be context switched. 2648 */ 2649 ld r5, VCPU_TFHAR(r3) 2650 ld r6, VCPU_TFIAR(r3) 2651 ld r7, VCPU_TEXASR(r3) 2652 mtspr SPRN_TFHAR, r5 2653 mtspr SPRN_TFIAR, r6 2654 mtspr SPRN_TEXASR, r7 2655 2656 rldicl. r5, r4, 64 - MSR_TS_S_LG, 62 2657 beqlr /* TM not active in guest */ 2658 2659 /* Make sure the failure summary is set */ 2660 oris r7, r7, (TEXASR_FS)@h 2661 mtspr SPRN_TEXASR, r7 2662 2663 cmpwi r5, 1 /* check for suspended state */ 2664 bgt 10f 2665 stb r5, HSTATE_FAKE_SUSPEND(r13) 2666 b 9f /* and return */ 2667 10: stdu r1, -PPC_MIN_STKFRM(r1) 2668 /* guest is in transactional state, so simulate rollback */ 2669 bl kvmhv_emulate_tm_rollback 2670 nop 2671 addi r1, r1, PPC_MIN_STKFRM 2672 9: ld r0, PPC_LR_STKOFF(r1) 2673 mtlr r0 2674 blr 2675 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ 2676 2677 /* 2678 * We come here if we get any exception or interrupt while we are 2679 * executing host real mode code while in guest MMU context. 2680 * r12 is (CR << 32) | vector 2681 * r13 points to our PACA 2682 * r12 is saved in HSTATE_SCRATCH0(r13) 2683 * r9 is saved in HSTATE_SCRATCH2(r13) 2684 * r13 is saved in HSPRG1 2685 * cfar is saved in HSTATE_CFAR(r13) 2686 * ppr is saved in HSTATE_PPR(r13) 2687 */ 2688 kvmppc_bad_host_intr: 2689 /* 2690 * Switch to the emergency stack, but start half-way down in 2691 * case we were already on it. 2692 */ 2693 mr r9, r1 2694 std r1, PACAR1(r13) 2695 ld r1, PACAEMERGSP(r13) 2696 subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE 2697 std r9, 0(r1) 2698 std r0, GPR0(r1) 2699 std r9, GPR1(r1) 2700 std r2, GPR2(r1) 2701 SAVE_GPRS(3, 8, r1) 2702 srdi r0, r12, 32 2703 clrldi r12, r12, 32 2704 std r0, _CCR(r1) 2705 std r12, _TRAP(r1) 2706 andi. r0, r12, 2 2707 beq 1f 2708 mfspr r3, SPRN_HSRR0 2709 mfspr r4, SPRN_HSRR1 2710 mfspr r5, SPRN_HDAR 2711 mfspr r6, SPRN_HDSISR 2712 b 2f 2713 1: mfspr r3, SPRN_SRR0 2714 mfspr r4, SPRN_SRR1 2715 mfspr r5, SPRN_DAR 2716 mfspr r6, SPRN_DSISR 2717 2: std r3, _NIP(r1) 2718 std r4, _MSR(r1) 2719 std r5, _DAR(r1) 2720 std r6, _DSISR(r1) 2721 ld r9, HSTATE_SCRATCH2(r13) 2722 ld r12, HSTATE_SCRATCH0(r13) 2723 GET_SCRATCH0(r0) 2724 SAVE_GPRS(9, 12, r1) 2725 std r0, GPR13(r1) 2726 SAVE_NVGPRS(r1) 2727 ld r5, HSTATE_CFAR(r13) 2728 std r5, ORIG_GPR3(r1) 2729 mflr r3 2730 mfctr r4 2731 mfxer r5 2732 lbz r6, PACAIRQSOFTMASK(r13) 2733 std r3, _LINK(r1) 2734 std r4, _CTR(r1) 2735 std r5, _XER(r1) 2736 std r6, SOFTE(r1) 2737 LOAD_PACA_TOC() 2738 LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER) 2739 std r3, STACK_INT_FRAME_MARKER(r1) 2740 2741 /* 2742 * XXX On POWER7 and POWER8, we just spin here since we don't 2743 * know what the other threads are doing (and we don't want to 2744 * coordinate with them) - but at least we now have register state 2745 * in memory that we might be able to look at from another CPU. 2746 */ 2747 b . 2748 2749 /* 2750 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken 2751 * from VCPU_INTR_MSR and is modified based on the required TM state changes. 2752 * r11 has the guest MSR value (in/out) 2753 * r9 has a vcpu pointer (in) 2754 * r0 is used as a scratch register 2755 */ 2756 SYM_FUNC_START_LOCAL(kvmppc_msr_interrupt) 2757 rldicl r0, r11, 64 - MSR_TS_S_LG, 62 2758 cmpwi r0, 2 /* Check if we are in transactional state.. */ 2759 ld r11, VCPU_INTR_MSR(r9) 2760 bne 1f 2761 /* ... if transactional, change to suspended */ 2762 li r0, 1 2763 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG 2764 blr 2765 SYM_FUNC_END(kvmppc_msr_interrupt) 2766 2767 /* 2768 * void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu) 2769 * 2770 * Load up guest PMU state. R3 points to the vcpu struct. 2771 */ 2772 SYM_FUNC_START_LOCAL(kvmhv_load_guest_pmu) 2773 mr r4, r3 2774 mflr r0 2775 li r3, 1 2776 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 2777 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 2778 isync 2779 BEGIN_FTR_SECTION 2780 ld r3, VCPU_MMCR(r4) 2781 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 2782 cmpwi r5, MMCR0_PMAO 2783 beql kvmppc_fix_pmao 2784 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 2785 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */ 2786 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */ 2787 lwz r6, VCPU_PMC + 8(r4) 2788 lwz r7, VCPU_PMC + 12(r4) 2789 lwz r8, VCPU_PMC + 16(r4) 2790 lwz r9, VCPU_PMC + 20(r4) 2791 mtspr SPRN_PMC1, r3 2792 mtspr SPRN_PMC2, r5 2793 mtspr SPRN_PMC3, r6 2794 mtspr SPRN_PMC4, r7 2795 mtspr SPRN_PMC5, r8 2796 mtspr SPRN_PMC6, r9 2797 ld r3, VCPU_MMCR(r4) 2798 ld r5, VCPU_MMCR + 8(r4) 2799 ld r6, VCPU_MMCRA(r4) 2800 ld r7, VCPU_SIAR(r4) 2801 ld r8, VCPU_SDAR(r4) 2802 mtspr SPRN_MMCR1, r5 2803 mtspr SPRN_MMCRA, r6 2804 mtspr SPRN_SIAR, r7 2805 mtspr SPRN_SDAR, r8 2806 BEGIN_FTR_SECTION 2807 ld r5, VCPU_MMCR + 16(r4) 2808 ld r6, VCPU_SIER(r4) 2809 mtspr SPRN_MMCR2, r5 2810 mtspr SPRN_SIER, r6 2811 lwz r7, VCPU_PMC + 24(r4) 2812 lwz r8, VCPU_PMC + 28(r4) 2813 ld r9, VCPU_MMCRS(r4) 2814 mtspr SPRN_SPMC1, r7 2815 mtspr SPRN_SPMC2, r8 2816 mtspr SPRN_MMCRS, r9 2817 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2818 mtspr SPRN_MMCR0, r3 2819 isync 2820 mtlr r0 2821 blr 2822 SYM_FUNC_END(kvmhv_load_guest_pmu) 2823 2824 /* 2825 * void kvmhv_load_host_pmu(void) 2826 * 2827 * Reload host PMU state saved in the PACA by kvmhv_save_host_pmu. 2828 */ 2829 SYM_FUNC_START_LOCAL(kvmhv_load_host_pmu) 2830 mflr r0 2831 lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */ 2832 cmpwi r4, 0 2833 beq 23f /* skip if not */ 2834 BEGIN_FTR_SECTION 2835 ld r3, HSTATE_MMCR0(r13) 2836 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO 2837 cmpwi r4, MMCR0_PMAO 2838 beql kvmppc_fix_pmao 2839 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG) 2840 lwz r3, HSTATE_PMC1(r13) 2841 lwz r4, HSTATE_PMC2(r13) 2842 lwz r5, HSTATE_PMC3(r13) 2843 lwz r6, HSTATE_PMC4(r13) 2844 lwz r8, HSTATE_PMC5(r13) 2845 lwz r9, HSTATE_PMC6(r13) 2846 mtspr SPRN_PMC1, r3 2847 mtspr SPRN_PMC2, r4 2848 mtspr SPRN_PMC3, r5 2849 mtspr SPRN_PMC4, r6 2850 mtspr SPRN_PMC5, r8 2851 mtspr SPRN_PMC6, r9 2852 ld r3, HSTATE_MMCR0(r13) 2853 ld r4, HSTATE_MMCR1(r13) 2854 ld r5, HSTATE_MMCRA(r13) 2855 ld r6, HSTATE_SIAR(r13) 2856 ld r7, HSTATE_SDAR(r13) 2857 mtspr SPRN_MMCR1, r4 2858 mtspr SPRN_MMCRA, r5 2859 mtspr SPRN_SIAR, r6 2860 mtspr SPRN_SDAR, r7 2861 BEGIN_FTR_SECTION 2862 ld r8, HSTATE_MMCR2(r13) 2863 ld r9, HSTATE_SIER(r13) 2864 mtspr SPRN_MMCR2, r8 2865 mtspr SPRN_SIER, r9 2866 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2867 mtspr SPRN_MMCR0, r3 2868 isync 2869 mtlr r0 2870 23: blr 2871 SYM_FUNC_END(kvmhv_load_host_pmu) 2872 2873 /* 2874 * void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use) 2875 * 2876 * Save guest PMU state into the vcpu struct. 2877 * r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA) 2878 */ 2879 SYM_FUNC_START_LOCAL(kvmhv_save_guest_pmu) 2880 mr r9, r3 2881 mr r8, r4 2882 BEGIN_FTR_SECTION 2883 /* 2884 * POWER8 seems to have a hardware bug where setting 2885 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE] 2886 * when some counters are already negative doesn't seem 2887 * to cause a performance monitor alert (and hence interrupt). 2888 * The effect of this is that when saving the PMU state, 2889 * if there is no PMU alert pending when we read MMCR0 2890 * before freezing the counters, but one becomes pending 2891 * before we read the counters, we lose it. 2892 * To work around this, we need a way to freeze the counters 2893 * before reading MMCR0. Normally, freezing the counters 2894 * is done by writing MMCR0 (to set MMCR0[FC]) which 2895 * unavoidably writes MMCR0[PMA0] as well. On POWER8, 2896 * we can also freeze the counters using MMCR2, by writing 2897 * 1s to all the counter freeze condition bits (there are 2898 * 9 bits each for 6 counters). 2899 */ 2900 li r3, -1 /* set all freeze bits */ 2901 clrrdi r3, r3, 10 2902 mfspr r10, SPRN_MMCR2 2903 mtspr SPRN_MMCR2, r3 2904 isync 2905 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2906 li r3, 1 2907 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */ 2908 mfspr r4, SPRN_MMCR0 /* save MMCR0 */ 2909 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */ 2910 mfspr r6, SPRN_MMCRA 2911 /* Clear MMCRA in order to disable SDAR updates */ 2912 li r7, 0 2913 mtspr SPRN_MMCRA, r7 2914 isync 2915 cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */ 2916 bne 21f 2917 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */ 2918 b 22f 2919 21: mfspr r5, SPRN_MMCR1 2920 mfspr r7, SPRN_SIAR 2921 mfspr r8, SPRN_SDAR 2922 std r4, VCPU_MMCR(r9) 2923 std r5, VCPU_MMCR + 8(r9) 2924 std r6, VCPU_MMCRA(r9) 2925 BEGIN_FTR_SECTION 2926 std r10, VCPU_MMCR + 16(r9) 2927 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2928 std r7, VCPU_SIAR(r9) 2929 std r8, VCPU_SDAR(r9) 2930 mfspr r3, SPRN_PMC1 2931 mfspr r4, SPRN_PMC2 2932 mfspr r5, SPRN_PMC3 2933 mfspr r6, SPRN_PMC4 2934 mfspr r7, SPRN_PMC5 2935 mfspr r8, SPRN_PMC6 2936 stw r3, VCPU_PMC(r9) 2937 stw r4, VCPU_PMC + 4(r9) 2938 stw r5, VCPU_PMC + 8(r9) 2939 stw r6, VCPU_PMC + 12(r9) 2940 stw r7, VCPU_PMC + 16(r9) 2941 stw r8, VCPU_PMC + 20(r9) 2942 BEGIN_FTR_SECTION 2943 mfspr r5, SPRN_SIER 2944 std r5, VCPU_SIER(r9) 2945 mfspr r6, SPRN_SPMC1 2946 mfspr r7, SPRN_SPMC2 2947 mfspr r8, SPRN_MMCRS 2948 stw r6, VCPU_PMC + 24(r9) 2949 stw r7, VCPU_PMC + 28(r9) 2950 std r8, VCPU_MMCRS(r9) 2951 lis r4, 0x8000 2952 mtspr SPRN_MMCRS, r4 2953 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 2954 22: blr 2955 SYM_FUNC_END(kvmhv_save_guest_pmu) 2956 2957 /* 2958 * This works around a hardware bug on POWER8E processors, where 2959 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a 2960 * performance monitor interrupt. Instead, when we need to have 2961 * an interrupt pending, we have to arrange for a counter to overflow. 2962 */ 2963 kvmppc_fix_pmao: 2964 li r3, 0 2965 mtspr SPRN_MMCR2, r3 2966 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h 2967 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN 2968 mtspr SPRN_MMCR0, r3 2969 lis r3, 0x7fff 2970 ori r3, r3, 0xffff 2971 mtspr SPRN_PMC6, r3 2972 isync 2973 blr 2974 2975 #ifdef CONFIG_KVM_BOOK3S_HV_P8_TIMING 2976 /* 2977 * Start timing an activity 2978 * r3 = pointer to time accumulation struct, r4 = vcpu 2979 */ 2980 kvmhv_start_timing: 2981 ld r5, HSTATE_KVM_VCORE(r13) 2982 ld r6, VCORE_TB_OFFSET_APPL(r5) 2983 mftb r5 2984 subf r5, r6, r5 /* subtract current timebase offset */ 2985 std r3, VCPU_CUR_ACTIVITY(r4) 2986 std r5, VCPU_ACTIVITY_START(r4) 2987 blr 2988 2989 /* 2990 * Accumulate time to one activity and start another. 2991 * r3 = pointer to new time accumulation struct, r4 = vcpu 2992 */ 2993 kvmhv_accumulate_time: 2994 ld r5, HSTATE_KVM_VCORE(r13) 2995 ld r8, VCORE_TB_OFFSET_APPL(r5) 2996 ld r5, VCPU_CUR_ACTIVITY(r4) 2997 ld r6, VCPU_ACTIVITY_START(r4) 2998 std r3, VCPU_CUR_ACTIVITY(r4) 2999 mftb r7 3000 subf r7, r8, r7 /* subtract current timebase offset */ 3001 std r7, VCPU_ACTIVITY_START(r4) 3002 cmpdi r5, 0 3003 beqlr 3004 subf r3, r6, r7 3005 ld r8, TAS_SEQCOUNT(r5) 3006 cmpdi r8, 0 3007 addi r8, r8, 1 3008 std r8, TAS_SEQCOUNT(r5) 3009 lwsync 3010 ld r7, TAS_TOTAL(r5) 3011 add r7, r7, r3 3012 std r7, TAS_TOTAL(r5) 3013 ld r6, TAS_MIN(r5) 3014 ld r7, TAS_MAX(r5) 3015 beq 3f 3016 cmpd r3, r6 3017 bge 1f 3018 3: std r3, TAS_MIN(r5) 3019 1: cmpd r3, r7 3020 ble 2f 3021 std r3, TAS_MAX(r5) 3022 2: lwsync 3023 addi r8, r8, 1 3024 std r8, TAS_SEQCOUNT(r5) 3025 blr 3026 #endif
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.