1 // SPDX-License-Identifier: GPL-2.0-only 1 2 /* 3 * Local APIC handling, local APIC timers 4 * 5 * (c) 1999, 2000, 2009 Ingo Molnar <ming 6 * 7 * Fixes 8 * Maciej W. Rozycki : Bits f 9 * thanks 10 * and Ro 11 * for te 12 * Maciej W. Rozycki : Variou 13 * Mikael Pettersson : Power 14 * Pavel Machek and 15 * Mikael Pettersson : PM con 16 */ 17 18 #include <linux/perf_event.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/mc146818rtc.h> 21 #include <linux/acpi_pmtmr.h> 22 #include <linux/bitmap.h> 23 #include <linux/clockchips.h> 24 #include <linux/interrupt.h> 25 #include <linux/memblock.h> 26 #include <linux/ftrace.h> 27 #include <linux/ioport.h> 28 #include <linux/export.h> 29 #include <linux/syscore_ops.h> 30 #include <linux/delay.h> 31 #include <linux/timex.h> 32 #include <linux/i8253.h> 33 #include <linux/dmar.h> 34 #include <linux/init.h> 35 #include <linux/cpu.h> 36 #include <linux/dmi.h> 37 #include <linux/smp.h> 38 #include <linux/mm.h> 39 40 #include <xen/xen.h> 41 42 #include <asm/trace/irq_vectors.h> 43 #include <asm/irq_remapping.h> 44 #include <asm/pc-conf-reg.h> 45 #include <asm/perf_event.h> 46 #include <asm/x86_init.h> 47 #include <linux/atomic.h> 48 #include <asm/barrier.h> 49 #include <asm/mpspec.h> 50 #include <asm/i8259.h> 51 #include <asm/proto.h> 52 #include <asm/traps.h> 53 #include <asm/apic.h> 54 #include <asm/acpi.h> 55 #include <asm/io_apic.h> 56 #include <asm/desc.h> 57 #include <asm/hpet.h> 58 #include <asm/mtrr.h> 59 #include <asm/time.h> 60 #include <asm/smp.h> 61 #include <asm/mce.h> 62 #include <asm/tsc.h> 63 #include <asm/hypervisor.h> 64 #include <asm/cpu_device_id.h> 65 #include <asm/intel-family.h> 66 #include <asm/irq_regs.h> 67 #include <asm/cpu.h> 68 69 #include "local.h" 70 71 /* Processor that is doing the boot up */ 72 u32 boot_cpu_physical_apicid __ro_after_init = 73 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid); 74 75 u8 boot_cpu_apic_version __ro_after_init; 76 77 /* 78 * This variable controls which CPUs receive e 79 * external NMIs are delivered only to the BSP 80 */ 81 static int apic_extnmi __ro_after_init = APIC_ 82 83 /* 84 * Hypervisor supports 15 bits of APIC ID in M 85 */ 86 static bool virt_ext_dest_id __ro_after_init; 87 88 /* For parallel bootup. */ 89 unsigned long apic_mmio_base __ro_after_init; 90 91 static inline bool apic_accessible(void) 92 { 93 return x2apic_mode || apic_mmio_base; 94 } 95 96 #ifdef CONFIG_X86_32 97 /* Local APIC was disabled by the BIOS and ena 98 static int enabled_via_apicbase __ro_after_ini 99 100 /* 101 * Handle interrupt mode configuration registe 102 * This register controls whether the interrup 103 * that reach the BSP come from the master PIC 104 * local APIC. Before entering Symmetric I/O M 105 * the BIOS or the operating system must switc 106 * PIC Mode by changing the IMCR. 107 */ 108 static inline void imcr_pic_to_apic(void) 109 { 110 /* NMI and 8259 INTR go through APIC * 111 pc_conf_set(PC_CONF_MPS_IMCR, 0x01); 112 } 113 114 static inline void imcr_apic_to_pic(void) 115 { 116 /* NMI and 8259 INTR go directly to BS 117 pc_conf_set(PC_CONF_MPS_IMCR, 0x00); 118 } 119 #endif 120 121 /* 122 * Knob to control our willingness to enable t 123 * 124 * +1=force-enable 125 */ 126 static int force_enable_local_apic __initdata; 127 128 /* 129 * APIC command line parameters 130 */ 131 static int __init parse_lapic(char *arg) 132 { 133 if (IS_ENABLED(CONFIG_X86_32) && !arg) 134 force_enable_local_apic = 1; 135 else if (arg && !strncmp(arg, "notscde 136 setup_clear_cpu_cap(X86_FEATUR 137 return 0; 138 } 139 early_param("lapic", parse_lapic); 140 141 #ifdef CONFIG_X86_64 142 static int apic_calibrate_pmtmr __initdata; 143 static __init int setup_apicpmtimer(char *s) 144 { 145 apic_calibrate_pmtmr = 1; 146 notsc_setup(NULL); 147 return 1; 148 } 149 __setup("apicpmtimer", setup_apicpmtimer); 150 #endif 151 152 static unsigned long mp_lapic_addr __ro_after_ 153 bool apic_is_disabled __ro_after_init; 154 /* Disable local APIC timer from the kernel co 155 static int disable_apic_timer __initdata; 156 /* Local APIC timer works in C2 */ 157 int local_apic_timer_c2_ok __ro_after_init; 158 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok); 159 160 /* 161 * Debug level, exported for io_apic.c 162 */ 163 int apic_verbosity __ro_after_init; 164 165 int pic_mode __ro_after_init; 166 167 /* Have we found an MP table */ 168 int smp_found_config __ro_after_init; 169 170 static struct resource lapic_resource = { 171 .name = "Local APIC", 172 .flags = IORESOURCE_MEM | IORESOURCE_B 173 }; 174 175 unsigned int lapic_timer_period = 0; 176 177 static void apic_pm_activate(void); 178 179 /* 180 * Get the LAPIC version 181 */ 182 static inline int lapic_get_version(void) 183 { 184 return GET_APIC_VERSION(apic_read(APIC 185 } 186 187 /* 188 * Check, if the APIC is integrated or a separ 189 */ 190 static inline int lapic_is_integrated(void) 191 { 192 return APIC_INTEGRATED(lapic_get_versi 193 } 194 195 /* 196 * Check, whether this is a modern or a first 197 */ 198 static int modern_apic(void) 199 { 200 /* AMD systems use old APIC versions, 201 if (boot_cpu_data.x86_vendor == X86_VE 202 boot_cpu_data.x86 >= 0xf) 203 return 1; 204 205 /* Hygon systems use modern APIC */ 206 if (boot_cpu_data.x86_vendor == X86_VE 207 return 1; 208 209 return lapic_get_version() >= 0x14; 210 } 211 212 /* 213 * right after this call apic become NOOP driv 214 * so apic->write/read doesn't do anything 215 */ 216 static void __init apic_disable(void) 217 { 218 apic_install_driver(&apic_noop); 219 } 220 221 void native_apic_icr_write(u32 low, u32 id) 222 { 223 unsigned long flags; 224 225 local_irq_save(flags); 226 apic_write(APIC_ICR2, SET_XAPIC_DEST_F 227 apic_write(APIC_ICR, low); 228 local_irq_restore(flags); 229 } 230 231 u64 native_apic_icr_read(void) 232 { 233 u32 icr1, icr2; 234 235 icr2 = apic_read(APIC_ICR2); 236 icr1 = apic_read(APIC_ICR); 237 238 return icr1 | ((u64)icr2 << 32); 239 } 240 241 /** 242 * lapic_get_maxlvt - get the maximum number o 243 */ 244 int lapic_get_maxlvt(void) 245 { 246 /* 247 * - we always have APIC integrated on 248 * - 82489DXs do not report # of LVT e 249 */ 250 return lapic_is_integrated() ? GET_API 251 } 252 253 /* 254 * Local APIC timer 255 */ 256 257 /* Clock divisor */ 258 #define APIC_DIVISOR 16 259 #define TSC_DIVISOR 8 260 261 /* i82489DX specific */ 262 #define I82489DX_BASE_DIVIDER 263 264 /* 265 * This function sets up the local APIC timer, 266 * 'clocks' APIC bus clock. During calibration 267 * this function twice on the boot CPU, once w 268 * value, second time for real. The other (non 269 * call this function only once, with the real 270 * 271 * We do reads before writes even if unnecessa 272 * P5 APIC double write bug. 273 */ 274 static void __setup_APIC_LVTT(unsigned int clo 275 { 276 unsigned int lvtt_value, tmp_value; 277 278 lvtt_value = LOCAL_TIMER_VECTOR; 279 if (!oneshot) 280 lvtt_value |= APIC_LVT_TIMER_P 281 else if (boot_cpu_has(X86_FEATURE_TSC_ 282 lvtt_value |= APIC_LVT_TIMER_T 283 284 /* 285 * The i82489DX APIC uses bit 18 and 1 286 * overlaps with bit 18 on integrated 287 * in the SDM. No problem though. i824 288 * have TSC deadline timer. 289 */ 290 if (!lapic_is_integrated()) 291 lvtt_value |= I82489DX_BASE_DI 292 293 if (!irqen) 294 lvtt_value |= APIC_LVT_MASKED; 295 296 apic_write(APIC_LVTT, lvtt_value); 297 298 if (lvtt_value & APIC_LVT_TIMER_TSCDEA 299 /* 300 * See Intel SDM: TSC-Deadline 301 * writing to the APIC LVTT an 302 * According to Intel, MFENCE 303 */ 304 asm volatile("mfence" : : : "m 305 return; 306 } 307 308 /* 309 * Divide PICLK by 16 310 */ 311 tmp_value = apic_read(APIC_TDCR); 312 apic_write(APIC_TDCR, 313 (tmp_value & ~(APIC_TDR_DIV_1 314 APIC_TDR_DIV_16); 315 316 if (!oneshot) 317 apic_write(APIC_TMICT, clocks 318 } 319 320 /* 321 * Setup extended LVT, AMD specific 322 * 323 * Software should use the LVT offsets the BIO 324 * are determined by the subsystems using it l 325 * threshold or IBS. On K8 only offset 0 (API 326 * are supported. Beginning with family 10h at 327 * available. 328 * 329 * Since the offsets must be consistent for al 330 * of the LVT offsets in software and reserve 331 * vector also to be used on other cores. An o 332 * setting the entry to APIC_EILVT_MASKED. 333 * 334 * If the BIOS is right, there should be no co 335 * "[Firmware Bug]: ..." error message is gene 336 * software does not properly determines the o 337 * necessarily a BIOS bug. 338 */ 339 340 static atomic_t eilvt_offsets[APIC_EILVT_NR_MA 341 342 static inline int eilvt_entry_is_changeable(un 343 { 344 return (old & APIC_EILVT_MASKED) 345 || (new == APIC_EILVT_MASKED) 346 || ((new & ~APIC_EILVT_MASKED) 347 } 348 349 static unsigned int reserve_eilvt_offset(int o 350 { 351 unsigned int rsvd, vector; 352 353 if (offset >= APIC_EILVT_NR_MAX) 354 return ~0; 355 356 rsvd = atomic_read(&eilvt_offsets[offs 357 do { 358 vector = rsvd & ~APIC_EILVT_MA 359 if (vector && !eilvt_entry_is_ 360 /* may not change if v 361 return rsvd; 362 } while (!atomic_try_cmpxchg(&eilvt_of 363 364 rsvd = new & ~APIC_EILVT_MASKED; 365 if (rsvd && rsvd != vector) 366 pr_info("LVT offset %d assigne 367 offset, rsvd); 368 369 return new; 370 } 371 372 /* 373 * If mask=1, the LVT entry does not generate 374 * enables the vector. See also the BKDGs. Mus 375 * preemption disabled. 376 */ 377 378 int setup_APIC_eilvt(u8 offset, u8 vector, u8 379 { 380 unsigned long reg = APIC_EILVTn(offset 381 unsigned int new, old, reserved; 382 383 new = (mask << 16) | (msg_type << 8) | 384 old = apic_read(reg); 385 reserved = reserve_eilvt_offset(offset 386 387 if (reserved != new) { 388 pr_err(FW_BUG "cpu %d, try to 389 "vector 0x%x, but the r 390 "vector 0x%x on another 391 smp_processor_id(), reg 392 return -EINVAL; 393 } 394 395 if (!eilvt_entry_is_changeable(old, ne 396 pr_err(FW_BUG "cpu %d, try to 397 "vector 0x%x, but the r 398 "vector 0x%x on this cp 399 smp_processor_id(), reg 400 return -EBUSY; 401 } 402 403 apic_write(reg, new); 404 405 return 0; 406 } 407 EXPORT_SYMBOL_GPL(setup_APIC_eilvt); 408 409 /* 410 * Program the next event, relative to now 411 */ 412 static int lapic_next_event(unsigned long delt 413 struct clock_event 414 { 415 apic_write(APIC_TMICT, delta); 416 return 0; 417 } 418 419 static int lapic_next_deadline(unsigned long d 420 struct clock_ev 421 { 422 u64 tsc; 423 424 /* This MSR is special and need a spec 425 weak_wrmsr_fence(); 426 427 tsc = rdtsc(); 428 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (( 429 return 0; 430 } 431 432 static int lapic_timer_shutdown(struct clock_e 433 { 434 unsigned int v; 435 436 /* Lapic used as dummy for broadcast ? 437 if (evt->features & CLOCK_EVT_FEAT_DUM 438 return 0; 439 440 v = apic_read(APIC_LVTT); 441 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VE 442 apic_write(APIC_LVTT, v); 443 apic_write(APIC_TMICT, 0); 444 return 0; 445 } 446 447 static inline int 448 lapic_timer_set_periodic_oneshot(struct clock_ 449 { 450 /* Lapic used as dummy for broadcast ? 451 if (evt->features & CLOCK_EVT_FEAT_DUM 452 return 0; 453 454 __setup_APIC_LVTT(lapic_timer_period, 455 return 0; 456 } 457 458 static int lapic_timer_set_periodic(struct clo 459 { 460 return lapic_timer_set_periodic_onesho 461 } 462 463 static int lapic_timer_set_oneshot(struct cloc 464 { 465 return lapic_timer_set_periodic_onesho 466 } 467 468 /* 469 * Local APIC timer broadcast function 470 */ 471 static void lapic_timer_broadcast(const struct 472 { 473 #ifdef CONFIG_SMP 474 __apic_send_IPI_mask(mask, LOCAL_TIMER 475 #endif 476 } 477 478 479 /* 480 * The local apic timer can be used for any fu 481 */ 482 static struct clock_event_device lapic_clockev 483 .name = "lap 484 .features = CLOC 485 CLOC 486 | CL 487 .shift = 32, 488 .set_state_shutdown = lapi 489 .set_state_periodic = lapi 490 .set_state_oneshot = lapi 491 .set_state_oneshot_stopped = lapi 492 .set_next_event = lapi 493 .broadcast = lapi 494 .rating = 100, 495 .irq = -1, 496 }; 497 static DEFINE_PER_CPU(struct clock_event_devic 498 499 static const struct x86_cpu_id deadline_match[ 500 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_ 501 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_ 502 503 X86_MATCH_VFM(INTEL_BROADWELL_X, 504 505 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL 506 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL 507 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL 508 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL 509 510 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_ 511 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_ 512 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_ 513 514 X86_MATCH_VFM(INTEL_HASWELL, 515 X86_MATCH_VFM(INTEL_HASWELL_L, 516 X86_MATCH_VFM(INTEL_HASWELL_G, 517 518 X86_MATCH_VFM(INTEL_BROADWELL, 519 X86_MATCH_VFM(INTEL_BROADWELL_G, 520 521 X86_MATCH_VFM(INTEL_SKYLAKE_L, 522 X86_MATCH_VFM(INTEL_SKYLAKE, 523 524 X86_MATCH_VFM(INTEL_KABYLAKE_L, 525 X86_MATCH_VFM(INTEL_KABYLAKE, 526 527 {}, 528 }; 529 530 static __init bool apic_validate_deadline_time 531 { 532 const struct x86_cpu_id *m; 533 u32 rev; 534 535 if (!boot_cpu_has(X86_FEATURE_TSC_DEAD 536 return false; 537 if (boot_cpu_has(X86_FEATURE_HYPERVISO 538 return true; 539 540 m = x86_match_cpu(deadline_match); 541 if (!m) 542 return true; 543 544 rev = (u32)m->driver_data; 545 546 if (boot_cpu_data.microcode >= rev) 547 return true; 548 549 setup_clear_cpu_cap(X86_FEATURE_TSC_DE 550 pr_err(FW_BUG "TSC_DEADLINE disabled d 551 "please update microcode to ver 552 return false; 553 } 554 555 /* 556 * Setup the local APIC timer for this CPU. Co 557 * of the boot CPU and register the clock even 558 */ 559 static void setup_APIC_timer(void) 560 { 561 struct clock_event_device *levt = this 562 563 if (this_cpu_has(X86_FEATURE_ARAT)) { 564 lapic_clockevent.features &= ~ 565 /* Make LAPIC timer preferable 566 lapic_clockevent.rating = 150; 567 } 568 569 memcpy(levt, &lapic_clockevent, sizeof 570 levt->cpumask = cpumask_of(smp_process 571 572 if (this_cpu_has(X86_FEATURE_TSC_DEADL 573 levt->name = "lapic-deadline"; 574 levt->features &= ~(CLOCK_EVT_ 575 CLOCK_EVT_ 576 levt->set_next_event = lapic_n 577 clockevents_config_and_registe 578 579 580 } else 581 clockevents_register_device(le 582 } 583 584 /* 585 * Install the updated TSC frequency from reca 586 * deadline clockevent devices. 587 */ 588 static void __lapic_update_tsc_freq(void *info 589 { 590 struct clock_event_device *levt = this 591 592 if (!this_cpu_has(X86_FEATURE_TSC_DEAD 593 return; 594 595 clockevents_update_freq(levt, tsc_khz 596 } 597 598 void lapic_update_tsc_freq(void) 599 { 600 /* 601 * The clockevent device's ->mult and 602 * changed. In order to avoid races, s 603 * update code on each CPU. 604 */ 605 on_each_cpu(__lapic_update_tsc_freq, N 606 } 607 608 /* 609 * In this functions we calibrate APIC bus clo 610 * 611 * We want to do the calibration only once sin 612 * irqs synchronous. CPUs connected by the sam 613 * frequency. 614 * 615 * This was previously done by reading the PIT 616 * around to find out, that a tick has elapsed 617 * readout is broken, so it never gets out of 618 * also reported by others. 619 * 620 * Monitoring the jiffies value is inaccurate 621 * infrastructure allows us to do a simple sub 622 * handler. 623 * 624 * The calibration routine also uses the pm_ti 625 * happens to run way too slow (factor 2.3 on 626 * back to normal later in the boot process). 627 */ 628 629 #define LAPIC_CAL_LOOPS (HZ/10) 630 631 static __initdata int lapic_cal_loops = -1; 632 static __initdata long lapic_cal_t1, lapic_cal 633 static __initdata unsigned long long lapic_cal 634 static __initdata u32 lapic_cal_pm1, lapic_cal 635 static __initdata unsigned long lapic_cal_j1, 636 637 /* 638 * Temporary interrupt handler and polled cali 639 */ 640 static void __init lapic_cal_handler(struct cl 641 { 642 unsigned long long tsc = 0; 643 long tapic = apic_read(APIC_TMCCT); 644 u32 pm = acpi_pm_read_early(); 645 646 if (boot_cpu_has(X86_FEATURE_TSC)) 647 tsc = rdtsc(); 648 649 switch (lapic_cal_loops++) { 650 case 0: 651 lapic_cal_t1 = tapic; 652 lapic_cal_tsc1 = tsc; 653 lapic_cal_pm1 = pm; 654 lapic_cal_j1 = jiffies; 655 break; 656 657 case LAPIC_CAL_LOOPS: 658 lapic_cal_t2 = tapic; 659 lapic_cal_tsc2 = tsc; 660 if (pm < lapic_cal_pm1) 661 pm += ACPI_PM_OVRRUN; 662 lapic_cal_pm2 = pm; 663 lapic_cal_j2 = jiffies; 664 break; 665 } 666 } 667 668 static int __init 669 calibrate_by_pmtimer(u32 deltapm, long *delta, 670 { 671 const long pm_100ms = PMTMR_TICKS_PER_ 672 const long pm_thresh = pm_100ms / 100; 673 unsigned long mult; 674 u64 res; 675 676 #ifndef CONFIG_X86_PM_TIMER 677 return -1; 678 #endif 679 680 apic_printk(APIC_VERBOSE, "... PM-Time 681 682 /* Check, if the PM timer is available 683 if (!deltapm) 684 return -1; 685 686 mult = clocksource_hz2mult(PMTMR_TICKS 687 688 if (deltapm > (pm_100ms - pm_thresh) & 689 deltapm < (pm_100ms + pm_thresh)) 690 apic_printk(APIC_VERBOSE, "... 691 return 0; 692 } 693 694 res = (((u64)deltapm) * mult) >> 22; 695 do_div(res, 1000000); 696 pr_warn("APIC calibration not consiste 697 "with PM-Timer: %ldms instead 698 699 /* Correct the lapic counter value */ 700 res = (((u64)(*delta)) * pm_100ms); 701 do_div(res, deltapm); 702 pr_info("APIC delta adjusted to PM-Tim 703 "%lu (%ld)\n", (unsigned long) 704 *delta = (long)res; 705 706 /* Correct the tsc counter value */ 707 if (boot_cpu_has(X86_FEATURE_TSC)) { 708 res = (((u64)(*deltatsc)) * pm 709 do_div(res, deltapm); 710 apic_printk(APIC_VERBOSE, "TSC 711 "PM- 712 (unsig 713 *deltatsc = (long)res; 714 } 715 716 return 0; 717 } 718 719 static int __init lapic_init_clockevent(void) 720 { 721 if (!lapic_timer_period) 722 return -1; 723 724 /* Calculate the scaled math multiplic 725 lapic_clockevent.mult = div_sc(lapic_t 726 TICK_N 727 lapic_clockevent.max_delta_ns = 728 clockevent_delta2ns(0x7FFFFFFF 729 lapic_clockevent.max_delta_ticks = 0x7 730 lapic_clockevent.min_delta_ns = 731 clockevent_delta2ns(0xF, &lapi 732 lapic_clockevent.min_delta_ticks = 0xF 733 734 return 0; 735 } 736 737 bool __init apic_needs_pit(void) 738 { 739 /* 740 * If the frequencies are not known, P 741 * and apic timer calibration. 742 */ 743 if (!tsc_khz || !cpu_khz) 744 return true; 745 746 /* Is there an APIC at all or is it di 747 if (!boot_cpu_has(X86_FEATURE_APIC) || 748 return true; 749 750 /* 751 * If interrupt delivery mode is legac 752 * configuration, the local APIC timer 753 * that the PIT is initialized. 754 */ 755 if (apic_intr_mode == APIC_PIC || 756 apic_intr_mode == APIC_VIRTUAL_WIR 757 return true; 758 759 /* Virt guests may lack ARAT, but stil 760 if (!boot_cpu_has(X86_FEATURE_ARAT)) 761 return true; 762 763 /* Deadline timer is based on TSC so n 764 if (boot_cpu_has(X86_FEATURE_TSC_DEADL 765 return false; 766 767 /* APIC timer disabled? */ 768 if (disable_apic_timer) 769 return true; 770 /* 771 * The APIC timer frequency is known a 772 * required. If unknown, let the PIT b 773 */ 774 return lapic_timer_period == 0; 775 } 776 777 static int __init calibrate_APIC_clock(void) 778 { 779 struct clock_event_device *levt = this 780 u64 tsc_perj = 0, tsc_start = 0; 781 unsigned long jif_start; 782 unsigned long deltaj; 783 long delta, deltatsc; 784 int pm_referenced = 0; 785 786 if (boot_cpu_has(X86_FEATURE_TSC_DEADL 787 return 0; 788 789 /* 790 * Check if lapic timer has already be 791 * specific routine, such as tsc calib 792 * in the clockevent structure and ret 793 */ 794 if (!lapic_init_clockevent()) { 795 apic_printk(APIC_VERBOSE, "lap 796 lapic_timer_period 797 /* 798 * Direct calibration methods 799 * local APIC timer, no need f 800 */ 801 lapic_clockevent.features &= ~ 802 return 0; 803 } 804 805 apic_printk(APIC_VERBOSE, "Using local 806 "calibrating APIC timer .. 807 808 /* 809 * There are platforms w/o global cloc 810 * making the calibration conditional 811 * approach everywhere. 812 */ 813 local_irq_disable(); 814 815 /* 816 * Setup the APIC counter to maximum. 817 * can underflow in the 100ms detectio 818 */ 819 __setup_APIC_LVTT(0xffffffff, 0, 0); 820 821 /* 822 * Methods to terminate the calibratio 823 * 1) Global clockevent if available 824 * 2) TSC if available and frequency 825 */ 826 jif_start = READ_ONCE(jiffies); 827 828 if (tsc_khz) { 829 tsc_start = rdtsc(); 830 tsc_perj = div_u64((u64)tsc_kh 831 } 832 833 /* 834 * Enable interrupts so the tick can f 835 * clockevent device is available 836 */ 837 local_irq_enable(); 838 839 while (lapic_cal_loops <= LAPIC_CAL_LO 840 /* Wait for a tick to elapse * 841 while (1) { 842 if (tsc_khz) { 843 u64 tsc_now = 844 if ((tsc_now - 845 tsc_st 846 break; 847 } 848 } else { 849 unsigned long 850 851 if (time_after 852 jif_st 853 break; 854 } 855 } 856 cpu_relax(); 857 } 858 859 /* Invoke the calibration rout 860 local_irq_disable(); 861 lapic_cal_handler(NULL); 862 local_irq_enable(); 863 } 864 865 local_irq_disable(); 866 867 /* Build delta t1-t2 as apic timer cou 868 delta = lapic_cal_t1 - lapic_cal_t2; 869 apic_printk(APIC_VERBOSE, "... lapic d 870 871 deltatsc = (long)(lapic_cal_tsc2 - lap 872 873 /* we trust the PM based calibration i 874 pm_referenced = !calibrate_by_pmtimer( 875 &delta 876 877 lapic_timer_period = (delta * APIC_DIV 878 lapic_init_clockevent(); 879 880 apic_printk(APIC_VERBOSE, "..... delta 881 apic_printk(APIC_VERBOSE, "..... mult: 882 apic_printk(APIC_VERBOSE, "..... calib 883 lapic_timer_period); 884 885 if (boot_cpu_has(X86_FEATURE_TSC)) { 886 apic_printk(APIC_VERBOSE, "... 887 "%ld.%04ld MHz.\n" 888 (deltatsc / LAPIC_ 889 (deltatsc / LAPIC_ 890 } 891 892 apic_printk(APIC_VERBOSE, "..... host 893 "%u.%04u MHz.\n", 894 lapic_timer_period / (1000 895 lapic_timer_period % (1000 896 897 /* 898 * Do a sanity check on the APIC calib 899 */ 900 if (lapic_timer_period < (1000000 / HZ 901 local_irq_enable(); 902 pr_warn("APIC frequency too sl 903 return -1; 904 } 905 906 levt->features &= ~CLOCK_EVT_FEAT_DUMM 907 908 /* 909 * PM timer calibration failed or not 910 * timer based calibration, if a globa 911 * available. 912 */ 913 if (!pm_referenced && global_clock_eve 914 apic_printk(APIC_VERBOSE, "... 915 916 /* 917 * Setup the apic timer manual 918 */ 919 levt->event_handler = lapic_ca 920 lapic_timer_set_periodic(levt) 921 lapic_cal_loops = -1; 922 923 /* Let the interrupts run */ 924 local_irq_enable(); 925 926 while (lapic_cal_loops <= LAPI 927 cpu_relax(); 928 929 /* Stop the lapic timer */ 930 local_irq_disable(); 931 lapic_timer_shutdown(levt); 932 933 /* Jiffies delta */ 934 deltaj = lapic_cal_j2 - lapic_ 935 apic_printk(APIC_VERBOSE, "... 936 937 /* Check, if the jiffies resul 938 if (deltaj >= LAPIC_CAL_LOOPS- 939 apic_printk(APIC_VERBO 940 else 941 levt->features |= CLOC 942 } 943 local_irq_enable(); 944 945 if (levt->features & CLOCK_EVT_FEAT_DU 946 pr_warn("APIC timer disabled d 947 return -1; 948 } 949 950 return 0; 951 } 952 953 /* 954 * Setup the boot APIC 955 * 956 * Calibrate and verify the result. 957 */ 958 void __init setup_boot_APIC_clock(void) 959 { 960 /* 961 * The local apic timer can be disable 962 * commandline or from the CPU detecti 963 * timer as a dummy clock event source 964 * broadcast mechanism is used. On UP 965 */ 966 if (disable_apic_timer) { 967 pr_info("Disabling APIC timer\ 968 /* No broadcast on UP ! */ 969 if (num_possible_cpus() > 1) { 970 lapic_clockevent.mult 971 setup_APIC_timer(); 972 } 973 return; 974 } 975 976 if (calibrate_APIC_clock()) { 977 /* No broadcast on UP ! */ 978 if (num_possible_cpus() > 1) 979 setup_APIC_timer(); 980 return; 981 } 982 983 /* 984 * If nmi_watchdog is set to IO_APIC, 985 * PIT/HPET going. Otherwise register 986 * device. 987 */ 988 lapic_clockevent.features &= ~CLOCK_EV 989 990 /* Setup the lapic or request the broa 991 setup_APIC_timer(); 992 amd_e400_c1e_apic_setup(); 993 } 994 995 void setup_secondary_APIC_clock(void) 996 { 997 setup_APIC_timer(); 998 amd_e400_c1e_apic_setup(); 999 } 1000 1001 /* 1002 * The guts of the apic timer interrupt 1003 */ 1004 static void local_apic_timer_interrupt(void) 1005 { 1006 struct clock_event_device *evt = this 1007 1008 /* 1009 * Normally we should not be here til 1010 * in some cases like kdump, its poss 1011 * timer interrupt from previous kern 1012 * new kernel the moment interrupts a 1013 * 1014 * Interrupts are enabled early and L 1015 * its possible that when we get here 1016 * Check for event_handler being NULL 1017 * spurious. 1018 */ 1019 if (!evt->event_handler) { 1020 pr_warn("Spurious LAPIC timer 1021 smp_processor_id()); 1022 /* Switch it off */ 1023 lapic_timer_shutdown(evt); 1024 return; 1025 } 1026 1027 /* 1028 * the NMI deadlock-detector uses thi 1029 */ 1030 inc_irq_stat(apic_timer_irqs); 1031 1032 evt->event_handler(evt); 1033 } 1034 1035 /* 1036 * Local APIC timer interrupt. This is the mo 1037 * local interrupts, but local timer interrup 1038 * broadcast interrupts too. [in case the hw 1039 * 1040 * [ if a single-CPU system runs an SMP kerne 1041 * interrupt as well. Thus we cannot inline 1042 */ 1043 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_inte 1044 { 1045 struct pt_regs *old_regs = set_irq_re 1046 1047 apic_eoi(); 1048 trace_local_timer_entry(LOCAL_TIMER_V 1049 local_apic_timer_interrupt(); 1050 trace_local_timer_exit(LOCAL_TIMER_VE 1051 1052 set_irq_regs(old_regs); 1053 } 1054 1055 /* 1056 * Local APIC start and shutdown 1057 */ 1058 1059 /** 1060 * clear_local_APIC - shutdown the local APIC 1061 * 1062 * This is called, when a CPU is disabled and 1063 * the local APIC has no dangling leftovers. 1064 * leftovers during boot. 1065 */ 1066 void clear_local_APIC(void) 1067 { 1068 int maxlvt; 1069 u32 v; 1070 1071 if (!apic_accessible()) 1072 return; 1073 1074 maxlvt = lapic_get_maxlvt(); 1075 /* 1076 * Masking an LVT entry can trigger a 1077 * if the vector is zero. Mask LVTERR 1078 */ 1079 if (maxlvt >= 3) { 1080 v = ERROR_APIC_VECTOR; /* any 1081 apic_write(APIC_LVTERR, v | A 1082 } 1083 /* 1084 * Careful: we have to set masks only 1085 * any level-triggered sources. 1086 */ 1087 v = apic_read(APIC_LVTT); 1088 apic_write(APIC_LVTT, v | APIC_LVT_MA 1089 v = apic_read(APIC_LVT0); 1090 apic_write(APIC_LVT0, v | APIC_LVT_MA 1091 v = apic_read(APIC_LVT1); 1092 apic_write(APIC_LVT1, v | APIC_LVT_MA 1093 if (maxlvt >= 4) { 1094 v = apic_read(APIC_LVTPC); 1095 apic_write(APIC_LVTPC, v | AP 1096 } 1097 1098 /* lets not touch this if we didn't f 1099 #ifdef CONFIG_X86_THERMAL_VECTOR 1100 if (maxlvt >= 5) { 1101 v = apic_read(APIC_LVTTHMR); 1102 apic_write(APIC_LVTTHMR, v | 1103 } 1104 #endif 1105 #ifdef CONFIG_X86_MCE_INTEL 1106 if (maxlvt >= 6) { 1107 v = apic_read(APIC_LVTCMCI); 1108 if (!(v & APIC_LVT_MASKED)) 1109 apic_write(APIC_LVTCM 1110 } 1111 #endif 1112 1113 /* 1114 * Clean APIC state for other OSs: 1115 */ 1116 apic_write(APIC_LVTT, APIC_LVT_MASKED 1117 apic_write(APIC_LVT0, APIC_LVT_MASKED 1118 apic_write(APIC_LVT1, APIC_LVT_MASKED 1119 if (maxlvt >= 3) 1120 apic_write(APIC_LVTERR, APIC_ 1121 if (maxlvt >= 4) 1122 apic_write(APIC_LVTPC, APIC_L 1123 1124 /* Integrated APIC (!82489DX) ? */ 1125 if (lapic_is_integrated()) { 1126 if (maxlvt > 3) 1127 /* Clear ESR due to P 1128 apic_write(APIC_ESR, 1129 apic_read(APIC_ESR); 1130 } 1131 } 1132 1133 /** 1134 * apic_soft_disable - Clears and software di 1135 * 1136 * Contrary to disable_local_APIC() this does 1137 * MSR_IA32_APICBASE. Clearing that bit on sy 1138 * bus would require a hardware reset as the 1139 * arbitration. On systems with FSB delivery 1140 * but it has to be guaranteed that no interr 1141 * in that state and it's not clear from the 1142 * to INIT/SIPI messages. Stay on the safe si 1143 */ 1144 void apic_soft_disable(void) 1145 { 1146 u32 value; 1147 1148 clear_local_APIC(); 1149 1150 /* Soft disable APIC (implies clearin 1151 value = apic_read(APIC_SPIV); 1152 value &= ~APIC_SPIV_APIC_ENABLED; 1153 apic_write(APIC_SPIV, value); 1154 } 1155 1156 /** 1157 * disable_local_APIC - clear and disable the 1158 */ 1159 void disable_local_APIC(void) 1160 { 1161 if (!apic_accessible()) 1162 return; 1163 1164 apic_soft_disable(); 1165 1166 #ifdef CONFIG_X86_32 1167 /* 1168 * When LAPIC was disabled by the BIO 1169 * restore the disabled state. 1170 */ 1171 if (enabled_via_apicbase) { 1172 unsigned int l, h; 1173 1174 rdmsr(MSR_IA32_APICBASE, l, h 1175 l &= ~MSR_IA32_APICBASE_ENABL 1176 wrmsr(MSR_IA32_APICBASE, l, h 1177 } 1178 #endif 1179 } 1180 1181 /* 1182 * If Linux enabled the LAPIC against the BIO 1183 * re-entering the BIOS on shutdown. Otherwi 1184 * not power-off. Additionally clear all LVT 1185 * for the case where Linux didn't enable the 1186 */ 1187 void lapic_shutdown(void) 1188 { 1189 unsigned long flags; 1190 1191 if (!boot_cpu_has(X86_FEATURE_APIC) & 1192 return; 1193 1194 local_irq_save(flags); 1195 1196 #ifdef CONFIG_X86_32 1197 if (!enabled_via_apicbase) 1198 clear_local_APIC(); 1199 else 1200 #endif 1201 disable_local_APIC(); 1202 1203 1204 local_irq_restore(flags); 1205 } 1206 1207 /** 1208 * sync_Arb_IDs - synchronize APIC bus arbitr 1209 */ 1210 void __init sync_Arb_IDs(void) 1211 { 1212 /* 1213 * Unsupported on P4 - see Intel Dev. 1214 * needed on AMD. 1215 */ 1216 if (modern_apic() || boot_cpu_data.x8 1217 return; 1218 1219 /* 1220 * Wait for idle. 1221 */ 1222 apic_wait_icr_idle(); 1223 1224 apic_printk(APIC_DEBUG, "Synchronizin 1225 apic_write(APIC_ICR, APIC_DEST_ALLINC 1226 APIC_INT_LEVELTRIG | 1227 } 1228 1229 enum apic_intr_mode_id apic_intr_mode __ro_af 1230 1231 static int __init __apic_intr_mode_select(voi 1232 { 1233 /* Check kernel option */ 1234 if (apic_is_disabled) { 1235 pr_info("APIC disabled via ke 1236 return APIC_PIC; 1237 } 1238 1239 /* Check BIOS */ 1240 #ifdef CONFIG_X86_64 1241 /* On 64-bit, the APIC must be integr 1242 if (!boot_cpu_has(X86_FEATURE_APIC)) 1243 apic_is_disabled = true; 1244 pr_info("APIC disabled by BIO 1245 return APIC_PIC; 1246 } 1247 #else 1248 /* On 32-bit, the APIC may be integra 1249 1250 /* Neither 82489DX nor integrated API 1251 if (!boot_cpu_has(X86_FEATURE_APIC) & 1252 apic_is_disabled = true; 1253 return APIC_PIC; 1254 } 1255 1256 /* If the BIOS pretends there is an i 1257 if (!boot_cpu_has(X86_FEATURE_APIC) & 1258 APIC_INTEGRATED(boot_cpu_apic 1259 apic_is_disabled = true; 1260 pr_err(FW_BUG "Local APIC not 1261 return APIC_PIC; 1262 } 1263 #endif 1264 1265 /* Check MP table or ACPI MADT config 1266 if (!smp_found_config) { 1267 disable_ioapic_support(); 1268 if (!acpi_lapic) { 1269 pr_info("APIC: ACPI M 1270 return APIC_VIRTUAL_W 1271 } 1272 return APIC_VIRTUAL_WIRE; 1273 } 1274 1275 #ifdef CONFIG_SMP 1276 /* If SMP should be disabled, then re 1277 if (!setup_max_cpus) { 1278 pr_info("APIC: SMP mode deact 1279 return APIC_SYMMETRIC_IO_NO_R 1280 } 1281 #endif 1282 1283 return APIC_SYMMETRIC_IO; 1284 } 1285 1286 /* Select the interrupt delivery mode for the 1287 void __init apic_intr_mode_select(void) 1288 { 1289 apic_intr_mode = __apic_intr_mode_sel 1290 } 1291 1292 /* 1293 * An initial setup of the virtual wire mode. 1294 */ 1295 void __init init_bsp_APIC(void) 1296 { 1297 unsigned int value; 1298 1299 /* 1300 * Don't do the setup now if we have 1301 * through-I/O-APIC virtual wire mode 1302 */ 1303 if (smp_found_config || !boot_cpu_has 1304 return; 1305 1306 /* 1307 * Do not trust the local APIC being 1308 */ 1309 clear_local_APIC(); 1310 1311 /* 1312 * Enable APIC. 1313 */ 1314 value = apic_read(APIC_SPIV); 1315 value &= ~APIC_VECTOR_MASK; 1316 value |= APIC_SPIV_APIC_ENABLED; 1317 1318 #ifdef CONFIG_X86_32 1319 /* This bit is reserved on P4/Xeon an 1320 if ((boot_cpu_data.x86_vendor == X86_ 1321 (boot_cpu_data.x86 == 15)) 1322 value &= ~APIC_SPIV_FOCUS_DIS 1323 else 1324 #endif 1325 value |= APIC_SPIV_FOCUS_DISA 1326 value |= SPURIOUS_APIC_VECTOR; 1327 apic_write(APIC_SPIV, value); 1328 1329 /* 1330 * Set up the virtual wire mode. 1331 */ 1332 apic_write(APIC_LVT0, APIC_DM_EXTINT) 1333 value = APIC_DM_NMI; 1334 if (!lapic_is_integrated()) 1335 value |= APIC_LVT_LEVEL_TRIGG 1336 if (apic_extnmi == APIC_EXTNMI_NONE) 1337 value |= APIC_LVT_MASKED; 1338 apic_write(APIC_LVT1, value); 1339 } 1340 1341 static void __init apic_bsp_setup(bool upmode 1342 1343 /* Init the interrupt delivery mode for the B 1344 void __init apic_intr_mode_init(void) 1345 { 1346 bool upmode = IS_ENABLED(CONFIG_UP_LA 1347 1348 switch (apic_intr_mode) { 1349 case APIC_PIC: 1350 pr_info("APIC: Keep in PIC mo 1351 return; 1352 case APIC_VIRTUAL_WIRE: 1353 pr_info("APIC: Switch to virt 1354 break; 1355 case APIC_VIRTUAL_WIRE_NO_CONFIG: 1356 pr_info("APIC: Switch to virt 1357 upmode = true; 1358 break; 1359 case APIC_SYMMETRIC_IO: 1360 pr_info("APIC: Switch to symm 1361 break; 1362 case APIC_SYMMETRIC_IO_NO_ROUTING: 1363 pr_info("APIC: Switch to symm 1364 break; 1365 } 1366 1367 x86_64_probe_apic(); 1368 1369 x86_32_install_bigsmp(); 1370 1371 if (x86_platform.apic_post_init) 1372 x86_platform.apic_post_init() 1373 1374 apic_bsp_setup(upmode); 1375 } 1376 1377 static void lapic_setup_esr(void) 1378 { 1379 unsigned int oldvalue, value, maxlvt; 1380 1381 if (!lapic_is_integrated()) { 1382 pr_info("No ESR for 82489DX.\ 1383 return; 1384 } 1385 1386 if (apic->disable_esr) { 1387 /* 1388 * Something untraceable is c 1389 * secondary quads ... for th 1390 * ESR disabled - we can't do 1391 * errors anyway - mbligh 1392 */ 1393 pr_info("Leaving ESR disabled 1394 return; 1395 } 1396 1397 maxlvt = lapic_get_maxlvt(); 1398 if (maxlvt > 3) /* Due to the 1399 apic_write(APIC_ESR, 0); 1400 oldvalue = apic_read(APIC_ESR); 1401 1402 /* enables sending errors */ 1403 value = ERROR_APIC_VECTOR; 1404 apic_write(APIC_LVTERR, value); 1405 1406 /* 1407 * spec says clear errors after enabl 1408 */ 1409 if (maxlvt > 3) 1410 apic_write(APIC_ESR, 0); 1411 value = apic_read(APIC_ESR); 1412 if (value != oldvalue) 1413 apic_printk(APIC_VERBOSE, "ES 1414 "vector: 0x%08x afte 1415 oldvalue, value); 1416 } 1417 1418 #define APIC_IR_REGS APIC_ISR_NR 1419 #define APIC_IR_BITS (APIC_IR_REGS 1420 #define APIC_IR_MAPSIZE (APIC_IR_BITS 1421 1422 union apic_ir { 1423 unsigned long map[APIC_IR_MAPSIZE]; 1424 u32 regs[APIC_IR_REGS]; 1425 }; 1426 1427 static bool apic_check_and_ack(union apic_ir 1428 { 1429 int i, bit; 1430 1431 /* Read the IRRs */ 1432 for (i = 0; i < APIC_IR_REGS; i++) 1433 irr->regs[i] = apic_read(APIC 1434 1435 /* Read the ISRs */ 1436 for (i = 0; i < APIC_IR_REGS; i++) 1437 isr->regs[i] = apic_read(APIC 1438 1439 /* 1440 * If the ISR map is not empty. ACK t 1441 * to verify whether a pending IRR ha 1442 * into a ISR. 1443 */ 1444 if (!bitmap_empty(isr->map, APIC_IR_B 1445 /* 1446 * There can be multiple ISR 1447 * interrupt preempted a lowe 1448 * per set bit. 1449 */ 1450 for_each_set_bit(bit, isr->ma 1451 apic_eoi(); 1452 return true; 1453 } 1454 1455 return !bitmap_empty(irr->map, APIC_I 1456 } 1457 1458 /* 1459 * After a crash, we no longer service the in 1460 * interrupt from previous kernel might still 1461 * 1462 * Most probably by now the CPU has serviced 1463 * might not have done the apic_eoi() because 1464 * came from i8259 as ExtInt. LAPIC did not g 1465 * the ISR bit and cpu thinks it has already 1466 * a vector might get locked. It was noticed 1467 * 0x31). Issue an extra EOI to clear ISR. 1468 * 1469 * If there are pending IRR bits they turn in 1470 * priority ISR bit has been acked. 1471 */ 1472 static void apic_pending_intr_clear(void) 1473 { 1474 union apic_ir irr, isr; 1475 unsigned int i; 1476 1477 /* 512 loops are way oversized and gi 1478 for (i = 0; i < 512; i++) { 1479 if (!apic_check_and_ack(&irr, 1480 return; 1481 } 1482 /* Dump the IRR/ISR content if that f 1483 pr_warn("APIC: Stale IRR: %256pb ISR: 1484 } 1485 1486 /** 1487 * setup_local_APIC - setup the local APIC 1488 * 1489 * Used to setup local APIC while initializin 1490 * Always called with preemption disabled. 1491 */ 1492 static void setup_local_APIC(void) 1493 { 1494 int cpu = smp_processor_id(); 1495 unsigned int value; 1496 1497 if (apic_is_disabled) { 1498 disable_ioapic_support(); 1499 return; 1500 } 1501 1502 /* 1503 * If this comes from kexec/kcrash th 1504 * SPIV. Soft disable it before doing 1505 */ 1506 value = apic_read(APIC_SPIV); 1507 value &= ~APIC_SPIV_APIC_ENABLED; 1508 apic_write(APIC_SPIV, value); 1509 1510 #ifdef CONFIG_X86_32 1511 /* Pound the ESR really hard over the 1512 if (lapic_is_integrated() && apic->di 1513 apic_write(APIC_ESR, 0); 1514 apic_write(APIC_ESR, 0); 1515 apic_write(APIC_ESR, 0); 1516 apic_write(APIC_ESR, 0); 1517 } 1518 #endif 1519 /* 1520 * Intel recommends to set DFR, LDR a 1521 * an APIC. See e.g. "AP-388 82489DX 1522 * document number 292116). 1523 * 1524 * Except for APICs which operate in 1525 */ 1526 if (apic->init_apic_ldr) 1527 apic->init_apic_ldr(); 1528 1529 /* 1530 * Set Task Priority to 'accept all e 1531 * vector in the 16-31 range could be 1532 * would think it's an exception and 1533 * never change this later on. 1534 */ 1535 value = apic_read(APIC_TASKPRI); 1536 value &= ~APIC_TPRI_MASK; 1537 value |= 0x10; 1538 apic_write(APIC_TASKPRI, value); 1539 1540 /* Clear eventually stale ISR/IRR bit 1541 apic_pending_intr_clear(); 1542 1543 /* 1544 * Now that we are all set up, enable 1545 */ 1546 value = apic_read(APIC_SPIV); 1547 value &= ~APIC_VECTOR_MASK; 1548 /* 1549 * Enable APIC 1550 */ 1551 value |= APIC_SPIV_APIC_ENABLED; 1552 1553 #ifdef CONFIG_X86_32 1554 /* 1555 * Some unknown Intel IO/APIC (or API 1556 * certain networking cards. If high 1557 * happening on a particular IOAPIC p 1558 * entry is masked/unmasked at a high 1559 * later IOAPIC line gets 'stuck', no 1560 * from the device. If focus CPU is d 1561 * away, oh well :-( 1562 * 1563 * [ This bug can be reproduced easil 1564 * PCI Ne2000 networking cards and 1565 * BX chipset. ] 1566 */ 1567 /* 1568 * Actually disabling the focus CPU c 1569 * frequent as it makes the interrupt 1570 * like LRU than MRU (the short-term 1571 */ 1572 1573 /* 1574 * - enable focus processor (bit==0) 1575 * - 64bit mode always use processor 1576 * so no need to set it 1577 */ 1578 value &= ~APIC_SPIV_FOCUS_DISABLED; 1579 #endif 1580 1581 /* 1582 * Set spurious IRQ vector 1583 */ 1584 value |= SPURIOUS_APIC_VECTOR; 1585 apic_write(APIC_SPIV, value); 1586 1587 perf_events_lapic_init(); 1588 1589 /* 1590 * Set up LVT0, LVT1: 1591 * 1592 * set up through-local-APIC on the b 1593 * strictly necessary in pure symmetr 1594 * we delegate interrupts to the 8259 1595 */ 1596 /* 1597 * TODO: set up through-local-APIC fr 1598 */ 1599 value = apic_read(APIC_LVT0) & APIC_L 1600 if (!cpu && (pic_mode || !value || io 1601 value = APIC_DM_EXTINT; 1602 apic_printk(APIC_VERBOSE, "en 1603 } else { 1604 value = APIC_DM_EXTINT | APIC 1605 apic_printk(APIC_VERBOSE, "ma 1606 } 1607 apic_write(APIC_LVT0, value); 1608 1609 /* 1610 * Only the BSP sees the LINT1 NMI si 1611 * modified by apic_extnmi= boot opti 1612 */ 1613 if ((!cpu && apic_extnmi != APIC_EXTN 1614 apic_extnmi == APIC_EXTNMI_ALL) 1615 value = APIC_DM_NMI; 1616 else 1617 value = APIC_DM_NMI | APIC_LV 1618 1619 /* Is 82489DX ? */ 1620 if (!lapic_is_integrated()) 1621 value |= APIC_LVT_LEVEL_TRIGG 1622 apic_write(APIC_LVT1, value); 1623 1624 #ifdef CONFIG_X86_MCE_INTEL 1625 /* Recheck CMCI information after loc 1626 if (!cpu) 1627 cmci_recheck(); 1628 #endif 1629 } 1630 1631 static void end_local_APIC_setup(void) 1632 { 1633 lapic_setup_esr(); 1634 1635 #ifdef CONFIG_X86_32 1636 { 1637 unsigned int value; 1638 /* Disable the local apic tim 1639 value = apic_read(APIC_LVTT); 1640 value |= (APIC_LVT_MASKED | L 1641 apic_write(APIC_LVTT, value); 1642 } 1643 #endif 1644 1645 apic_pm_activate(); 1646 } 1647 1648 /* 1649 * APIC setup function for application proces 1650 */ 1651 void apic_ap_setup(void) 1652 { 1653 setup_local_APIC(); 1654 end_local_APIC_setup(); 1655 } 1656 1657 static __init void apic_read_boot_cpu_id(bool 1658 { 1659 /* 1660 * This can be invoked from check_x2a 1661 * selected. But that code knows for 1662 * X2APIC. 1663 */ 1664 if (x2apic) { 1665 boot_cpu_physical_apicid = na 1666 boot_cpu_apic_version = GET_A 1667 } else { 1668 boot_cpu_physical_apicid = re 1669 boot_cpu_apic_version = GET_A 1670 } 1671 topology_register_boot_apic(boot_cpu_ 1672 x86_32_probe_bigsmp_early(); 1673 } 1674 1675 #ifdef CONFIG_X86_X2APIC 1676 int x2apic_mode; 1677 EXPORT_SYMBOL_GPL(x2apic_mode); 1678 1679 enum { 1680 X2APIC_OFF, 1681 X2APIC_DISABLED, 1682 /* All states below here have X2APIC 1683 X2APIC_ON, 1684 X2APIC_ON_LOCKED 1685 }; 1686 static int x2apic_state; 1687 1688 static bool x2apic_hw_locked(void) 1689 { 1690 u64 x86_arch_cap_msr; 1691 u64 msr; 1692 1693 x86_arch_cap_msr = x86_read_arch_cap_ 1694 if (x86_arch_cap_msr & ARCH_CAP_XAPIC 1695 rdmsrl(MSR_IA32_XAPIC_DISABLE 1696 return (msr & LEGACY_XAPIC_DI 1697 } 1698 return false; 1699 } 1700 1701 static void __x2apic_disable(void) 1702 { 1703 u64 msr; 1704 1705 if (!boot_cpu_has(X86_FEATURE_APIC)) 1706 return; 1707 1708 rdmsrl(MSR_IA32_APICBASE, msr); 1709 if (!(msr & X2APIC_ENABLE)) 1710 return; 1711 /* Disable xapic and x2apic first and 1712 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2A 1713 wrmsrl(MSR_IA32_APICBASE, msr & ~X2AP 1714 printk_once(KERN_INFO "x2apic disable 1715 } 1716 1717 static void __x2apic_enable(void) 1718 { 1719 u64 msr; 1720 1721 rdmsrl(MSR_IA32_APICBASE, msr); 1722 if (msr & X2APIC_ENABLE) 1723 return; 1724 wrmsrl(MSR_IA32_APICBASE, msr | X2API 1725 printk_once(KERN_INFO "x2apic enabled 1726 } 1727 1728 static int __init setup_nox2apic(char *str) 1729 { 1730 if (x2apic_enabled()) { 1731 u32 apicid = native_apic_msr_ 1732 1733 if (apicid >= 255) { 1734 pr_warn("Apicid: %08x 1735 apicid); 1736 return 0; 1737 } 1738 if (x2apic_hw_locked()) { 1739 pr_warn("APIC locked 1740 return 0; 1741 } 1742 pr_warn("x2apic already enabl 1743 __x2apic_disable(); 1744 } 1745 setup_clear_cpu_cap(X86_FEATURE_X2API 1746 x2apic_state = X2APIC_DISABLED; 1747 x2apic_mode = 0; 1748 return 0; 1749 } 1750 early_param("nox2apic", setup_nox2apic); 1751 1752 /* Called from cpu_init() to enable x2apic on 1753 void x2apic_setup(void) 1754 { 1755 /* 1756 * Try to make the AP's APIC state ma 1757 * BSP is unlocked and the AP is lock 1758 * Warn about the mismatch in case a 1759 * trying to be turned off. 1760 */ 1761 if (x2apic_state != X2APIC_ON_LOCKED 1762 pr_warn("x2apic lock mismatch 1763 /* 1764 * If x2apic is not in ON or LOCKED s 1765 * from BIOS. 1766 */ 1767 if (x2apic_state < X2APIC_ON) { 1768 __x2apic_disable(); 1769 return; 1770 } 1771 __x2apic_enable(); 1772 } 1773 1774 static __init void apic_set_fixmap(bool read_ 1775 1776 static __init void x2apic_disable(void) 1777 { 1778 u32 x2apic_id, state = x2apic_state; 1779 1780 x2apic_mode = 0; 1781 x2apic_state = X2APIC_DISABLED; 1782 1783 if (state != X2APIC_ON) 1784 return; 1785 1786 x2apic_id = read_apic_id(); 1787 if (x2apic_id >= 255) 1788 panic("Cannot disable x2apic, 1789 1790 if (x2apic_hw_locked()) { 1791 pr_warn("Cannot disable locke 1792 return; 1793 } 1794 1795 __x2apic_disable(); 1796 /* 1797 * Don't reread the APIC ID as it was 1798 * check_x2apic() and the APIC driver 1799 * which fails to do the read after x 1800 */ 1801 apic_set_fixmap(false); 1802 } 1803 1804 static __init void x2apic_enable(void) 1805 { 1806 if (x2apic_state != X2APIC_OFF) 1807 return; 1808 1809 x2apic_mode = 1; 1810 x2apic_state = X2APIC_ON; 1811 __x2apic_enable(); 1812 } 1813 1814 static __init void try_to_enable_x2apic(int r 1815 { 1816 if (x2apic_state == X2APIC_DISABLED) 1817 return; 1818 1819 if (remap_mode != IRQ_REMAP_X2APIC_MO 1820 u32 apic_limit = 255; 1821 1822 /* 1823 * Using X2APIC without IR is 1824 * on bare metal but may be s 1825 */ 1826 if (!x86_init.hyper.x2apic_av 1827 pr_info("x2apic: IRQ 1828 x2apic_disable(); 1829 return; 1830 } 1831 1832 /* 1833 * If the hypervisor supports 1834 * MSI, that increases the ma 1835 * used for non-remapped IRQ 1836 */ 1837 if (x86_init.hyper.msi_ext_de 1838 virt_ext_dest_id = 1; 1839 apic_limit = 32767; 1840 } 1841 1842 /* 1843 * Without IR, all CPUs can b 1844 * in physical mode, and CPUs 1845 * be addressed must not be b 1846 */ 1847 x2apic_set_max_apicid(apic_li 1848 x2apic_phys = 1; 1849 } 1850 x2apic_enable(); 1851 } 1852 1853 void __init check_x2apic(void) 1854 { 1855 if (x2apic_enabled()) { 1856 pr_info("x2apic: enabled by B 1857 x2apic_mode = 1; 1858 if (x2apic_hw_locked()) 1859 x2apic_state = X2APIC 1860 else 1861 x2apic_state = X2APIC 1862 apic_read_boot_cpu_id(true); 1863 } else if (!boot_cpu_has(X86_FEATURE_ 1864 x2apic_state = X2APIC_DISABLE 1865 } 1866 } 1867 #else /* CONFIG_X86_X2APIC */ 1868 void __init check_x2apic(void) 1869 { 1870 if (!apic_is_x2apic_enabled()) 1871 return; 1872 /* 1873 * Checkme: Can we simply turn off x2 1874 */ 1875 pr_err("Kernel does not support x2API 1876 pr_err("Disabling APIC, expect reduce 1877 1878 apic_is_disabled = true; 1879 setup_clear_cpu_cap(X86_FEATURE_APIC) 1880 } 1881 1882 static inline void try_to_enable_x2apic(int r 1883 static inline void __x2apic_enable(void) { } 1884 #endif /* !CONFIG_X86_X2APIC */ 1885 1886 void __init enable_IR_x2apic(void) 1887 { 1888 unsigned long flags; 1889 int ret, ir_stat; 1890 1891 if (ioapic_is_disabled) { 1892 pr_info("Not enabling interru 1893 return; 1894 } 1895 1896 ir_stat = irq_remapping_prepare(); 1897 if (ir_stat < 0 && !x2apic_supported( 1898 return; 1899 1900 ret = save_ioapic_entries(); 1901 if (ret) { 1902 pr_info("Saving IO-APIC state 1903 return; 1904 } 1905 1906 local_irq_save(flags); 1907 legacy_pic->mask_all(); 1908 mask_ioapic_entries(); 1909 1910 /* If irq_remapping_prepare() succeed 1911 if (ir_stat >= 0) 1912 ir_stat = irq_remapping_enabl 1913 /* ir_stat contains the remap mode or 1914 try_to_enable_x2apic(ir_stat); 1915 1916 if (ir_stat < 0) 1917 restore_ioapic_entries(); 1918 legacy_pic->restore_mask(); 1919 local_irq_restore(flags); 1920 } 1921 1922 #ifdef CONFIG_X86_64 1923 /* 1924 * Detect and enable local APICs on non-SMP b 1925 * Original code written by Keir Fraser. 1926 * On AMD64 we trust the BIOS - if it says no 1927 * not correctly set up (usually the APIC tim 1928 */ 1929 static bool __init detect_init_APIC(void) 1930 { 1931 if (!boot_cpu_has(X86_FEATURE_APIC)) 1932 pr_info("No local APIC presen 1933 return false; 1934 } 1935 1936 register_lapic_address(APIC_DEFAULT_P 1937 return true; 1938 } 1939 #else 1940 1941 static bool __init apic_verify(unsigned long 1942 { 1943 u32 features, h, l; 1944 1945 /* 1946 * The APIC feature bit should now be 1947 * in `cpuid' 1948 */ 1949 features = cpuid_edx(1); 1950 if (!(features & (1 << X86_FEATURE_AP 1951 pr_warn("Could not enable API 1952 return false; 1953 } 1954 set_cpu_cap(&boot_cpu_data, X86_FEATU 1955 1956 /* The BIOS may have set up the APIC 1957 if (boot_cpu_data.x86 >= 6) { 1958 rdmsr(MSR_IA32_APICBASE, l, h 1959 if (l & MSR_IA32_APICBASE_ENA 1960 addr = l & MSR_IA32_A 1961 } 1962 1963 register_lapic_address(addr); 1964 pr_info("Found and enabled local APIC 1965 return true; 1966 } 1967 1968 bool __init apic_force_enable(unsigned long a 1969 { 1970 u32 h, l; 1971 1972 if (apic_is_disabled) 1973 return false; 1974 1975 /* 1976 * Some BIOSes disable the local APIC 1977 * MSR. This can only be done in soft 1978 * and AMD K7 (Model > 1) or later. 1979 */ 1980 if (boot_cpu_data.x86 >= 6) { 1981 rdmsr(MSR_IA32_APICBASE, l, h 1982 if (!(l & MSR_IA32_APICBASE_E 1983 pr_info("Local APIC d 1984 l &= ~MSR_IA32_APICBA 1985 l |= MSR_IA32_APICBAS 1986 wrmsr(MSR_IA32_APICBA 1987 enabled_via_apicbase 1988 } 1989 } 1990 return apic_verify(addr); 1991 } 1992 1993 /* 1994 * Detect and initialize APIC 1995 */ 1996 static bool __init detect_init_APIC(void) 1997 { 1998 /* Disabled by kernel option? */ 1999 if (apic_is_disabled) 2000 return false; 2001 2002 switch (boot_cpu_data.x86_vendor) { 2003 case X86_VENDOR_AMD: 2004 if ((boot_cpu_data.x86 == 6 & 2005 (boot_cpu_data.x86 >= 15) 2006 break; 2007 goto no_apic; 2008 case X86_VENDOR_HYGON: 2009 break; 2010 case X86_VENDOR_INTEL: 2011 if (boot_cpu_data.x86 == 6 || 2012 (boot_cpu_data.x86 == 5 & 2013 break; 2014 goto no_apic; 2015 default: 2016 goto no_apic; 2017 } 2018 2019 if (!boot_cpu_has(X86_FEATURE_APIC)) 2020 /* 2021 * Over-ride BIOS and try to 2022 * "lapic" specified. 2023 */ 2024 if (!force_enable_local_apic) 2025 pr_info("Local APIC d 2026 "you can enab 2027 return false; 2028 } 2029 if (!apic_force_enable(APIC_D 2030 return false; 2031 } else { 2032 if (!apic_verify(APIC_DEFAULT 2033 return false; 2034 } 2035 2036 apic_pm_activate(); 2037 2038 return true; 2039 2040 no_apic: 2041 pr_info("No local APIC present or har 2042 return false; 2043 } 2044 #endif 2045 2046 /** 2047 * init_apic_mappings - initialize APIC mappi 2048 */ 2049 void __init init_apic_mappings(void) 2050 { 2051 if (apic_validate_deadline_timer()) 2052 pr_info("TSC deadline timer a 2053 2054 if (x2apic_mode) 2055 return; 2056 2057 if (!smp_found_config) { 2058 if (!detect_init_APIC()) { 2059 pr_info("APIC: disabl 2060 apic_disable(); 2061 } 2062 } 2063 } 2064 2065 static __init void apic_set_fixmap(bool read_ 2066 { 2067 set_fixmap_nocache(FIX_APIC_BASE, mp_ 2068 apic_mmio_base = APIC_BASE; 2069 apic_printk(APIC_VERBOSE, "mapped API 2070 apic_mmio_base, mp_lapic_ 2071 if (read_apic) 2072 apic_read_boot_cpu_id(false); 2073 } 2074 2075 void __init register_lapic_address(unsigned l 2076 { 2077 /* This should only happen once */ 2078 WARN_ON_ONCE(mp_lapic_addr); 2079 mp_lapic_addr = address; 2080 2081 if (!x2apic_mode) 2082 apic_set_fixmap(true); 2083 } 2084 2085 /* 2086 * Local APIC interrupts 2087 */ 2088 2089 /* 2090 * Common handling code for spurious_interrup 2091 * points below. No point in allowing the com 2092 */ 2093 static noinline void handle_spurious_interrup 2094 { 2095 u32 v; 2096 2097 trace_spurious_apic_entry(vector); 2098 2099 inc_irq_stat(irq_spurious_count); 2100 2101 /* 2102 * If this is a spurious interrupt th 2103 */ 2104 if (vector == SPURIOUS_APIC_VECTOR) { 2105 /* See SDM vol 3 */ 2106 pr_info("Spurious APIC interr 2107 smp_processor_id()); 2108 goto out; 2109 } 2110 2111 /* 2112 * If it is a vectored one, verify it 2113 * acknowledge it. 2114 */ 2115 v = apic_read(APIC_ISR + ((vector & ~ 2116 if (v & (1 << (vector & 0x1f))) { 2117 pr_info("Spurious interrupt ( 2118 vector, smp_processor 2119 apic_eoi(); 2120 } else { 2121 pr_info("Spurious interrupt ( 2122 vector, smp_processor 2123 } 2124 out: 2125 trace_spurious_apic_exit(vector); 2126 } 2127 2128 /** 2129 * spurious_interrupt - Catch all for interru 2130 * @regs: Pointer to pt_regs on stack 2131 * @vector: The vector number 2132 * 2133 * This is invoked from ASM entry code to cat 2134 * trigger on an entry which is routed to the 2135 * point. 2136 */ 2137 DEFINE_IDTENTRY_IRQ(spurious_interrupt) 2138 { 2139 handle_spurious_interrupt(vector); 2140 } 2141 2142 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_i 2143 { 2144 handle_spurious_interrupt(SPURIOUS_AP 2145 } 2146 2147 /* 2148 * This interrupt should never happen with ou 2149 */ 2150 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt 2151 { 2152 static const char * const error_inter 2153 "Send CS error", 2154 "Receive CS error", 2155 "Send accept error", 2156 "Receive accept error", 2157 "Redirectable IPI", 2158 "Send illegal vector", 2159 "Received illegal vector", 2160 "Illegal register address", 2161 }; 2162 u32 v, i = 0; 2163 2164 trace_error_apic_entry(ERROR_APIC_VEC 2165 2166 /* First tickle the hardware, only th 2167 if (lapic_get_maxlvt() > 3) /* Du 2168 apic_write(APIC_ESR, 0); 2169 v = apic_read(APIC_ESR); 2170 apic_eoi(); 2171 atomic_inc(&irq_err_count); 2172 2173 apic_printk(APIC_DEBUG, KERN_DEBUG "A 2174 smp_processor_id(), v); 2175 2176 v &= 0xff; 2177 while (v) { 2178 if (v & 0x1) 2179 apic_printk(APIC_DEBU 2180 i++; 2181 v >>= 1; 2182 } 2183 2184 apic_printk(APIC_DEBUG, KERN_CONT "\n 2185 2186 trace_error_apic_exit(ERROR_APIC_VECT 2187 } 2188 2189 /** 2190 * connect_bsp_APIC - attach the APIC to the 2191 */ 2192 static void __init connect_bsp_APIC(void) 2193 { 2194 #ifdef CONFIG_X86_32 2195 if (pic_mode) { 2196 /* 2197 * Do not trust the local API 2198 */ 2199 clear_local_APIC(); 2200 /* 2201 * PIC mode, enable APIC mode 2202 * local APIC to INT and NMI 2203 */ 2204 apic_printk(APIC_VERBOSE, "le 2205 "enabling API 2206 imcr_pic_to_apic(); 2207 } 2208 #endif 2209 } 2210 2211 /** 2212 * disconnect_bsp_APIC - detach the APIC from 2213 * @virt_wire_setup: indicates, whether vi 2214 * 2215 * Virtual wire mode is necessary to deliver 2216 * APIC is disabled. 2217 */ 2218 void disconnect_bsp_APIC(int virt_wire_setup) 2219 { 2220 unsigned int value; 2221 2222 #ifdef CONFIG_X86_32 2223 if (pic_mode) { 2224 /* 2225 * Put the board back into PI 2226 * certain older boards). No 2227 * IPIs, won't work beyond th 2228 * INIT IPIs. 2229 */ 2230 apic_printk(APIC_VERBOSE, "di 2231 "entering PIC 2232 imcr_apic_to_pic(); 2233 return; 2234 } 2235 #endif 2236 2237 /* Go back to Virtual Wire compatibil 2238 2239 /* For the spurious interrupt use vec 2240 value = apic_read(APIC_SPIV); 2241 value &= ~APIC_VECTOR_MASK; 2242 value |= APIC_SPIV_APIC_ENABLED; 2243 value |= 0xf; 2244 apic_write(APIC_SPIV, value); 2245 2246 if (!virt_wire_setup) { 2247 /* 2248 * For LVT0 make it edge trig 2249 * external and enabled 2250 */ 2251 value = apic_read(APIC_LVT0); 2252 value &= ~(APIC_MODE_MASK | A 2253 APIC_INPUT_POLARITY | 2254 APIC_LVT_LEVEL_TRIGGE 2255 value |= APIC_LVT_REMOTE_IRR 2256 value = SET_APIC_DELIVERY_MOD 2257 apic_write(APIC_LVT0, value); 2258 } else { 2259 /* Disable LVT0 */ 2260 apic_write(APIC_LVT0, APIC_LV 2261 } 2262 2263 /* 2264 * For LVT1 make it edge triggered, a 2265 * nmi and enabled 2266 */ 2267 value = apic_read(APIC_LVT1); 2268 value &= ~(APIC_MODE_MASK | APIC_SEND 2269 APIC_INPUT_POLARITY | 2270 APIC_LVT_LEVEL_TRIGGE 2271 value |= APIC_LVT_REMOTE_IRR | APIC_S 2272 value = SET_APIC_DELIVERY_MODE(value, 2273 apic_write(APIC_LVT1, value); 2274 } 2275 2276 void __irq_msi_compose_msg(struct irq_cfg *cf 2277 bool dmar) 2278 { 2279 memset(msg, 0, sizeof(*msg)); 2280 2281 msg->arch_addr_lo.base_address = X86_ 2282 msg->arch_addr_lo.dest_mode_logical = 2283 msg->arch_addr_lo.destid_0_7 = cfg->d 2284 2285 msg->arch_data.delivery_mode = APIC_D 2286 msg->arch_data.vector = cfg->vector; 2287 2288 msg->address_hi = X86_MSI_BASE_ADDRES 2289 /* 2290 * Only the IOMMU itself can use the 2291 * APIC ID into the high bits of the 2292 * just be writing to memory if it tr 2293 * address APICs which can't be addre 2294 * address range at 0xFFExxxxx. That 2295 * some hypervisors allow the extende 2296 * 5-11 to be used, giving support fo 2297 */ 2298 if (dmar) 2299 msg->arch_addr_hi.destid_8_31 2300 else if (virt_ext_dest_id && cfg->des 2301 msg->arch_addr_lo.virt_destid 2302 else 2303 WARN_ON_ONCE(cfg->dest_apicid 2304 } 2305 2306 u32 x86_msi_msg_get_destid(struct msi_msg *ms 2307 { 2308 u32 dest = msg->arch_addr_lo.destid_0 2309 2310 if (extid) 2311 dest |= msg->arch_addr_hi.des 2312 return dest; 2313 } 2314 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); 2315 2316 static void __init apic_bsp_up_setup(void) 2317 { 2318 reset_phys_cpu_present_map(boot_cpu_p 2319 } 2320 2321 /** 2322 * apic_bsp_setup - Setup function for local 2323 * @upmode: Force UP mode (for AP 2324 */ 2325 static void __init apic_bsp_setup(bool upmode 2326 { 2327 connect_bsp_APIC(); 2328 if (upmode) 2329 apic_bsp_up_setup(); 2330 setup_local_APIC(); 2331 2332 enable_IO_APIC(); 2333 end_local_APIC_setup(); 2334 irq_remap_enable_fault_handling(); 2335 setup_IO_APIC(); 2336 lapic_update_legacy_vectors(); 2337 } 2338 2339 #ifdef CONFIG_UP_LATE_INIT 2340 void __init up_late_init(void) 2341 { 2342 if (apic_intr_mode == APIC_PIC) 2343 return; 2344 2345 /* Setup local timer */ 2346 x86_init.timers.setup_percpu_clockev( 2347 } 2348 #endif 2349 2350 /* 2351 * Power management 2352 */ 2353 #ifdef CONFIG_PM 2354 2355 static struct { 2356 /* 2357 * 'active' is true if the local APIC 2358 * not the BIOS; this signifies that 2359 * for disabling it before entering a 2360 */ 2361 int active; 2362 /* r/w apic fields */ 2363 u32 apic_id; 2364 unsigned int apic_taskpri; 2365 unsigned int apic_ldr; 2366 unsigned int apic_dfr; 2367 unsigned int apic_spiv; 2368 unsigned int apic_lvtt; 2369 unsigned int apic_lvtpc; 2370 unsigned int apic_lvt0; 2371 unsigned int apic_lvt1; 2372 unsigned int apic_lvterr; 2373 unsigned int apic_tmict; 2374 unsigned int apic_tdcr; 2375 unsigned int apic_thmr; 2376 unsigned int apic_cmci; 2377 } apic_pm_state; 2378 2379 static int lapic_suspend(void) 2380 { 2381 unsigned long flags; 2382 int maxlvt; 2383 2384 if (!apic_pm_state.active) 2385 return 0; 2386 2387 maxlvt = lapic_get_maxlvt(); 2388 2389 apic_pm_state.apic_id = apic_read(API 2390 apic_pm_state.apic_taskpri = apic_rea 2391 apic_pm_state.apic_ldr = apic_read(AP 2392 apic_pm_state.apic_dfr = apic_read(AP 2393 apic_pm_state.apic_spiv = apic_read(A 2394 apic_pm_state.apic_lvtt = apic_read(A 2395 if (maxlvt >= 4) 2396 apic_pm_state.apic_lvtpc = ap 2397 apic_pm_state.apic_lvt0 = apic_read(A 2398 apic_pm_state.apic_lvt1 = apic_read(A 2399 apic_pm_state.apic_lvterr = apic_read 2400 apic_pm_state.apic_tmict = apic_read( 2401 apic_pm_state.apic_tdcr = apic_read(A 2402 #ifdef CONFIG_X86_THERMAL_VECTOR 2403 if (maxlvt >= 5) 2404 apic_pm_state.apic_thmr = api 2405 #endif 2406 #ifdef CONFIG_X86_MCE_INTEL 2407 if (maxlvt >= 6) 2408 apic_pm_state.apic_cmci = api 2409 #endif 2410 2411 local_irq_save(flags); 2412 2413 /* 2414 * Mask IOAPIC before disabling the l 2415 * entries on some implementations. 2416 */ 2417 mask_ioapic_entries(); 2418 2419 disable_local_APIC(); 2420 2421 irq_remapping_disable(); 2422 2423 local_irq_restore(flags); 2424 return 0; 2425 } 2426 2427 static void lapic_resume(void) 2428 { 2429 unsigned int l, h; 2430 unsigned long flags; 2431 int maxlvt; 2432 2433 if (!apic_pm_state.active) 2434 return; 2435 2436 local_irq_save(flags); 2437 2438 /* 2439 * IO-APIC and PIC have their own res 2440 * We just mask them here to make sur 2441 * subsystem is completely quiet whil 2442 * and interrupt-remapping. 2443 */ 2444 mask_ioapic_entries(); 2445 legacy_pic->mask_all(); 2446 2447 if (x2apic_mode) { 2448 __x2apic_enable(); 2449 } else { 2450 /* 2451 * Make sure the APICBASE poi 2452 * 2453 * FIXME! This will be wrong 2454 * SMP! We'll need to do this 2455 */ 2456 if (boot_cpu_data.x86 >= 6) { 2457 rdmsr(MSR_IA32_APICBA 2458 l &= ~MSR_IA32_APICBA 2459 l |= MSR_IA32_APICBAS 2460 wrmsr(MSR_IA32_APICBA 2461 } 2462 } 2463 2464 maxlvt = lapic_get_maxlvt(); 2465 apic_write(APIC_LVTERR, ERROR_APIC_VE 2466 apic_write(APIC_ID, apic_pm_state.api 2467 apic_write(APIC_DFR, apic_pm_state.ap 2468 apic_write(APIC_LDR, apic_pm_state.ap 2469 apic_write(APIC_TASKPRI, apic_pm_stat 2470 apic_write(APIC_SPIV, apic_pm_state.a 2471 apic_write(APIC_LVT0, apic_pm_state.a 2472 apic_write(APIC_LVT1, apic_pm_state.a 2473 #ifdef CONFIG_X86_THERMAL_VECTOR 2474 if (maxlvt >= 5) 2475 apic_write(APIC_LVTTHMR, apic 2476 #endif 2477 #ifdef CONFIG_X86_MCE_INTEL 2478 if (maxlvt >= 6) 2479 apic_write(APIC_LVTCMCI, apic 2480 #endif 2481 if (maxlvt >= 4) 2482 apic_write(APIC_LVTPC, apic_p 2483 apic_write(APIC_LVTT, apic_pm_state.a 2484 apic_write(APIC_TDCR, apic_pm_state.a 2485 apic_write(APIC_TMICT, apic_pm_state. 2486 apic_write(APIC_ESR, 0); 2487 apic_read(APIC_ESR); 2488 apic_write(APIC_LVTERR, apic_pm_state 2489 apic_write(APIC_ESR, 0); 2490 apic_read(APIC_ESR); 2491 2492 irq_remapping_reenable(x2apic_mode); 2493 2494 local_irq_restore(flags); 2495 } 2496 2497 /* 2498 * This device has no shutdown method - fully 2499 * are needed on every CPU up until machine_h 2500 */ 2501 2502 static struct syscore_ops lapic_syscore_ops = 2503 .resume = lapic_resume, 2504 .suspend = lapic_suspend, 2505 }; 2506 2507 static void apic_pm_activate(void) 2508 { 2509 apic_pm_state.active = 1; 2510 } 2511 2512 static int __init init_lapic_sysfs(void) 2513 { 2514 /* XXX: remove suspend/resume procs i 2515 if (boot_cpu_has(X86_FEATURE_APIC)) 2516 register_syscore_ops(&lapic_s 2517 2518 return 0; 2519 } 2520 2521 /* local apic needs to resume before other de 2522 core_initcall(init_lapic_sysfs); 2523 2524 #else /* CONFIG_PM */ 2525 2526 static void apic_pm_activate(void) { } 2527 2528 #endif /* CONFIG_PM */ 2529 2530 #ifdef CONFIG_X86_64 2531 2532 static int multi_checked; 2533 static int multi; 2534 2535 static int set_multi(const struct dmi_system_ 2536 { 2537 if (multi) 2538 return 0; 2539 pr_info("APIC: %s detected, Multi Cha 2540 multi = 1; 2541 return 0; 2542 } 2543 2544 static const struct dmi_system_id multi_dmi_t 2545 { 2546 .callback = set_multi, 2547 .ident = "IBM System Summit2" 2548 .matches = { 2549 DMI_MATCH(DMI_SYS_VEN 2550 DMI_MATCH(DMI_PRODUCT 2551 }, 2552 }, 2553 {} 2554 }; 2555 2556 static void dmi_check_multi(void) 2557 { 2558 if (multi_checked) 2559 return; 2560 2561 dmi_check_system(multi_dmi_table); 2562 multi_checked = 1; 2563 } 2564 2565 /* 2566 * apic_is_clustered_box() -- Check if we can 2567 * 2568 * Thus far, the major user of this is IBM's 2569 * Clustered boxes may have unsynced TSC prob 2570 * multi-chassis. 2571 * Use DMI to check them 2572 */ 2573 int apic_is_clustered_box(void) 2574 { 2575 dmi_check_multi(); 2576 return multi; 2577 } 2578 #endif 2579 2580 /* 2581 * APIC command line parameters 2582 */ 2583 static int __init setup_disableapic(char *arg 2584 { 2585 apic_is_disabled = true; 2586 setup_clear_cpu_cap(X86_FEATURE_APIC) 2587 return 0; 2588 } 2589 early_param("disableapic", setup_disableapic) 2590 2591 /* same as disableapic, for compatibility */ 2592 static int __init setup_nolapic(char *arg) 2593 { 2594 return setup_disableapic(arg); 2595 } 2596 early_param("nolapic", setup_nolapic); 2597 2598 static int __init parse_lapic_timer_c2_ok(cha 2599 { 2600 local_apic_timer_c2_ok = 1; 2601 return 0; 2602 } 2603 early_param("lapic_timer_c2_ok", parse_lapic_ 2604 2605 static int __init parse_disable_apic_timer(ch 2606 { 2607 disable_apic_timer = 1; 2608 return 0; 2609 } 2610 early_param("noapictimer", parse_disable_apic 2611 2612 static int __init parse_nolapic_timer(char *a 2613 { 2614 disable_apic_timer = 1; 2615 return 0; 2616 } 2617 early_param("nolapic_timer", parse_nolapic_ti 2618 2619 static int __init apic_set_verbosity(char *ar 2620 { 2621 if (!arg) { 2622 if (IS_ENABLED(CONFIG_X86_32) 2623 return -EINVAL; 2624 2625 ioapic_is_disabled = false; 2626 return 0; 2627 } 2628 2629 if (strcmp("debug", arg) == 0) 2630 apic_verbosity = APIC_DEBUG; 2631 else if (strcmp("verbose", arg) == 0) 2632 apic_verbosity = APIC_VERBOSE 2633 #ifdef CONFIG_X86_64 2634 else { 2635 pr_warn("APIC Verbosity level 2636 " use apic=verbose or 2637 return -EINVAL; 2638 } 2639 #endif 2640 2641 return 0; 2642 } 2643 early_param("apic", apic_set_verbosity); 2644 2645 static int __init lapic_insert_resource(void) 2646 { 2647 if (!apic_mmio_base) 2648 return -1; 2649 2650 /* Put local APIC into the resource m 2651 lapic_resource.start = apic_mmio_base 2652 lapic_resource.end = lapic_resource.s 2653 insert_resource(&iomem_resource, &lap 2654 2655 return 0; 2656 } 2657 2658 /* 2659 * need call insert after e820__reserve_resou 2660 * that is using request_resource 2661 */ 2662 late_initcall(lapic_insert_resource); 2663 2664 static int __init apic_set_extnmi(char *arg) 2665 { 2666 if (!arg) 2667 return -EINVAL; 2668 2669 if (!strncmp("all", arg, 3)) 2670 apic_extnmi = APIC_EXTNMI_ALL 2671 else if (!strncmp("none", arg, 4)) 2672 apic_extnmi = APIC_EXTNMI_NON 2673 else if (!strncmp("bsp", arg, 3)) 2674 apic_extnmi = APIC_EXTNMI_BSP 2675 else { 2676 pr_warn("Unknown external NMI 2677 return -EINVAL; 2678 } 2679 2680 return 0; 2681 } 2682 early_param("apic_extnmi", apic_set_extnmi); 2683
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.