1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * Local APIC handling, local APIC timers
4 *
5 * (c) 1999, 2000, 2009 Ingo Molnar <ming
6 *
7 * Fixes
8 * Maciej W. Rozycki : Bits f
9 * thanks
10 * and Ro
11 * for te
12 * Maciej W. Rozycki : Variou
13 * Mikael Pettersson : Power
14 * Pavel Machek and
15 * Mikael Pettersson : PM con
16 */
17
18 #include <linux/perf_event.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mc146818rtc.h>
21 #include <linux/acpi_pmtmr.h>
22 #include <linux/bitmap.h>
23 #include <linux/clockchips.h>
24 #include <linux/interrupt.h>
25 #include <linux/memblock.h>
26 #include <linux/ftrace.h>
27 #include <linux/ioport.h>
28 #include <linux/export.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/delay.h>
31 #include <linux/timex.h>
32 #include <linux/i8253.h>
33 #include <linux/dmar.h>
34 #include <linux/init.h>
35 #include <linux/cpu.h>
36 #include <linux/dmi.h>
37 #include <linux/smp.h>
38 #include <linux/mm.h>
39
40 #include <xen/xen.h>
41
42 #include <asm/trace/irq_vectors.h>
43 #include <asm/irq_remapping.h>
44 #include <asm/pc-conf-reg.h>
45 #include <asm/perf_event.h>
46 #include <asm/x86_init.h>
47 #include <linux/atomic.h>
48 #include <asm/barrier.h>
49 #include <asm/mpspec.h>
50 #include <asm/i8259.h>
51 #include <asm/proto.h>
52 #include <asm/traps.h>
53 #include <asm/apic.h>
54 #include <asm/acpi.h>
55 #include <asm/io_apic.h>
56 #include <asm/desc.h>
57 #include <asm/hpet.h>
58 #include <asm/mtrr.h>
59 #include <asm/time.h>
60 #include <asm/smp.h>
61 #include <asm/mce.h>
62 #include <asm/tsc.h>
63 #include <asm/hypervisor.h>
64 #include <asm/cpu_device_id.h>
65 #include <asm/intel-family.h>
66 #include <asm/irq_regs.h>
67 #include <asm/cpu.h>
68
69 #include "local.h"
70
71 /* Processor that is doing the boot up */
72 u32 boot_cpu_physical_apicid __ro_after_init =
73 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
74
75 u8 boot_cpu_apic_version __ro_after_init;
76
77 /*
78 * This variable controls which CPUs receive e
79 * external NMIs are delivered only to the BSP
80 */
81 static int apic_extnmi __ro_after_init = APIC_
82
83 /*
84 * Hypervisor supports 15 bits of APIC ID in M
85 */
86 static bool virt_ext_dest_id __ro_after_init;
87
88 /* For parallel bootup. */
89 unsigned long apic_mmio_base __ro_after_init;
90
91 static inline bool apic_accessible(void)
92 {
93 return x2apic_mode || apic_mmio_base;
94 }
95
96 #ifdef CONFIG_X86_32
97 /* Local APIC was disabled by the BIOS and ena
98 static int enabled_via_apicbase __ro_after_ini
99
100 /*
101 * Handle interrupt mode configuration registe
102 * This register controls whether the interrup
103 * that reach the BSP come from the master PIC
104 * local APIC. Before entering Symmetric I/O M
105 * the BIOS or the operating system must switc
106 * PIC Mode by changing the IMCR.
107 */
108 static inline void imcr_pic_to_apic(void)
109 {
110 /* NMI and 8259 INTR go through APIC *
111 pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
112 }
113
114 static inline void imcr_apic_to_pic(void)
115 {
116 /* NMI and 8259 INTR go directly to BS
117 pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
118 }
119 #endif
120
121 /*
122 * Knob to control our willingness to enable t
123 *
124 * +1=force-enable
125 */
126 static int force_enable_local_apic __initdata;
127
128 /*
129 * APIC command line parameters
130 */
131 static int __init parse_lapic(char *arg)
132 {
133 if (IS_ENABLED(CONFIG_X86_32) && !arg)
134 force_enable_local_apic = 1;
135 else if (arg && !strncmp(arg, "notscde
136 setup_clear_cpu_cap(X86_FEATUR
137 return 0;
138 }
139 early_param("lapic", parse_lapic);
140
141 #ifdef CONFIG_X86_64
142 static int apic_calibrate_pmtmr __initdata;
143 static __init int setup_apicpmtimer(char *s)
144 {
145 apic_calibrate_pmtmr = 1;
146 notsc_setup(NULL);
147 return 1;
148 }
149 __setup("apicpmtimer", setup_apicpmtimer);
150 #endif
151
152 static unsigned long mp_lapic_addr __ro_after_
153 bool apic_is_disabled __ro_after_init;
154 /* Disable local APIC timer from the kernel co
155 static int disable_apic_timer __initdata;
156 /* Local APIC timer works in C2 */
157 int local_apic_timer_c2_ok __ro_after_init;
158 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
159
160 /*
161 * Debug level, exported for io_apic.c
162 */
163 int apic_verbosity __ro_after_init;
164
165 int pic_mode __ro_after_init;
166
167 /* Have we found an MP table */
168 int smp_found_config __ro_after_init;
169
170 static struct resource lapic_resource = {
171 .name = "Local APIC",
172 .flags = IORESOURCE_MEM | IORESOURCE_B
173 };
174
175 unsigned int lapic_timer_period = 0;
176
177 static void apic_pm_activate(void);
178
179 /*
180 * Get the LAPIC version
181 */
182 static inline int lapic_get_version(void)
183 {
184 return GET_APIC_VERSION(apic_read(APIC
185 }
186
187 /*
188 * Check, if the APIC is integrated or a separ
189 */
190 static inline int lapic_is_integrated(void)
191 {
192 return APIC_INTEGRATED(lapic_get_versi
193 }
194
195 /*
196 * Check, whether this is a modern or a first
197 */
198 static int modern_apic(void)
199 {
200 /* AMD systems use old APIC versions,
201 if (boot_cpu_data.x86_vendor == X86_VE
202 boot_cpu_data.x86 >= 0xf)
203 return 1;
204
205 /* Hygon systems use modern APIC */
206 if (boot_cpu_data.x86_vendor == X86_VE
207 return 1;
208
209 return lapic_get_version() >= 0x14;
210 }
211
212 /*
213 * right after this call apic become NOOP driv
214 * so apic->write/read doesn't do anything
215 */
216 static void __init apic_disable(void)
217 {
218 apic_install_driver(&apic_noop);
219 }
220
221 void native_apic_icr_write(u32 low, u32 id)
222 {
223 unsigned long flags;
224
225 local_irq_save(flags);
226 apic_write(APIC_ICR2, SET_XAPIC_DEST_F
227 apic_write(APIC_ICR, low);
228 local_irq_restore(flags);
229 }
230
231 u64 native_apic_icr_read(void)
232 {
233 u32 icr1, icr2;
234
235 icr2 = apic_read(APIC_ICR2);
236 icr1 = apic_read(APIC_ICR);
237
238 return icr1 | ((u64)icr2 << 32);
239 }
240
241 /**
242 * lapic_get_maxlvt - get the maximum number o
243 */
244 int lapic_get_maxlvt(void)
245 {
246 /*
247 * - we always have APIC integrated on
248 * - 82489DXs do not report # of LVT e
249 */
250 return lapic_is_integrated() ? GET_API
251 }
252
253 /*
254 * Local APIC timer
255 */
256
257 /* Clock divisor */
258 #define APIC_DIVISOR 16
259 #define TSC_DIVISOR 8
260
261 /* i82489DX specific */
262 #define I82489DX_BASE_DIVIDER
263
264 /*
265 * This function sets up the local APIC timer,
266 * 'clocks' APIC bus clock. During calibration
267 * this function twice on the boot CPU, once w
268 * value, second time for real. The other (non
269 * call this function only once, with the real
270 *
271 * We do reads before writes even if unnecessa
272 * P5 APIC double write bug.
273 */
274 static void __setup_APIC_LVTT(unsigned int clo
275 {
276 unsigned int lvtt_value, tmp_value;
277
278 lvtt_value = LOCAL_TIMER_VECTOR;
279 if (!oneshot)
280 lvtt_value |= APIC_LVT_TIMER_P
281 else if (boot_cpu_has(X86_FEATURE_TSC_
282 lvtt_value |= APIC_LVT_TIMER_T
283
284 /*
285 * The i82489DX APIC uses bit 18 and 1
286 * overlaps with bit 18 on integrated
287 * in the SDM. No problem though. i824
288 * have TSC deadline timer.
289 */
290 if (!lapic_is_integrated())
291 lvtt_value |= I82489DX_BASE_DI
292
293 if (!irqen)
294 lvtt_value |= APIC_LVT_MASKED;
295
296 apic_write(APIC_LVTT, lvtt_value);
297
298 if (lvtt_value & APIC_LVT_TIMER_TSCDEA
299 /*
300 * See Intel SDM: TSC-Deadline
301 * writing to the APIC LVTT an
302 * According to Intel, MFENCE
303 */
304 asm volatile("mfence" : : : "m
305 return;
306 }
307
308 /*
309 * Divide PICLK by 16
310 */
311 tmp_value = apic_read(APIC_TDCR);
312 apic_write(APIC_TDCR,
313 (tmp_value & ~(APIC_TDR_DIV_1
314 APIC_TDR_DIV_16);
315
316 if (!oneshot)
317 apic_write(APIC_TMICT, clocks
318 }
319
320 /*
321 * Setup extended LVT, AMD specific
322 *
323 * Software should use the LVT offsets the BIO
324 * are determined by the subsystems using it l
325 * threshold or IBS. On K8 only offset 0 (API
326 * are supported. Beginning with family 10h at
327 * available.
328 *
329 * Since the offsets must be consistent for al
330 * of the LVT offsets in software and reserve
331 * vector also to be used on other cores. An o
332 * setting the entry to APIC_EILVT_MASKED.
333 *
334 * If the BIOS is right, there should be no co
335 * "[Firmware Bug]: ..." error message is gene
336 * software does not properly determines the o
337 * necessarily a BIOS bug.
338 */
339
340 static atomic_t eilvt_offsets[APIC_EILVT_NR_MA
341
342 static inline int eilvt_entry_is_changeable(un
343 {
344 return (old & APIC_EILVT_MASKED)
345 || (new == APIC_EILVT_MASKED)
346 || ((new & ~APIC_EILVT_MASKED)
347 }
348
349 static unsigned int reserve_eilvt_offset(int o
350 {
351 unsigned int rsvd, vector;
352
353 if (offset >= APIC_EILVT_NR_MAX)
354 return ~0;
355
356 rsvd = atomic_read(&eilvt_offsets[offs
357 do {
358 vector = rsvd & ~APIC_EILVT_MA
359 if (vector && !eilvt_entry_is_
360 /* may not change if v
361 return rsvd;
362 } while (!atomic_try_cmpxchg(&eilvt_of
363
364 rsvd = new & ~APIC_EILVT_MASKED;
365 if (rsvd && rsvd != vector)
366 pr_info("LVT offset %d assigne
367 offset, rsvd);
368
369 return new;
370 }
371
372 /*
373 * If mask=1, the LVT entry does not generate
374 * enables the vector. See also the BKDGs. Mus
375 * preemption disabled.
376 */
377
378 int setup_APIC_eilvt(u8 offset, u8 vector, u8
379 {
380 unsigned long reg = APIC_EILVTn(offset
381 unsigned int new, old, reserved;
382
383 new = (mask << 16) | (msg_type << 8) |
384 old = apic_read(reg);
385 reserved = reserve_eilvt_offset(offset
386
387 if (reserved != new) {
388 pr_err(FW_BUG "cpu %d, try to
389 "vector 0x%x, but the r
390 "vector 0x%x on another
391 smp_processor_id(), reg
392 return -EINVAL;
393 }
394
395 if (!eilvt_entry_is_changeable(old, ne
396 pr_err(FW_BUG "cpu %d, try to
397 "vector 0x%x, but the r
398 "vector 0x%x on this cp
399 smp_processor_id(), reg
400 return -EBUSY;
401 }
402
403 apic_write(reg, new);
404
405 return 0;
406 }
407 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
408
409 /*
410 * Program the next event, relative to now
411 */
412 static int lapic_next_event(unsigned long delt
413 struct clock_event
414 {
415 apic_write(APIC_TMICT, delta);
416 return 0;
417 }
418
419 static int lapic_next_deadline(unsigned long d
420 struct clock_ev
421 {
422 u64 tsc;
423
424 /* This MSR is special and need a spec
425 weak_wrmsr_fence();
426
427 tsc = rdtsc();
428 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + ((
429 return 0;
430 }
431
432 static int lapic_timer_shutdown(struct clock_e
433 {
434 unsigned int v;
435
436 /* Lapic used as dummy for broadcast ?
437 if (evt->features & CLOCK_EVT_FEAT_DUM
438 return 0;
439
440 v = apic_read(APIC_LVTT);
441 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VE
442 apic_write(APIC_LVTT, v);
443 apic_write(APIC_TMICT, 0);
444 return 0;
445 }
446
447 static inline int
448 lapic_timer_set_periodic_oneshot(struct clock_
449 {
450 /* Lapic used as dummy for broadcast ?
451 if (evt->features & CLOCK_EVT_FEAT_DUM
452 return 0;
453
454 __setup_APIC_LVTT(lapic_timer_period,
455 return 0;
456 }
457
458 static int lapic_timer_set_periodic(struct clo
459 {
460 return lapic_timer_set_periodic_onesho
461 }
462
463 static int lapic_timer_set_oneshot(struct cloc
464 {
465 return lapic_timer_set_periodic_onesho
466 }
467
468 /*
469 * Local APIC timer broadcast function
470 */
471 static void lapic_timer_broadcast(const struct
472 {
473 #ifdef CONFIG_SMP
474 __apic_send_IPI_mask(mask, LOCAL_TIMER
475 #endif
476 }
477
478
479 /*
480 * The local apic timer can be used for any fu
481 */
482 static struct clock_event_device lapic_clockev
483 .name = "lap
484 .features = CLOC
485 CLOC
486 | CL
487 .shift = 32,
488 .set_state_shutdown = lapi
489 .set_state_periodic = lapi
490 .set_state_oneshot = lapi
491 .set_state_oneshot_stopped = lapi
492 .set_next_event = lapi
493 .broadcast = lapi
494 .rating = 100,
495 .irq = -1,
496 };
497 static DEFINE_PER_CPU(struct clock_event_devic
498
499 static const struct x86_cpu_id deadline_match[
500 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_
501 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_
502
503 X86_MATCH_VFM(INTEL_BROADWELL_X,
504
505 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
506 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
507 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
508 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
509
510 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
511 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
512 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
513
514 X86_MATCH_VFM(INTEL_HASWELL,
515 X86_MATCH_VFM(INTEL_HASWELL_L,
516 X86_MATCH_VFM(INTEL_HASWELL_G,
517
518 X86_MATCH_VFM(INTEL_BROADWELL,
519 X86_MATCH_VFM(INTEL_BROADWELL_G,
520
521 X86_MATCH_VFM(INTEL_SKYLAKE_L,
522 X86_MATCH_VFM(INTEL_SKYLAKE,
523
524 X86_MATCH_VFM(INTEL_KABYLAKE_L,
525 X86_MATCH_VFM(INTEL_KABYLAKE,
526
527 {},
528 };
529
530 static __init bool apic_validate_deadline_time
531 {
532 const struct x86_cpu_id *m;
533 u32 rev;
534
535 if (!boot_cpu_has(X86_FEATURE_TSC_DEAD
536 return false;
537 if (boot_cpu_has(X86_FEATURE_HYPERVISO
538 return true;
539
540 m = x86_match_cpu(deadline_match);
541 if (!m)
542 return true;
543
544 rev = (u32)m->driver_data;
545
546 if (boot_cpu_data.microcode >= rev)
547 return true;
548
549 setup_clear_cpu_cap(X86_FEATURE_TSC_DE
550 pr_err(FW_BUG "TSC_DEADLINE disabled d
551 "please update microcode to ver
552 return false;
553 }
554
555 /*
556 * Setup the local APIC timer for this CPU. Co
557 * of the boot CPU and register the clock even
558 */
559 static void setup_APIC_timer(void)
560 {
561 struct clock_event_device *levt = this
562
563 if (this_cpu_has(X86_FEATURE_ARAT)) {
564 lapic_clockevent.features &= ~
565 /* Make LAPIC timer preferable
566 lapic_clockevent.rating = 150;
567 }
568
569 memcpy(levt, &lapic_clockevent, sizeof
570 levt->cpumask = cpumask_of(smp_process
571
572 if (this_cpu_has(X86_FEATURE_TSC_DEADL
573 levt->name = "lapic-deadline";
574 levt->features &= ~(CLOCK_EVT_
575 CLOCK_EVT_
576 levt->set_next_event = lapic_n
577 clockevents_config_and_registe
578
579
580 } else
581 clockevents_register_device(le
582 }
583
584 /*
585 * Install the updated TSC frequency from reca
586 * deadline clockevent devices.
587 */
588 static void __lapic_update_tsc_freq(void *info
589 {
590 struct clock_event_device *levt = this
591
592 if (!this_cpu_has(X86_FEATURE_TSC_DEAD
593 return;
594
595 clockevents_update_freq(levt, tsc_khz
596 }
597
598 void lapic_update_tsc_freq(void)
599 {
600 /*
601 * The clockevent device's ->mult and
602 * changed. In order to avoid races, s
603 * update code on each CPU.
604 */
605 on_each_cpu(__lapic_update_tsc_freq, N
606 }
607
608 /*
609 * In this functions we calibrate APIC bus clo
610 *
611 * We want to do the calibration only once sin
612 * irqs synchronous. CPUs connected by the sam
613 * frequency.
614 *
615 * This was previously done by reading the PIT
616 * around to find out, that a tick has elapsed
617 * readout is broken, so it never gets out of
618 * also reported by others.
619 *
620 * Monitoring the jiffies value is inaccurate
621 * infrastructure allows us to do a simple sub
622 * handler.
623 *
624 * The calibration routine also uses the pm_ti
625 * happens to run way too slow (factor 2.3 on
626 * back to normal later in the boot process).
627 */
628
629 #define LAPIC_CAL_LOOPS (HZ/10)
630
631 static __initdata int lapic_cal_loops = -1;
632 static __initdata long lapic_cal_t1, lapic_cal
633 static __initdata unsigned long long lapic_cal
634 static __initdata u32 lapic_cal_pm1, lapic_cal
635 static __initdata unsigned long lapic_cal_j1,
636
637 /*
638 * Temporary interrupt handler and polled cali
639 */
640 static void __init lapic_cal_handler(struct cl
641 {
642 unsigned long long tsc = 0;
643 long tapic = apic_read(APIC_TMCCT);
644 u32 pm = acpi_pm_read_early();
645
646 if (boot_cpu_has(X86_FEATURE_TSC))
647 tsc = rdtsc();
648
649 switch (lapic_cal_loops++) {
650 case 0:
651 lapic_cal_t1 = tapic;
652 lapic_cal_tsc1 = tsc;
653 lapic_cal_pm1 = pm;
654 lapic_cal_j1 = jiffies;
655 break;
656
657 case LAPIC_CAL_LOOPS:
658 lapic_cal_t2 = tapic;
659 lapic_cal_tsc2 = tsc;
660 if (pm < lapic_cal_pm1)
661 pm += ACPI_PM_OVRRUN;
662 lapic_cal_pm2 = pm;
663 lapic_cal_j2 = jiffies;
664 break;
665 }
666 }
667
668 static int __init
669 calibrate_by_pmtimer(u32 deltapm, long *delta,
670 {
671 const long pm_100ms = PMTMR_TICKS_PER_
672 const long pm_thresh = pm_100ms / 100;
673 unsigned long mult;
674 u64 res;
675
676 #ifndef CONFIG_X86_PM_TIMER
677 return -1;
678 #endif
679
680 apic_printk(APIC_VERBOSE, "... PM-Time
681
682 /* Check, if the PM timer is available
683 if (!deltapm)
684 return -1;
685
686 mult = clocksource_hz2mult(PMTMR_TICKS
687
688 if (deltapm > (pm_100ms - pm_thresh) &
689 deltapm < (pm_100ms + pm_thresh))
690 apic_printk(APIC_VERBOSE, "...
691 return 0;
692 }
693
694 res = (((u64)deltapm) * mult) >> 22;
695 do_div(res, 1000000);
696 pr_warn("APIC calibration not consiste
697 "with PM-Timer: %ldms instead
698
699 /* Correct the lapic counter value */
700 res = (((u64)(*delta)) * pm_100ms);
701 do_div(res, deltapm);
702 pr_info("APIC delta adjusted to PM-Tim
703 "%lu (%ld)\n", (unsigned long)
704 *delta = (long)res;
705
706 /* Correct the tsc counter value */
707 if (boot_cpu_has(X86_FEATURE_TSC)) {
708 res = (((u64)(*deltatsc)) * pm
709 do_div(res, deltapm);
710 apic_printk(APIC_VERBOSE, "TSC
711 "PM-
712 (unsig
713 *deltatsc = (long)res;
714 }
715
716 return 0;
717 }
718
719 static int __init lapic_init_clockevent(void)
720 {
721 if (!lapic_timer_period)
722 return -1;
723
724 /* Calculate the scaled math multiplic
725 lapic_clockevent.mult = div_sc(lapic_t
726 TICK_N
727 lapic_clockevent.max_delta_ns =
728 clockevent_delta2ns(0x7FFFFFFF
729 lapic_clockevent.max_delta_ticks = 0x7
730 lapic_clockevent.min_delta_ns =
731 clockevent_delta2ns(0xF, &lapi
732 lapic_clockevent.min_delta_ticks = 0xF
733
734 return 0;
735 }
736
737 bool __init apic_needs_pit(void)
738 {
739 /*
740 * If the frequencies are not known, P
741 * and apic timer calibration.
742 */
743 if (!tsc_khz || !cpu_khz)
744 return true;
745
746 /* Is there an APIC at all or is it di
747 if (!boot_cpu_has(X86_FEATURE_APIC) ||
748 return true;
749
750 /*
751 * If interrupt delivery mode is legac
752 * configuration, the local APIC timer
753 * that the PIT is initialized.
754 */
755 if (apic_intr_mode == APIC_PIC ||
756 apic_intr_mode == APIC_VIRTUAL_WIR
757 return true;
758
759 /* Virt guests may lack ARAT, but stil
760 if (!boot_cpu_has(X86_FEATURE_ARAT))
761 return true;
762
763 /* Deadline timer is based on TSC so n
764 if (boot_cpu_has(X86_FEATURE_TSC_DEADL
765 return false;
766
767 /* APIC timer disabled? */
768 if (disable_apic_timer)
769 return true;
770 /*
771 * The APIC timer frequency is known a
772 * required. If unknown, let the PIT b
773 */
774 return lapic_timer_period == 0;
775 }
776
777 static int __init calibrate_APIC_clock(void)
778 {
779 struct clock_event_device *levt = this
780 u64 tsc_perj = 0, tsc_start = 0;
781 unsigned long jif_start;
782 unsigned long deltaj;
783 long delta, deltatsc;
784 int pm_referenced = 0;
785
786 if (boot_cpu_has(X86_FEATURE_TSC_DEADL
787 return 0;
788
789 /*
790 * Check if lapic timer has already be
791 * specific routine, such as tsc calib
792 * in the clockevent structure and ret
793 */
794 if (!lapic_init_clockevent()) {
795 apic_printk(APIC_VERBOSE, "lap
796 lapic_timer_period
797 /*
798 * Direct calibration methods
799 * local APIC timer, no need f
800 */
801 lapic_clockevent.features &= ~
802 return 0;
803 }
804
805 apic_printk(APIC_VERBOSE, "Using local
806 "calibrating APIC timer ..
807
808 /*
809 * There are platforms w/o global cloc
810 * making the calibration conditional
811 * approach everywhere.
812 */
813 local_irq_disable();
814
815 /*
816 * Setup the APIC counter to maximum.
817 * can underflow in the 100ms detectio
818 */
819 __setup_APIC_LVTT(0xffffffff, 0, 0);
820
821 /*
822 * Methods to terminate the calibratio
823 * 1) Global clockevent if available
824 * 2) TSC if available and frequency
825 */
826 jif_start = READ_ONCE(jiffies);
827
828 if (tsc_khz) {
829 tsc_start = rdtsc();
830 tsc_perj = div_u64((u64)tsc_kh
831 }
832
833 /*
834 * Enable interrupts so the tick can f
835 * clockevent device is available
836 */
837 local_irq_enable();
838
839 while (lapic_cal_loops <= LAPIC_CAL_LO
840 /* Wait for a tick to elapse *
841 while (1) {
842 if (tsc_khz) {
843 u64 tsc_now =
844 if ((tsc_now -
845 tsc_st
846 break;
847 }
848 } else {
849 unsigned long
850
851 if (time_after
852 jif_st
853 break;
854 }
855 }
856 cpu_relax();
857 }
858
859 /* Invoke the calibration rout
860 local_irq_disable();
861 lapic_cal_handler(NULL);
862 local_irq_enable();
863 }
864
865 local_irq_disable();
866
867 /* Build delta t1-t2 as apic timer cou
868 delta = lapic_cal_t1 - lapic_cal_t2;
869 apic_printk(APIC_VERBOSE, "... lapic d
870
871 deltatsc = (long)(lapic_cal_tsc2 - lap
872
873 /* we trust the PM based calibration i
874 pm_referenced = !calibrate_by_pmtimer(
875 &delta
876
877 lapic_timer_period = (delta * APIC_DIV
878 lapic_init_clockevent();
879
880 apic_printk(APIC_VERBOSE, "..... delta
881 apic_printk(APIC_VERBOSE, "..... mult:
882 apic_printk(APIC_VERBOSE, "..... calib
883 lapic_timer_period);
884
885 if (boot_cpu_has(X86_FEATURE_TSC)) {
886 apic_printk(APIC_VERBOSE, "...
887 "%ld.%04ld MHz.\n"
888 (deltatsc / LAPIC_
889 (deltatsc / LAPIC_
890 }
891
892 apic_printk(APIC_VERBOSE, "..... host
893 "%u.%04u MHz.\n",
894 lapic_timer_period / (1000
895 lapic_timer_period % (1000
896
897 /*
898 * Do a sanity check on the APIC calib
899 */
900 if (lapic_timer_period < (1000000 / HZ
901 local_irq_enable();
902 pr_warn("APIC frequency too sl
903 return -1;
904 }
905
906 levt->features &= ~CLOCK_EVT_FEAT_DUMM
907
908 /*
909 * PM timer calibration failed or not
910 * timer based calibration, if a globa
911 * available.
912 */
913 if (!pm_referenced && global_clock_eve
914 apic_printk(APIC_VERBOSE, "...
915
916 /*
917 * Setup the apic timer manual
918 */
919 levt->event_handler = lapic_ca
920 lapic_timer_set_periodic(levt)
921 lapic_cal_loops = -1;
922
923 /* Let the interrupts run */
924 local_irq_enable();
925
926 while (lapic_cal_loops <= LAPI
927 cpu_relax();
928
929 /* Stop the lapic timer */
930 local_irq_disable();
931 lapic_timer_shutdown(levt);
932
933 /* Jiffies delta */
934 deltaj = lapic_cal_j2 - lapic_
935 apic_printk(APIC_VERBOSE, "...
936
937 /* Check, if the jiffies resul
938 if (deltaj >= LAPIC_CAL_LOOPS-
939 apic_printk(APIC_VERBO
940 else
941 levt->features |= CLOC
942 }
943 local_irq_enable();
944
945 if (levt->features & CLOCK_EVT_FEAT_DU
946 pr_warn("APIC timer disabled d
947 return -1;
948 }
949
950 return 0;
951 }
952
953 /*
954 * Setup the boot APIC
955 *
956 * Calibrate and verify the result.
957 */
958 void __init setup_boot_APIC_clock(void)
959 {
960 /*
961 * The local apic timer can be disable
962 * commandline or from the CPU detecti
963 * timer as a dummy clock event source
964 * broadcast mechanism is used. On UP
965 */
966 if (disable_apic_timer) {
967 pr_info("Disabling APIC timer\
968 /* No broadcast on UP ! */
969 if (num_possible_cpus() > 1) {
970 lapic_clockevent.mult
971 setup_APIC_timer();
972 }
973 return;
974 }
975
976 if (calibrate_APIC_clock()) {
977 /* No broadcast on UP ! */
978 if (num_possible_cpus() > 1)
979 setup_APIC_timer();
980 return;
981 }
982
983 /*
984 * If nmi_watchdog is set to IO_APIC,
985 * PIT/HPET going. Otherwise register
986 * device.
987 */
988 lapic_clockevent.features &= ~CLOCK_EV
989
990 /* Setup the lapic or request the broa
991 setup_APIC_timer();
992 amd_e400_c1e_apic_setup();
993 }
994
995 void setup_secondary_APIC_clock(void)
996 {
997 setup_APIC_timer();
998 amd_e400_c1e_apic_setup();
999 }
1000
1001 /*
1002 * The guts of the apic timer interrupt
1003 */
1004 static void local_apic_timer_interrupt(void)
1005 {
1006 struct clock_event_device *evt = this
1007
1008 /*
1009 * Normally we should not be here til
1010 * in some cases like kdump, its poss
1011 * timer interrupt from previous kern
1012 * new kernel the moment interrupts a
1013 *
1014 * Interrupts are enabled early and L
1015 * its possible that when we get here
1016 * Check for event_handler being NULL
1017 * spurious.
1018 */
1019 if (!evt->event_handler) {
1020 pr_warn("Spurious LAPIC timer
1021 smp_processor_id());
1022 /* Switch it off */
1023 lapic_timer_shutdown(evt);
1024 return;
1025 }
1026
1027 /*
1028 * the NMI deadlock-detector uses thi
1029 */
1030 inc_irq_stat(apic_timer_irqs);
1031
1032 evt->event_handler(evt);
1033 }
1034
1035 /*
1036 * Local APIC timer interrupt. This is the mo
1037 * local interrupts, but local timer interrup
1038 * broadcast interrupts too. [in case the hw
1039 *
1040 * [ if a single-CPU system runs an SMP kerne
1041 * interrupt as well. Thus we cannot inline
1042 */
1043 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_inte
1044 {
1045 struct pt_regs *old_regs = set_irq_re
1046
1047 apic_eoi();
1048 trace_local_timer_entry(LOCAL_TIMER_V
1049 local_apic_timer_interrupt();
1050 trace_local_timer_exit(LOCAL_TIMER_VE
1051
1052 set_irq_regs(old_regs);
1053 }
1054
1055 /*
1056 * Local APIC start and shutdown
1057 */
1058
1059 /**
1060 * clear_local_APIC - shutdown the local APIC
1061 *
1062 * This is called, when a CPU is disabled and
1063 * the local APIC has no dangling leftovers.
1064 * leftovers during boot.
1065 */
1066 void clear_local_APIC(void)
1067 {
1068 int maxlvt;
1069 u32 v;
1070
1071 if (!apic_accessible())
1072 return;
1073
1074 maxlvt = lapic_get_maxlvt();
1075 /*
1076 * Masking an LVT entry can trigger a
1077 * if the vector is zero. Mask LVTERR
1078 */
1079 if (maxlvt >= 3) {
1080 v = ERROR_APIC_VECTOR; /* any
1081 apic_write(APIC_LVTERR, v | A
1082 }
1083 /*
1084 * Careful: we have to set masks only
1085 * any level-triggered sources.
1086 */
1087 v = apic_read(APIC_LVTT);
1088 apic_write(APIC_LVTT, v | APIC_LVT_MA
1089 v = apic_read(APIC_LVT0);
1090 apic_write(APIC_LVT0, v | APIC_LVT_MA
1091 v = apic_read(APIC_LVT1);
1092 apic_write(APIC_LVT1, v | APIC_LVT_MA
1093 if (maxlvt >= 4) {
1094 v = apic_read(APIC_LVTPC);
1095 apic_write(APIC_LVTPC, v | AP
1096 }
1097
1098 /* lets not touch this if we didn't f
1099 #ifdef CONFIG_X86_THERMAL_VECTOR
1100 if (maxlvt >= 5) {
1101 v = apic_read(APIC_LVTTHMR);
1102 apic_write(APIC_LVTTHMR, v |
1103 }
1104 #endif
1105 #ifdef CONFIG_X86_MCE_INTEL
1106 if (maxlvt >= 6) {
1107 v = apic_read(APIC_LVTCMCI);
1108 if (!(v & APIC_LVT_MASKED))
1109 apic_write(APIC_LVTCM
1110 }
1111 #endif
1112
1113 /*
1114 * Clean APIC state for other OSs:
1115 */
1116 apic_write(APIC_LVTT, APIC_LVT_MASKED
1117 apic_write(APIC_LVT0, APIC_LVT_MASKED
1118 apic_write(APIC_LVT1, APIC_LVT_MASKED
1119 if (maxlvt >= 3)
1120 apic_write(APIC_LVTERR, APIC_
1121 if (maxlvt >= 4)
1122 apic_write(APIC_LVTPC, APIC_L
1123
1124 /* Integrated APIC (!82489DX) ? */
1125 if (lapic_is_integrated()) {
1126 if (maxlvt > 3)
1127 /* Clear ESR due to P
1128 apic_write(APIC_ESR,
1129 apic_read(APIC_ESR);
1130 }
1131 }
1132
1133 /**
1134 * apic_soft_disable - Clears and software di
1135 *
1136 * Contrary to disable_local_APIC() this does
1137 * MSR_IA32_APICBASE. Clearing that bit on sy
1138 * bus would require a hardware reset as the
1139 * arbitration. On systems with FSB delivery
1140 * but it has to be guaranteed that no interr
1141 * in that state and it's not clear from the
1142 * to INIT/SIPI messages. Stay on the safe si
1143 */
1144 void apic_soft_disable(void)
1145 {
1146 u32 value;
1147
1148 clear_local_APIC();
1149
1150 /* Soft disable APIC (implies clearin
1151 value = apic_read(APIC_SPIV);
1152 value &= ~APIC_SPIV_APIC_ENABLED;
1153 apic_write(APIC_SPIV, value);
1154 }
1155
1156 /**
1157 * disable_local_APIC - clear and disable the
1158 */
1159 void disable_local_APIC(void)
1160 {
1161 if (!apic_accessible())
1162 return;
1163
1164 apic_soft_disable();
1165
1166 #ifdef CONFIG_X86_32
1167 /*
1168 * When LAPIC was disabled by the BIO
1169 * restore the disabled state.
1170 */
1171 if (enabled_via_apicbase) {
1172 unsigned int l, h;
1173
1174 rdmsr(MSR_IA32_APICBASE, l, h
1175 l &= ~MSR_IA32_APICBASE_ENABL
1176 wrmsr(MSR_IA32_APICBASE, l, h
1177 }
1178 #endif
1179 }
1180
1181 /*
1182 * If Linux enabled the LAPIC against the BIO
1183 * re-entering the BIOS on shutdown. Otherwi
1184 * not power-off. Additionally clear all LVT
1185 * for the case where Linux didn't enable the
1186 */
1187 void lapic_shutdown(void)
1188 {
1189 unsigned long flags;
1190
1191 if (!boot_cpu_has(X86_FEATURE_APIC) &
1192 return;
1193
1194 local_irq_save(flags);
1195
1196 #ifdef CONFIG_X86_32
1197 if (!enabled_via_apicbase)
1198 clear_local_APIC();
1199 else
1200 #endif
1201 disable_local_APIC();
1202
1203
1204 local_irq_restore(flags);
1205 }
1206
1207 /**
1208 * sync_Arb_IDs - synchronize APIC bus arbitr
1209 */
1210 void __init sync_Arb_IDs(void)
1211 {
1212 /*
1213 * Unsupported on P4 - see Intel Dev.
1214 * needed on AMD.
1215 */
1216 if (modern_apic() || boot_cpu_data.x8
1217 return;
1218
1219 /*
1220 * Wait for idle.
1221 */
1222 apic_wait_icr_idle();
1223
1224 apic_printk(APIC_DEBUG, "Synchronizin
1225 apic_write(APIC_ICR, APIC_DEST_ALLINC
1226 APIC_INT_LEVELTRIG |
1227 }
1228
1229 enum apic_intr_mode_id apic_intr_mode __ro_af
1230
1231 static int __init __apic_intr_mode_select(voi
1232 {
1233 /* Check kernel option */
1234 if (apic_is_disabled) {
1235 pr_info("APIC disabled via ke
1236 return APIC_PIC;
1237 }
1238
1239 /* Check BIOS */
1240 #ifdef CONFIG_X86_64
1241 /* On 64-bit, the APIC must be integr
1242 if (!boot_cpu_has(X86_FEATURE_APIC))
1243 apic_is_disabled = true;
1244 pr_info("APIC disabled by BIO
1245 return APIC_PIC;
1246 }
1247 #else
1248 /* On 32-bit, the APIC may be integra
1249
1250 /* Neither 82489DX nor integrated API
1251 if (!boot_cpu_has(X86_FEATURE_APIC) &
1252 apic_is_disabled = true;
1253 return APIC_PIC;
1254 }
1255
1256 /* If the BIOS pretends there is an i
1257 if (!boot_cpu_has(X86_FEATURE_APIC) &
1258 APIC_INTEGRATED(boot_cpu_apic
1259 apic_is_disabled = true;
1260 pr_err(FW_BUG "Local APIC not
1261 return APIC_PIC;
1262 }
1263 #endif
1264
1265 /* Check MP table or ACPI MADT config
1266 if (!smp_found_config) {
1267 disable_ioapic_support();
1268 if (!acpi_lapic) {
1269 pr_info("APIC: ACPI M
1270 return APIC_VIRTUAL_W
1271 }
1272 return APIC_VIRTUAL_WIRE;
1273 }
1274
1275 #ifdef CONFIG_SMP
1276 /* If SMP should be disabled, then re
1277 if (!setup_max_cpus) {
1278 pr_info("APIC: SMP mode deact
1279 return APIC_SYMMETRIC_IO_NO_R
1280 }
1281 #endif
1282
1283 return APIC_SYMMETRIC_IO;
1284 }
1285
1286 /* Select the interrupt delivery mode for the
1287 void __init apic_intr_mode_select(void)
1288 {
1289 apic_intr_mode = __apic_intr_mode_sel
1290 }
1291
1292 /*
1293 * An initial setup of the virtual wire mode.
1294 */
1295 void __init init_bsp_APIC(void)
1296 {
1297 unsigned int value;
1298
1299 /*
1300 * Don't do the setup now if we have
1301 * through-I/O-APIC virtual wire mode
1302 */
1303 if (smp_found_config || !boot_cpu_has
1304 return;
1305
1306 /*
1307 * Do not trust the local APIC being
1308 */
1309 clear_local_APIC();
1310
1311 /*
1312 * Enable APIC.
1313 */
1314 value = apic_read(APIC_SPIV);
1315 value &= ~APIC_VECTOR_MASK;
1316 value |= APIC_SPIV_APIC_ENABLED;
1317
1318 #ifdef CONFIG_X86_32
1319 /* This bit is reserved on P4/Xeon an
1320 if ((boot_cpu_data.x86_vendor == X86_
1321 (boot_cpu_data.x86 == 15))
1322 value &= ~APIC_SPIV_FOCUS_DIS
1323 else
1324 #endif
1325 value |= APIC_SPIV_FOCUS_DISA
1326 value |= SPURIOUS_APIC_VECTOR;
1327 apic_write(APIC_SPIV, value);
1328
1329 /*
1330 * Set up the virtual wire mode.
1331 */
1332 apic_write(APIC_LVT0, APIC_DM_EXTINT)
1333 value = APIC_DM_NMI;
1334 if (!lapic_is_integrated())
1335 value |= APIC_LVT_LEVEL_TRIGG
1336 if (apic_extnmi == APIC_EXTNMI_NONE)
1337 value |= APIC_LVT_MASKED;
1338 apic_write(APIC_LVT1, value);
1339 }
1340
1341 static void __init apic_bsp_setup(bool upmode
1342
1343 /* Init the interrupt delivery mode for the B
1344 void __init apic_intr_mode_init(void)
1345 {
1346 bool upmode = IS_ENABLED(CONFIG_UP_LA
1347
1348 switch (apic_intr_mode) {
1349 case APIC_PIC:
1350 pr_info("APIC: Keep in PIC mo
1351 return;
1352 case APIC_VIRTUAL_WIRE:
1353 pr_info("APIC: Switch to virt
1354 break;
1355 case APIC_VIRTUAL_WIRE_NO_CONFIG:
1356 pr_info("APIC: Switch to virt
1357 upmode = true;
1358 break;
1359 case APIC_SYMMETRIC_IO:
1360 pr_info("APIC: Switch to symm
1361 break;
1362 case APIC_SYMMETRIC_IO_NO_ROUTING:
1363 pr_info("APIC: Switch to symm
1364 break;
1365 }
1366
1367 x86_64_probe_apic();
1368
1369 x86_32_install_bigsmp();
1370
1371 if (x86_platform.apic_post_init)
1372 x86_platform.apic_post_init()
1373
1374 apic_bsp_setup(upmode);
1375 }
1376
1377 static void lapic_setup_esr(void)
1378 {
1379 unsigned int oldvalue, value, maxlvt;
1380
1381 if (!lapic_is_integrated()) {
1382 pr_info("No ESR for 82489DX.\
1383 return;
1384 }
1385
1386 if (apic->disable_esr) {
1387 /*
1388 * Something untraceable is c
1389 * secondary quads ... for th
1390 * ESR disabled - we can't do
1391 * errors anyway - mbligh
1392 */
1393 pr_info("Leaving ESR disabled
1394 return;
1395 }
1396
1397 maxlvt = lapic_get_maxlvt();
1398 if (maxlvt > 3) /* Due to the
1399 apic_write(APIC_ESR, 0);
1400 oldvalue = apic_read(APIC_ESR);
1401
1402 /* enables sending errors */
1403 value = ERROR_APIC_VECTOR;
1404 apic_write(APIC_LVTERR, value);
1405
1406 /*
1407 * spec says clear errors after enabl
1408 */
1409 if (maxlvt > 3)
1410 apic_write(APIC_ESR, 0);
1411 value = apic_read(APIC_ESR);
1412 if (value != oldvalue)
1413 apic_printk(APIC_VERBOSE, "ES
1414 "vector: 0x%08x afte
1415 oldvalue, value);
1416 }
1417
1418 #define APIC_IR_REGS APIC_ISR_NR
1419 #define APIC_IR_BITS (APIC_IR_REGS
1420 #define APIC_IR_MAPSIZE (APIC_IR_BITS
1421
1422 union apic_ir {
1423 unsigned long map[APIC_IR_MAPSIZE];
1424 u32 regs[APIC_IR_REGS];
1425 };
1426
1427 static bool apic_check_and_ack(union apic_ir
1428 {
1429 int i, bit;
1430
1431 /* Read the IRRs */
1432 for (i = 0; i < APIC_IR_REGS; i++)
1433 irr->regs[i] = apic_read(APIC
1434
1435 /* Read the ISRs */
1436 for (i = 0; i < APIC_IR_REGS; i++)
1437 isr->regs[i] = apic_read(APIC
1438
1439 /*
1440 * If the ISR map is not empty. ACK t
1441 * to verify whether a pending IRR ha
1442 * into a ISR.
1443 */
1444 if (!bitmap_empty(isr->map, APIC_IR_B
1445 /*
1446 * There can be multiple ISR
1447 * interrupt preempted a lowe
1448 * per set bit.
1449 */
1450 for_each_set_bit(bit, isr->ma
1451 apic_eoi();
1452 return true;
1453 }
1454
1455 return !bitmap_empty(irr->map, APIC_I
1456 }
1457
1458 /*
1459 * After a crash, we no longer service the in
1460 * interrupt from previous kernel might still
1461 *
1462 * Most probably by now the CPU has serviced
1463 * might not have done the apic_eoi() because
1464 * came from i8259 as ExtInt. LAPIC did not g
1465 * the ISR bit and cpu thinks it has already
1466 * a vector might get locked. It was noticed
1467 * 0x31). Issue an extra EOI to clear ISR.
1468 *
1469 * If there are pending IRR bits they turn in
1470 * priority ISR bit has been acked.
1471 */
1472 static void apic_pending_intr_clear(void)
1473 {
1474 union apic_ir irr, isr;
1475 unsigned int i;
1476
1477 /* 512 loops are way oversized and gi
1478 for (i = 0; i < 512; i++) {
1479 if (!apic_check_and_ack(&irr,
1480 return;
1481 }
1482 /* Dump the IRR/ISR content if that f
1483 pr_warn("APIC: Stale IRR: %256pb ISR:
1484 }
1485
1486 /**
1487 * setup_local_APIC - setup the local APIC
1488 *
1489 * Used to setup local APIC while initializin
1490 * Always called with preemption disabled.
1491 */
1492 static void setup_local_APIC(void)
1493 {
1494 int cpu = smp_processor_id();
1495 unsigned int value;
1496
1497 if (apic_is_disabled) {
1498 disable_ioapic_support();
1499 return;
1500 }
1501
1502 /*
1503 * If this comes from kexec/kcrash th
1504 * SPIV. Soft disable it before doing
1505 */
1506 value = apic_read(APIC_SPIV);
1507 value &= ~APIC_SPIV_APIC_ENABLED;
1508 apic_write(APIC_SPIV, value);
1509
1510 #ifdef CONFIG_X86_32
1511 /* Pound the ESR really hard over the
1512 if (lapic_is_integrated() && apic->di
1513 apic_write(APIC_ESR, 0);
1514 apic_write(APIC_ESR, 0);
1515 apic_write(APIC_ESR, 0);
1516 apic_write(APIC_ESR, 0);
1517 }
1518 #endif
1519 /*
1520 * Intel recommends to set DFR, LDR a
1521 * an APIC. See e.g. "AP-388 82489DX
1522 * document number 292116).
1523 *
1524 * Except for APICs which operate in
1525 */
1526 if (apic->init_apic_ldr)
1527 apic->init_apic_ldr();
1528
1529 /*
1530 * Set Task Priority to 'accept all e
1531 * vector in the 16-31 range could be
1532 * would think it's an exception and
1533 * never change this later on.
1534 */
1535 value = apic_read(APIC_TASKPRI);
1536 value &= ~APIC_TPRI_MASK;
1537 value |= 0x10;
1538 apic_write(APIC_TASKPRI, value);
1539
1540 /* Clear eventually stale ISR/IRR bit
1541 apic_pending_intr_clear();
1542
1543 /*
1544 * Now that we are all set up, enable
1545 */
1546 value = apic_read(APIC_SPIV);
1547 value &= ~APIC_VECTOR_MASK;
1548 /*
1549 * Enable APIC
1550 */
1551 value |= APIC_SPIV_APIC_ENABLED;
1552
1553 #ifdef CONFIG_X86_32
1554 /*
1555 * Some unknown Intel IO/APIC (or API
1556 * certain networking cards. If high
1557 * happening on a particular IOAPIC p
1558 * entry is masked/unmasked at a high
1559 * later IOAPIC line gets 'stuck', no
1560 * from the device. If focus CPU is d
1561 * away, oh well :-(
1562 *
1563 * [ This bug can be reproduced easil
1564 * PCI Ne2000 networking cards and
1565 * BX chipset. ]
1566 */
1567 /*
1568 * Actually disabling the focus CPU c
1569 * frequent as it makes the interrupt
1570 * like LRU than MRU (the short-term
1571 */
1572
1573 /*
1574 * - enable focus processor (bit==0)
1575 * - 64bit mode always use processor
1576 * so no need to set it
1577 */
1578 value &= ~APIC_SPIV_FOCUS_DISABLED;
1579 #endif
1580
1581 /*
1582 * Set spurious IRQ vector
1583 */
1584 value |= SPURIOUS_APIC_VECTOR;
1585 apic_write(APIC_SPIV, value);
1586
1587 perf_events_lapic_init();
1588
1589 /*
1590 * Set up LVT0, LVT1:
1591 *
1592 * set up through-local-APIC on the b
1593 * strictly necessary in pure symmetr
1594 * we delegate interrupts to the 8259
1595 */
1596 /*
1597 * TODO: set up through-local-APIC fr
1598 */
1599 value = apic_read(APIC_LVT0) & APIC_L
1600 if (!cpu && (pic_mode || !value || io
1601 value = APIC_DM_EXTINT;
1602 apic_printk(APIC_VERBOSE, "en
1603 } else {
1604 value = APIC_DM_EXTINT | APIC
1605 apic_printk(APIC_VERBOSE, "ma
1606 }
1607 apic_write(APIC_LVT0, value);
1608
1609 /*
1610 * Only the BSP sees the LINT1 NMI si
1611 * modified by apic_extnmi= boot opti
1612 */
1613 if ((!cpu && apic_extnmi != APIC_EXTN
1614 apic_extnmi == APIC_EXTNMI_ALL)
1615 value = APIC_DM_NMI;
1616 else
1617 value = APIC_DM_NMI | APIC_LV
1618
1619 /* Is 82489DX ? */
1620 if (!lapic_is_integrated())
1621 value |= APIC_LVT_LEVEL_TRIGG
1622 apic_write(APIC_LVT1, value);
1623
1624 #ifdef CONFIG_X86_MCE_INTEL
1625 /* Recheck CMCI information after loc
1626 if (!cpu)
1627 cmci_recheck();
1628 #endif
1629 }
1630
1631 static void end_local_APIC_setup(void)
1632 {
1633 lapic_setup_esr();
1634
1635 #ifdef CONFIG_X86_32
1636 {
1637 unsigned int value;
1638 /* Disable the local apic tim
1639 value = apic_read(APIC_LVTT);
1640 value |= (APIC_LVT_MASKED | L
1641 apic_write(APIC_LVTT, value);
1642 }
1643 #endif
1644
1645 apic_pm_activate();
1646 }
1647
1648 /*
1649 * APIC setup function for application proces
1650 */
1651 void apic_ap_setup(void)
1652 {
1653 setup_local_APIC();
1654 end_local_APIC_setup();
1655 }
1656
1657 static __init void apic_read_boot_cpu_id(bool
1658 {
1659 /*
1660 * This can be invoked from check_x2a
1661 * selected. But that code knows for
1662 * X2APIC.
1663 */
1664 if (x2apic) {
1665 boot_cpu_physical_apicid = na
1666 boot_cpu_apic_version = GET_A
1667 } else {
1668 boot_cpu_physical_apicid = re
1669 boot_cpu_apic_version = GET_A
1670 }
1671 topology_register_boot_apic(boot_cpu_
1672 x86_32_probe_bigsmp_early();
1673 }
1674
1675 #ifdef CONFIG_X86_X2APIC
1676 int x2apic_mode;
1677 EXPORT_SYMBOL_GPL(x2apic_mode);
1678
1679 enum {
1680 X2APIC_OFF,
1681 X2APIC_DISABLED,
1682 /* All states below here have X2APIC
1683 X2APIC_ON,
1684 X2APIC_ON_LOCKED
1685 };
1686 static int x2apic_state;
1687
1688 static bool x2apic_hw_locked(void)
1689 {
1690 u64 x86_arch_cap_msr;
1691 u64 msr;
1692
1693 x86_arch_cap_msr = x86_read_arch_cap_
1694 if (x86_arch_cap_msr & ARCH_CAP_XAPIC
1695 rdmsrl(MSR_IA32_XAPIC_DISABLE
1696 return (msr & LEGACY_XAPIC_DI
1697 }
1698 return false;
1699 }
1700
1701 static void __x2apic_disable(void)
1702 {
1703 u64 msr;
1704
1705 if (!boot_cpu_has(X86_FEATURE_APIC))
1706 return;
1707
1708 rdmsrl(MSR_IA32_APICBASE, msr);
1709 if (!(msr & X2APIC_ENABLE))
1710 return;
1711 /* Disable xapic and x2apic first and
1712 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2A
1713 wrmsrl(MSR_IA32_APICBASE, msr & ~X2AP
1714 printk_once(KERN_INFO "x2apic disable
1715 }
1716
1717 static void __x2apic_enable(void)
1718 {
1719 u64 msr;
1720
1721 rdmsrl(MSR_IA32_APICBASE, msr);
1722 if (msr & X2APIC_ENABLE)
1723 return;
1724 wrmsrl(MSR_IA32_APICBASE, msr | X2API
1725 printk_once(KERN_INFO "x2apic enabled
1726 }
1727
1728 static int __init setup_nox2apic(char *str)
1729 {
1730 if (x2apic_enabled()) {
1731 u32 apicid = native_apic_msr_
1732
1733 if (apicid >= 255) {
1734 pr_warn("Apicid: %08x
1735 apicid);
1736 return 0;
1737 }
1738 if (x2apic_hw_locked()) {
1739 pr_warn("APIC locked
1740 return 0;
1741 }
1742 pr_warn("x2apic already enabl
1743 __x2apic_disable();
1744 }
1745 setup_clear_cpu_cap(X86_FEATURE_X2API
1746 x2apic_state = X2APIC_DISABLED;
1747 x2apic_mode = 0;
1748 return 0;
1749 }
1750 early_param("nox2apic", setup_nox2apic);
1751
1752 /* Called from cpu_init() to enable x2apic on
1753 void x2apic_setup(void)
1754 {
1755 /*
1756 * Try to make the AP's APIC state ma
1757 * BSP is unlocked and the AP is lock
1758 * Warn about the mismatch in case a
1759 * trying to be turned off.
1760 */
1761 if (x2apic_state != X2APIC_ON_LOCKED
1762 pr_warn("x2apic lock mismatch
1763 /*
1764 * If x2apic is not in ON or LOCKED s
1765 * from BIOS.
1766 */
1767 if (x2apic_state < X2APIC_ON) {
1768 __x2apic_disable();
1769 return;
1770 }
1771 __x2apic_enable();
1772 }
1773
1774 static __init void apic_set_fixmap(bool read_
1775
1776 static __init void x2apic_disable(void)
1777 {
1778 u32 x2apic_id, state = x2apic_state;
1779
1780 x2apic_mode = 0;
1781 x2apic_state = X2APIC_DISABLED;
1782
1783 if (state != X2APIC_ON)
1784 return;
1785
1786 x2apic_id = read_apic_id();
1787 if (x2apic_id >= 255)
1788 panic("Cannot disable x2apic,
1789
1790 if (x2apic_hw_locked()) {
1791 pr_warn("Cannot disable locke
1792 return;
1793 }
1794
1795 __x2apic_disable();
1796 /*
1797 * Don't reread the APIC ID as it was
1798 * check_x2apic() and the APIC driver
1799 * which fails to do the read after x
1800 */
1801 apic_set_fixmap(false);
1802 }
1803
1804 static __init void x2apic_enable(void)
1805 {
1806 if (x2apic_state != X2APIC_OFF)
1807 return;
1808
1809 x2apic_mode = 1;
1810 x2apic_state = X2APIC_ON;
1811 __x2apic_enable();
1812 }
1813
1814 static __init void try_to_enable_x2apic(int r
1815 {
1816 if (x2apic_state == X2APIC_DISABLED)
1817 return;
1818
1819 if (remap_mode != IRQ_REMAP_X2APIC_MO
1820 u32 apic_limit = 255;
1821
1822 /*
1823 * Using X2APIC without IR is
1824 * on bare metal but may be s
1825 */
1826 if (!x86_init.hyper.x2apic_av
1827 pr_info("x2apic: IRQ
1828 x2apic_disable();
1829 return;
1830 }
1831
1832 /*
1833 * If the hypervisor supports
1834 * MSI, that increases the ma
1835 * used for non-remapped IRQ
1836 */
1837 if (x86_init.hyper.msi_ext_de
1838 virt_ext_dest_id = 1;
1839 apic_limit = 32767;
1840 }
1841
1842 /*
1843 * Without IR, all CPUs can b
1844 * in physical mode, and CPUs
1845 * be addressed must not be b
1846 */
1847 x2apic_set_max_apicid(apic_li
1848 x2apic_phys = 1;
1849 }
1850 x2apic_enable();
1851 }
1852
1853 void __init check_x2apic(void)
1854 {
1855 if (x2apic_enabled()) {
1856 pr_info("x2apic: enabled by B
1857 x2apic_mode = 1;
1858 if (x2apic_hw_locked())
1859 x2apic_state = X2APIC
1860 else
1861 x2apic_state = X2APIC
1862 apic_read_boot_cpu_id(true);
1863 } else if (!boot_cpu_has(X86_FEATURE_
1864 x2apic_state = X2APIC_DISABLE
1865 }
1866 }
1867 #else /* CONFIG_X86_X2APIC */
1868 void __init check_x2apic(void)
1869 {
1870 if (!apic_is_x2apic_enabled())
1871 return;
1872 /*
1873 * Checkme: Can we simply turn off x2
1874 */
1875 pr_err("Kernel does not support x2API
1876 pr_err("Disabling APIC, expect reduce
1877
1878 apic_is_disabled = true;
1879 setup_clear_cpu_cap(X86_FEATURE_APIC)
1880 }
1881
1882 static inline void try_to_enable_x2apic(int r
1883 static inline void __x2apic_enable(void) { }
1884 #endif /* !CONFIG_X86_X2APIC */
1885
1886 void __init enable_IR_x2apic(void)
1887 {
1888 unsigned long flags;
1889 int ret, ir_stat;
1890
1891 if (ioapic_is_disabled) {
1892 pr_info("Not enabling interru
1893 return;
1894 }
1895
1896 ir_stat = irq_remapping_prepare();
1897 if (ir_stat < 0 && !x2apic_supported(
1898 return;
1899
1900 ret = save_ioapic_entries();
1901 if (ret) {
1902 pr_info("Saving IO-APIC state
1903 return;
1904 }
1905
1906 local_irq_save(flags);
1907 legacy_pic->mask_all();
1908 mask_ioapic_entries();
1909
1910 /* If irq_remapping_prepare() succeed
1911 if (ir_stat >= 0)
1912 ir_stat = irq_remapping_enabl
1913 /* ir_stat contains the remap mode or
1914 try_to_enable_x2apic(ir_stat);
1915
1916 if (ir_stat < 0)
1917 restore_ioapic_entries();
1918 legacy_pic->restore_mask();
1919 local_irq_restore(flags);
1920 }
1921
1922 #ifdef CONFIG_X86_64
1923 /*
1924 * Detect and enable local APICs on non-SMP b
1925 * Original code written by Keir Fraser.
1926 * On AMD64 we trust the BIOS - if it says no
1927 * not correctly set up (usually the APIC tim
1928 */
1929 static bool __init detect_init_APIC(void)
1930 {
1931 if (!boot_cpu_has(X86_FEATURE_APIC))
1932 pr_info("No local APIC presen
1933 return false;
1934 }
1935
1936 register_lapic_address(APIC_DEFAULT_P
1937 return true;
1938 }
1939 #else
1940
1941 static bool __init apic_verify(unsigned long
1942 {
1943 u32 features, h, l;
1944
1945 /*
1946 * The APIC feature bit should now be
1947 * in `cpuid'
1948 */
1949 features = cpuid_edx(1);
1950 if (!(features & (1 << X86_FEATURE_AP
1951 pr_warn("Could not enable API
1952 return false;
1953 }
1954 set_cpu_cap(&boot_cpu_data, X86_FEATU
1955
1956 /* The BIOS may have set up the APIC
1957 if (boot_cpu_data.x86 >= 6) {
1958 rdmsr(MSR_IA32_APICBASE, l, h
1959 if (l & MSR_IA32_APICBASE_ENA
1960 addr = l & MSR_IA32_A
1961 }
1962
1963 register_lapic_address(addr);
1964 pr_info("Found and enabled local APIC
1965 return true;
1966 }
1967
1968 bool __init apic_force_enable(unsigned long a
1969 {
1970 u32 h, l;
1971
1972 if (apic_is_disabled)
1973 return false;
1974
1975 /*
1976 * Some BIOSes disable the local APIC
1977 * MSR. This can only be done in soft
1978 * and AMD K7 (Model > 1) or later.
1979 */
1980 if (boot_cpu_data.x86 >= 6) {
1981 rdmsr(MSR_IA32_APICBASE, l, h
1982 if (!(l & MSR_IA32_APICBASE_E
1983 pr_info("Local APIC d
1984 l &= ~MSR_IA32_APICBA
1985 l |= MSR_IA32_APICBAS
1986 wrmsr(MSR_IA32_APICBA
1987 enabled_via_apicbase
1988 }
1989 }
1990 return apic_verify(addr);
1991 }
1992
1993 /*
1994 * Detect and initialize APIC
1995 */
1996 static bool __init detect_init_APIC(void)
1997 {
1998 /* Disabled by kernel option? */
1999 if (apic_is_disabled)
2000 return false;
2001
2002 switch (boot_cpu_data.x86_vendor) {
2003 case X86_VENDOR_AMD:
2004 if ((boot_cpu_data.x86 == 6 &
2005 (boot_cpu_data.x86 >= 15)
2006 break;
2007 goto no_apic;
2008 case X86_VENDOR_HYGON:
2009 break;
2010 case X86_VENDOR_INTEL:
2011 if (boot_cpu_data.x86 == 6 ||
2012 (boot_cpu_data.x86 == 5 &
2013 break;
2014 goto no_apic;
2015 default:
2016 goto no_apic;
2017 }
2018
2019 if (!boot_cpu_has(X86_FEATURE_APIC))
2020 /*
2021 * Over-ride BIOS and try to
2022 * "lapic" specified.
2023 */
2024 if (!force_enable_local_apic)
2025 pr_info("Local APIC d
2026 "you can enab
2027 return false;
2028 }
2029 if (!apic_force_enable(APIC_D
2030 return false;
2031 } else {
2032 if (!apic_verify(APIC_DEFAULT
2033 return false;
2034 }
2035
2036 apic_pm_activate();
2037
2038 return true;
2039
2040 no_apic:
2041 pr_info("No local APIC present or har
2042 return false;
2043 }
2044 #endif
2045
2046 /**
2047 * init_apic_mappings - initialize APIC mappi
2048 */
2049 void __init init_apic_mappings(void)
2050 {
2051 if (apic_validate_deadline_timer())
2052 pr_info("TSC deadline timer a
2053
2054 if (x2apic_mode)
2055 return;
2056
2057 if (!smp_found_config) {
2058 if (!detect_init_APIC()) {
2059 pr_info("APIC: disabl
2060 apic_disable();
2061 }
2062 }
2063 }
2064
2065 static __init void apic_set_fixmap(bool read_
2066 {
2067 set_fixmap_nocache(FIX_APIC_BASE, mp_
2068 apic_mmio_base = APIC_BASE;
2069 apic_printk(APIC_VERBOSE, "mapped API
2070 apic_mmio_base, mp_lapic_
2071 if (read_apic)
2072 apic_read_boot_cpu_id(false);
2073 }
2074
2075 void __init register_lapic_address(unsigned l
2076 {
2077 /* This should only happen once */
2078 WARN_ON_ONCE(mp_lapic_addr);
2079 mp_lapic_addr = address;
2080
2081 if (!x2apic_mode)
2082 apic_set_fixmap(true);
2083 }
2084
2085 /*
2086 * Local APIC interrupts
2087 */
2088
2089 /*
2090 * Common handling code for spurious_interrup
2091 * points below. No point in allowing the com
2092 */
2093 static noinline void handle_spurious_interrup
2094 {
2095 u32 v;
2096
2097 trace_spurious_apic_entry(vector);
2098
2099 inc_irq_stat(irq_spurious_count);
2100
2101 /*
2102 * If this is a spurious interrupt th
2103 */
2104 if (vector == SPURIOUS_APIC_VECTOR) {
2105 /* See SDM vol 3 */
2106 pr_info("Spurious APIC interr
2107 smp_processor_id());
2108 goto out;
2109 }
2110
2111 /*
2112 * If it is a vectored one, verify it
2113 * acknowledge it.
2114 */
2115 v = apic_read(APIC_ISR + ((vector & ~
2116 if (v & (1 << (vector & 0x1f))) {
2117 pr_info("Spurious interrupt (
2118 vector, smp_processor
2119 apic_eoi();
2120 } else {
2121 pr_info("Spurious interrupt (
2122 vector, smp_processor
2123 }
2124 out:
2125 trace_spurious_apic_exit(vector);
2126 }
2127
2128 /**
2129 * spurious_interrupt - Catch all for interru
2130 * @regs: Pointer to pt_regs on stack
2131 * @vector: The vector number
2132 *
2133 * This is invoked from ASM entry code to cat
2134 * trigger on an entry which is routed to the
2135 * point.
2136 */
2137 DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2138 {
2139 handle_spurious_interrupt(vector);
2140 }
2141
2142 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_i
2143 {
2144 handle_spurious_interrupt(SPURIOUS_AP
2145 }
2146
2147 /*
2148 * This interrupt should never happen with ou
2149 */
2150 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt
2151 {
2152 static const char * const error_inter
2153 "Send CS error",
2154 "Receive CS error",
2155 "Send accept error",
2156 "Receive accept error",
2157 "Redirectable IPI",
2158 "Send illegal vector",
2159 "Received illegal vector",
2160 "Illegal register address",
2161 };
2162 u32 v, i = 0;
2163
2164 trace_error_apic_entry(ERROR_APIC_VEC
2165
2166 /* First tickle the hardware, only th
2167 if (lapic_get_maxlvt() > 3) /* Du
2168 apic_write(APIC_ESR, 0);
2169 v = apic_read(APIC_ESR);
2170 apic_eoi();
2171 atomic_inc(&irq_err_count);
2172
2173 apic_printk(APIC_DEBUG, KERN_DEBUG "A
2174 smp_processor_id(), v);
2175
2176 v &= 0xff;
2177 while (v) {
2178 if (v & 0x1)
2179 apic_printk(APIC_DEBU
2180 i++;
2181 v >>= 1;
2182 }
2183
2184 apic_printk(APIC_DEBUG, KERN_CONT "\n
2185
2186 trace_error_apic_exit(ERROR_APIC_VECT
2187 }
2188
2189 /**
2190 * connect_bsp_APIC - attach the APIC to the
2191 */
2192 static void __init connect_bsp_APIC(void)
2193 {
2194 #ifdef CONFIG_X86_32
2195 if (pic_mode) {
2196 /*
2197 * Do not trust the local API
2198 */
2199 clear_local_APIC();
2200 /*
2201 * PIC mode, enable APIC mode
2202 * local APIC to INT and NMI
2203 */
2204 apic_printk(APIC_VERBOSE, "le
2205 "enabling API
2206 imcr_pic_to_apic();
2207 }
2208 #endif
2209 }
2210
2211 /**
2212 * disconnect_bsp_APIC - detach the APIC from
2213 * @virt_wire_setup: indicates, whether vi
2214 *
2215 * Virtual wire mode is necessary to deliver
2216 * APIC is disabled.
2217 */
2218 void disconnect_bsp_APIC(int virt_wire_setup)
2219 {
2220 unsigned int value;
2221
2222 #ifdef CONFIG_X86_32
2223 if (pic_mode) {
2224 /*
2225 * Put the board back into PI
2226 * certain older boards). No
2227 * IPIs, won't work beyond th
2228 * INIT IPIs.
2229 */
2230 apic_printk(APIC_VERBOSE, "di
2231 "entering PIC
2232 imcr_apic_to_pic();
2233 return;
2234 }
2235 #endif
2236
2237 /* Go back to Virtual Wire compatibil
2238
2239 /* For the spurious interrupt use vec
2240 value = apic_read(APIC_SPIV);
2241 value &= ~APIC_VECTOR_MASK;
2242 value |= APIC_SPIV_APIC_ENABLED;
2243 value |= 0xf;
2244 apic_write(APIC_SPIV, value);
2245
2246 if (!virt_wire_setup) {
2247 /*
2248 * For LVT0 make it edge trig
2249 * external and enabled
2250 */
2251 value = apic_read(APIC_LVT0);
2252 value &= ~(APIC_MODE_MASK | A
2253 APIC_INPUT_POLARITY |
2254 APIC_LVT_LEVEL_TRIGGE
2255 value |= APIC_LVT_REMOTE_IRR
2256 value = SET_APIC_DELIVERY_MOD
2257 apic_write(APIC_LVT0, value);
2258 } else {
2259 /* Disable LVT0 */
2260 apic_write(APIC_LVT0, APIC_LV
2261 }
2262
2263 /*
2264 * For LVT1 make it edge triggered, a
2265 * nmi and enabled
2266 */
2267 value = apic_read(APIC_LVT1);
2268 value &= ~(APIC_MODE_MASK | APIC_SEND
2269 APIC_INPUT_POLARITY |
2270 APIC_LVT_LEVEL_TRIGGE
2271 value |= APIC_LVT_REMOTE_IRR | APIC_S
2272 value = SET_APIC_DELIVERY_MODE(value,
2273 apic_write(APIC_LVT1, value);
2274 }
2275
2276 void __irq_msi_compose_msg(struct irq_cfg *cf
2277 bool dmar)
2278 {
2279 memset(msg, 0, sizeof(*msg));
2280
2281 msg->arch_addr_lo.base_address = X86_
2282 msg->arch_addr_lo.dest_mode_logical =
2283 msg->arch_addr_lo.destid_0_7 = cfg->d
2284
2285 msg->arch_data.delivery_mode = APIC_D
2286 msg->arch_data.vector = cfg->vector;
2287
2288 msg->address_hi = X86_MSI_BASE_ADDRES
2289 /*
2290 * Only the IOMMU itself can use the
2291 * APIC ID into the high bits of the
2292 * just be writing to memory if it tr
2293 * address APICs which can't be addre
2294 * address range at 0xFFExxxxx. That
2295 * some hypervisors allow the extende
2296 * 5-11 to be used, giving support fo
2297 */
2298 if (dmar)
2299 msg->arch_addr_hi.destid_8_31
2300 else if (virt_ext_dest_id && cfg->des
2301 msg->arch_addr_lo.virt_destid
2302 else
2303 WARN_ON_ONCE(cfg->dest_apicid
2304 }
2305
2306 u32 x86_msi_msg_get_destid(struct msi_msg *ms
2307 {
2308 u32 dest = msg->arch_addr_lo.destid_0
2309
2310 if (extid)
2311 dest |= msg->arch_addr_hi.des
2312 return dest;
2313 }
2314 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2315
2316 static void __init apic_bsp_up_setup(void)
2317 {
2318 reset_phys_cpu_present_map(boot_cpu_p
2319 }
2320
2321 /**
2322 * apic_bsp_setup - Setup function for local
2323 * @upmode: Force UP mode (for AP
2324 */
2325 static void __init apic_bsp_setup(bool upmode
2326 {
2327 connect_bsp_APIC();
2328 if (upmode)
2329 apic_bsp_up_setup();
2330 setup_local_APIC();
2331
2332 enable_IO_APIC();
2333 end_local_APIC_setup();
2334 irq_remap_enable_fault_handling();
2335 setup_IO_APIC();
2336 lapic_update_legacy_vectors();
2337 }
2338
2339 #ifdef CONFIG_UP_LATE_INIT
2340 void __init up_late_init(void)
2341 {
2342 if (apic_intr_mode == APIC_PIC)
2343 return;
2344
2345 /* Setup local timer */
2346 x86_init.timers.setup_percpu_clockev(
2347 }
2348 #endif
2349
2350 /*
2351 * Power management
2352 */
2353 #ifdef CONFIG_PM
2354
2355 static struct {
2356 /*
2357 * 'active' is true if the local APIC
2358 * not the BIOS; this signifies that
2359 * for disabling it before entering a
2360 */
2361 int active;
2362 /* r/w apic fields */
2363 u32 apic_id;
2364 unsigned int apic_taskpri;
2365 unsigned int apic_ldr;
2366 unsigned int apic_dfr;
2367 unsigned int apic_spiv;
2368 unsigned int apic_lvtt;
2369 unsigned int apic_lvtpc;
2370 unsigned int apic_lvt0;
2371 unsigned int apic_lvt1;
2372 unsigned int apic_lvterr;
2373 unsigned int apic_tmict;
2374 unsigned int apic_tdcr;
2375 unsigned int apic_thmr;
2376 unsigned int apic_cmci;
2377 } apic_pm_state;
2378
2379 static int lapic_suspend(void)
2380 {
2381 unsigned long flags;
2382 int maxlvt;
2383
2384 if (!apic_pm_state.active)
2385 return 0;
2386
2387 maxlvt = lapic_get_maxlvt();
2388
2389 apic_pm_state.apic_id = apic_read(API
2390 apic_pm_state.apic_taskpri = apic_rea
2391 apic_pm_state.apic_ldr = apic_read(AP
2392 apic_pm_state.apic_dfr = apic_read(AP
2393 apic_pm_state.apic_spiv = apic_read(A
2394 apic_pm_state.apic_lvtt = apic_read(A
2395 if (maxlvt >= 4)
2396 apic_pm_state.apic_lvtpc = ap
2397 apic_pm_state.apic_lvt0 = apic_read(A
2398 apic_pm_state.apic_lvt1 = apic_read(A
2399 apic_pm_state.apic_lvterr = apic_read
2400 apic_pm_state.apic_tmict = apic_read(
2401 apic_pm_state.apic_tdcr = apic_read(A
2402 #ifdef CONFIG_X86_THERMAL_VECTOR
2403 if (maxlvt >= 5)
2404 apic_pm_state.apic_thmr = api
2405 #endif
2406 #ifdef CONFIG_X86_MCE_INTEL
2407 if (maxlvt >= 6)
2408 apic_pm_state.apic_cmci = api
2409 #endif
2410
2411 local_irq_save(flags);
2412
2413 /*
2414 * Mask IOAPIC before disabling the l
2415 * entries on some implementations.
2416 */
2417 mask_ioapic_entries();
2418
2419 disable_local_APIC();
2420
2421 irq_remapping_disable();
2422
2423 local_irq_restore(flags);
2424 return 0;
2425 }
2426
2427 static void lapic_resume(void)
2428 {
2429 unsigned int l, h;
2430 unsigned long flags;
2431 int maxlvt;
2432
2433 if (!apic_pm_state.active)
2434 return;
2435
2436 local_irq_save(flags);
2437
2438 /*
2439 * IO-APIC and PIC have their own res
2440 * We just mask them here to make sur
2441 * subsystem is completely quiet whil
2442 * and interrupt-remapping.
2443 */
2444 mask_ioapic_entries();
2445 legacy_pic->mask_all();
2446
2447 if (x2apic_mode) {
2448 __x2apic_enable();
2449 } else {
2450 /*
2451 * Make sure the APICBASE poi
2452 *
2453 * FIXME! This will be wrong
2454 * SMP! We'll need to do this
2455 */
2456 if (boot_cpu_data.x86 >= 6) {
2457 rdmsr(MSR_IA32_APICBA
2458 l &= ~MSR_IA32_APICBA
2459 l |= MSR_IA32_APICBAS
2460 wrmsr(MSR_IA32_APICBA
2461 }
2462 }
2463
2464 maxlvt = lapic_get_maxlvt();
2465 apic_write(APIC_LVTERR, ERROR_APIC_VE
2466 apic_write(APIC_ID, apic_pm_state.api
2467 apic_write(APIC_DFR, apic_pm_state.ap
2468 apic_write(APIC_LDR, apic_pm_state.ap
2469 apic_write(APIC_TASKPRI, apic_pm_stat
2470 apic_write(APIC_SPIV, apic_pm_state.a
2471 apic_write(APIC_LVT0, apic_pm_state.a
2472 apic_write(APIC_LVT1, apic_pm_state.a
2473 #ifdef CONFIG_X86_THERMAL_VECTOR
2474 if (maxlvt >= 5)
2475 apic_write(APIC_LVTTHMR, apic
2476 #endif
2477 #ifdef CONFIG_X86_MCE_INTEL
2478 if (maxlvt >= 6)
2479 apic_write(APIC_LVTCMCI, apic
2480 #endif
2481 if (maxlvt >= 4)
2482 apic_write(APIC_LVTPC, apic_p
2483 apic_write(APIC_LVTT, apic_pm_state.a
2484 apic_write(APIC_TDCR, apic_pm_state.a
2485 apic_write(APIC_TMICT, apic_pm_state.
2486 apic_write(APIC_ESR, 0);
2487 apic_read(APIC_ESR);
2488 apic_write(APIC_LVTERR, apic_pm_state
2489 apic_write(APIC_ESR, 0);
2490 apic_read(APIC_ESR);
2491
2492 irq_remapping_reenable(x2apic_mode);
2493
2494 local_irq_restore(flags);
2495 }
2496
2497 /*
2498 * This device has no shutdown method - fully
2499 * are needed on every CPU up until machine_h
2500 */
2501
2502 static struct syscore_ops lapic_syscore_ops =
2503 .resume = lapic_resume,
2504 .suspend = lapic_suspend,
2505 };
2506
2507 static void apic_pm_activate(void)
2508 {
2509 apic_pm_state.active = 1;
2510 }
2511
2512 static int __init init_lapic_sysfs(void)
2513 {
2514 /* XXX: remove suspend/resume procs i
2515 if (boot_cpu_has(X86_FEATURE_APIC))
2516 register_syscore_ops(&lapic_s
2517
2518 return 0;
2519 }
2520
2521 /* local apic needs to resume before other de
2522 core_initcall(init_lapic_sysfs);
2523
2524 #else /* CONFIG_PM */
2525
2526 static void apic_pm_activate(void) { }
2527
2528 #endif /* CONFIG_PM */
2529
2530 #ifdef CONFIG_X86_64
2531
2532 static int multi_checked;
2533 static int multi;
2534
2535 static int set_multi(const struct dmi_system_
2536 {
2537 if (multi)
2538 return 0;
2539 pr_info("APIC: %s detected, Multi Cha
2540 multi = 1;
2541 return 0;
2542 }
2543
2544 static const struct dmi_system_id multi_dmi_t
2545 {
2546 .callback = set_multi,
2547 .ident = "IBM System Summit2"
2548 .matches = {
2549 DMI_MATCH(DMI_SYS_VEN
2550 DMI_MATCH(DMI_PRODUCT
2551 },
2552 },
2553 {}
2554 };
2555
2556 static void dmi_check_multi(void)
2557 {
2558 if (multi_checked)
2559 return;
2560
2561 dmi_check_system(multi_dmi_table);
2562 multi_checked = 1;
2563 }
2564
2565 /*
2566 * apic_is_clustered_box() -- Check if we can
2567 *
2568 * Thus far, the major user of this is IBM's
2569 * Clustered boxes may have unsynced TSC prob
2570 * multi-chassis.
2571 * Use DMI to check them
2572 */
2573 int apic_is_clustered_box(void)
2574 {
2575 dmi_check_multi();
2576 return multi;
2577 }
2578 #endif
2579
2580 /*
2581 * APIC command line parameters
2582 */
2583 static int __init setup_disableapic(char *arg
2584 {
2585 apic_is_disabled = true;
2586 setup_clear_cpu_cap(X86_FEATURE_APIC)
2587 return 0;
2588 }
2589 early_param("disableapic", setup_disableapic)
2590
2591 /* same as disableapic, for compatibility */
2592 static int __init setup_nolapic(char *arg)
2593 {
2594 return setup_disableapic(arg);
2595 }
2596 early_param("nolapic", setup_nolapic);
2597
2598 static int __init parse_lapic_timer_c2_ok(cha
2599 {
2600 local_apic_timer_c2_ok = 1;
2601 return 0;
2602 }
2603 early_param("lapic_timer_c2_ok", parse_lapic_
2604
2605 static int __init parse_disable_apic_timer(ch
2606 {
2607 disable_apic_timer = 1;
2608 return 0;
2609 }
2610 early_param("noapictimer", parse_disable_apic
2611
2612 static int __init parse_nolapic_timer(char *a
2613 {
2614 disable_apic_timer = 1;
2615 return 0;
2616 }
2617 early_param("nolapic_timer", parse_nolapic_ti
2618
2619 static int __init apic_set_verbosity(char *ar
2620 {
2621 if (!arg) {
2622 if (IS_ENABLED(CONFIG_X86_32)
2623 return -EINVAL;
2624
2625 ioapic_is_disabled = false;
2626 return 0;
2627 }
2628
2629 if (strcmp("debug", arg) == 0)
2630 apic_verbosity = APIC_DEBUG;
2631 else if (strcmp("verbose", arg) == 0)
2632 apic_verbosity = APIC_VERBOSE
2633 #ifdef CONFIG_X86_64
2634 else {
2635 pr_warn("APIC Verbosity level
2636 " use apic=verbose or
2637 return -EINVAL;
2638 }
2639 #endif
2640
2641 return 0;
2642 }
2643 early_param("apic", apic_set_verbosity);
2644
2645 static int __init lapic_insert_resource(void)
2646 {
2647 if (!apic_mmio_base)
2648 return -1;
2649
2650 /* Put local APIC into the resource m
2651 lapic_resource.start = apic_mmio_base
2652 lapic_resource.end = lapic_resource.s
2653 insert_resource(&iomem_resource, &lap
2654
2655 return 0;
2656 }
2657
2658 /*
2659 * need call insert after e820__reserve_resou
2660 * that is using request_resource
2661 */
2662 late_initcall(lapic_insert_resource);
2663
2664 static int __init apic_set_extnmi(char *arg)
2665 {
2666 if (!arg)
2667 return -EINVAL;
2668
2669 if (!strncmp("all", arg, 3))
2670 apic_extnmi = APIC_EXTNMI_ALL
2671 else if (!strncmp("none", arg, 4))
2672 apic_extnmi = APIC_EXTNMI_NON
2673 else if (!strncmp("bsp", arg, 3))
2674 apic_extnmi = APIC_EXTNMI_BSP
2675 else {
2676 pr_warn("Unknown external NMI
2677 return -EINVAL;
2678 }
2679
2680 return 0;
2681 }
2682 early_param("apic_extnmi", apic_set_extnmi);
2683
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.