1 // SPDX-License-Identifier: GPL-2.0-only 1
2 /*
3 * Local APIC handling, local APIC timers
4 *
5 * (c) 1999, 2000, 2009 Ingo Molnar <ming
6 *
7 * Fixes
8 * Maciej W. Rozycki : Bits f
9 * thanks
10 * and Ro
11 * for te
12 * Maciej W. Rozycki : Variou
13 * Mikael Pettersson : Power
14 * Pavel Machek and
15 * Mikael Pettersson : PM con
16 */
17
18 #include <linux/perf_event.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/mc146818rtc.h>
21 #include <linux/acpi_pmtmr.h>
22 #include <linux/bitmap.h>
23 #include <linux/clockchips.h>
24 #include <linux/interrupt.h>
25 #include <linux/memblock.h>
26 #include <linux/ftrace.h>
27 #include <linux/ioport.h>
28 #include <linux/export.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/delay.h>
31 #include <linux/timex.h>
32 #include <linux/i8253.h>
33 #include <linux/dmar.h>
34 #include <linux/init.h>
35 #include <linux/cpu.h>
36 #include <linux/dmi.h>
37 #include <linux/smp.h>
38 #include <linux/mm.h>
39
40 #include <xen/xen.h>
41
42 #include <asm/trace/irq_vectors.h>
43 #include <asm/irq_remapping.h>
44 #include <asm/pc-conf-reg.h>
45 #include <asm/perf_event.h>
46 #include <asm/x86_init.h>
47 #include <linux/atomic.h>
48 #include <asm/barrier.h>
49 #include <asm/mpspec.h>
50 #include <asm/i8259.h>
51 #include <asm/proto.h>
52 #include <asm/traps.h>
53 #include <asm/apic.h>
54 #include <asm/acpi.h>
55 #include <asm/io_apic.h>
56 #include <asm/desc.h>
57 #include <asm/hpet.h>
58 #include <asm/mtrr.h>
59 #include <asm/time.h>
60 #include <asm/smp.h>
61 #include <asm/mce.h>
62 #include <asm/tsc.h>
63 #include <asm/hypervisor.h>
64 #include <asm/cpu_device_id.h>
65 #include <asm/intel-family.h>
66 #include <asm/irq_regs.h>
67 #include <asm/cpu.h>
68
69 #include "local.h"
70
71 /* Processor that is doing the boot up */
72 u32 boot_cpu_physical_apicid __ro_after_init =
73 EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
74
75 u8 boot_cpu_apic_version __ro_after_init;
76
77 /*
78 * This variable controls which CPUs receive e
79 * external NMIs are delivered only to the BSP
80 */
81 static int apic_extnmi __ro_after_init = APIC_
82
83 /*
84 * Hypervisor supports 15 bits of APIC ID in M
85 */
86 static bool virt_ext_dest_id __ro_after_init;
87
88 /* For parallel bootup. */
89 unsigned long apic_mmio_base __ro_after_init;
90
91 static inline bool apic_accessible(void)
92 {
93 return x2apic_mode || apic_mmio_base;
94 }
95
96 #ifdef CONFIG_X86_32
97 /* Local APIC was disabled by the BIOS and ena
98 static int enabled_via_apicbase __ro_after_ini
99
100 /*
101 * Handle interrupt mode configuration registe
102 * This register controls whether the interrup
103 * that reach the BSP come from the master PIC
104 * local APIC. Before entering Symmetric I/O M
105 * the BIOS or the operating system must switc
106 * PIC Mode by changing the IMCR.
107 */
108 static inline void imcr_pic_to_apic(void)
109 {
110 /* NMI and 8259 INTR go through APIC *
111 pc_conf_set(PC_CONF_MPS_IMCR, 0x01);
112 }
113
114 static inline void imcr_apic_to_pic(void)
115 {
116 /* NMI and 8259 INTR go directly to BS
117 pc_conf_set(PC_CONF_MPS_IMCR, 0x00);
118 }
119 #endif
120
121 /*
122 * Knob to control our willingness to enable t
123 *
124 * +1=force-enable
125 */
126 static int force_enable_local_apic __initdata;
127
128 /*
129 * APIC command line parameters
130 */
131 static int __init parse_lapic(char *arg)
132 {
133 if (IS_ENABLED(CONFIG_X86_32) && !arg)
134 force_enable_local_apic = 1;
135 else if (arg && !strncmp(arg, "notscde
136 setup_clear_cpu_cap(X86_FEATUR
137 return 0;
138 }
139 early_param("lapic", parse_lapic);
140
141 #ifdef CONFIG_X86_64
142 static int apic_calibrate_pmtmr __initdata;
143 static __init int setup_apicpmtimer(char *s)
144 {
145 apic_calibrate_pmtmr = 1;
146 notsc_setup(NULL);
147 return 1;
148 }
149 __setup("apicpmtimer", setup_apicpmtimer);
150 #endif
151
152 static unsigned long mp_lapic_addr __ro_after_
153 bool apic_is_disabled __ro_after_init;
154 /* Disable local APIC timer from the kernel co
155 static int disable_apic_timer __initdata;
156 /* Local APIC timer works in C2 */
157 int local_apic_timer_c2_ok __ro_after_init;
158 EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
159
160 /*
161 * Debug level, exported for io_apic.c
162 */
163 int apic_verbosity __ro_after_init;
164
165 int pic_mode __ro_after_init;
166
167 /* Have we found an MP table */
168 int smp_found_config __ro_after_init;
169
170 static struct resource lapic_resource = {
171 .name = "Local APIC",
172 .flags = IORESOURCE_MEM | IORESOURCE_B
173 };
174
175 unsigned int lapic_timer_period = 0;
176
177 static void apic_pm_activate(void);
178
179 /*
180 * Get the LAPIC version
181 */
182 static inline int lapic_get_version(void)
183 {
184 return GET_APIC_VERSION(apic_read(APIC
185 }
186
187 /*
188 * Check, if the APIC is integrated or a separ
189 */
190 static inline int lapic_is_integrated(void)
191 {
192 return APIC_INTEGRATED(lapic_get_versi
193 }
194
195 /*
196 * Check, whether this is a modern or a first
197 */
198 static int modern_apic(void)
199 {
200 /* AMD systems use old APIC versions,
201 if (boot_cpu_data.x86_vendor == X86_VE
202 boot_cpu_data.x86 >= 0xf)
203 return 1;
204
205 /* Hygon systems use modern APIC */
206 if (boot_cpu_data.x86_vendor == X86_VE
207 return 1;
208
209 return lapic_get_version() >= 0x14;
210 }
211
212 /*
213 * right after this call apic become NOOP driv
214 * so apic->write/read doesn't do anything
215 */
216 static void __init apic_disable(void)
217 {
218 apic_install_driver(&apic_noop);
219 }
220
221 void native_apic_icr_write(u32 low, u32 id)
222 {
223 unsigned long flags;
224
225 local_irq_save(flags);
226 apic_write(APIC_ICR2, SET_XAPIC_DEST_F
227 apic_write(APIC_ICR, low);
228 local_irq_restore(flags);
229 }
230
231 u64 native_apic_icr_read(void)
232 {
233 u32 icr1, icr2;
234
235 icr2 = apic_read(APIC_ICR2);
236 icr1 = apic_read(APIC_ICR);
237
238 return icr1 | ((u64)icr2 << 32);
239 }
240
241 /**
242 * lapic_get_maxlvt - get the maximum number o
243 */
244 int lapic_get_maxlvt(void)
245 {
246 /*
247 * - we always have APIC integrated on
248 * - 82489DXs do not report # of LVT e
249 */
250 return lapic_is_integrated() ? GET_API
251 }
252
253 /*
254 * Local APIC timer
255 */
256
257 /* Clock divisor */
258 #define APIC_DIVISOR 16
259 #define TSC_DIVISOR 8
260
261 /* i82489DX specific */
262 #define I82489DX_BASE_DIVIDER
263
264 /*
265 * This function sets up the local APIC timer,
266 * 'clocks' APIC bus clock. During calibration
267 * this function twice on the boot CPU, once w
268 * value, second time for real. The other (non
269 * call this function only once, with the real
270 *
271 * We do reads before writes even if unnecessa
272 * P5 APIC double write bug.
273 */
274 static void __setup_APIC_LVTT(unsigned int clo
275 {
276 unsigned int lvtt_value, tmp_value;
277
278 lvtt_value = LOCAL_TIMER_VECTOR;
279 if (!oneshot)
280 lvtt_value |= APIC_LVT_TIMER_P
281 else if (boot_cpu_has(X86_FEATURE_TSC_
282 lvtt_value |= APIC_LVT_TIMER_T
283
284 /*
285 * The i82489DX APIC uses bit 18 and 1
286 * overlaps with bit 18 on integrated
287 * in the SDM. No problem though. i824
288 * have TSC deadline timer.
289 */
290 if (!lapic_is_integrated())
291 lvtt_value |= I82489DX_BASE_DI
292
293 if (!irqen)
294 lvtt_value |= APIC_LVT_MASKED;
295
296 apic_write(APIC_LVTT, lvtt_value);
297
298 if (lvtt_value & APIC_LVT_TIMER_TSCDEA
299 /*
300 * See Intel SDM: TSC-Deadline
301 * writing to the APIC LVTT an
302 * According to Intel, MFENCE
303 */
304 asm volatile("mfence" : : : "m
305 return;
306 }
307
308 /*
309 * Divide PICLK by 16
310 */
311 tmp_value = apic_read(APIC_TDCR);
312 apic_write(APIC_TDCR,
313 (tmp_value & ~(APIC_TDR_DIV_1
314 APIC_TDR_DIV_16);
315
316 if (!oneshot)
317 apic_write(APIC_TMICT, clocks
318 }
319
320 /*
321 * Setup extended LVT, AMD specific
322 *
323 * Software should use the LVT offsets the BIO
324 * are determined by the subsystems using it l
325 * threshold or IBS. On K8 only offset 0 (API
326 * are supported. Beginning with family 10h at
327 * available.
328 *
329 * Since the offsets must be consistent for al
330 * of the LVT offsets in software and reserve
331 * vector also to be used on other cores. An o
332 * setting the entry to APIC_EILVT_MASKED.
333 *
334 * If the BIOS is right, there should be no co
335 * "[Firmware Bug]: ..." error message is gene
336 * software does not properly determines the o
337 * necessarily a BIOS bug.
338 */
339
340 static atomic_t eilvt_offsets[APIC_EILVT_NR_MA
341
342 static inline int eilvt_entry_is_changeable(un
343 {
344 return (old & APIC_EILVT_MASKED)
345 || (new == APIC_EILVT_MASKED)
346 || ((new & ~APIC_EILVT_MASKED)
347 }
348
349 static unsigned int reserve_eilvt_offset(int o
350 {
351 unsigned int rsvd, vector;
352
353 if (offset >= APIC_EILVT_NR_MAX)
354 return ~0;
355
356 rsvd = atomic_read(&eilvt_offsets[offs
357 do {
358 vector = rsvd & ~APIC_EILVT_MA
359 if (vector && !eilvt_entry_is_
360 /* may not change if v
361 return rsvd;
362 } while (!atomic_try_cmpxchg(&eilvt_of
363
364 rsvd = new & ~APIC_EILVT_MASKED;
365 if (rsvd && rsvd != vector)
366 pr_info("LVT offset %d assigne
367 offset, rsvd);
368
369 return new;
370 }
371
372 /*
373 * If mask=1, the LVT entry does not generate
374 * enables the vector. See also the BKDGs. Mus
375 * preemption disabled.
376 */
377
378 int setup_APIC_eilvt(u8 offset, u8 vector, u8
379 {
380 unsigned long reg = APIC_EILVTn(offset
381 unsigned int new, old, reserved;
382
383 new = (mask << 16) | (msg_type << 8) |
384 old = apic_read(reg);
385 reserved = reserve_eilvt_offset(offset
386
387 if (reserved != new) {
388 pr_err(FW_BUG "cpu %d, try to
389 "vector 0x%x, but the r
390 "vector 0x%x on another
391 smp_processor_id(), reg
392 return -EINVAL;
393 }
394
395 if (!eilvt_entry_is_changeable(old, ne
396 pr_err(FW_BUG "cpu %d, try to
397 "vector 0x%x, but the r
398 "vector 0x%x on this cp
399 smp_processor_id(), reg
400 return -EBUSY;
401 }
402
403 apic_write(reg, new);
404
405 return 0;
406 }
407 EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
408
409 /*
410 * Program the next event, relative to now
411 */
412 static int lapic_next_event(unsigned long delt
413 struct clock_event
414 {
415 apic_write(APIC_TMICT, delta);
416 return 0;
417 }
418
419 static int lapic_next_deadline(unsigned long d
420 struct clock_ev
421 {
422 u64 tsc;
423
424 /* This MSR is special and need a spec
425 weak_wrmsr_fence();
426
427 tsc = rdtsc();
428 wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + ((
429 return 0;
430 }
431
432 static int lapic_timer_shutdown(struct clock_e
433 {
434 unsigned int v;
435
436 /* Lapic used as dummy for broadcast ?
437 if (evt->features & CLOCK_EVT_FEAT_DUM
438 return 0;
439
440 v = apic_read(APIC_LVTT);
441 v |= (APIC_LVT_MASKED | LOCAL_TIMER_VE
442 apic_write(APIC_LVTT, v);
443
444 /*
445 * Setting APIC_LVT_MASKED (above) sho
446 * the hardware that this timer will n
447 * erratum 411 and some Intel CPU beha
448 * otherwise. Time for belt and suspe
449 * the timer _and_ zero the counter re
450 */
451 if (v & APIC_LVT_TIMER_TSCDEADLINE)
452 wrmsrl(MSR_IA32_TSC_DEADLINE,
453 else
454 apic_write(APIC_TMICT, 0);
455
456 return 0;
457 }
458
459 static inline int
460 lapic_timer_set_periodic_oneshot(struct clock_
461 {
462 /* Lapic used as dummy for broadcast ?
463 if (evt->features & CLOCK_EVT_FEAT_DUM
464 return 0;
465
466 __setup_APIC_LVTT(lapic_timer_period,
467 return 0;
468 }
469
470 static int lapic_timer_set_periodic(struct clo
471 {
472 return lapic_timer_set_periodic_onesho
473 }
474
475 static int lapic_timer_set_oneshot(struct cloc
476 {
477 return lapic_timer_set_periodic_onesho
478 }
479
480 /*
481 * Local APIC timer broadcast function
482 */
483 static void lapic_timer_broadcast(const struct
484 {
485 #ifdef CONFIG_SMP
486 __apic_send_IPI_mask(mask, LOCAL_TIMER
487 #endif
488 }
489
490
491 /*
492 * The local apic timer can be used for any fu
493 */
494 static struct clock_event_device lapic_clockev
495 .name = "lap
496 .features = CLOC
497 CLOC
498 | CL
499 .shift = 32,
500 .set_state_shutdown = lapi
501 .set_state_periodic = lapi
502 .set_state_oneshot = lapi
503 .set_state_oneshot_stopped = lapi
504 .set_next_event = lapi
505 .broadcast = lapi
506 .rating = 100,
507 .irq = -1,
508 };
509 static DEFINE_PER_CPU(struct clock_event_devic
510
511 static const struct x86_cpu_id deadline_match[
512 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_
513 X86_MATCH_VFM_STEPPINGS(INTEL_HASWELL_
514
515 X86_MATCH_VFM(INTEL_BROADWELL_X,
516
517 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
518 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
519 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
520 X86_MATCH_VFM_STEPPINGS(INTEL_BROADWEL
521
522 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
523 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
524 X86_MATCH_VFM_STEPPINGS(INTEL_SKYLAKE_
525
526 X86_MATCH_VFM(INTEL_HASWELL,
527 X86_MATCH_VFM(INTEL_HASWELL_L,
528 X86_MATCH_VFM(INTEL_HASWELL_G,
529
530 X86_MATCH_VFM(INTEL_BROADWELL,
531 X86_MATCH_VFM(INTEL_BROADWELL_G,
532
533 X86_MATCH_VFM(INTEL_SKYLAKE_L,
534 X86_MATCH_VFM(INTEL_SKYLAKE,
535
536 X86_MATCH_VFM(INTEL_KABYLAKE_L,
537 X86_MATCH_VFM(INTEL_KABYLAKE,
538
539 {},
540 };
541
542 static __init bool apic_validate_deadline_time
543 {
544 const struct x86_cpu_id *m;
545 u32 rev;
546
547 if (!boot_cpu_has(X86_FEATURE_TSC_DEAD
548 return false;
549 if (boot_cpu_has(X86_FEATURE_HYPERVISO
550 return true;
551
552 m = x86_match_cpu(deadline_match);
553 if (!m)
554 return true;
555
556 rev = (u32)m->driver_data;
557
558 if (boot_cpu_data.microcode >= rev)
559 return true;
560
561 setup_clear_cpu_cap(X86_FEATURE_TSC_DE
562 pr_err(FW_BUG "TSC_DEADLINE disabled d
563 "please update microcode to ver
564 return false;
565 }
566
567 /*
568 * Setup the local APIC timer for this CPU. Co
569 * of the boot CPU and register the clock even
570 */
571 static void setup_APIC_timer(void)
572 {
573 struct clock_event_device *levt = this
574
575 if (this_cpu_has(X86_FEATURE_ARAT)) {
576 lapic_clockevent.features &= ~
577 /* Make LAPIC timer preferable
578 lapic_clockevent.rating = 150;
579 }
580
581 memcpy(levt, &lapic_clockevent, sizeof
582 levt->cpumask = cpumask_of(smp_process
583
584 if (this_cpu_has(X86_FEATURE_TSC_DEADL
585 levt->name = "lapic-deadline";
586 levt->features &= ~(CLOCK_EVT_
587 CLOCK_EVT_
588 levt->set_next_event = lapic_n
589 clockevents_config_and_registe
590
591
592 } else
593 clockevents_register_device(le
594 }
595
596 /*
597 * Install the updated TSC frequency from reca
598 * deadline clockevent devices.
599 */
600 static void __lapic_update_tsc_freq(void *info
601 {
602 struct clock_event_device *levt = this
603
604 if (!this_cpu_has(X86_FEATURE_TSC_DEAD
605 return;
606
607 clockevents_update_freq(levt, tsc_khz
608 }
609
610 void lapic_update_tsc_freq(void)
611 {
612 /*
613 * The clockevent device's ->mult and
614 * changed. In order to avoid races, s
615 * update code on each CPU.
616 */
617 on_each_cpu(__lapic_update_tsc_freq, N
618 }
619
620 /*
621 * In this functions we calibrate APIC bus clo
622 *
623 * We want to do the calibration only once sin
624 * irqs synchronous. CPUs connected by the sam
625 * frequency.
626 *
627 * This was previously done by reading the PIT
628 * around to find out, that a tick has elapsed
629 * readout is broken, so it never gets out of
630 * also reported by others.
631 *
632 * Monitoring the jiffies value is inaccurate
633 * infrastructure allows us to do a simple sub
634 * handler.
635 *
636 * The calibration routine also uses the pm_ti
637 * happens to run way too slow (factor 2.3 on
638 * back to normal later in the boot process).
639 */
640
641 #define LAPIC_CAL_LOOPS (HZ/10)
642
643 static __initdata int lapic_cal_loops = -1;
644 static __initdata long lapic_cal_t1, lapic_cal
645 static __initdata unsigned long long lapic_cal
646 static __initdata u32 lapic_cal_pm1, lapic_cal
647 static __initdata unsigned long lapic_cal_j1,
648
649 /*
650 * Temporary interrupt handler and polled cali
651 */
652 static void __init lapic_cal_handler(struct cl
653 {
654 unsigned long long tsc = 0;
655 long tapic = apic_read(APIC_TMCCT);
656 u32 pm = acpi_pm_read_early();
657
658 if (boot_cpu_has(X86_FEATURE_TSC))
659 tsc = rdtsc();
660
661 switch (lapic_cal_loops++) {
662 case 0:
663 lapic_cal_t1 = tapic;
664 lapic_cal_tsc1 = tsc;
665 lapic_cal_pm1 = pm;
666 lapic_cal_j1 = jiffies;
667 break;
668
669 case LAPIC_CAL_LOOPS:
670 lapic_cal_t2 = tapic;
671 lapic_cal_tsc2 = tsc;
672 if (pm < lapic_cal_pm1)
673 pm += ACPI_PM_OVRRUN;
674 lapic_cal_pm2 = pm;
675 lapic_cal_j2 = jiffies;
676 break;
677 }
678 }
679
680 static int __init
681 calibrate_by_pmtimer(u32 deltapm, long *delta,
682 {
683 const long pm_100ms = PMTMR_TICKS_PER_
684 const long pm_thresh = pm_100ms / 100;
685 unsigned long mult;
686 u64 res;
687
688 #ifndef CONFIG_X86_PM_TIMER
689 return -1;
690 #endif
691
692 apic_printk(APIC_VERBOSE, "... PM-Time
693
694 /* Check, if the PM timer is available
695 if (!deltapm)
696 return -1;
697
698 mult = clocksource_hz2mult(PMTMR_TICKS
699
700 if (deltapm > (pm_100ms - pm_thresh) &
701 deltapm < (pm_100ms + pm_thresh))
702 apic_printk(APIC_VERBOSE, "...
703 return 0;
704 }
705
706 res = (((u64)deltapm) * mult) >> 22;
707 do_div(res, 1000000);
708 pr_warn("APIC calibration not consiste
709 "with PM-Timer: %ldms instead
710
711 /* Correct the lapic counter value */
712 res = (((u64)(*delta)) * pm_100ms);
713 do_div(res, deltapm);
714 pr_info("APIC delta adjusted to PM-Tim
715 "%lu (%ld)\n", (unsigned long)
716 *delta = (long)res;
717
718 /* Correct the tsc counter value */
719 if (boot_cpu_has(X86_FEATURE_TSC)) {
720 res = (((u64)(*deltatsc)) * pm
721 do_div(res, deltapm);
722 apic_printk(APIC_VERBOSE, "TSC
723 "PM-
724 (unsig
725 *deltatsc = (long)res;
726 }
727
728 return 0;
729 }
730
731 static int __init lapic_init_clockevent(void)
732 {
733 if (!lapic_timer_period)
734 return -1;
735
736 /* Calculate the scaled math multiplic
737 lapic_clockevent.mult = div_sc(lapic_t
738 TICK_N
739 lapic_clockevent.max_delta_ns =
740 clockevent_delta2ns(0x7FFFFFFF
741 lapic_clockevent.max_delta_ticks = 0x7
742 lapic_clockevent.min_delta_ns =
743 clockevent_delta2ns(0xF, &lapi
744 lapic_clockevent.min_delta_ticks = 0xF
745
746 return 0;
747 }
748
749 bool __init apic_needs_pit(void)
750 {
751 /*
752 * If the frequencies are not known, P
753 * and apic timer calibration.
754 */
755 if (!tsc_khz || !cpu_khz)
756 return true;
757
758 /* Is there an APIC at all or is it di
759 if (!boot_cpu_has(X86_FEATURE_APIC) ||
760 return true;
761
762 /*
763 * If interrupt delivery mode is legac
764 * configuration, the local APIC timer
765 * that the PIT is initialized.
766 */
767 if (apic_intr_mode == APIC_PIC ||
768 apic_intr_mode == APIC_VIRTUAL_WIR
769 return true;
770
771 /* Virt guests may lack ARAT, but stil
772 if (!boot_cpu_has(X86_FEATURE_ARAT))
773 return true;
774
775 /* Deadline timer is based on TSC so n
776 if (boot_cpu_has(X86_FEATURE_TSC_DEADL
777 return false;
778
779 /* APIC timer disabled? */
780 if (disable_apic_timer)
781 return true;
782 /*
783 * The APIC timer frequency is known a
784 * required. If unknown, let the PIT b
785 */
786 return lapic_timer_period == 0;
787 }
788
789 static int __init calibrate_APIC_clock(void)
790 {
791 struct clock_event_device *levt = this
792 u64 tsc_perj = 0, tsc_start = 0;
793 unsigned long jif_start;
794 unsigned long deltaj;
795 long delta, deltatsc;
796 int pm_referenced = 0;
797
798 if (boot_cpu_has(X86_FEATURE_TSC_DEADL
799 return 0;
800
801 /*
802 * Check if lapic timer has already be
803 * specific routine, such as tsc calib
804 * in the clockevent structure and ret
805 */
806 if (!lapic_init_clockevent()) {
807 apic_printk(APIC_VERBOSE, "lap
808 lapic_timer_period
809 /*
810 * Direct calibration methods
811 * local APIC timer, no need f
812 */
813 lapic_clockevent.features &= ~
814 return 0;
815 }
816
817 apic_printk(APIC_VERBOSE, "Using local
818 "calibrating APIC timer ..
819
820 /*
821 * There are platforms w/o global cloc
822 * making the calibration conditional
823 * approach everywhere.
824 */
825 local_irq_disable();
826
827 /*
828 * Setup the APIC counter to maximum.
829 * can underflow in the 100ms detectio
830 */
831 __setup_APIC_LVTT(0xffffffff, 0, 0);
832
833 /*
834 * Methods to terminate the calibratio
835 * 1) Global clockevent if available
836 * 2) TSC if available and frequency
837 */
838 jif_start = READ_ONCE(jiffies);
839
840 if (tsc_khz) {
841 tsc_start = rdtsc();
842 tsc_perj = div_u64((u64)tsc_kh
843 }
844
845 /*
846 * Enable interrupts so the tick can f
847 * clockevent device is available
848 */
849 local_irq_enable();
850
851 while (lapic_cal_loops <= LAPIC_CAL_LO
852 /* Wait for a tick to elapse *
853 while (1) {
854 if (tsc_khz) {
855 u64 tsc_now =
856 if ((tsc_now -
857 tsc_st
858 break;
859 }
860 } else {
861 unsigned long
862
863 if (time_after
864 jif_st
865 break;
866 }
867 }
868 cpu_relax();
869 }
870
871 /* Invoke the calibration rout
872 local_irq_disable();
873 lapic_cal_handler(NULL);
874 local_irq_enable();
875 }
876
877 local_irq_disable();
878
879 /* Build delta t1-t2 as apic timer cou
880 delta = lapic_cal_t1 - lapic_cal_t2;
881 apic_printk(APIC_VERBOSE, "... lapic d
882
883 deltatsc = (long)(lapic_cal_tsc2 - lap
884
885 /* we trust the PM based calibration i
886 pm_referenced = !calibrate_by_pmtimer(
887 &delta
888
889 lapic_timer_period = (delta * APIC_DIV
890 lapic_init_clockevent();
891
892 apic_printk(APIC_VERBOSE, "..... delta
893 apic_printk(APIC_VERBOSE, "..... mult:
894 apic_printk(APIC_VERBOSE, "..... calib
895 lapic_timer_period);
896
897 if (boot_cpu_has(X86_FEATURE_TSC)) {
898 apic_printk(APIC_VERBOSE, "...
899 "%ld.%04ld MHz.\n"
900 (deltatsc / LAPIC_
901 (deltatsc / LAPIC_
902 }
903
904 apic_printk(APIC_VERBOSE, "..... host
905 "%u.%04u MHz.\n",
906 lapic_timer_period / (1000
907 lapic_timer_period % (1000
908
909 /*
910 * Do a sanity check on the APIC calib
911 */
912 if (lapic_timer_period < (1000000 / HZ
913 local_irq_enable();
914 pr_warn("APIC frequency too sl
915 return -1;
916 }
917
918 levt->features &= ~CLOCK_EVT_FEAT_DUMM
919
920 /*
921 * PM timer calibration failed or not
922 * timer based calibration, if a globa
923 * available.
924 */
925 if (!pm_referenced && global_clock_eve
926 apic_printk(APIC_VERBOSE, "...
927
928 /*
929 * Setup the apic timer manual
930 */
931 levt->event_handler = lapic_ca
932 lapic_timer_set_periodic(levt)
933 lapic_cal_loops = -1;
934
935 /* Let the interrupts run */
936 local_irq_enable();
937
938 while (lapic_cal_loops <= LAPI
939 cpu_relax();
940
941 /* Stop the lapic timer */
942 local_irq_disable();
943 lapic_timer_shutdown(levt);
944
945 /* Jiffies delta */
946 deltaj = lapic_cal_j2 - lapic_
947 apic_printk(APIC_VERBOSE, "...
948
949 /* Check, if the jiffies resul
950 if (deltaj >= LAPIC_CAL_LOOPS-
951 apic_printk(APIC_VERBO
952 else
953 levt->features |= CLOC
954 }
955 local_irq_enable();
956
957 if (levt->features & CLOCK_EVT_FEAT_DU
958 pr_warn("APIC timer disabled d
959 return -1;
960 }
961
962 return 0;
963 }
964
965 /*
966 * Setup the boot APIC
967 *
968 * Calibrate and verify the result.
969 */
970 void __init setup_boot_APIC_clock(void)
971 {
972 /*
973 * The local apic timer can be disable
974 * commandline or from the CPU detecti
975 * timer as a dummy clock event source
976 * broadcast mechanism is used. On UP
977 */
978 if (disable_apic_timer) {
979 pr_info("Disabling APIC timer\
980 /* No broadcast on UP ! */
981 if (num_possible_cpus() > 1) {
982 lapic_clockevent.mult
983 setup_APIC_timer();
984 }
985 return;
986 }
987
988 if (calibrate_APIC_clock()) {
989 /* No broadcast on UP ! */
990 if (num_possible_cpus() > 1)
991 setup_APIC_timer();
992 return;
993 }
994
995 /*
996 * If nmi_watchdog is set to IO_APIC,
997 * PIT/HPET going. Otherwise register
998 * device.
999 */
1000 lapic_clockevent.features &= ~CLOCK_E
1001
1002 /* Setup the lapic or request the bro
1003 setup_APIC_timer();
1004 amd_e400_c1e_apic_setup();
1005 }
1006
1007 void setup_secondary_APIC_clock(void)
1008 {
1009 setup_APIC_timer();
1010 amd_e400_c1e_apic_setup();
1011 }
1012
1013 /*
1014 * The guts of the apic timer interrupt
1015 */
1016 static void local_apic_timer_interrupt(void)
1017 {
1018 struct clock_event_device *evt = this
1019
1020 /*
1021 * Normally we should not be here til
1022 * in some cases like kdump, its poss
1023 * timer interrupt from previous kern
1024 * new kernel the moment interrupts a
1025 *
1026 * Interrupts are enabled early and L
1027 * its possible that when we get here
1028 * Check for event_handler being NULL
1029 * spurious.
1030 */
1031 if (!evt->event_handler) {
1032 pr_warn("Spurious LAPIC timer
1033 smp_processor_id());
1034 /* Switch it off */
1035 lapic_timer_shutdown(evt);
1036 return;
1037 }
1038
1039 /*
1040 * the NMI deadlock-detector uses thi
1041 */
1042 inc_irq_stat(apic_timer_irqs);
1043
1044 evt->event_handler(evt);
1045 }
1046
1047 /*
1048 * Local APIC timer interrupt. This is the mo
1049 * local interrupts, but local timer interrup
1050 * broadcast interrupts too. [in case the hw
1051 *
1052 * [ if a single-CPU system runs an SMP kerne
1053 * interrupt as well. Thus we cannot inline
1054 */
1055 DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_inte
1056 {
1057 struct pt_regs *old_regs = set_irq_re
1058
1059 apic_eoi();
1060 trace_local_timer_entry(LOCAL_TIMER_V
1061 local_apic_timer_interrupt();
1062 trace_local_timer_exit(LOCAL_TIMER_VE
1063
1064 set_irq_regs(old_regs);
1065 }
1066
1067 /*
1068 * Local APIC start and shutdown
1069 */
1070
1071 /**
1072 * clear_local_APIC - shutdown the local APIC
1073 *
1074 * This is called, when a CPU is disabled and
1075 * the local APIC has no dangling leftovers.
1076 * leftovers during boot.
1077 */
1078 void clear_local_APIC(void)
1079 {
1080 int maxlvt;
1081 u32 v;
1082
1083 if (!apic_accessible())
1084 return;
1085
1086 maxlvt = lapic_get_maxlvt();
1087 /*
1088 * Masking an LVT entry can trigger a
1089 * if the vector is zero. Mask LVTERR
1090 */
1091 if (maxlvt >= 3) {
1092 v = ERROR_APIC_VECTOR; /* any
1093 apic_write(APIC_LVTERR, v | A
1094 }
1095 /*
1096 * Careful: we have to set masks only
1097 * any level-triggered sources.
1098 */
1099 v = apic_read(APIC_LVTT);
1100 apic_write(APIC_LVTT, v | APIC_LVT_MA
1101 v = apic_read(APIC_LVT0);
1102 apic_write(APIC_LVT0, v | APIC_LVT_MA
1103 v = apic_read(APIC_LVT1);
1104 apic_write(APIC_LVT1, v | APIC_LVT_MA
1105 if (maxlvt >= 4) {
1106 v = apic_read(APIC_LVTPC);
1107 apic_write(APIC_LVTPC, v | AP
1108 }
1109
1110 /* lets not touch this if we didn't f
1111 #ifdef CONFIG_X86_THERMAL_VECTOR
1112 if (maxlvt >= 5) {
1113 v = apic_read(APIC_LVTTHMR);
1114 apic_write(APIC_LVTTHMR, v |
1115 }
1116 #endif
1117 #ifdef CONFIG_X86_MCE_INTEL
1118 if (maxlvt >= 6) {
1119 v = apic_read(APIC_LVTCMCI);
1120 if (!(v & APIC_LVT_MASKED))
1121 apic_write(APIC_LVTCM
1122 }
1123 #endif
1124
1125 /*
1126 * Clean APIC state for other OSs:
1127 */
1128 apic_write(APIC_LVTT, APIC_LVT_MASKED
1129 apic_write(APIC_LVT0, APIC_LVT_MASKED
1130 apic_write(APIC_LVT1, APIC_LVT_MASKED
1131 if (maxlvt >= 3)
1132 apic_write(APIC_LVTERR, APIC_
1133 if (maxlvt >= 4)
1134 apic_write(APIC_LVTPC, APIC_L
1135
1136 /* Integrated APIC (!82489DX) ? */
1137 if (lapic_is_integrated()) {
1138 if (maxlvt > 3)
1139 /* Clear ESR due to P
1140 apic_write(APIC_ESR,
1141 apic_read(APIC_ESR);
1142 }
1143 }
1144
1145 /**
1146 * apic_soft_disable - Clears and software di
1147 *
1148 * Contrary to disable_local_APIC() this does
1149 * MSR_IA32_APICBASE. Clearing that bit on sy
1150 * bus would require a hardware reset as the
1151 * arbitration. On systems with FSB delivery
1152 * but it has to be guaranteed that no interr
1153 * in that state and it's not clear from the
1154 * to INIT/SIPI messages. Stay on the safe si
1155 */
1156 void apic_soft_disable(void)
1157 {
1158 u32 value;
1159
1160 clear_local_APIC();
1161
1162 /* Soft disable APIC (implies clearin
1163 value = apic_read(APIC_SPIV);
1164 value &= ~APIC_SPIV_APIC_ENABLED;
1165 apic_write(APIC_SPIV, value);
1166 }
1167
1168 /**
1169 * disable_local_APIC - clear and disable the
1170 */
1171 void disable_local_APIC(void)
1172 {
1173 if (!apic_accessible())
1174 return;
1175
1176 apic_soft_disable();
1177
1178 #ifdef CONFIG_X86_32
1179 /*
1180 * When LAPIC was disabled by the BIO
1181 * restore the disabled state.
1182 */
1183 if (enabled_via_apicbase) {
1184 unsigned int l, h;
1185
1186 rdmsr(MSR_IA32_APICBASE, l, h
1187 l &= ~MSR_IA32_APICBASE_ENABL
1188 wrmsr(MSR_IA32_APICBASE, l, h
1189 }
1190 #endif
1191 }
1192
1193 /*
1194 * If Linux enabled the LAPIC against the BIO
1195 * re-entering the BIOS on shutdown. Otherwi
1196 * not power-off. Additionally clear all LVT
1197 * for the case where Linux didn't enable the
1198 */
1199 void lapic_shutdown(void)
1200 {
1201 unsigned long flags;
1202
1203 if (!boot_cpu_has(X86_FEATURE_APIC) &
1204 return;
1205
1206 local_irq_save(flags);
1207
1208 #ifdef CONFIG_X86_32
1209 if (!enabled_via_apicbase)
1210 clear_local_APIC();
1211 else
1212 #endif
1213 disable_local_APIC();
1214
1215
1216 local_irq_restore(flags);
1217 }
1218
1219 /**
1220 * sync_Arb_IDs - synchronize APIC bus arbitr
1221 */
1222 void __init sync_Arb_IDs(void)
1223 {
1224 /*
1225 * Unsupported on P4 - see Intel Dev.
1226 * needed on AMD.
1227 */
1228 if (modern_apic() || boot_cpu_data.x8
1229 return;
1230
1231 /*
1232 * Wait for idle.
1233 */
1234 apic_wait_icr_idle();
1235
1236 apic_printk(APIC_DEBUG, "Synchronizin
1237 apic_write(APIC_ICR, APIC_DEST_ALLINC
1238 APIC_INT_LEVELTRIG |
1239 }
1240
1241 enum apic_intr_mode_id apic_intr_mode __ro_af
1242
1243 static int __init __apic_intr_mode_select(voi
1244 {
1245 /* Check kernel option */
1246 if (apic_is_disabled) {
1247 pr_info("APIC disabled via ke
1248 return APIC_PIC;
1249 }
1250
1251 /* Check BIOS */
1252 #ifdef CONFIG_X86_64
1253 /* On 64-bit, the APIC must be integr
1254 if (!boot_cpu_has(X86_FEATURE_APIC))
1255 apic_is_disabled = true;
1256 pr_info("APIC disabled by BIO
1257 return APIC_PIC;
1258 }
1259 #else
1260 /* On 32-bit, the APIC may be integra
1261
1262 /* Neither 82489DX nor integrated API
1263 if (!boot_cpu_has(X86_FEATURE_APIC) &
1264 apic_is_disabled = true;
1265 return APIC_PIC;
1266 }
1267
1268 /* If the BIOS pretends there is an i
1269 if (!boot_cpu_has(X86_FEATURE_APIC) &
1270 APIC_INTEGRATED(boot_cpu_apic
1271 apic_is_disabled = true;
1272 pr_err(FW_BUG "Local APIC not
1273 return APIC_PIC;
1274 }
1275 #endif
1276
1277 /* Check MP table or ACPI MADT config
1278 if (!smp_found_config) {
1279 disable_ioapic_support();
1280 if (!acpi_lapic) {
1281 pr_info("APIC: ACPI M
1282 return APIC_VIRTUAL_W
1283 }
1284 return APIC_VIRTUAL_WIRE;
1285 }
1286
1287 #ifdef CONFIG_SMP
1288 /* If SMP should be disabled, then re
1289 if (!setup_max_cpus) {
1290 pr_info("APIC: SMP mode deact
1291 return APIC_SYMMETRIC_IO_NO_R
1292 }
1293 #endif
1294
1295 return APIC_SYMMETRIC_IO;
1296 }
1297
1298 /* Select the interrupt delivery mode for the
1299 void __init apic_intr_mode_select(void)
1300 {
1301 apic_intr_mode = __apic_intr_mode_sel
1302 }
1303
1304 /*
1305 * An initial setup of the virtual wire mode.
1306 */
1307 void __init init_bsp_APIC(void)
1308 {
1309 unsigned int value;
1310
1311 /*
1312 * Don't do the setup now if we have
1313 * through-I/O-APIC virtual wire mode
1314 */
1315 if (smp_found_config || !boot_cpu_has
1316 return;
1317
1318 /*
1319 * Do not trust the local APIC being
1320 */
1321 clear_local_APIC();
1322
1323 /*
1324 * Enable APIC.
1325 */
1326 value = apic_read(APIC_SPIV);
1327 value &= ~APIC_VECTOR_MASK;
1328 value |= APIC_SPIV_APIC_ENABLED;
1329
1330 #ifdef CONFIG_X86_32
1331 /* This bit is reserved on P4/Xeon an
1332 if ((boot_cpu_data.x86_vendor == X86_
1333 (boot_cpu_data.x86 == 15))
1334 value &= ~APIC_SPIV_FOCUS_DIS
1335 else
1336 #endif
1337 value |= APIC_SPIV_FOCUS_DISA
1338 value |= SPURIOUS_APIC_VECTOR;
1339 apic_write(APIC_SPIV, value);
1340
1341 /*
1342 * Set up the virtual wire mode.
1343 */
1344 apic_write(APIC_LVT0, APIC_DM_EXTINT)
1345 value = APIC_DM_NMI;
1346 if (!lapic_is_integrated())
1347 value |= APIC_LVT_LEVEL_TRIGG
1348 if (apic_extnmi == APIC_EXTNMI_NONE)
1349 value |= APIC_LVT_MASKED;
1350 apic_write(APIC_LVT1, value);
1351 }
1352
1353 static void __init apic_bsp_setup(bool upmode
1354
1355 /* Init the interrupt delivery mode for the B
1356 void __init apic_intr_mode_init(void)
1357 {
1358 bool upmode = IS_ENABLED(CONFIG_UP_LA
1359
1360 switch (apic_intr_mode) {
1361 case APIC_PIC:
1362 pr_info("APIC: Keep in PIC mo
1363 return;
1364 case APIC_VIRTUAL_WIRE:
1365 pr_info("APIC: Switch to virt
1366 break;
1367 case APIC_VIRTUAL_WIRE_NO_CONFIG:
1368 pr_info("APIC: Switch to virt
1369 upmode = true;
1370 break;
1371 case APIC_SYMMETRIC_IO:
1372 pr_info("APIC: Switch to symm
1373 break;
1374 case APIC_SYMMETRIC_IO_NO_ROUTING:
1375 pr_info("APIC: Switch to symm
1376 break;
1377 }
1378
1379 x86_64_probe_apic();
1380
1381 x86_32_install_bigsmp();
1382
1383 if (x86_platform.apic_post_init)
1384 x86_platform.apic_post_init()
1385
1386 apic_bsp_setup(upmode);
1387 }
1388
1389 static void lapic_setup_esr(void)
1390 {
1391 unsigned int oldvalue, value, maxlvt;
1392
1393 if (!lapic_is_integrated()) {
1394 pr_info("No ESR for 82489DX.\
1395 return;
1396 }
1397
1398 if (apic->disable_esr) {
1399 /*
1400 * Something untraceable is c
1401 * secondary quads ... for th
1402 * ESR disabled - we can't do
1403 * errors anyway - mbligh
1404 */
1405 pr_info("Leaving ESR disabled
1406 return;
1407 }
1408
1409 maxlvt = lapic_get_maxlvt();
1410 if (maxlvt > 3) /* Due to the
1411 apic_write(APIC_ESR, 0);
1412 oldvalue = apic_read(APIC_ESR);
1413
1414 /* enables sending errors */
1415 value = ERROR_APIC_VECTOR;
1416 apic_write(APIC_LVTERR, value);
1417
1418 /*
1419 * spec says clear errors after enabl
1420 */
1421 if (maxlvt > 3)
1422 apic_write(APIC_ESR, 0);
1423 value = apic_read(APIC_ESR);
1424 if (value != oldvalue)
1425 apic_printk(APIC_VERBOSE, "ES
1426 "vector: 0x%08x afte
1427 oldvalue, value);
1428 }
1429
1430 #define APIC_IR_REGS APIC_ISR_NR
1431 #define APIC_IR_BITS (APIC_IR_REGS
1432 #define APIC_IR_MAPSIZE (APIC_IR_BITS
1433
1434 union apic_ir {
1435 unsigned long map[APIC_IR_MAPSIZE];
1436 u32 regs[APIC_IR_REGS];
1437 };
1438
1439 static bool apic_check_and_ack(union apic_ir
1440 {
1441 int i, bit;
1442
1443 /* Read the IRRs */
1444 for (i = 0; i < APIC_IR_REGS; i++)
1445 irr->regs[i] = apic_read(APIC
1446
1447 /* Read the ISRs */
1448 for (i = 0; i < APIC_IR_REGS; i++)
1449 isr->regs[i] = apic_read(APIC
1450
1451 /*
1452 * If the ISR map is not empty. ACK t
1453 * to verify whether a pending IRR ha
1454 * into a ISR.
1455 */
1456 if (!bitmap_empty(isr->map, APIC_IR_B
1457 /*
1458 * There can be multiple ISR
1459 * interrupt preempted a lowe
1460 * per set bit.
1461 */
1462 for_each_set_bit(bit, isr->ma
1463 apic_eoi();
1464 return true;
1465 }
1466
1467 return !bitmap_empty(irr->map, APIC_I
1468 }
1469
1470 /*
1471 * After a crash, we no longer service the in
1472 * interrupt from previous kernel might still
1473 *
1474 * Most probably by now the CPU has serviced
1475 * might not have done the apic_eoi() because
1476 * came from i8259 as ExtInt. LAPIC did not g
1477 * the ISR bit and cpu thinks it has already
1478 * a vector might get locked. It was noticed
1479 * 0x31). Issue an extra EOI to clear ISR.
1480 *
1481 * If there are pending IRR bits they turn in
1482 * priority ISR bit has been acked.
1483 */
1484 static void apic_pending_intr_clear(void)
1485 {
1486 union apic_ir irr, isr;
1487 unsigned int i;
1488
1489 /* 512 loops are way oversized and gi
1490 for (i = 0; i < 512; i++) {
1491 if (!apic_check_and_ack(&irr,
1492 return;
1493 }
1494 /* Dump the IRR/ISR content if that f
1495 pr_warn("APIC: Stale IRR: %256pb ISR:
1496 }
1497
1498 /**
1499 * setup_local_APIC - setup the local APIC
1500 *
1501 * Used to setup local APIC while initializin
1502 * Always called with preemption disabled.
1503 */
1504 static void setup_local_APIC(void)
1505 {
1506 int cpu = smp_processor_id();
1507 unsigned int value;
1508
1509 if (apic_is_disabled) {
1510 disable_ioapic_support();
1511 return;
1512 }
1513
1514 /*
1515 * If this comes from kexec/kcrash th
1516 * SPIV. Soft disable it before doing
1517 */
1518 value = apic_read(APIC_SPIV);
1519 value &= ~APIC_SPIV_APIC_ENABLED;
1520 apic_write(APIC_SPIV, value);
1521
1522 #ifdef CONFIG_X86_32
1523 /* Pound the ESR really hard over the
1524 if (lapic_is_integrated() && apic->di
1525 apic_write(APIC_ESR, 0);
1526 apic_write(APIC_ESR, 0);
1527 apic_write(APIC_ESR, 0);
1528 apic_write(APIC_ESR, 0);
1529 }
1530 #endif
1531 /*
1532 * Intel recommends to set DFR, LDR a
1533 * an APIC. See e.g. "AP-388 82489DX
1534 * document number 292116).
1535 *
1536 * Except for APICs which operate in
1537 */
1538 if (apic->init_apic_ldr)
1539 apic->init_apic_ldr();
1540
1541 /*
1542 * Set Task Priority to 'accept all e
1543 * vector in the 16-31 range could be
1544 * would think it's an exception and
1545 * never change this later on.
1546 */
1547 value = apic_read(APIC_TASKPRI);
1548 value &= ~APIC_TPRI_MASK;
1549 value |= 0x10;
1550 apic_write(APIC_TASKPRI, value);
1551
1552 /* Clear eventually stale ISR/IRR bit
1553 apic_pending_intr_clear();
1554
1555 /*
1556 * Now that we are all set up, enable
1557 */
1558 value = apic_read(APIC_SPIV);
1559 value &= ~APIC_VECTOR_MASK;
1560 /*
1561 * Enable APIC
1562 */
1563 value |= APIC_SPIV_APIC_ENABLED;
1564
1565 #ifdef CONFIG_X86_32
1566 /*
1567 * Some unknown Intel IO/APIC (or API
1568 * certain networking cards. If high
1569 * happening on a particular IOAPIC p
1570 * entry is masked/unmasked at a high
1571 * later IOAPIC line gets 'stuck', no
1572 * from the device. If focus CPU is d
1573 * away, oh well :-(
1574 *
1575 * [ This bug can be reproduced easil
1576 * PCI Ne2000 networking cards and
1577 * BX chipset. ]
1578 */
1579 /*
1580 * Actually disabling the focus CPU c
1581 * frequent as it makes the interrupt
1582 * like LRU than MRU (the short-term
1583 */
1584
1585 /*
1586 * - enable focus processor (bit==0)
1587 * - 64bit mode always use processor
1588 * so no need to set it
1589 */
1590 value &= ~APIC_SPIV_FOCUS_DISABLED;
1591 #endif
1592
1593 /*
1594 * Set spurious IRQ vector
1595 */
1596 value |= SPURIOUS_APIC_VECTOR;
1597 apic_write(APIC_SPIV, value);
1598
1599 perf_events_lapic_init();
1600
1601 /*
1602 * Set up LVT0, LVT1:
1603 *
1604 * set up through-local-APIC on the b
1605 * strictly necessary in pure symmetr
1606 * we delegate interrupts to the 8259
1607 */
1608 /*
1609 * TODO: set up through-local-APIC fr
1610 */
1611 value = apic_read(APIC_LVT0) & APIC_L
1612 if (!cpu && (pic_mode || !value || io
1613 value = APIC_DM_EXTINT;
1614 apic_printk(APIC_VERBOSE, "en
1615 } else {
1616 value = APIC_DM_EXTINT | APIC
1617 apic_printk(APIC_VERBOSE, "ma
1618 }
1619 apic_write(APIC_LVT0, value);
1620
1621 /*
1622 * Only the BSP sees the LINT1 NMI si
1623 * modified by apic_extnmi= boot opti
1624 */
1625 if ((!cpu && apic_extnmi != APIC_EXTN
1626 apic_extnmi == APIC_EXTNMI_ALL)
1627 value = APIC_DM_NMI;
1628 else
1629 value = APIC_DM_NMI | APIC_LV
1630
1631 /* Is 82489DX ? */
1632 if (!lapic_is_integrated())
1633 value |= APIC_LVT_LEVEL_TRIGG
1634 apic_write(APIC_LVT1, value);
1635
1636 #ifdef CONFIG_X86_MCE_INTEL
1637 /* Recheck CMCI information after loc
1638 if (!cpu)
1639 cmci_recheck();
1640 #endif
1641 }
1642
1643 static void end_local_APIC_setup(void)
1644 {
1645 lapic_setup_esr();
1646
1647 #ifdef CONFIG_X86_32
1648 {
1649 unsigned int value;
1650 /* Disable the local apic tim
1651 value = apic_read(APIC_LVTT);
1652 value |= (APIC_LVT_MASKED | L
1653 apic_write(APIC_LVTT, value);
1654 }
1655 #endif
1656
1657 apic_pm_activate();
1658 }
1659
1660 /*
1661 * APIC setup function for application proces
1662 */
1663 void apic_ap_setup(void)
1664 {
1665 setup_local_APIC();
1666 end_local_APIC_setup();
1667 }
1668
1669 static __init void apic_read_boot_cpu_id(bool
1670 {
1671 /*
1672 * This can be invoked from check_x2a
1673 * selected. But that code knows for
1674 * X2APIC.
1675 */
1676 if (x2apic) {
1677 boot_cpu_physical_apicid = na
1678 boot_cpu_apic_version = GET_A
1679 } else {
1680 boot_cpu_physical_apicid = re
1681 boot_cpu_apic_version = GET_A
1682 }
1683 topology_register_boot_apic(boot_cpu_
1684 x86_32_probe_bigsmp_early();
1685 }
1686
1687 #ifdef CONFIG_X86_X2APIC
1688 int x2apic_mode;
1689 EXPORT_SYMBOL_GPL(x2apic_mode);
1690
1691 enum {
1692 X2APIC_OFF,
1693 X2APIC_DISABLED,
1694 /* All states below here have X2APIC
1695 X2APIC_ON,
1696 X2APIC_ON_LOCKED
1697 };
1698 static int x2apic_state;
1699
1700 static bool x2apic_hw_locked(void)
1701 {
1702 u64 x86_arch_cap_msr;
1703 u64 msr;
1704
1705 x86_arch_cap_msr = x86_read_arch_cap_
1706 if (x86_arch_cap_msr & ARCH_CAP_XAPIC
1707 rdmsrl(MSR_IA32_XAPIC_DISABLE
1708 return (msr & LEGACY_XAPIC_DI
1709 }
1710 return false;
1711 }
1712
1713 static void __x2apic_disable(void)
1714 {
1715 u64 msr;
1716
1717 if (!boot_cpu_has(X86_FEATURE_APIC))
1718 return;
1719
1720 rdmsrl(MSR_IA32_APICBASE, msr);
1721 if (!(msr & X2APIC_ENABLE))
1722 return;
1723 /* Disable xapic and x2apic first and
1724 wrmsrl(MSR_IA32_APICBASE, msr & ~(X2A
1725 wrmsrl(MSR_IA32_APICBASE, msr & ~X2AP
1726 printk_once(KERN_INFO "x2apic disable
1727 }
1728
1729 static void __x2apic_enable(void)
1730 {
1731 u64 msr;
1732
1733 rdmsrl(MSR_IA32_APICBASE, msr);
1734 if (msr & X2APIC_ENABLE)
1735 return;
1736 wrmsrl(MSR_IA32_APICBASE, msr | X2API
1737 printk_once(KERN_INFO "x2apic enabled
1738 }
1739
1740 static int __init setup_nox2apic(char *str)
1741 {
1742 if (x2apic_enabled()) {
1743 u32 apicid = native_apic_msr_
1744
1745 if (apicid >= 255) {
1746 pr_warn("Apicid: %08x
1747 apicid);
1748 return 0;
1749 }
1750 if (x2apic_hw_locked()) {
1751 pr_warn("APIC locked
1752 return 0;
1753 }
1754 pr_warn("x2apic already enabl
1755 __x2apic_disable();
1756 }
1757 setup_clear_cpu_cap(X86_FEATURE_X2API
1758 x2apic_state = X2APIC_DISABLED;
1759 x2apic_mode = 0;
1760 return 0;
1761 }
1762 early_param("nox2apic", setup_nox2apic);
1763
1764 /* Called from cpu_init() to enable x2apic on
1765 void x2apic_setup(void)
1766 {
1767 /*
1768 * Try to make the AP's APIC state ma
1769 * BSP is unlocked and the AP is lock
1770 * Warn about the mismatch in case a
1771 * trying to be turned off.
1772 */
1773 if (x2apic_state != X2APIC_ON_LOCKED
1774 pr_warn("x2apic lock mismatch
1775 /*
1776 * If x2apic is not in ON or LOCKED s
1777 * from BIOS.
1778 */
1779 if (x2apic_state < X2APIC_ON) {
1780 __x2apic_disable();
1781 return;
1782 }
1783 __x2apic_enable();
1784 }
1785
1786 static __init void apic_set_fixmap(bool read_
1787
1788 static __init void x2apic_disable(void)
1789 {
1790 u32 x2apic_id;
1791
1792 if (x2apic_state < X2APIC_ON)
1793 return;
1794
1795 x2apic_id = read_apic_id();
1796 if (x2apic_id >= 255)
1797 panic("Cannot disable x2apic,
1798
1799 if (x2apic_hw_locked()) {
1800 pr_warn("Cannot disable locke
1801 return;
1802 }
1803
1804 __x2apic_disable();
1805
1806 x2apic_mode = 0;
1807 x2apic_state = X2APIC_DISABLED;
1808
1809 /*
1810 * Don't reread the APIC ID as it was
1811 * check_x2apic() and the APIC driver
1812 * which fails to do the read after x
1813 */
1814 apic_set_fixmap(false);
1815 }
1816
1817 static __init void x2apic_enable(void)
1818 {
1819 if (x2apic_state != X2APIC_OFF)
1820 return;
1821
1822 x2apic_mode = 1;
1823 x2apic_state = X2APIC_ON;
1824 __x2apic_enable();
1825 }
1826
1827 static __init void try_to_enable_x2apic(int r
1828 {
1829 if (x2apic_state == X2APIC_DISABLED)
1830 return;
1831
1832 if (remap_mode != IRQ_REMAP_X2APIC_MO
1833 u32 apic_limit = 255;
1834
1835 /*
1836 * Using X2APIC without IR is
1837 * on bare metal but may be s
1838 */
1839 if (!x86_init.hyper.x2apic_av
1840 pr_info("x2apic: IRQ
1841 x2apic_disable();
1842 return;
1843 }
1844
1845 /*
1846 * If the hypervisor supports
1847 * MSI, that increases the ma
1848 * used for non-remapped IRQ
1849 */
1850 if (x86_init.hyper.msi_ext_de
1851 virt_ext_dest_id = 1;
1852 apic_limit = 32767;
1853 }
1854
1855 /*
1856 * Without IR, all CPUs can b
1857 * in physical mode, and CPUs
1858 * be addressed must not be b
1859 */
1860 x2apic_set_max_apicid(apic_li
1861 x2apic_phys = 1;
1862 }
1863 x2apic_enable();
1864 }
1865
1866 void __init check_x2apic(void)
1867 {
1868 if (x2apic_enabled()) {
1869 pr_info("x2apic: enabled by B
1870 x2apic_mode = 1;
1871 if (x2apic_hw_locked())
1872 x2apic_state = X2APIC
1873 else
1874 x2apic_state = X2APIC
1875 apic_read_boot_cpu_id(true);
1876 } else if (!boot_cpu_has(X86_FEATURE_
1877 x2apic_state = X2APIC_DISABLE
1878 }
1879 }
1880 #else /* CONFIG_X86_X2APIC */
1881 void __init check_x2apic(void)
1882 {
1883 if (!apic_is_x2apic_enabled())
1884 return;
1885 /*
1886 * Checkme: Can we simply turn off x2
1887 */
1888 pr_err("Kernel does not support x2API
1889 pr_err("Disabling APIC, expect reduce
1890
1891 apic_is_disabled = true;
1892 setup_clear_cpu_cap(X86_FEATURE_APIC)
1893 }
1894
1895 static inline void try_to_enable_x2apic(int r
1896 static inline void __x2apic_enable(void) { }
1897 #endif /* !CONFIG_X86_X2APIC */
1898
1899 void __init enable_IR_x2apic(void)
1900 {
1901 unsigned long flags;
1902 int ret, ir_stat;
1903
1904 if (ioapic_is_disabled) {
1905 pr_info("Not enabling interru
1906 return;
1907 }
1908
1909 ir_stat = irq_remapping_prepare();
1910 if (ir_stat < 0 && !x2apic_supported(
1911 return;
1912
1913 ret = save_ioapic_entries();
1914 if (ret) {
1915 pr_info("Saving IO-APIC state
1916 return;
1917 }
1918
1919 local_irq_save(flags);
1920 legacy_pic->mask_all();
1921 mask_ioapic_entries();
1922
1923 /* If irq_remapping_prepare() succeed
1924 if (ir_stat >= 0)
1925 ir_stat = irq_remapping_enabl
1926 /* ir_stat contains the remap mode or
1927 try_to_enable_x2apic(ir_stat);
1928
1929 if (ir_stat < 0)
1930 restore_ioapic_entries();
1931 legacy_pic->restore_mask();
1932 local_irq_restore(flags);
1933 }
1934
1935 #ifdef CONFIG_X86_64
1936 /*
1937 * Detect and enable local APICs on non-SMP b
1938 * Original code written by Keir Fraser.
1939 * On AMD64 we trust the BIOS - if it says no
1940 * not correctly set up (usually the APIC tim
1941 */
1942 static bool __init detect_init_APIC(void)
1943 {
1944 if (!boot_cpu_has(X86_FEATURE_APIC))
1945 pr_info("No local APIC presen
1946 return false;
1947 }
1948
1949 register_lapic_address(APIC_DEFAULT_P
1950 return true;
1951 }
1952 #else
1953
1954 static bool __init apic_verify(unsigned long
1955 {
1956 u32 features, h, l;
1957
1958 /*
1959 * The APIC feature bit should now be
1960 * in `cpuid'
1961 */
1962 features = cpuid_edx(1);
1963 if (!(features & (1 << X86_FEATURE_AP
1964 pr_warn("Could not enable API
1965 return false;
1966 }
1967 set_cpu_cap(&boot_cpu_data, X86_FEATU
1968
1969 /* The BIOS may have set up the APIC
1970 if (boot_cpu_data.x86 >= 6) {
1971 rdmsr(MSR_IA32_APICBASE, l, h
1972 if (l & MSR_IA32_APICBASE_ENA
1973 addr = l & MSR_IA32_A
1974 }
1975
1976 register_lapic_address(addr);
1977 pr_info("Found and enabled local APIC
1978 return true;
1979 }
1980
1981 bool __init apic_force_enable(unsigned long a
1982 {
1983 u32 h, l;
1984
1985 if (apic_is_disabled)
1986 return false;
1987
1988 /*
1989 * Some BIOSes disable the local APIC
1990 * MSR. This can only be done in soft
1991 * and AMD K7 (Model > 1) or later.
1992 */
1993 if (boot_cpu_data.x86 >= 6) {
1994 rdmsr(MSR_IA32_APICBASE, l, h
1995 if (!(l & MSR_IA32_APICBASE_E
1996 pr_info("Local APIC d
1997 l &= ~MSR_IA32_APICBA
1998 l |= MSR_IA32_APICBAS
1999 wrmsr(MSR_IA32_APICBA
2000 enabled_via_apicbase
2001 }
2002 }
2003 return apic_verify(addr);
2004 }
2005
2006 /*
2007 * Detect and initialize APIC
2008 */
2009 static bool __init detect_init_APIC(void)
2010 {
2011 /* Disabled by kernel option? */
2012 if (apic_is_disabled)
2013 return false;
2014
2015 switch (boot_cpu_data.x86_vendor) {
2016 case X86_VENDOR_AMD:
2017 if ((boot_cpu_data.x86 == 6 &
2018 (boot_cpu_data.x86 >= 15)
2019 break;
2020 goto no_apic;
2021 case X86_VENDOR_HYGON:
2022 break;
2023 case X86_VENDOR_INTEL:
2024 if (boot_cpu_data.x86 == 6 ||
2025 (boot_cpu_data.x86 == 5 &
2026 break;
2027 goto no_apic;
2028 default:
2029 goto no_apic;
2030 }
2031
2032 if (!boot_cpu_has(X86_FEATURE_APIC))
2033 /*
2034 * Over-ride BIOS and try to
2035 * "lapic" specified.
2036 */
2037 if (!force_enable_local_apic)
2038 pr_info("Local APIC d
2039 "you can enab
2040 return false;
2041 }
2042 if (!apic_force_enable(APIC_D
2043 return false;
2044 } else {
2045 if (!apic_verify(APIC_DEFAULT
2046 return false;
2047 }
2048
2049 apic_pm_activate();
2050
2051 return true;
2052
2053 no_apic:
2054 pr_info("No local APIC present or har
2055 return false;
2056 }
2057 #endif
2058
2059 /**
2060 * init_apic_mappings - initialize APIC mappi
2061 */
2062 void __init init_apic_mappings(void)
2063 {
2064 if (apic_validate_deadline_timer())
2065 pr_info("TSC deadline timer a
2066
2067 if (x2apic_mode)
2068 return;
2069
2070 if (!smp_found_config) {
2071 if (!detect_init_APIC()) {
2072 pr_info("APIC: disabl
2073 apic_disable();
2074 }
2075 }
2076 }
2077
2078 static __init void apic_set_fixmap(bool read_
2079 {
2080 set_fixmap_nocache(FIX_APIC_BASE, mp_
2081 apic_mmio_base = APIC_BASE;
2082 apic_printk(APIC_VERBOSE, "mapped API
2083 apic_mmio_base, mp_lapic_
2084 if (read_apic)
2085 apic_read_boot_cpu_id(false);
2086 }
2087
2088 void __init register_lapic_address(unsigned l
2089 {
2090 /* This should only happen once */
2091 WARN_ON_ONCE(mp_lapic_addr);
2092 mp_lapic_addr = address;
2093
2094 if (!x2apic_mode)
2095 apic_set_fixmap(true);
2096 }
2097
2098 /*
2099 * Local APIC interrupts
2100 */
2101
2102 /*
2103 * Common handling code for spurious_interrup
2104 * points below. No point in allowing the com
2105 */
2106 static noinline void handle_spurious_interrup
2107 {
2108 u32 v;
2109
2110 trace_spurious_apic_entry(vector);
2111
2112 inc_irq_stat(irq_spurious_count);
2113
2114 /*
2115 * If this is a spurious interrupt th
2116 */
2117 if (vector == SPURIOUS_APIC_VECTOR) {
2118 /* See SDM vol 3 */
2119 pr_info("Spurious APIC interr
2120 smp_processor_id());
2121 goto out;
2122 }
2123
2124 /*
2125 * If it is a vectored one, verify it
2126 * acknowledge it.
2127 */
2128 v = apic_read(APIC_ISR + ((vector & ~
2129 if (v & (1 << (vector & 0x1f))) {
2130 pr_info("Spurious interrupt (
2131 vector, smp_processor
2132 apic_eoi();
2133 } else {
2134 pr_info("Spurious interrupt (
2135 vector, smp_processor
2136 }
2137 out:
2138 trace_spurious_apic_exit(vector);
2139 }
2140
2141 /**
2142 * spurious_interrupt - Catch all for interru
2143 * @regs: Pointer to pt_regs on stack
2144 * @vector: The vector number
2145 *
2146 * This is invoked from ASM entry code to cat
2147 * trigger on an entry which is routed to the
2148 * point.
2149 */
2150 DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2151 {
2152 handle_spurious_interrupt(vector);
2153 }
2154
2155 DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_i
2156 {
2157 handle_spurious_interrupt(SPURIOUS_AP
2158 }
2159
2160 /*
2161 * This interrupt should never happen with ou
2162 */
2163 DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt
2164 {
2165 static const char * const error_inter
2166 "Send CS error",
2167 "Receive CS error",
2168 "Send accept error",
2169 "Receive accept error",
2170 "Redirectable IPI",
2171 "Send illegal vector",
2172 "Received illegal vector",
2173 "Illegal register address",
2174 };
2175 u32 v, i = 0;
2176
2177 trace_error_apic_entry(ERROR_APIC_VEC
2178
2179 /* First tickle the hardware, only th
2180 if (lapic_get_maxlvt() > 3) /* Du
2181 apic_write(APIC_ESR, 0);
2182 v = apic_read(APIC_ESR);
2183 apic_eoi();
2184 atomic_inc(&irq_err_count);
2185
2186 apic_printk(APIC_DEBUG, KERN_DEBUG "A
2187 smp_processor_id(), v);
2188
2189 v &= 0xff;
2190 while (v) {
2191 if (v & 0x1)
2192 apic_printk(APIC_DEBU
2193 i++;
2194 v >>= 1;
2195 }
2196
2197 apic_printk(APIC_DEBUG, KERN_CONT "\n
2198
2199 trace_error_apic_exit(ERROR_APIC_VECT
2200 }
2201
2202 /**
2203 * connect_bsp_APIC - attach the APIC to the
2204 */
2205 static void __init connect_bsp_APIC(void)
2206 {
2207 #ifdef CONFIG_X86_32
2208 if (pic_mode) {
2209 /*
2210 * Do not trust the local API
2211 */
2212 clear_local_APIC();
2213 /*
2214 * PIC mode, enable APIC mode
2215 * local APIC to INT and NMI
2216 */
2217 apic_printk(APIC_VERBOSE, "le
2218 "enabling API
2219 imcr_pic_to_apic();
2220 }
2221 #endif
2222 }
2223
2224 /**
2225 * disconnect_bsp_APIC - detach the APIC from
2226 * @virt_wire_setup: indicates, whether vi
2227 *
2228 * Virtual wire mode is necessary to deliver
2229 * APIC is disabled.
2230 */
2231 void disconnect_bsp_APIC(int virt_wire_setup)
2232 {
2233 unsigned int value;
2234
2235 #ifdef CONFIG_X86_32
2236 if (pic_mode) {
2237 /*
2238 * Put the board back into PI
2239 * certain older boards). No
2240 * IPIs, won't work beyond th
2241 * INIT IPIs.
2242 */
2243 apic_printk(APIC_VERBOSE, "di
2244 "entering PIC
2245 imcr_apic_to_pic();
2246 return;
2247 }
2248 #endif
2249
2250 /* Go back to Virtual Wire compatibil
2251
2252 /* For the spurious interrupt use vec
2253 value = apic_read(APIC_SPIV);
2254 value &= ~APIC_VECTOR_MASK;
2255 value |= APIC_SPIV_APIC_ENABLED;
2256 value |= 0xf;
2257 apic_write(APIC_SPIV, value);
2258
2259 if (!virt_wire_setup) {
2260 /*
2261 * For LVT0 make it edge trig
2262 * external and enabled
2263 */
2264 value = apic_read(APIC_LVT0);
2265 value &= ~(APIC_MODE_MASK | A
2266 APIC_INPUT_POLARITY |
2267 APIC_LVT_LEVEL_TRIGGE
2268 value |= APIC_LVT_REMOTE_IRR
2269 value = SET_APIC_DELIVERY_MOD
2270 apic_write(APIC_LVT0, value);
2271 } else {
2272 /* Disable LVT0 */
2273 apic_write(APIC_LVT0, APIC_LV
2274 }
2275
2276 /*
2277 * For LVT1 make it edge triggered, a
2278 * nmi and enabled
2279 */
2280 value = apic_read(APIC_LVT1);
2281 value &= ~(APIC_MODE_MASK | APIC_SEND
2282 APIC_INPUT_POLARITY |
2283 APIC_LVT_LEVEL_TRIGGE
2284 value |= APIC_LVT_REMOTE_IRR | APIC_S
2285 value = SET_APIC_DELIVERY_MODE(value,
2286 apic_write(APIC_LVT1, value);
2287 }
2288
2289 void __irq_msi_compose_msg(struct irq_cfg *cf
2290 bool dmar)
2291 {
2292 memset(msg, 0, sizeof(*msg));
2293
2294 msg->arch_addr_lo.base_address = X86_
2295 msg->arch_addr_lo.dest_mode_logical =
2296 msg->arch_addr_lo.destid_0_7 = cfg->d
2297
2298 msg->arch_data.delivery_mode = APIC_D
2299 msg->arch_data.vector = cfg->vector;
2300
2301 msg->address_hi = X86_MSI_BASE_ADDRES
2302 /*
2303 * Only the IOMMU itself can use the
2304 * APIC ID into the high bits of the
2305 * just be writing to memory if it tr
2306 * address APICs which can't be addre
2307 * address range at 0xFFExxxxx. That
2308 * some hypervisors allow the extende
2309 * 5-11 to be used, giving support fo
2310 */
2311 if (dmar)
2312 msg->arch_addr_hi.destid_8_31
2313 else if (virt_ext_dest_id && cfg->des
2314 msg->arch_addr_lo.virt_destid
2315 else
2316 WARN_ON_ONCE(cfg->dest_apicid
2317 }
2318
2319 u32 x86_msi_msg_get_destid(struct msi_msg *ms
2320 {
2321 u32 dest = msg->arch_addr_lo.destid_0
2322
2323 if (extid)
2324 dest |= msg->arch_addr_hi.des
2325 return dest;
2326 }
2327 EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2328
2329 static void __init apic_bsp_up_setup(void)
2330 {
2331 reset_phys_cpu_present_map(boot_cpu_p
2332 }
2333
2334 /**
2335 * apic_bsp_setup - Setup function for local
2336 * @upmode: Force UP mode (for AP
2337 */
2338 static void __init apic_bsp_setup(bool upmode
2339 {
2340 connect_bsp_APIC();
2341 if (upmode)
2342 apic_bsp_up_setup();
2343 setup_local_APIC();
2344
2345 enable_IO_APIC();
2346 end_local_APIC_setup();
2347 irq_remap_enable_fault_handling();
2348 setup_IO_APIC();
2349 lapic_update_legacy_vectors();
2350 }
2351
2352 #ifdef CONFIG_UP_LATE_INIT
2353 void __init up_late_init(void)
2354 {
2355 if (apic_intr_mode == APIC_PIC)
2356 return;
2357
2358 /* Setup local timer */
2359 x86_init.timers.setup_percpu_clockev(
2360 }
2361 #endif
2362
2363 /*
2364 * Power management
2365 */
2366 #ifdef CONFIG_PM
2367
2368 static struct {
2369 /*
2370 * 'active' is true if the local APIC
2371 * not the BIOS; this signifies that
2372 * for disabling it before entering a
2373 */
2374 int active;
2375 /* r/w apic fields */
2376 u32 apic_id;
2377 unsigned int apic_taskpri;
2378 unsigned int apic_ldr;
2379 unsigned int apic_dfr;
2380 unsigned int apic_spiv;
2381 unsigned int apic_lvtt;
2382 unsigned int apic_lvtpc;
2383 unsigned int apic_lvt0;
2384 unsigned int apic_lvt1;
2385 unsigned int apic_lvterr;
2386 unsigned int apic_tmict;
2387 unsigned int apic_tdcr;
2388 unsigned int apic_thmr;
2389 unsigned int apic_cmci;
2390 } apic_pm_state;
2391
2392 static int lapic_suspend(void)
2393 {
2394 unsigned long flags;
2395 int maxlvt;
2396
2397 if (!apic_pm_state.active)
2398 return 0;
2399
2400 maxlvt = lapic_get_maxlvt();
2401
2402 apic_pm_state.apic_id = apic_read(API
2403 apic_pm_state.apic_taskpri = apic_rea
2404 apic_pm_state.apic_ldr = apic_read(AP
2405 apic_pm_state.apic_dfr = apic_read(AP
2406 apic_pm_state.apic_spiv = apic_read(A
2407 apic_pm_state.apic_lvtt = apic_read(A
2408 if (maxlvt >= 4)
2409 apic_pm_state.apic_lvtpc = ap
2410 apic_pm_state.apic_lvt0 = apic_read(A
2411 apic_pm_state.apic_lvt1 = apic_read(A
2412 apic_pm_state.apic_lvterr = apic_read
2413 apic_pm_state.apic_tmict = apic_read(
2414 apic_pm_state.apic_tdcr = apic_read(A
2415 #ifdef CONFIG_X86_THERMAL_VECTOR
2416 if (maxlvt >= 5)
2417 apic_pm_state.apic_thmr = api
2418 #endif
2419 #ifdef CONFIG_X86_MCE_INTEL
2420 if (maxlvt >= 6)
2421 apic_pm_state.apic_cmci = api
2422 #endif
2423
2424 local_irq_save(flags);
2425
2426 /*
2427 * Mask IOAPIC before disabling the l
2428 * entries on some implementations.
2429 */
2430 mask_ioapic_entries();
2431
2432 disable_local_APIC();
2433
2434 irq_remapping_disable();
2435
2436 local_irq_restore(flags);
2437 return 0;
2438 }
2439
2440 static void lapic_resume(void)
2441 {
2442 unsigned int l, h;
2443 unsigned long flags;
2444 int maxlvt;
2445
2446 if (!apic_pm_state.active)
2447 return;
2448
2449 local_irq_save(flags);
2450
2451 /*
2452 * IO-APIC and PIC have their own res
2453 * We just mask them here to make sur
2454 * subsystem is completely quiet whil
2455 * and interrupt-remapping.
2456 */
2457 mask_ioapic_entries();
2458 legacy_pic->mask_all();
2459
2460 if (x2apic_mode) {
2461 __x2apic_enable();
2462 } else {
2463 /*
2464 * Make sure the APICBASE poi
2465 *
2466 * FIXME! This will be wrong
2467 * SMP! We'll need to do this
2468 */
2469 if (boot_cpu_data.x86 >= 6) {
2470 rdmsr(MSR_IA32_APICBA
2471 l &= ~MSR_IA32_APICBA
2472 l |= MSR_IA32_APICBAS
2473 wrmsr(MSR_IA32_APICBA
2474 }
2475 }
2476
2477 maxlvt = lapic_get_maxlvt();
2478 apic_write(APIC_LVTERR, ERROR_APIC_VE
2479 apic_write(APIC_ID, apic_pm_state.api
2480 apic_write(APIC_DFR, apic_pm_state.ap
2481 apic_write(APIC_LDR, apic_pm_state.ap
2482 apic_write(APIC_TASKPRI, apic_pm_stat
2483 apic_write(APIC_SPIV, apic_pm_state.a
2484 apic_write(APIC_LVT0, apic_pm_state.a
2485 apic_write(APIC_LVT1, apic_pm_state.a
2486 #ifdef CONFIG_X86_THERMAL_VECTOR
2487 if (maxlvt >= 5)
2488 apic_write(APIC_LVTTHMR, apic
2489 #endif
2490 #ifdef CONFIG_X86_MCE_INTEL
2491 if (maxlvt >= 6)
2492 apic_write(APIC_LVTCMCI, apic
2493 #endif
2494 if (maxlvt >= 4)
2495 apic_write(APIC_LVTPC, apic_p
2496 apic_write(APIC_LVTT, apic_pm_state.a
2497 apic_write(APIC_TDCR, apic_pm_state.a
2498 apic_write(APIC_TMICT, apic_pm_state.
2499 apic_write(APIC_ESR, 0);
2500 apic_read(APIC_ESR);
2501 apic_write(APIC_LVTERR, apic_pm_state
2502 apic_write(APIC_ESR, 0);
2503 apic_read(APIC_ESR);
2504
2505 irq_remapping_reenable(x2apic_mode);
2506
2507 local_irq_restore(flags);
2508 }
2509
2510 /*
2511 * This device has no shutdown method - fully
2512 * are needed on every CPU up until machine_h
2513 */
2514
2515 static struct syscore_ops lapic_syscore_ops =
2516 .resume = lapic_resume,
2517 .suspend = lapic_suspend,
2518 };
2519
2520 static void apic_pm_activate(void)
2521 {
2522 apic_pm_state.active = 1;
2523 }
2524
2525 static int __init init_lapic_sysfs(void)
2526 {
2527 /* XXX: remove suspend/resume procs i
2528 if (boot_cpu_has(X86_FEATURE_APIC))
2529 register_syscore_ops(&lapic_s
2530
2531 return 0;
2532 }
2533
2534 /* local apic needs to resume before other de
2535 core_initcall(init_lapic_sysfs);
2536
2537 #else /* CONFIG_PM */
2538
2539 static void apic_pm_activate(void) { }
2540
2541 #endif /* CONFIG_PM */
2542
2543 #ifdef CONFIG_X86_64
2544
2545 static int multi_checked;
2546 static int multi;
2547
2548 static int set_multi(const struct dmi_system_
2549 {
2550 if (multi)
2551 return 0;
2552 pr_info("APIC: %s detected, Multi Cha
2553 multi = 1;
2554 return 0;
2555 }
2556
2557 static const struct dmi_system_id multi_dmi_t
2558 {
2559 .callback = set_multi,
2560 .ident = "IBM System Summit2"
2561 .matches = {
2562 DMI_MATCH(DMI_SYS_VEN
2563 DMI_MATCH(DMI_PRODUCT
2564 },
2565 },
2566 {}
2567 };
2568
2569 static void dmi_check_multi(void)
2570 {
2571 if (multi_checked)
2572 return;
2573
2574 dmi_check_system(multi_dmi_table);
2575 multi_checked = 1;
2576 }
2577
2578 /*
2579 * apic_is_clustered_box() -- Check if we can
2580 *
2581 * Thus far, the major user of this is IBM's
2582 * Clustered boxes may have unsynced TSC prob
2583 * multi-chassis.
2584 * Use DMI to check them
2585 */
2586 int apic_is_clustered_box(void)
2587 {
2588 dmi_check_multi();
2589 return multi;
2590 }
2591 #endif
2592
2593 /*
2594 * APIC command line parameters
2595 */
2596 static int __init setup_disableapic(char *arg
2597 {
2598 apic_is_disabled = true;
2599 setup_clear_cpu_cap(X86_FEATURE_APIC)
2600 return 0;
2601 }
2602 early_param("disableapic", setup_disableapic)
2603
2604 /* same as disableapic, for compatibility */
2605 static int __init setup_nolapic(char *arg)
2606 {
2607 return setup_disableapic(arg);
2608 }
2609 early_param("nolapic", setup_nolapic);
2610
2611 static int __init parse_lapic_timer_c2_ok(cha
2612 {
2613 local_apic_timer_c2_ok = 1;
2614 return 0;
2615 }
2616 early_param("lapic_timer_c2_ok", parse_lapic_
2617
2618 static int __init parse_disable_apic_timer(ch
2619 {
2620 disable_apic_timer = 1;
2621 return 0;
2622 }
2623 early_param("noapictimer", parse_disable_apic
2624
2625 static int __init parse_nolapic_timer(char *a
2626 {
2627 disable_apic_timer = 1;
2628 return 0;
2629 }
2630 early_param("nolapic_timer", parse_nolapic_ti
2631
2632 static int __init apic_set_verbosity(char *ar
2633 {
2634 if (!arg) {
2635 if (IS_ENABLED(CONFIG_X86_32)
2636 return -EINVAL;
2637
2638 ioapic_is_disabled = false;
2639 return 0;
2640 }
2641
2642 if (strcmp("debug", arg) == 0)
2643 apic_verbosity = APIC_DEBUG;
2644 else if (strcmp("verbose", arg) == 0)
2645 apic_verbosity = APIC_VERBOSE
2646 #ifdef CONFIG_X86_64
2647 else {
2648 pr_warn("APIC Verbosity level
2649 " use apic=verbose or
2650 return -EINVAL;
2651 }
2652 #endif
2653
2654 return 0;
2655 }
2656 early_param("apic", apic_set_verbosity);
2657
2658 static int __init lapic_insert_resource(void)
2659 {
2660 if (!apic_mmio_base)
2661 return -1;
2662
2663 /* Put local APIC into the resource m
2664 lapic_resource.start = apic_mmio_base
2665 lapic_resource.end = lapic_resource.s
2666 insert_resource(&iomem_resource, &lap
2667
2668 return 0;
2669 }
2670
2671 /*
2672 * need call insert after e820__reserve_resou
2673 * that is using request_resource
2674 */
2675 late_initcall(lapic_insert_resource);
2676
2677 static int __init apic_set_extnmi(char *arg)
2678 {
2679 if (!arg)
2680 return -EINVAL;
2681
2682 if (!strncmp("all", arg, 3))
2683 apic_extnmi = APIC_EXTNMI_ALL
2684 else if (!strncmp("none", arg, 4))
2685 apic_extnmi = APIC_EXTNMI_NON
2686 else if (!strncmp("bsp", arg, 3))
2687 apic_extnmi = APIC_EXTNMI_BSP
2688 else {
2689 pr_warn("Unknown external NMI
2690 return -EINVAL;
2691 }
2692
2693 return 0;
2694 }
2695 early_param("apic_extnmi", apic_set_extnmi);
2696
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.