1 // SPDX-License-Identifier: GPL-2.0 1
2 /*
3 * Copyright (C) 1994 Linus Torvalds
4 *
5 * Cyrix stuff, June 1998 by:
6 * - Rafael R. Reilova (moved everything
7 * <rreilova@ececs.uc.edu>
8 * - Channing Corn (tests & fixes),
9 * - Andrew D. Balsa (code cleanup).
10 */
11 #include <linux/init.h>
12 #include <linux/cpu.h>
13 #include <linux/module.h>
14 #include <linux/nospec.h>
15 #include <linux/prctl.h>
16 #include <linux/sched/smt.h>
17 #include <linux/pgtable.h>
18 #include <linux/bpf.h>
19
20 #include <asm/spec-ctrl.h>
21 #include <asm/cmdline.h>
22 #include <asm/bugs.h>
23 #include <asm/processor.h>
24 #include <asm/processor-flags.h>
25 #include <asm/fpu/api.h>
26 #include <asm/msr.h>
27 #include <asm/vmx.h>
28 #include <asm/paravirt.h>
29 #include <asm/cpu_device_id.h>
30 #include <asm/e820/api.h>
31 #include <asm/hypervisor.h>
32 #include <asm/tlbflush.h>
33 #include <asm/cpu.h>
34
35 #include "cpu.h"
36
37 static void __init spectre_v1_select_mitigatio
38 static void __init spectre_v2_select_mitigatio
39 static void __init retbleed_select_mitigation(
40 static void __init spectre_v2_user_select_miti
41 static void __init ssb_select_mitigation(void)
42 static void __init l1tf_select_mitigation(void
43 static void __init mds_select_mitigation(void)
44 static void __init md_clear_update_mitigation(
45 static void __init md_clear_select_mitigation(
46 static void __init taa_select_mitigation(void)
47 static void __init mmio_select_mitigation(void
48 static void __init srbds_select_mitigation(voi
49 static void __init l1d_flush_select_mitigation
50 static void __init srso_select_mitigation(void
51 static void __init gds_select_mitigation(void)
52
53 /* The base value of the SPEC_CTRL MSR without
54 u64 x86_spec_ctrl_base;
55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
56
57 /* The current value of the SPEC_CTRL MSR with
58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
59 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_curren
60
61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IB
62 EXPORT_SYMBOL_GPL(x86_pred_cmd);
63
64 static u64 __ro_after_init x86_arch_cap_msr;
65
66 static DEFINE_MUTEX(spec_ctrl_mutex);
67
68 void (*x86_return_thunk)(void) __ro_after_init
69
70 /* Update SPEC_CTRL MSR and its cached copy un
71 static void update_spec_ctrl(u64 val)
72 {
73 this_cpu_write(x86_spec_ctrl_current,
74 wrmsrl(MSR_IA32_SPEC_CTRL, val);
75 }
76
77 /*
78 * Keep track of the SPEC_CTRL MSR value for t
79 * from x86_spec_ctrl_base due to STIBP/SSB in
80 */
81 void update_spec_ctrl_cond(u64 val)
82 {
83 if (this_cpu_read(x86_spec_ctrl_curren
84 return;
85
86 this_cpu_write(x86_spec_ctrl_current,
87
88 /*
89 * When KERNEL_IBRS this MSR is writte
90 * forced the update can be delayed un
91 */
92 if (!cpu_feature_enabled(X86_FEATURE_K
93 wrmsrl(MSR_IA32_SPEC_CTRL, val
94 }
95
96 noinstr u64 spec_ctrl_current(void)
97 {
98 return this_cpu_read(x86_spec_ctrl_cur
99 }
100 EXPORT_SYMBOL_GPL(spec_ctrl_current);
101
102 /*
103 * AMD specific MSR info for Speculative Store
104 * x86_amd_ls_cfg_ssbd_mask is initialized in
105 */
106 u64 __ro_after_init x86_amd_ls_cfg_base;
107 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
108
109 /* Control conditional STIBP in switch_to() */
110 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
111 /* Control conditional IBPB in switch_mm() */
112 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
113 /* Control unconditional IBPB in switch_mm() *
114 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb)
115
116 /* Control MDS CPU buffer clear before idling
117 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
118 EXPORT_SYMBOL_GPL(mds_idle_clear);
119
120 /*
121 * Controls whether l1d flush based mitigation
122 * based on hw features and admin setting via
123 * defaults to false
124 */
125 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flu
126
127 /* Controls CPU Fill buffer clear before KVM g
128 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear)
129 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
130
131 void __init cpu_select_mitigations(void)
132 {
133 /*
134 * Read the SPEC_CTRL MSR to account f
135 * have unknown values. AMD64_LS_CFG M
136 * init code as it is not enumerated a
137 */
138 if (cpu_feature_enabled(X86_FEATURE_MS
139 rdmsrl(MSR_IA32_SPEC_CTRL, x86
140
141 /*
142 * Previously running kernel (
143 * turned ON. Clear them and l
144 * rediscover them based on co
145 */
146 x86_spec_ctrl_base &= ~SPEC_CT
147 }
148
149 x86_arch_cap_msr = x86_read_arch_cap_m
150
151 /* Select the proper CPU mitigations b
152 spectre_v1_select_mitigation();
153 spectre_v2_select_mitigation();
154 /*
155 * retbleed_select_mitigation() relies
156 * spectre_v2_select_mitigation(); spe
157 * spectre_v2=ibrs.
158 */
159 retbleed_select_mitigation();
160 /*
161 * spectre_v2_user_select_mitigation()
162 * retbleed_select_mitigation(); speci
163 * forced for UNRET or IBPB.
164 */
165 spectre_v2_user_select_mitigation();
166 ssb_select_mitigation();
167 l1tf_select_mitigation();
168 md_clear_select_mitigation();
169 srbds_select_mitigation();
170 l1d_flush_select_mitigation();
171
172 /*
173 * srso_select_mitigation() depends an
174 * retbleed_select_mitigation().
175 */
176 srso_select_mitigation();
177 gds_select_mitigation();
178 }
179
180 /*
181 * NOTE: This function is *only* called for SV
182 * MSR_IA32_SPEC_CTRL for SSBD.
183 */
184 void
185 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, b
186 {
187 u64 guestval, hostval;
188 struct thread_info *ti = current_threa
189
190 /*
191 * If SSBD is not handled in MSR_SPEC_
192 * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_C
193 */
194 if (!static_cpu_has(X86_FEATURE_LS_CFG
195 !static_cpu_has(X86_FEATURE_VIRT_S
196 return;
197
198 /*
199 * If the host has SSBD mitigation ena
200 * virtual MSR value. If its not perma
201 * current's TIF_SSBD thread flag.
202 */
203 if (static_cpu_has(X86_FEATURE_SPEC_ST
204 hostval = SPEC_CTRL_SSBD;
205 else
206 hostval = ssbd_tif_to_spec_ctr
207
208 /* Sanitize the guest value */
209 guestval = guest_virt_spec_ctrl & SPEC
210
211 if (hostval != guestval) {
212 unsigned long tif;
213
214 tif = setguest ? ssbd_spec_ctr
215 ssbd_spec_ctr
216
217 speculation_ctrl_update(tif);
218 }
219 }
220 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
221
222 static void x86_amd_ssb_disable(void)
223 {
224 u64 msrval = x86_amd_ls_cfg_base | x86
225
226 if (boot_cpu_has(X86_FEATURE_VIRT_SSBD
227 wrmsrl(MSR_AMD64_VIRT_SPEC_CTR
228 else if (boot_cpu_has(X86_FEATURE_LS_C
229 wrmsrl(MSR_AMD64_LS_CFG, msrva
230 }
231
232 #undef pr_fmt
233 #define pr_fmt(fmt) "MDS: " fmt
234
235 /* Default mitigation for MDS-affected CPUs */
236 static enum mds_mitigations mds_mitigation __r
237 IS_ENABLED(CONFIG_MITIGATION_MDS) ? MD
238 static bool mds_nosmt __ro_after_init = false;
239
240 static const char * const mds_strings[] = {
241 [MDS_MITIGATION_OFF] = "Vulnerable"
242 [MDS_MITIGATION_FULL] = "Mitigation:
243 [MDS_MITIGATION_VMWERV] = "Vulnerable:
244 };
245
246 static void __init mds_select_mitigation(void)
247 {
248 if (!boot_cpu_has_bug(X86_BUG_MDS) ||
249 mds_mitigation = MDS_MITIGATIO
250 return;
251 }
252
253 if (mds_mitigation == MDS_MITIGATION_F
254 if (!boot_cpu_has(X86_FEATURE_
255 mds_mitigation = MDS_M
256
257 setup_force_cpu_cap(X86_FEATUR
258
259 if (!boot_cpu_has(X86_BUG_MSBD
260 (mds_nosmt || cpu_mitigati
261 cpu_smt_disable(false)
262 }
263 }
264
265 static int __init mds_cmdline(char *str)
266 {
267 if (!boot_cpu_has_bug(X86_BUG_MDS))
268 return 0;
269
270 if (!str)
271 return -EINVAL;
272
273 if (!strcmp(str, "off"))
274 mds_mitigation = MDS_MITIGATIO
275 else if (!strcmp(str, "full"))
276 mds_mitigation = MDS_MITIGATIO
277 else if (!strcmp(str, "full,nosmt")) {
278 mds_mitigation = MDS_MITIGATIO
279 mds_nosmt = true;
280 }
281
282 return 0;
283 }
284 early_param("mds", mds_cmdline);
285
286 #undef pr_fmt
287 #define pr_fmt(fmt) "TAA: " fmt
288
289 enum taa_mitigations {
290 TAA_MITIGATION_OFF,
291 TAA_MITIGATION_UCODE_NEEDED,
292 TAA_MITIGATION_VERW,
293 TAA_MITIGATION_TSX_DISABLED,
294 };
295
296 /* Default mitigation for TAA-affected CPUs */
297 static enum taa_mitigations taa_mitigation __r
298 IS_ENABLED(CONFIG_MITIGATION_TAA) ? TA
299 static bool taa_nosmt __ro_after_init;
300
301 static const char * const taa_strings[] = {
302 [TAA_MITIGATION_OFF] = "Vul
303 [TAA_MITIGATION_UCODE_NEEDED] = "Vul
304 [TAA_MITIGATION_VERW] = "Mit
305 [TAA_MITIGATION_TSX_DISABLED] = "Mit
306 };
307
308 static void __init taa_select_mitigation(void)
309 {
310 if (!boot_cpu_has_bug(X86_BUG_TAA)) {
311 taa_mitigation = TAA_MITIGATIO
312 return;
313 }
314
315 /* TSX previously disabled by tsx=off
316 if (!boot_cpu_has(X86_FEATURE_RTM)) {
317 taa_mitigation = TAA_MITIGATIO
318 return;
319 }
320
321 if (cpu_mitigations_off()) {
322 taa_mitigation = TAA_MITIGATIO
323 return;
324 }
325
326 /*
327 * TAA mitigation via VERW is turned o
328 * tsx_async_abort=off and mds=off are
329 */
330 if (taa_mitigation == TAA_MITIGATION_O
331 mds_mitigation == MDS_MITIGATION_O
332 return;
333
334 if (boot_cpu_has(X86_FEATURE_MD_CLEAR)
335 taa_mitigation = TAA_MITIGATIO
336 else
337 taa_mitigation = TAA_MITIGATIO
338
339 /*
340 * VERW doesn't clear the CPU buffers
341 * A microcode update fixes this behav
342 * adds support for MSR_IA32_TSX_CTRL
343 * ARCH_CAP_TSX_CTRL_MSR bit.
344 *
345 * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CT
346 * update is required.
347 */
348 if ( (x86_arch_cap_msr & ARCH_CAP_MDS_
349 !(x86_arch_cap_msr & ARCH_CAP_TSX_
350 taa_mitigation = TAA_MITIGATIO
351
352 /*
353 * TSX is enabled, select alternate mi
354 * the same as MDS. Enable MDS static
355 *
356 * For guests that can't determine whe
357 * present on host, enable the mitigat
358 */
359 setup_force_cpu_cap(X86_FEATURE_CLEAR_
360
361 if (taa_nosmt || cpu_mitigations_auto_
362 cpu_smt_disable(false);
363 }
364
365 static int __init tsx_async_abort_parse_cmdlin
366 {
367 if (!boot_cpu_has_bug(X86_BUG_TAA))
368 return 0;
369
370 if (!str)
371 return -EINVAL;
372
373 if (!strcmp(str, "off")) {
374 taa_mitigation = TAA_MITIGATIO
375 } else if (!strcmp(str, "full")) {
376 taa_mitigation = TAA_MITIGATIO
377 } else if (!strcmp(str, "full,nosmt"))
378 taa_mitigation = TAA_MITIGATIO
379 taa_nosmt = true;
380 }
381
382 return 0;
383 }
384 early_param("tsx_async_abort", tsx_async_abort
385
386 #undef pr_fmt
387 #define pr_fmt(fmt) "MMIO Stale Data: " fm
388
389 enum mmio_mitigations {
390 MMIO_MITIGATION_OFF,
391 MMIO_MITIGATION_UCODE_NEEDED,
392 MMIO_MITIGATION_VERW,
393 };
394
395 /* Default mitigation for Processor MMIO Stale
396 static enum mmio_mitigations mmio_mitigation _
397 IS_ENABLED(CONFIG_MITIGATION_MMIO_STAL
398 static bool mmio_nosmt __ro_after_init = false
399
400 static const char * const mmio_strings[] = {
401 [MMIO_MITIGATION_OFF] = "Vul
402 [MMIO_MITIGATION_UCODE_NEEDED] = "Vul
403 [MMIO_MITIGATION_VERW] = "Mit
404 };
405
406 static void __init mmio_select_mitigation(void
407 {
408 if (!boot_cpu_has_bug(X86_BUG_MMIO_STA
409 boot_cpu_has_bug(X86_BUG_MMIO_UNK
410 cpu_mitigations_off()) {
411 mmio_mitigation = MMIO_MITIGAT
412 return;
413 }
414
415 if (mmio_mitigation == MMIO_MITIGATION
416 return;
417
418 /*
419 * Enable CPU buffer clear mitigation
420 * by MDS or TAA. Otherwise, enable mi
421 */
422 if (boot_cpu_has_bug(X86_BUG_MDS) || (
423
424 setup_force_cpu_cap(X86_FEATUR
425
426 /*
427 * X86_FEATURE_CLEAR_CPU_BUF could be
428 * mitigations, disable KVM-only mitig
429 */
430 if (boot_cpu_has(X86_FEATURE_CLEAR_CPU
431 static_branch_disable(&mmio_st
432 else
433 static_branch_enable(&mmio_sta
434
435 /*
436 * If Processor-MMIO-Stale-Data bug is
437 * be propagated to uncore buffers, cl
438 * is required irrespective of SMT sta
439 */
440 if (!(x86_arch_cap_msr & ARCH_CAP_FBSD
441 static_branch_enable(&mds_idle
442
443 /*
444 * Check if the system has the right m
445 *
446 * CPU Fill buffer clear mitigation is
447 * FB_CLEAR or by the presence of both
448 * affected systems.
449 */
450 if ((x86_arch_cap_msr & ARCH_CAP_FB_CL
451 (boot_cpu_has(X86_FEATURE_MD_CLEAR
452 boot_cpu_has(X86_FEATURE_FLUSH_L1
453 !(x86_arch_cap_msr & ARCH_CAP_MDS
454 mmio_mitigation = MMIO_MITIGAT
455 else
456 mmio_mitigation = MMIO_MITIGAT
457
458 if (mmio_nosmt || cpu_mitigations_auto
459 cpu_smt_disable(false);
460 }
461
462 static int __init mmio_stale_data_parse_cmdlin
463 {
464 if (!boot_cpu_has_bug(X86_BUG_MMIO_STA
465 return 0;
466
467 if (!str)
468 return -EINVAL;
469
470 if (!strcmp(str, "off")) {
471 mmio_mitigation = MMIO_MITIGAT
472 } else if (!strcmp(str, "full")) {
473 mmio_mitigation = MMIO_MITIGAT
474 } else if (!strcmp(str, "full,nosmt"))
475 mmio_mitigation = MMIO_MITIGAT
476 mmio_nosmt = true;
477 }
478
479 return 0;
480 }
481 early_param("mmio_stale_data", mmio_stale_data
482
483 #undef pr_fmt
484 #define pr_fmt(fmt) "Register File Data Sa
485
486 enum rfds_mitigations {
487 RFDS_MITIGATION_OFF,
488 RFDS_MITIGATION_VERW,
489 RFDS_MITIGATION_UCODE_NEEDED,
490 };
491
492 /* Default mitigation for Register File Data S
493 static enum rfds_mitigations rfds_mitigation _
494 IS_ENABLED(CONFIG_MITIGATION_RFDS) ? R
495
496 static const char * const rfds_strings[] = {
497 [RFDS_MITIGATION_OFF]
498 [RFDS_MITIGATION_VERW]
499 [RFDS_MITIGATION_UCODE_NEEDED]
500 };
501
502 static void __init rfds_select_mitigation(void
503 {
504 if (!boot_cpu_has_bug(X86_BUG_RFDS) ||
505 rfds_mitigation = RFDS_MITIGAT
506 return;
507 }
508 if (rfds_mitigation == RFDS_MITIGATION
509 return;
510
511 if (x86_arch_cap_msr & ARCH_CAP_RFDS_C
512 setup_force_cpu_cap(X86_FEATUR
513 else
514 rfds_mitigation = RFDS_MITIGAT
515 }
516
517 static __init int rfds_parse_cmdline(char *str
518 {
519 if (!str)
520 return -EINVAL;
521
522 if (!boot_cpu_has_bug(X86_BUG_RFDS))
523 return 0;
524
525 if (!strcmp(str, "off"))
526 rfds_mitigation = RFDS_MITIGAT
527 else if (!strcmp(str, "on"))
528 rfds_mitigation = RFDS_MITIGAT
529
530 return 0;
531 }
532 early_param("reg_file_data_sampling", rfds_par
533
534 #undef pr_fmt
535 #define pr_fmt(fmt) "" fmt
536
537 static void __init md_clear_update_mitigation(
538 {
539 if (cpu_mitigations_off())
540 return;
541
542 if (!boot_cpu_has(X86_FEATURE_CLEAR_CP
543 goto out;
544
545 /*
546 * X86_FEATURE_CLEAR_CPU_BUF is now en
547 * Stale Data mitigation, if necessary
548 */
549 if (mds_mitigation == MDS_MITIGATION_O
550 boot_cpu_has_bug(X86_BUG_MDS)) {
551 mds_mitigation = MDS_MITIGATIO
552 mds_select_mitigation();
553 }
554 if (taa_mitigation == TAA_MITIGATION_O
555 boot_cpu_has_bug(X86_BUG_TAA)) {
556 taa_mitigation = TAA_MITIGATIO
557 taa_select_mitigation();
558 }
559 /*
560 * MMIO_MITIGATION_OFF is not checked
561 * gets updated correctly as per X86_F
562 */
563 if (boot_cpu_has_bug(X86_BUG_MMIO_STAL
564 mmio_mitigation = MMIO_MITIGAT
565 mmio_select_mitigation();
566 }
567 if (rfds_mitigation == RFDS_MITIGATION
568 boot_cpu_has_bug(X86_BUG_RFDS)) {
569 rfds_mitigation = RFDS_MITIGAT
570 rfds_select_mitigation();
571 }
572 out:
573 if (boot_cpu_has_bug(X86_BUG_MDS))
574 pr_info("MDS: %s\n", mds_strin
575 if (boot_cpu_has_bug(X86_BUG_TAA))
576 pr_info("TAA: %s\n", taa_strin
577 if (boot_cpu_has_bug(X86_BUG_MMIO_STAL
578 pr_info("MMIO Stale Data: %s\n
579 else if (boot_cpu_has_bug(X86_BUG_MMIO
580 pr_info("MMIO Stale Data: Unkn
581 if (boot_cpu_has_bug(X86_BUG_RFDS))
582 pr_info("Register File Data Sa
583 }
584
585 static void __init md_clear_select_mitigation(
586 {
587 mds_select_mitigation();
588 taa_select_mitigation();
589 mmio_select_mitigation();
590 rfds_select_mitigation();
591
592 /*
593 * As these mitigations are inter-rela
594 * to clear the microarchitural buffer
595 * after mitigation selection is done
596 */
597 md_clear_update_mitigation();
598 }
599
600 #undef pr_fmt
601 #define pr_fmt(fmt) "SRBDS: " fmt
602
603 enum srbds_mitigations {
604 SRBDS_MITIGATION_OFF,
605 SRBDS_MITIGATION_UCODE_NEEDED,
606 SRBDS_MITIGATION_FULL,
607 SRBDS_MITIGATION_TSX_OFF,
608 SRBDS_MITIGATION_HYPERVISOR,
609 };
610
611 static enum srbds_mitigations srbds_mitigation
612 IS_ENABLED(CONFIG_MITIGATION_SRBDS) ?
613
614 static const char * const srbds_strings[] = {
615 [SRBDS_MITIGATION_OFF] = "Vul
616 [SRBDS_MITIGATION_UCODE_NEEDED] = "Vul
617 [SRBDS_MITIGATION_FULL] = "Mit
618 [SRBDS_MITIGATION_TSX_OFF] = "Mit
619 [SRBDS_MITIGATION_HYPERVISOR] = "Unk
620 };
621
622 static bool srbds_off;
623
624 void update_srbds_msr(void)
625 {
626 u64 mcu_ctrl;
627
628 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
629 return;
630
631 if (boot_cpu_has(X86_FEATURE_HYPERVISO
632 return;
633
634 if (srbds_mitigation == SRBDS_MITIGATI
635 return;
636
637 /*
638 * A MDS_NO CPU for which SRBDS mitiga
639 * being disabled and it hasn't receiv
640 */
641 if (!boot_cpu_has(X86_FEATURE_SRBDS_CT
642 return;
643
644 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl
645
646 switch (srbds_mitigation) {
647 case SRBDS_MITIGATION_OFF:
648 case SRBDS_MITIGATION_TSX_OFF:
649 mcu_ctrl |= RNGDS_MITG_DIS;
650 break;
651 case SRBDS_MITIGATION_FULL:
652 mcu_ctrl &= ~RNGDS_MITG_DIS;
653 break;
654 default:
655 break;
656 }
657
658 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl
659 }
660
661 static void __init srbds_select_mitigation(voi
662 {
663 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
664 return;
665
666 /*
667 * Check to see if this is one of the
668 * are only exposed to SRBDS when TSX
669 * by Processor MMIO Stale Data vulner
670 */
671 if ((x86_arch_cap_msr & ARCH_CAP_MDS_N
672 !boot_cpu_has_bug(X86_BUG_MMIO_STA
673 srbds_mitigation = SRBDS_MITIG
674 else if (boot_cpu_has(X86_FEATURE_HYPE
675 srbds_mitigation = SRBDS_MITIG
676 else if (!boot_cpu_has(X86_FEATURE_SRB
677 srbds_mitigation = SRBDS_MITIG
678 else if (cpu_mitigations_off() || srbd
679 srbds_mitigation = SRBDS_MITIG
680
681 update_srbds_msr();
682 pr_info("%s\n", srbds_strings[srbds_mi
683 }
684
685 static int __init srbds_parse_cmdline(char *st
686 {
687 if (!str)
688 return -EINVAL;
689
690 if (!boot_cpu_has_bug(X86_BUG_SRBDS))
691 return 0;
692
693 srbds_off = !strcmp(str, "off");
694 return 0;
695 }
696 early_param("srbds", srbds_parse_cmdline);
697
698 #undef pr_fmt
699 #define pr_fmt(fmt) "L1D Flush : " fmt
700
701 enum l1d_flush_mitigations {
702 L1D_FLUSH_OFF = 0,
703 L1D_FLUSH_ON,
704 };
705
706 static enum l1d_flush_mitigations l1d_flush_mi
707
708 static void __init l1d_flush_select_mitigation
709 {
710 if (!l1d_flush_mitigation || !boot_cpu
711 return;
712
713 static_branch_enable(&switch_mm_cond_l
714 pr_info("Conditional flush on switch_m
715 }
716
717 static int __init l1d_flush_parse_cmdline(char
718 {
719 if (!strcmp(str, "on"))
720 l1d_flush_mitigation = L1D_FLU
721
722 return 0;
723 }
724 early_param("l1d_flush", l1d_flush_parse_cmdli
725
726 #undef pr_fmt
727 #define pr_fmt(fmt) "GDS: " fmt
728
729 enum gds_mitigations {
730 GDS_MITIGATION_OFF,
731 GDS_MITIGATION_UCODE_NEEDED,
732 GDS_MITIGATION_FORCE,
733 GDS_MITIGATION_FULL,
734 GDS_MITIGATION_FULL_LOCKED,
735 GDS_MITIGATION_HYPERVISOR,
736 };
737
738 static enum gds_mitigations gds_mitigation __r
739 IS_ENABLED(CONFIG_MITIGATION_GDS) ? GD
740
741 static const char * const gds_strings[] = {
742 [GDS_MITIGATION_OFF] = "Vul
743 [GDS_MITIGATION_UCODE_NEEDED] = "Vul
744 [GDS_MITIGATION_FORCE] = "Mit
745 [GDS_MITIGATION_FULL] = "Mit
746 [GDS_MITIGATION_FULL_LOCKED] = "Mit
747 [GDS_MITIGATION_HYPERVISOR] = "Unk
748 };
749
750 bool gds_ucode_mitigated(void)
751 {
752 return (gds_mitigation == GDS_MITIGATI
753 gds_mitigation == GDS_MITIGATI
754 }
755 EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
756
757 void update_gds_msr(void)
758 {
759 u64 mcu_ctrl_after;
760 u64 mcu_ctrl;
761
762 switch (gds_mitigation) {
763 case GDS_MITIGATION_OFF:
764 rdmsrl(MSR_IA32_MCU_OPT_CTRL,
765 mcu_ctrl |= GDS_MITG_DIS;
766 break;
767 case GDS_MITIGATION_FULL_LOCKED:
768 /*
769 * The LOCKED state comes from
770 * the same state. Make sure t
771 * CPUs.
772 */
773 case GDS_MITIGATION_FULL:
774 rdmsrl(MSR_IA32_MCU_OPT_CTRL,
775 mcu_ctrl &= ~GDS_MITG_DIS;
776 break;
777 case GDS_MITIGATION_FORCE:
778 case GDS_MITIGATION_UCODE_NEEDED:
779 case GDS_MITIGATION_HYPERVISOR:
780 return;
781 }
782
783 wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl
784
785 /*
786 * Check to make sure that the WRMSR v
787 * GDS_MITG_DIS will be ignored if thi
788 * processor was not.
789 */
790 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl
791 WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_afte
792 }
793
794 static void __init gds_select_mitigation(void)
795 {
796 u64 mcu_ctrl;
797
798 if (!boot_cpu_has_bug(X86_BUG_GDS))
799 return;
800
801 if (boot_cpu_has(X86_FEATURE_HYPERVISO
802 gds_mitigation = GDS_MITIGATIO
803 goto out;
804 }
805
806 if (cpu_mitigations_off())
807 gds_mitigation = GDS_MITIGATIO
808 /* Will verify below that mitigation _
809
810 /* No microcode */
811 if (!(x86_arch_cap_msr & ARCH_CAP_GDS_
812 if (gds_mitigation == GDS_MITI
813 /*
814 * This only needs to
815 * here rather than in
816 */
817 setup_clear_cpu_cap(X8
818 pr_warn("Microcode upd
819 } else {
820 gds_mitigation = GDS_M
821 }
822 goto out;
823 }
824
825 /* Microcode has mitigation, use it */
826 if (gds_mitigation == GDS_MITIGATION_F
827 gds_mitigation = GDS_MITIGATIO
828
829 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl
830 if (mcu_ctrl & GDS_MITG_LOCKED) {
831 if (gds_mitigation == GDS_MITI
832 pr_warn("Mitigation lo
833
834 /*
835 * The mitigation is selected
836 * _should_ have the same stat
837 * but others are then update_
838 * mismatch. If the boot CPU i
839 * ensure the other CPUs have
840 */
841 gds_mitigation = GDS_MITIGATIO
842 }
843
844 update_gds_msr();
845 out:
846 pr_info("%s\n", gds_strings[gds_mitiga
847 }
848
849 static int __init gds_parse_cmdline(char *str)
850 {
851 if (!str)
852 return -EINVAL;
853
854 if (!boot_cpu_has_bug(X86_BUG_GDS))
855 return 0;
856
857 if (!strcmp(str, "off"))
858 gds_mitigation = GDS_MITIGATIO
859 else if (!strcmp(str, "force"))
860 gds_mitigation = GDS_MITIGATIO
861
862 return 0;
863 }
864 early_param("gather_data_sampling", gds_parse_
865
866 #undef pr_fmt
867 #define pr_fmt(fmt) "Spectre V1 : " fmt
868
869 enum spectre_v1_mitigation {
870 SPECTRE_V1_MITIGATION_NONE,
871 SPECTRE_V1_MITIGATION_AUTO,
872 };
873
874 static enum spectre_v1_mitigation spectre_v1_m
875 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_V
876 SPECTRE_V1_MITIGATION_AUTO : S
877
878 static const char * const spectre_v1_strings[]
879 [SPECTRE_V1_MITIGATION_NONE] = "Vulner
880 [SPECTRE_V1_MITIGATION_AUTO] = "Mitiga
881 };
882
883 /*
884 * Does SMAP provide full mitigation against s
885 * userspace?
886 */
887 static bool smap_works_speculatively(void)
888 {
889 if (!boot_cpu_has(X86_FEATURE_SMAP))
890 return false;
891
892 /*
893 * On CPUs which are vulnerable to Mel
894 * prevent speculative access to user
895 * Consider SMAP to be non-functional
896 * CPUs.
897 */
898 if (boot_cpu_has(X86_BUG_CPU_MELTDOWN)
899 return false;
900
901 return true;
902 }
903
904 static void __init spectre_v1_select_mitigatio
905 {
906 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_
907 spectre_v1_mitigation = SPECTR
908 return;
909 }
910
911 if (spectre_v1_mitigation == SPECTRE_V
912 /*
913 * With Spectre v1, a user can
914 * path of a conditional swapg
915 * value. The mitigation is t
916 *
917 * If FSGSBASE is enabled, the
918 * GS, in which case SMAP prov
919 *
920 * If FSGSBASE is disabled, th
921 * address in GS. That makes
922 * possible if there's no SMAP
923 */
924 if (boot_cpu_has(X86_FEATURE_F
925 !smap_works_speculatively(
926 /*
927 * Mitigation can be p
928 * PTI as the CR3 writ
929 * is serializing.
930 *
931 * If neither is there
932 * stop speculation th
933 */
934 if (boot_cpu_has_bug(X
935 !boot_cpu_has(X86_
936 setup_force_cp
937
938 /*
939 * Enable lfences in t
940 * paths, to prevent u
941 * skipping swapgs.
942 */
943 setup_force_cpu_cap(X8
944 }
945 }
946
947 pr_info("%s\n", spectre_v1_strings[spe
948 }
949
950 static int __init nospectre_v1_cmdline(char *s
951 {
952 spectre_v1_mitigation = SPECTRE_V1_MIT
953 return 0;
954 }
955 early_param("nospectre_v1", nospectre_v1_cmdli
956
957 enum spectre_v2_mitigation spectre_v2_enabled
958
959 #undef pr_fmt
960 #define pr_fmt(fmt) "RETBleed: " fmt
961
962 enum retbleed_mitigation {
963 RETBLEED_MITIGATION_NONE,
964 RETBLEED_MITIGATION_UNRET,
965 RETBLEED_MITIGATION_IBPB,
966 RETBLEED_MITIGATION_IBRS,
967 RETBLEED_MITIGATION_EIBRS,
968 RETBLEED_MITIGATION_STUFF,
969 };
970
971 enum retbleed_mitigation_cmd {
972 RETBLEED_CMD_OFF,
973 RETBLEED_CMD_AUTO,
974 RETBLEED_CMD_UNRET,
975 RETBLEED_CMD_IBPB,
976 RETBLEED_CMD_STUFF,
977 };
978
979 static const char * const retbleed_strings[] =
980 [RETBLEED_MITIGATION_NONE] = "Vul
981 [RETBLEED_MITIGATION_UNRET] = "Mit
982 [RETBLEED_MITIGATION_IBPB] = "Mit
983 [RETBLEED_MITIGATION_IBRS] = "Mit
984 [RETBLEED_MITIGATION_EIBRS] = "Mit
985 [RETBLEED_MITIGATION_STUFF] = "Mit
986 };
987
988 static enum retbleed_mitigation retbleed_mitig
989 RETBLEED_MITIGATION_NONE;
990 static enum retbleed_mitigation_cmd retbleed_c
991 IS_ENABLED(CONFIG_MITIGATION_RETBLEED)
992
993 static int __ro_after_init retbleed_nosmt = fa
994
995 static int __init retbleed_parse_cmdline(char
996 {
997 if (!str)
998 return -EINVAL;
999
1000 while (str) {
1001 char *next = strchr(str, ',')
1002 if (next) {
1003 *next = 0;
1004 next++;
1005 }
1006
1007 if (!strcmp(str, "off")) {
1008 retbleed_cmd = RETBLE
1009 } else if (!strcmp(str, "auto
1010 retbleed_cmd = RETBLE
1011 } else if (!strcmp(str, "unre
1012 retbleed_cmd = RETBLE
1013 } else if (!strcmp(str, "ibpb
1014 retbleed_cmd = RETBLE
1015 } else if (!strcmp(str, "stuf
1016 retbleed_cmd = RETBLE
1017 } else if (!strcmp(str, "nosm
1018 retbleed_nosmt = true
1019 } else if (!strcmp(str, "forc
1020 setup_force_cpu_bug(X
1021 } else {
1022 pr_err("Ignoring unkn
1023 }
1024
1025 str = next;
1026 }
1027
1028 return 0;
1029 }
1030 early_param("retbleed", retbleed_parse_cmdlin
1031
1032 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB un
1033 #define RETBLEED_INTEL_MSG "WARNING: Spectre
1034
1035 static void __init retbleed_select_mitigation
1036 {
1037 bool mitigate_smt = false;
1038
1039 if (!boot_cpu_has_bug(X86_BUG_RETBLEE
1040 return;
1041
1042 switch (retbleed_cmd) {
1043 case RETBLEED_CMD_OFF:
1044 return;
1045
1046 case RETBLEED_CMD_UNRET:
1047 if (IS_ENABLED(CONFIG_MITIGAT
1048 retbleed_mitigation =
1049 } else {
1050 pr_err("WARNING: kern
1051 goto do_cmd_auto;
1052 }
1053 break;
1054
1055 case RETBLEED_CMD_IBPB:
1056 if (!boot_cpu_has(X86_FEATURE
1057 pr_err("WARNING: CPU
1058 goto do_cmd_auto;
1059 } else if (IS_ENABLED(CONFIG_
1060 retbleed_mitigation =
1061 } else {
1062 pr_err("WARNING: kern
1063 goto do_cmd_auto;
1064 }
1065 break;
1066
1067 case RETBLEED_CMD_STUFF:
1068 if (IS_ENABLED(CONFIG_MITIGAT
1069 spectre_v2_enabled == SPE
1070 retbleed_mitigation =
1071
1072 } else {
1073 if (IS_ENABLED(CONFIG
1074 pr_err("WARNI
1075 else
1076 pr_err("WARNI
1077
1078 goto do_cmd_auto;
1079 }
1080 break;
1081
1082 do_cmd_auto:
1083 case RETBLEED_CMD_AUTO:
1084 if (boot_cpu_data.x86_vendor
1085 boot_cpu_data.x86_vendor
1086 if (IS_ENABLED(CONFIG
1087 retbleed_miti
1088 else if (IS_ENABLED(C
1089 boot_cpu_has
1090 retbleed_miti
1091 }
1092
1093 /*
1094 * The Intel mitigation (IBRS
1095 * spectre_v2_select_mitigati
1096 * be set accordingly below.
1097 */
1098
1099 break;
1100 }
1101
1102 switch (retbleed_mitigation) {
1103 case RETBLEED_MITIGATION_UNRET:
1104 setup_force_cpu_cap(X86_FEATU
1105 setup_force_cpu_cap(X86_FEATU
1106
1107 x86_return_thunk = retbleed_r
1108
1109 if (boot_cpu_data.x86_vendor
1110 boot_cpu_data.x86_vendor
1111 pr_err(RETBLEED_UNTRA
1112
1113 mitigate_smt = true;
1114 break;
1115
1116 case RETBLEED_MITIGATION_IBPB:
1117 setup_force_cpu_cap(X86_FEATU
1118
1119 /*
1120 * IBPB on entry already obvi
1121 * software-based untraining
1122 * other mitigation like SRSO
1123 */
1124 setup_clear_cpu_cap(X86_FEATU
1125 setup_clear_cpu_cap(X86_FEATU
1126
1127 setup_force_cpu_cap(X86_FEATU
1128 mitigate_smt = true;
1129
1130 /*
1131 * There is no need for RSB f
1132 * all predictions, including
1133 * regardless of IBPB impleme
1134 */
1135 setup_clear_cpu_cap(X86_FEATU
1136
1137 break;
1138
1139 case RETBLEED_MITIGATION_STUFF:
1140 setup_force_cpu_cap(X86_FEATU
1141 setup_force_cpu_cap(X86_FEATU
1142
1143 x86_return_thunk = call_depth
1144 break;
1145
1146 default:
1147 break;
1148 }
1149
1150 if (mitigate_smt && !boot_cpu_has(X86
1151 (retbleed_nosmt || cpu_mitigation
1152 cpu_smt_disable(false);
1153
1154 /*
1155 * Let IBRS trump all on Intel withou
1156 * retbleed= cmdline option except fo
1157 */
1158 if (boot_cpu_data.x86_vendor == X86_V
1159 switch (spectre_v2_enabled) {
1160 case SPECTRE_V2_IBRS:
1161 retbleed_mitigation =
1162 break;
1163 case SPECTRE_V2_EIBRS:
1164 case SPECTRE_V2_EIBRS_RETPOLI
1165 case SPECTRE_V2_EIBRS_LFENCE:
1166 retbleed_mitigation =
1167 break;
1168 default:
1169 if (retbleed_mitigati
1170 pr_err(RETBLE
1171 }
1172 }
1173
1174 pr_info("%s\n", retbleed_strings[retb
1175 }
1176
1177 #undef pr_fmt
1178 #define pr_fmt(fmt) "Spectre V2 : " fmt
1179
1180 static enum spectre_v2_user_mitigation spectr
1181 SPECTRE_V2_USER_NONE;
1182 static enum spectre_v2_user_mitigation spectr
1183 SPECTRE_V2_USER_NONE;
1184
1185 #ifdef CONFIG_MITIGATION_RETPOLINE
1186 static bool spectre_v2_bad_module;
1187
1188 bool retpoline_module_ok(bool has_retpoline)
1189 {
1190 if (spectre_v2_enabled == SPECTRE_V2_
1191 return true;
1192
1193 pr_err("System may be vulnerable to s
1194 spectre_v2_bad_module = true;
1195 return false;
1196 }
1197
1198 static inline const char *spectre_v2_module_s
1199 {
1200 return spectre_v2_bad_module ? " - vu
1201 }
1202 #else
1203 static inline const char *spectre_v2_module_s
1204 #endif
1205
1206 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENC
1207 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: U
1208 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG
1209 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IB
1210
1211 #ifdef CONFIG_BPF_SYSCALL
1212 void unpriv_ebpf_notify(int new_state)
1213 {
1214 if (new_state)
1215 return;
1216
1217 /* Unprivileged eBPF is enabled */
1218
1219 switch (spectre_v2_enabled) {
1220 case SPECTRE_V2_EIBRS:
1221 pr_err(SPECTRE_V2_EIBRS_EBPF_
1222 break;
1223 case SPECTRE_V2_EIBRS_LFENCE:
1224 if (sched_smt_active())
1225 pr_err(SPECTRE_V2_EIB
1226 break;
1227 default:
1228 break;
1229 }
1230 }
1231 #endif
1232
1233 static inline bool match_option(const char *a
1234 {
1235 int len = strlen(opt);
1236
1237 return len == arglen && !strncmp(arg,
1238 }
1239
1240 /* The kernel command line selection for spec
1241 enum spectre_v2_mitigation_cmd {
1242 SPECTRE_V2_CMD_NONE,
1243 SPECTRE_V2_CMD_AUTO,
1244 SPECTRE_V2_CMD_FORCE,
1245 SPECTRE_V2_CMD_RETPOLINE,
1246 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1247 SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1248 SPECTRE_V2_CMD_EIBRS,
1249 SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1250 SPECTRE_V2_CMD_EIBRS_LFENCE,
1251 SPECTRE_V2_CMD_IBRS,
1252 };
1253
1254 enum spectre_v2_user_cmd {
1255 SPECTRE_V2_USER_CMD_NONE,
1256 SPECTRE_V2_USER_CMD_AUTO,
1257 SPECTRE_V2_USER_CMD_FORCE,
1258 SPECTRE_V2_USER_CMD_PRCTL,
1259 SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1260 SPECTRE_V2_USER_CMD_SECCOMP,
1261 SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1262 };
1263
1264 static const char * const spectre_v2_user_str
1265 [SPECTRE_V2_USER_NONE]
1266 [SPECTRE_V2_USER_STRICT]
1267 [SPECTRE_V2_USER_STRICT_PREFERRED]
1268 [SPECTRE_V2_USER_PRCTL]
1269 [SPECTRE_V2_USER_SECCOMP]
1270 };
1271
1272 static const struct {
1273 const char *opti
1274 enum spectre_v2_user_cmd cmd;
1275 bool secur
1276 } v2_user_options[] __initconst = {
1277 { "auto", SPECTRE_V2_US
1278 { "off", SPECTRE_V2_US
1279 { "on", SPECTRE_V2_US
1280 { "prctl", SPECTRE_V2_US
1281 { "prctl,ibpb", SPECTRE_V2_US
1282 { "seccomp", SPECTRE_V2_US
1283 { "seccomp,ibpb", SPECTRE_V2_US
1284 };
1285
1286 static void __init spec_v2_user_print_cond(co
1287 {
1288 if (boot_cpu_has_bug(X86_BUG_SPECTRE_
1289 pr_info("spectre_v2_user=%s f
1290 }
1291
1292 static __ro_after_init enum spectre_v2_mitiga
1293
1294 static enum spectre_v2_user_cmd __init
1295 spectre_v2_parse_user_cmdline(void)
1296 {
1297 char arg[20];
1298 int ret, i;
1299
1300 switch (spectre_v2_cmd) {
1301 case SPECTRE_V2_CMD_NONE:
1302 return SPECTRE_V2_USER_CMD_NO
1303 case SPECTRE_V2_CMD_FORCE:
1304 return SPECTRE_V2_USER_CMD_FO
1305 default:
1306 break;
1307 }
1308
1309 ret = cmdline_find_option(boot_comman
1310 arg, sizeof
1311 if (ret < 0)
1312 return SPECTRE_V2_USER_CMD_AU
1313
1314 for (i = 0; i < ARRAY_SIZE(v2_user_op
1315 if (match_option(arg, ret, v2
1316 spec_v2_user_print_co
1317
1318 return v2_user_option
1319 }
1320 }
1321
1322 pr_err("Unknown user space protection
1323 return SPECTRE_V2_USER_CMD_AUTO;
1324 }
1325
1326 static inline bool spectre_v2_in_ibrs_mode(en
1327 {
1328 return spectre_v2_in_eibrs_mode(mode)
1329 }
1330
1331 static void __init
1332 spectre_v2_user_select_mitigation(void)
1333 {
1334 enum spectre_v2_user_mitigation mode
1335 bool smt_possible = IS_ENABLED(CONFIG
1336 enum spectre_v2_user_cmd cmd;
1337
1338 if (!boot_cpu_has(X86_FEATURE_IBPB) &
1339 return;
1340
1341 if (cpu_smt_control == CPU_SMT_FORCE_
1342 cpu_smt_control == CPU_SMT_NOT_SU
1343 smt_possible = false;
1344
1345 cmd = spectre_v2_parse_user_cmdline()
1346 switch (cmd) {
1347 case SPECTRE_V2_USER_CMD_NONE:
1348 goto set_mode;
1349 case SPECTRE_V2_USER_CMD_FORCE:
1350 mode = SPECTRE_V2_USER_STRICT
1351 break;
1352 case SPECTRE_V2_USER_CMD_AUTO:
1353 case SPECTRE_V2_USER_CMD_PRCTL:
1354 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1355 mode = SPECTRE_V2_USER_PRCTL;
1356 break;
1357 case SPECTRE_V2_USER_CMD_SECCOMP:
1358 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB
1359 if (IS_ENABLED(CONFIG_SECCOMP
1360 mode = SPECTRE_V2_USE
1361 else
1362 mode = SPECTRE_V2_USE
1363 break;
1364 }
1365
1366 /* Initialize Indirect Branch Predict
1367 if (boot_cpu_has(X86_FEATURE_IBPB)) {
1368 setup_force_cpu_cap(X86_FEATU
1369
1370 spectre_v2_user_ibpb = mode;
1371 switch (cmd) {
1372 case SPECTRE_V2_USER_CMD_NONE
1373 break;
1374 case SPECTRE_V2_USER_CMD_FORC
1375 case SPECTRE_V2_USER_CMD_PRCT
1376 case SPECTRE_V2_USER_CMD_SECC
1377 static_branch_enable(
1378 spectre_v2_user_ibpb
1379 break;
1380 case SPECTRE_V2_USER_CMD_PRCT
1381 case SPECTRE_V2_USER_CMD_AUTO
1382 case SPECTRE_V2_USER_CMD_SECC
1383 static_branch_enable(
1384 break;
1385 }
1386
1387 pr_info("mitigation: Enabling
1388 static_key_enabled(&s
1389 "always-on" : "condit
1390 }
1391
1392 /*
1393 * If no STIBP, Intel enhanced IBRS i
1394 * is not required.
1395 *
1396 * Intel's Enhanced IBRS also protect
1397 * injection in user-mode as the IBRS
1398 * implicitly enables cross-thread pr
1399 * mode, the IBRS bit is set only on
1400 * to userspace. AMD Automatic IBRS
1401 * These modes therefore disable the
1402 * so allow for STIBP to be selected
1403 */
1404 if (!boot_cpu_has(X86_FEATURE_STIBP)
1405 !smt_possible ||
1406 (spectre_v2_in_eibrs_mode(spectre
1407 !boot_cpu_has(X86_FEATURE_AUTOIB
1408 return;
1409
1410 /*
1411 * At this point, an STIBP mode other
1412 * If STIBP support is not being forc
1413 * is preferred.
1414 */
1415 if (mode != SPECTRE_V2_USER_STRICT &&
1416 boot_cpu_has(X86_FEATURE_AMD_STIB
1417 mode = SPECTRE_V2_USER_STRICT
1418
1419 if (retbleed_mitigation == RETBLEED_M
1420 retbleed_mitigation == RETBLEED_M
1421 if (mode != SPECTRE_V2_USER_S
1422 mode != SPECTRE_V2_USER_S
1423 pr_info("Selecting ST
1424 mode = SPECTRE_V2_USER_STRICT
1425 }
1426
1427 spectre_v2_user_stibp = mode;
1428
1429 set_mode:
1430 pr_info("%s\n", spectre_v2_user_strin
1431 }
1432
1433 static const char * const spectre_v2_strings[
1434 [SPECTRE_V2_NONE]
1435 [SPECTRE_V2_RETPOLINE]
1436 [SPECTRE_V2_LFENCE]
1437 [SPECTRE_V2_EIBRS]
1438 [SPECTRE_V2_EIBRS_LFENCE]
1439 [SPECTRE_V2_EIBRS_RETPOLINE]
1440 [SPECTRE_V2_IBRS]
1441 };
1442
1443 static const struct {
1444 const char *option;
1445 enum spectre_v2_mitigation_cmd cmd;
1446 bool secure;
1447 } mitigation_options[] __initconst = {
1448 { "off", SPECTRE_V2_CM
1449 { "on", SPECTRE_V2_CM
1450 { "retpoline", SPECTRE_V2_CM
1451 { "retpoline,amd", SPECTRE_V2_CM
1452 { "retpoline,lfence", SPECTRE_V2_CM
1453 { "retpoline,generic", SPECTRE_V2_CM
1454 { "eibrs", SPECTRE_V2_CM
1455 { "eibrs,lfence", SPECTRE_V2_CM
1456 { "eibrs,retpoline", SPECTRE_V2_CM
1457 { "auto", SPECTRE_V2_CM
1458 { "ibrs", SPECTRE_V2_CM
1459 };
1460
1461 static void __init spec_v2_print_cond(const c
1462 {
1463 if (boot_cpu_has_bug(X86_BUG_SPECTRE_
1464 pr_info("%s selected on comma
1465 }
1466
1467 static enum spectre_v2_mitigation_cmd __init
1468 {
1469 enum spectre_v2_mitigation_cmd cmd;
1470 char arg[20];
1471 int ret, i;
1472
1473 cmd = IS_ENABLED(CONFIG_MITIGATION_SP
1474 if (cmdline_find_option_bool(boot_com
1475 cpu_mitigations_off())
1476 return SPECTRE_V2_CMD_NONE;
1477
1478 ret = cmdline_find_option(boot_comman
1479 if (ret < 0)
1480 return cmd;
1481
1482 for (i = 0; i < ARRAY_SIZE(mitigation
1483 if (!match_option(arg, ret, m
1484 continue;
1485 cmd = mitigation_options[i].c
1486 break;
1487 }
1488
1489 if (i >= ARRAY_SIZE(mitigation_option
1490 pr_err("unknown option (%s).
1491 return cmd;
1492 }
1493
1494 if ((cmd == SPECTRE_V2_CMD_RETPOLINE
1495 cmd == SPECTRE_V2_CMD_RETPOLINE_
1496 cmd == SPECTRE_V2_CMD_RETPOLINE_
1497 cmd == SPECTRE_V2_CMD_EIBRS_LFEN
1498 cmd == SPECTRE_V2_CMD_EIBRS_RETP
1499 !IS_ENABLED(CONFIG_MITIGATION_RET
1500 pr_err("%s selected but not c
1501 mitigation_options[i].
1502 return SPECTRE_V2_CMD_AUTO;
1503 }
1504
1505 if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1506 cmd == SPECTRE_V2_CMD_EIBRS_LFEN
1507 cmd == SPECTRE_V2_CMD_EIBRS_RETP
1508 !boot_cpu_has(X86_FEATURE_IBRS_EN
1509 pr_err("%s selected but CPU d
1510 mitigation_options[i].
1511 return SPECTRE_V2_CMD_AUTO;
1512 }
1513
1514 if ((cmd == SPECTRE_V2_CMD_RETPOLINE_
1515 cmd == SPECTRE_V2_CMD_EIBRS_LFEN
1516 !boot_cpu_has(X86_FEATURE_LFENCE_
1517 pr_err("%s selected, but CPU
1518 mitigation_options[i].
1519 return SPECTRE_V2_CMD_AUTO;
1520 }
1521
1522 if (cmd == SPECTRE_V2_CMD_IBRS && !IS
1523 pr_err("%s selected but not c
1524 mitigation_options[i].
1525 return SPECTRE_V2_CMD_AUTO;
1526 }
1527
1528 if (cmd == SPECTRE_V2_CMD_IBRS && boo
1529 pr_err("%s selected but not I
1530 mitigation_options[i].
1531 return SPECTRE_V2_CMD_AUTO;
1532 }
1533
1534 if (cmd == SPECTRE_V2_CMD_IBRS && !bo
1535 pr_err("%s selected but CPU d
1536 mitigation_options[i].
1537 return SPECTRE_V2_CMD_AUTO;
1538 }
1539
1540 if (cmd == SPECTRE_V2_CMD_IBRS && cpu
1541 pr_err("%s selected but runni
1542 mitigation_options[i].
1543 return SPECTRE_V2_CMD_AUTO;
1544 }
1545
1546 spec_v2_print_cond(mitigation_options
1547 mitigation_options
1548 return cmd;
1549 }
1550
1551 static enum spectre_v2_mitigation __init spec
1552 {
1553 if (!IS_ENABLED(CONFIG_MITIGATION_RET
1554 pr_err("Kernel not compiled w
1555 return SPECTRE_V2_NONE;
1556 }
1557
1558 return SPECTRE_V2_RETPOLINE;
1559 }
1560
1561 static bool __ro_after_init rrsba_disabled;
1562
1563 /* Disable in-kernel use of non-RSB RET predi
1564 static void __init spec_ctrl_disable_kernel_r
1565 {
1566 if (rrsba_disabled)
1567 return;
1568
1569 if (!(x86_arch_cap_msr & ARCH_CAP_RRS
1570 rrsba_disabled = true;
1571 return;
1572 }
1573
1574 if (!boot_cpu_has(X86_FEATURE_RRSBA_C
1575 return;
1576
1577 x86_spec_ctrl_base |= SPEC_CTRL_RRSBA
1578 update_spec_ctrl(x86_spec_ctrl_base);
1579 rrsba_disabled = true;
1580 }
1581
1582 static void __init spectre_v2_determine_rsb_f
1583 {
1584 /*
1585 * Similar to context switches, there
1586 * after VM exit:
1587 *
1588 * 1) RSB underflow
1589 *
1590 * 2) Poisoned RSB entry
1591 *
1592 * When retpoline is enabled, both ar
1593 * the RSB.
1594 *
1595 * When IBRS is enabled, while #1 wou
1596 * prediction isolation protections,
1597 * because of #2. Note that SMEP pro
1598 * user-space-poisoned RSB entries.
1599 *
1600 * eIBRS should protect against RSB p
1601 * bug is present then a LITE version
1602 * just a single call needs to retire
1603 */
1604 switch (mode) {
1605 case SPECTRE_V2_NONE:
1606 return;
1607
1608 case SPECTRE_V2_EIBRS_LFENCE:
1609 case SPECTRE_V2_EIBRS:
1610 if (boot_cpu_has_bug(X86_BUG_
1611 setup_force_cpu_cap(X
1612 pr_info("Spectre v2 /
1613 }
1614 return;
1615
1616 case SPECTRE_V2_EIBRS_RETPOLINE:
1617 case SPECTRE_V2_RETPOLINE:
1618 case SPECTRE_V2_LFENCE:
1619 case SPECTRE_V2_IBRS:
1620 setup_force_cpu_cap(X86_FEATU
1621 pr_info("Spectre v2 / Spectre
1622 return;
1623 }
1624
1625 pr_warn_once("Unknown Spectre v2 mode
1626 dump_stack();
1627 }
1628
1629 /*
1630 * Set BHI_DIS_S to prevent indirect branches
1631 * branch history in userspace. Not needed if
1632 */
1633 static bool __init spec_ctrl_bhi_dis(void)
1634 {
1635 if (!boot_cpu_has(X86_FEATURE_BHI_CTR
1636 return false;
1637
1638 x86_spec_ctrl_base |= SPEC_CTRL_BHI_D
1639 update_spec_ctrl(x86_spec_ctrl_base);
1640 setup_force_cpu_cap(X86_FEATURE_CLEAR
1641
1642 return true;
1643 }
1644
1645 enum bhi_mitigations {
1646 BHI_MITIGATION_OFF,
1647 BHI_MITIGATION_ON,
1648 BHI_MITIGATION_VMEXIT_ONLY,
1649 };
1650
1651 static enum bhi_mitigations bhi_mitigation __
1652 IS_ENABLED(CONFIG_MITIGATION_SPECTRE_
1653
1654 static int __init spectre_bhi_parse_cmdline(c
1655 {
1656 if (!str)
1657 return -EINVAL;
1658
1659 if (!strcmp(str, "off"))
1660 bhi_mitigation = BHI_MITIGATI
1661 else if (!strcmp(str, "on"))
1662 bhi_mitigation = BHI_MITIGATI
1663 else if (!strcmp(str, "vmexit"))
1664 bhi_mitigation = BHI_MITIGATI
1665 else
1666 pr_err("Ignoring unknown spec
1667
1668 return 0;
1669 }
1670 early_param("spectre_bhi", spectre_bhi_parse_
1671
1672 static void __init bhi_select_mitigation(void
1673 {
1674 if (bhi_mitigation == BHI_MITIGATION_
1675 return;
1676
1677 /* Retpoline mitigates against BHI un
1678 if (boot_cpu_has(X86_FEATURE_RETPOLIN
1679 !boot_cpu_has(X86_FEATURE_RETPOLI
1680 spec_ctrl_disable_kernel_rrsb
1681 if (rrsba_disabled)
1682 return;
1683 }
1684
1685 /* Mitigate in hardware if supported
1686 if (spec_ctrl_bhi_dis())
1687 return;
1688
1689 if (!IS_ENABLED(CONFIG_X86_64))
1690 return;
1691
1692 if (bhi_mitigation == BHI_MITIGATION_
1693 pr_info("Spectre BHI mitigati
1694 setup_force_cpu_cap(X86_FEATU
1695 return;
1696 }
1697
1698 pr_info("Spectre BHI mitigation: SW B
1699 setup_force_cpu_cap(X86_FEATURE_CLEAR
1700 setup_force_cpu_cap(X86_FEATURE_CLEAR
1701 }
1702
1703 static void __init spectre_v2_select_mitigati
1704 {
1705 enum spectre_v2_mitigation_cmd cmd =
1706 enum spectre_v2_mitigation mode = SPE
1707
1708 /*
1709 * If the CPU is not affected and the
1710 * then nothing to do.
1711 */
1712 if (!boot_cpu_has_bug(X86_BUG_SPECTRE
1713 (cmd == SPECTRE_V2_CMD_NONE || cm
1714 return;
1715
1716 switch (cmd) {
1717 case SPECTRE_V2_CMD_NONE:
1718 return;
1719
1720 case SPECTRE_V2_CMD_FORCE:
1721 case SPECTRE_V2_CMD_AUTO:
1722 if (boot_cpu_has(X86_FEATURE_
1723 mode = SPECTRE_V2_EIB
1724 break;
1725 }
1726
1727 if (IS_ENABLED(CONFIG_MITIGAT
1728 boot_cpu_has_bug(X86_BUG_
1729 retbleed_cmd != RETBLEED_
1730 retbleed_cmd != RETBLEED_
1731 boot_cpu_has(X86_FEATURE_
1732 boot_cpu_data.x86_vendor
1733 mode = SPECTRE_V2_IBR
1734 break;
1735 }
1736
1737 mode = spectre_v2_select_retp
1738 break;
1739
1740 case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1741 pr_err(SPECTRE_V2_LFENCE_MSG)
1742 mode = SPECTRE_V2_LFENCE;
1743 break;
1744
1745 case SPECTRE_V2_CMD_RETPOLINE_GENERIC
1746 mode = SPECTRE_V2_RETPOLINE;
1747 break;
1748
1749 case SPECTRE_V2_CMD_RETPOLINE:
1750 mode = spectre_v2_select_retp
1751 break;
1752
1753 case SPECTRE_V2_CMD_IBRS:
1754 mode = SPECTRE_V2_IBRS;
1755 break;
1756
1757 case SPECTRE_V2_CMD_EIBRS:
1758 mode = SPECTRE_V2_EIBRS;
1759 break;
1760
1761 case SPECTRE_V2_CMD_EIBRS_LFENCE:
1762 mode = SPECTRE_V2_EIBRS_LFENC
1763 break;
1764
1765 case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1766 mode = SPECTRE_V2_EIBRS_RETPO
1767 break;
1768 }
1769
1770 if (mode == SPECTRE_V2_EIBRS && unpri
1771 pr_err(SPECTRE_V2_EIBRS_EBPF_
1772
1773 if (spectre_v2_in_ibrs_mode(mode)) {
1774 if (boot_cpu_has(X86_FEATURE_
1775 msr_set_bit(MSR_EFER,
1776 } else {
1777 x86_spec_ctrl_base |=
1778 update_spec_ctrl(x86_
1779 }
1780 }
1781
1782 switch (mode) {
1783 case SPECTRE_V2_NONE:
1784 case SPECTRE_V2_EIBRS:
1785 break;
1786
1787 case SPECTRE_V2_IBRS:
1788 setup_force_cpu_cap(X86_FEATU
1789 if (boot_cpu_has(X86_FEATURE_
1790 pr_warn(SPECTRE_V2_IB
1791 break;
1792
1793 case SPECTRE_V2_LFENCE:
1794 case SPECTRE_V2_EIBRS_LFENCE:
1795 setup_force_cpu_cap(X86_FEATU
1796 fallthrough;
1797
1798 case SPECTRE_V2_RETPOLINE:
1799 case SPECTRE_V2_EIBRS_RETPOLINE:
1800 setup_force_cpu_cap(X86_FEATU
1801 break;
1802 }
1803
1804 /*
1805 * Disable alternate RSB predictions
1806 * JMPs gets protection against BHI a
1807 * prediction from a non-RSB predicto
1808 */
1809 if (mode == SPECTRE_V2_EIBRS_LFENCE |
1810 mode == SPECTRE_V2_EIBRS_RETPOLIN
1811 mode == SPECTRE_V2_RETPOLINE)
1812 spec_ctrl_disable_kernel_rrsb
1813
1814 if (boot_cpu_has(X86_BUG_BHI))
1815 bhi_select_mitigation();
1816
1817 spectre_v2_enabled = mode;
1818 pr_info("%s\n", spectre_v2_strings[mo
1819
1820 /*
1821 * If Spectre v2 protection has been
1822 * context switch. In general there
1823 * across context switches, for which
1824 *
1825 * 1) RSB underflow
1826 *
1827 * Some Intel parts have "bottomle
1828 * speculated return targets may c
1829 * which could have a user-poisone
1830 *
1831 * AMD has it even worse: *all* re
1832 * regardless of the state of the
1833 *
1834 * When IBRS or eIBRS is enabled,
1835 * scenario is mitigated by the IB
1836 * properties, so the RSB buffer f
1837 * protect against this type of at
1838 *
1839 * The "user -> user" attack scena
1840 *
1841 * 2) Poisoned RSB entry
1842 *
1843 * If the 'next' in-kernel return
1844 * 'next' could be tricked into sp
1845 * entry.
1846 *
1847 * The "user -> kernel" attack sce
1848 * eIBRS.
1849 *
1850 * The "user -> user" scenario, al
1851 * RSB clearing.
1852 *
1853 * So to mitigate all cases, uncondit
1854 * switches.
1855 *
1856 * FIXME: Is this pointless for retbl
1857 */
1858 setup_force_cpu_cap(X86_FEATURE_RSB_C
1859 pr_info("Spectre v2 / SpectreRSB miti
1860
1861 spectre_v2_determine_rsb_fill_type_at
1862
1863 /*
1864 * Retpoline protects the kernel, but
1865 * and Enhanced IBRS protect firmware
1866 * firmware calls only when IBRS / En
1867 * otherwise enabled.
1868 *
1869 * Use "mode" to check Enhanced IBRS
1870 * the user might select retpoline on
1871 * the CPU supports Enhanced IBRS, ke
1872 * enable IBRS around firmware calls.
1873 */
1874 if (boot_cpu_has_bug(X86_BUG_RETBLEED
1875 boot_cpu_has(X86_FEATURE_IBPB) &&
1876 (boot_cpu_data.x86_vendor == X86_
1877 boot_cpu_data.x86_vendor == X86_
1878
1879 if (retbleed_cmd != RETBLEED_
1880 setup_force_cpu_cap(X
1881 pr_info("Enabling Spe
1882 }
1883
1884 } else if (boot_cpu_has(X86_FEATURE_I
1885 setup_force_cpu_cap(X86_FEATU
1886 pr_info("Enabling Restricted
1887 }
1888
1889 /* Set up IBPB and STIBP depending on
1890 spectre_v2_cmd = cmd;
1891 }
1892
1893 static void update_stibp_msr(void * __unused)
1894 {
1895 u64 val = spec_ctrl_current() | (x86_
1896 update_spec_ctrl(val);
1897 }
1898
1899 /* Update x86_spec_ctrl_base in case SMT stat
1900 static void update_stibp_strict(void)
1901 {
1902 u64 mask = x86_spec_ctrl_base & ~SPEC
1903
1904 if (sched_smt_active())
1905 mask |= SPEC_CTRL_STIBP;
1906
1907 if (mask == x86_spec_ctrl_base)
1908 return;
1909
1910 pr_info("Update user space SMT mitiga
1911 mask & SPEC_CTRL_STIBP ? "alw
1912 x86_spec_ctrl_base = mask;
1913 on_each_cpu(update_stibp_msr, NULL, 1
1914 }
1915
1916 /* Update the static key controlling the eval
1917 static void update_indir_branch_cond(void)
1918 {
1919 if (sched_smt_active())
1920 static_branch_enable(&switch_
1921 else
1922 static_branch_disable(&switch
1923 }
1924
1925 #undef pr_fmt
1926 #define pr_fmt(fmt) fmt
1927
1928 /* Update the static key controlling the MDS
1929 static void update_mds_branch_idle(void)
1930 {
1931 /*
1932 * Enable the idle clearing if SMT is
1933 * affected only by MSBDS and not any
1934 *
1935 * The other variants cannot be mitig
1936 * clearing the buffers on idle just
1937 * repartitioning leak would be a win
1938 */
1939 if (!boot_cpu_has_bug(X86_BUG_MSBDS_O
1940 return;
1941
1942 if (sched_smt_active()) {
1943 static_branch_enable(&mds_idl
1944 } else if (mmio_mitigation == MMIO_MI
1945 (x86_arch_cap_msr & ARCH_C
1946 static_branch_disable(&mds_id
1947 }
1948 }
1949
1950 #define MDS_MSG_SMT "MDS CPU bug present and
1951 #define TAA_MSG_SMT "TAA CPU bug present and
1952 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug
1953
1954 void cpu_bugs_smt_update(void)
1955 {
1956 mutex_lock(&spec_ctrl_mutex);
1957
1958 if (sched_smt_active() && unprivilege
1959 spectre_v2_enabled == SPECTRE_V2_
1960 pr_warn_once(SPECTRE_V2_EIBRS
1961
1962 switch (spectre_v2_user_stibp) {
1963 case SPECTRE_V2_USER_NONE:
1964 break;
1965 case SPECTRE_V2_USER_STRICT:
1966 case SPECTRE_V2_USER_STRICT_PREFERRED
1967 update_stibp_strict();
1968 break;
1969 case SPECTRE_V2_USER_PRCTL:
1970 case SPECTRE_V2_USER_SECCOMP:
1971 update_indir_branch_cond();
1972 break;
1973 }
1974
1975 switch (mds_mitigation) {
1976 case MDS_MITIGATION_FULL:
1977 case MDS_MITIGATION_VMWERV:
1978 if (sched_smt_active() && !bo
1979 pr_warn_once(MDS_MSG_
1980 update_mds_branch_idle();
1981 break;
1982 case MDS_MITIGATION_OFF:
1983 break;
1984 }
1985
1986 switch (taa_mitigation) {
1987 case TAA_MITIGATION_VERW:
1988 case TAA_MITIGATION_UCODE_NEEDED:
1989 if (sched_smt_active())
1990 pr_warn_once(TAA_MSG_
1991 break;
1992 case TAA_MITIGATION_TSX_DISABLED:
1993 case TAA_MITIGATION_OFF:
1994 break;
1995 }
1996
1997 switch (mmio_mitigation) {
1998 case MMIO_MITIGATION_VERW:
1999 case MMIO_MITIGATION_UCODE_NEEDED:
2000 if (sched_smt_active())
2001 pr_warn_once(MMIO_MSG
2002 break;
2003 case MMIO_MITIGATION_OFF:
2004 break;
2005 }
2006
2007 mutex_unlock(&spec_ctrl_mutex);
2008 }
2009
2010 #undef pr_fmt
2011 #define pr_fmt(fmt) "Speculative Store By
2012
2013 static enum ssb_mitigation ssb_mode __ro_afte
2014
2015 /* The kernel command line selection */
2016 enum ssb_mitigation_cmd {
2017 SPEC_STORE_BYPASS_CMD_NONE,
2018 SPEC_STORE_BYPASS_CMD_AUTO,
2019 SPEC_STORE_BYPASS_CMD_ON,
2020 SPEC_STORE_BYPASS_CMD_PRCTL,
2021 SPEC_STORE_BYPASS_CMD_SECCOMP,
2022 };
2023
2024 static const char * const ssb_strings[] = {
2025 [SPEC_STORE_BYPASS_NONE] = "Vu
2026 [SPEC_STORE_BYPASS_DISABLE] = "Mi
2027 [SPEC_STORE_BYPASS_PRCTL] = "Mi
2028 [SPEC_STORE_BYPASS_SECCOMP] = "Mi
2029 };
2030
2031 static const struct {
2032 const char *option;
2033 enum ssb_mitigation_cmd cmd;
2034 } ssb_mitigation_options[] __initconst = {
2035 { "auto", SPEC_STORE_BYPASS_CMD
2036 { "on", SPEC_STORE_BYPASS_CMD
2037 { "off", SPEC_STORE_BYPASS_CMD
2038 { "prctl", SPEC_STORE_BYPASS_CMD
2039 { "seccomp", SPEC_STORE_BYPASS_CMD
2040 };
2041
2042 static enum ssb_mitigation_cmd __init ssb_par
2043 {
2044 enum ssb_mitigation_cmd cmd;
2045 char arg[20];
2046 int ret, i;
2047
2048 cmd = IS_ENABLED(CONFIG_MITIGATION_SS
2049 SPEC_STORE_BYPASS_CMD_AUTO :
2050 if (cmdline_find_option_bool(boot_com
2051 cpu_mitigations_off()) {
2052 return SPEC_STORE_BYPASS_CMD_
2053 } else {
2054 ret = cmdline_find_option(boo
2055 arg
2056 if (ret < 0)
2057 return cmd;
2058
2059 for (i = 0; i < ARRAY_SIZE(ss
2060 if (!match_option(arg
2061 continue;
2062
2063 cmd = ssb_mitigation_
2064 break;
2065 }
2066
2067 if (i >= ARRAY_SIZE(ssb_mitig
2068 pr_err("unknown optio
2069 return cmd;
2070 }
2071 }
2072
2073 return cmd;
2074 }
2075
2076 static enum ssb_mitigation __init __ssb_selec
2077 {
2078 enum ssb_mitigation mode = SPEC_STORE
2079 enum ssb_mitigation_cmd cmd;
2080
2081 if (!boot_cpu_has(X86_FEATURE_SSBD))
2082 return mode;
2083
2084 cmd = ssb_parse_cmdline();
2085 if (!boot_cpu_has_bug(X86_BUG_SPEC_ST
2086 (cmd == SPEC_STORE_BYPASS_CMD_NON
2087 cmd == SPEC_STORE_BYPASS_CMD_AUT
2088 return mode;
2089
2090 switch (cmd) {
2091 case SPEC_STORE_BYPASS_CMD_SECCOMP:
2092 /*
2093 * Choose prctl+seccomp as th
2094 * enabled.
2095 */
2096 if (IS_ENABLED(CONFIG_SECCOMP
2097 mode = SPEC_STORE_BYP
2098 else
2099 mode = SPEC_STORE_BYP
2100 break;
2101 case SPEC_STORE_BYPASS_CMD_ON:
2102 mode = SPEC_STORE_BYPASS_DISA
2103 break;
2104 case SPEC_STORE_BYPASS_CMD_AUTO:
2105 case SPEC_STORE_BYPASS_CMD_PRCTL:
2106 mode = SPEC_STORE_BYPASS_PRCT
2107 break;
2108 case SPEC_STORE_BYPASS_CMD_NONE:
2109 break;
2110 }
2111
2112 /*
2113 * We have three CPU feature flags th
2114 * - X86_BUG_SPEC_STORE_BYPASS - CPU
2115 * - X86_FEATURE_SSBD - CPU is able
2116 * - X86_FEATURE_SPEC_STORE_BYPASS_D
2117 */
2118 if (mode == SPEC_STORE_BYPASS_DISABLE
2119 setup_force_cpu_cap(X86_FEATU
2120 /*
2121 * Intel uses the SPEC CTRL M
2122 * use a completely different
2123 */
2124 if (!static_cpu_has(X86_FEATU
2125 !static_cpu_has(X86_FEATU
2126 x86_amd_ssb_disable()
2127 } else {
2128 x86_spec_ctrl_base |=
2129 update_spec_ctrl(x86_
2130 }
2131 }
2132
2133 return mode;
2134 }
2135
2136 static void ssb_select_mitigation(void)
2137 {
2138 ssb_mode = __ssb_select_mitigation();
2139
2140 if (boot_cpu_has_bug(X86_BUG_SPEC_STO
2141 pr_info("%s\n", ssb_strings[s
2142 }
2143
2144 #undef pr_fmt
2145 #define pr_fmt(fmt) "Speculation prctl: "
2146
2147 static void task_update_spec_tif(struct task_
2148 {
2149 /* Force the update of the real TIF b
2150 set_tsk_thread_flag(tsk, TIF_SPEC_FOR
2151
2152 /*
2153 * Immediately update the speculation
2154 * task, but for a non-current task d
2155 * mitigation until it is scheduled n
2156 *
2157 * This can only happen for SECCOMP m
2158 * always the current task.
2159 */
2160 if (tsk == current)
2161 speculation_ctrl_update_curre
2162 }
2163
2164 static int l1d_flush_prctl_set(struct task_st
2165 {
2166
2167 if (!static_branch_unlikely(&switch_m
2168 return -EPERM;
2169
2170 switch (ctrl) {
2171 case PR_SPEC_ENABLE:
2172 set_ti_thread_flag(&task->thr
2173 return 0;
2174 case PR_SPEC_DISABLE:
2175 clear_ti_thread_flag(&task->t
2176 return 0;
2177 default:
2178 return -ERANGE;
2179 }
2180 }
2181
2182 static int ssb_prctl_set(struct task_struct *
2183 {
2184 if (ssb_mode != SPEC_STORE_BYPASS_PRC
2185 ssb_mode != SPEC_STORE_BYPASS_SEC
2186 return -ENXIO;
2187
2188 switch (ctrl) {
2189 case PR_SPEC_ENABLE:
2190 /* If speculation is force di
2191 if (task_spec_ssb_force_disab
2192 return -EPERM;
2193 task_clear_spec_ssb_disable(t
2194 task_clear_spec_ssb_noexec(ta
2195 task_update_spec_tif(task);
2196 break;
2197 case PR_SPEC_DISABLE:
2198 task_set_spec_ssb_disable(tas
2199 task_clear_spec_ssb_noexec(ta
2200 task_update_spec_tif(task);
2201 break;
2202 case PR_SPEC_FORCE_DISABLE:
2203 task_set_spec_ssb_disable(tas
2204 task_set_spec_ssb_force_disab
2205 task_clear_spec_ssb_noexec(ta
2206 task_update_spec_tif(task);
2207 break;
2208 case PR_SPEC_DISABLE_NOEXEC:
2209 if (task_spec_ssb_force_disab
2210 return -EPERM;
2211 task_set_spec_ssb_disable(tas
2212 task_set_spec_ssb_noexec(task
2213 task_update_spec_tif(task);
2214 break;
2215 default:
2216 return -ERANGE;
2217 }
2218 return 0;
2219 }
2220
2221 static bool is_spec_ib_user_controlled(void)
2222 {
2223 return spectre_v2_user_ibpb == SPECTR
2224 spectre_v2_user_ibpb == SPECT
2225 spectre_v2_user_stibp == SPEC
2226 spectre_v2_user_stibp == SPEC
2227 }
2228
2229 static int ib_prctl_set(struct task_struct *t
2230 {
2231 switch (ctrl) {
2232 case PR_SPEC_ENABLE:
2233 if (spectre_v2_user_ibpb == S
2234 spectre_v2_user_stibp ==
2235 return 0;
2236
2237 /*
2238 * With strict mode for both
2239 * code paths avoid checking
2240 * unconditionally run the in
2241 * are independent and either
2242 * enabled regardless of the
2243 *
2244 * If either is set to condit
2245 * updated, unless it was for
2246 * call. Currently, this is p
2247 * feature X86_FEATURE_AMD_ST
2248 * kernel is booted with 'spe
2249 * spectre_v2_user_ibpb == SP
2250 * spectre_v2_user_stibp == S
2251 */
2252 if (!is_spec_ib_user_controll
2253 task_spec_ib_force_disabl
2254 return -EPERM;
2255
2256 task_clear_spec_ib_disable(ta
2257 task_update_spec_tif(task);
2258 break;
2259 case PR_SPEC_DISABLE:
2260 case PR_SPEC_FORCE_DISABLE:
2261 /*
2262 * Indirect branch speculatio
2263 * mitigation is force disabl
2264 */
2265 if (spectre_v2_user_ibpb == S
2266 spectre_v2_user_stibp ==
2267 return -EPERM;
2268
2269 if (!is_spec_ib_user_controll
2270 return 0;
2271
2272 task_set_spec_ib_disable(task
2273 if (ctrl == PR_SPEC_FORCE_DIS
2274 task_set_spec_ib_forc
2275 task_update_spec_tif(task);
2276 if (task == current)
2277 indirect_branch_predi
2278 break;
2279 default:
2280 return -ERANGE;
2281 }
2282 return 0;
2283 }
2284
2285 int arch_prctl_spec_ctrl_set(struct task_stru
2286 unsigned long ct
2287 {
2288 switch (which) {
2289 case PR_SPEC_STORE_BYPASS:
2290 return ssb_prctl_set(task, ct
2291 case PR_SPEC_INDIRECT_BRANCH:
2292 return ib_prctl_set(task, ctr
2293 case PR_SPEC_L1D_FLUSH:
2294 return l1d_flush_prctl_set(ta
2295 default:
2296 return -ENODEV;
2297 }
2298 }
2299
2300 #ifdef CONFIG_SECCOMP
2301 void arch_seccomp_spec_mitigate(struct task_s
2302 {
2303 if (ssb_mode == SPEC_STORE_BYPASS_SEC
2304 ssb_prctl_set(task, PR_SPEC_F
2305 if (spectre_v2_user_ibpb == SPECTRE_V
2306 spectre_v2_user_stibp == SPECTRE_
2307 ib_prctl_set(task, PR_SPEC_FO
2308 }
2309 #endif
2310
2311 static int l1d_flush_prctl_get(struct task_st
2312 {
2313 if (!static_branch_unlikely(&switch_m
2314 return PR_SPEC_FORCE_DISABLE;
2315
2316 if (test_ti_thread_flag(&task->thread
2317 return PR_SPEC_PRCTL | PR_SPE
2318 else
2319 return PR_SPEC_PRCTL | PR_SPE
2320 }
2321
2322 static int ssb_prctl_get(struct task_struct *
2323 {
2324 switch (ssb_mode) {
2325 case SPEC_STORE_BYPASS_NONE:
2326 if (boot_cpu_has_bug(X86_BUG_
2327 return PR_SPEC_ENABLE
2328 return PR_SPEC_NOT_AFFECTED;
2329 case SPEC_STORE_BYPASS_DISABLE:
2330 return PR_SPEC_DISABLE;
2331 case SPEC_STORE_BYPASS_SECCOMP:
2332 case SPEC_STORE_BYPASS_PRCTL:
2333 if (task_spec_ssb_force_disab
2334 return PR_SPEC_PRCTL
2335 if (task_spec_ssb_noexec(task
2336 return PR_SPEC_PRCTL
2337 if (task_spec_ssb_disable(tas
2338 return PR_SPEC_PRCTL
2339 return PR_SPEC_PRCTL | PR_SPE
2340 }
2341 BUG();
2342 }
2343
2344 static int ib_prctl_get(struct task_struct *t
2345 {
2346 if (!boot_cpu_has_bug(X86_BUG_SPECTRE
2347 return PR_SPEC_NOT_AFFECTED;
2348
2349 if (spectre_v2_user_ibpb == SPECTRE_V
2350 spectre_v2_user_stibp == SPECTRE_
2351 return PR_SPEC_ENABLE;
2352 else if (is_spec_ib_user_controlled()
2353 if (task_spec_ib_force_disabl
2354 return PR_SPEC_PRCTL
2355 if (task_spec_ib_disable(task
2356 return PR_SPEC_PRCTL
2357 return PR_SPEC_PRCTL | PR_SPE
2358 } else if (spectre_v2_user_ibpb == SP
2359 spectre_v2_user_stibp == SPECTRE_
2360 spectre_v2_user_stibp == SPECTRE_
2361 return PR_SPEC_DISABLE;
2362 else
2363 return PR_SPEC_NOT_AFFECTED;
2364 }
2365
2366 int arch_prctl_spec_ctrl_get(struct task_stru
2367 {
2368 switch (which) {
2369 case PR_SPEC_STORE_BYPASS:
2370 return ssb_prctl_get(task);
2371 case PR_SPEC_INDIRECT_BRANCH:
2372 return ib_prctl_get(task);
2373 case PR_SPEC_L1D_FLUSH:
2374 return l1d_flush_prctl_get(ta
2375 default:
2376 return -ENODEV;
2377 }
2378 }
2379
2380 void x86_spec_ctrl_setup_ap(void)
2381 {
2382 if (boot_cpu_has(X86_FEATURE_MSR_SPEC
2383 update_spec_ctrl(x86_spec_ctr
2384
2385 if (ssb_mode == SPEC_STORE_BYPASS_DIS
2386 x86_amd_ssb_disable();
2387 }
2388
2389 bool itlb_multihit_kvm_mitigation;
2390 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigatio
2391
2392 #undef pr_fmt
2393 #define pr_fmt(fmt) "L1TF: " fmt
2394
2395 /* Default mitigation for L1TF-affected CPUs
2396 enum l1tf_mitigations l1tf_mitigation __ro_af
2397 IS_ENABLED(CONFIG_MITIGATION_L1TF) ?
2398 #if IS_ENABLED(CONFIG_KVM_INTEL)
2399 EXPORT_SYMBOL_GPL(l1tf_mitigation);
2400 #endif
2401 enum vmx_l1d_flush_state l1tf_vmx_mitigation
2402 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
2403
2404 /*
2405 * These CPUs all support 44bits physical add
2406 * cache but CPUID can report a smaller numbe
2407 *
2408 * The L1TF mitigation uses the top most addr
2409 * non present PTEs. When the installed memor
2410 * address bit due to memory holes, which has
2411 * which report 36bits physical address bits
2412 * then the mitigation range check in l1tf_se
2413 * This is a false positive because the mitig
2414 * the fact that the cache uses 44bit interna
2415 * instead of the reported physical bits and
2416 * machines to 44bit if the reported bits are
2417 */
2418 static void override_cache_bits(struct cpuinf
2419 {
2420 if (c->x86 != 6)
2421 return;
2422
2423 switch (c->x86_vfm) {
2424 case INTEL_NEHALEM:
2425 case INTEL_WESTMERE:
2426 case INTEL_SANDYBRIDGE:
2427 case INTEL_IVYBRIDGE:
2428 case INTEL_HASWELL:
2429 case INTEL_HASWELL_L:
2430 case INTEL_HASWELL_G:
2431 case INTEL_BROADWELL:
2432 case INTEL_BROADWELL_G:
2433 case INTEL_SKYLAKE_L:
2434 case INTEL_SKYLAKE:
2435 case INTEL_KABYLAKE_L:
2436 case INTEL_KABYLAKE:
2437 if (c->x86_cache_bits < 44)
2438 c->x86_cache_bits = 4
2439 break;
2440 }
2441 }
2442
2443 static void __init l1tf_select_mitigation(voi
2444 {
2445 u64 half_pa;
2446
2447 if (!boot_cpu_has_bug(X86_BUG_L1TF))
2448 return;
2449
2450 if (cpu_mitigations_off())
2451 l1tf_mitigation = L1TF_MITIGA
2452 else if (cpu_mitigations_auto_nosmt()
2453 l1tf_mitigation = L1TF_MITIGA
2454
2455 override_cache_bits(&boot_cpu_data);
2456
2457 switch (l1tf_mitigation) {
2458 case L1TF_MITIGATION_OFF:
2459 case L1TF_MITIGATION_FLUSH_NOWARN:
2460 case L1TF_MITIGATION_FLUSH:
2461 break;
2462 case L1TF_MITIGATION_FLUSH_NOSMT:
2463 case L1TF_MITIGATION_FULL:
2464 cpu_smt_disable(false);
2465 break;
2466 case L1TF_MITIGATION_FULL_FORCE:
2467 cpu_smt_disable(true);
2468 break;
2469 }
2470
2471 #if CONFIG_PGTABLE_LEVELS == 2
2472 pr_warn("Kernel not compiled for PAE.
2473 return;
2474 #endif
2475
2476 half_pa = (u64)l1tf_pfn_limit() << PA
2477 if (l1tf_mitigation != L1TF_MITIGATIO
2478 e820__mapped_any(half
2479 pr_warn("System has more than
2480 pr_info("You may make it effe
2481 half_pa);
2482 pr_info("However, doing so wi
2483 pr_info("Reading https://www.
2484 return;
2485 }
2486
2487 setup_force_cpu_cap(X86_FEATURE_L1TF_
2488 }
2489
2490 static int __init l1tf_cmdline(char *str)
2491 {
2492 if (!boot_cpu_has_bug(X86_BUG_L1TF))
2493 return 0;
2494
2495 if (!str)
2496 return -EINVAL;
2497
2498 if (!strcmp(str, "off"))
2499 l1tf_mitigation = L1TF_MITIGA
2500 else if (!strcmp(str, "flush,nowarn")
2501 l1tf_mitigation = L1TF_MITIGA
2502 else if (!strcmp(str, "flush"))
2503 l1tf_mitigation = L1TF_MITIGA
2504 else if (!strcmp(str, "flush,nosmt"))
2505 l1tf_mitigation = L1TF_MITIGA
2506 else if (!strcmp(str, "full"))
2507 l1tf_mitigation = L1TF_MITIGA
2508 else if (!strcmp(str, "full,force"))
2509 l1tf_mitigation = L1TF_MITIGA
2510
2511 return 0;
2512 }
2513 early_param("l1tf", l1tf_cmdline);
2514
2515 #undef pr_fmt
2516 #define pr_fmt(fmt) "Speculative Return S
2517
2518 enum srso_mitigation {
2519 SRSO_MITIGATION_NONE,
2520 SRSO_MITIGATION_UCODE_NEEDED,
2521 SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED
2522 SRSO_MITIGATION_MICROCODE,
2523 SRSO_MITIGATION_SAFE_RET,
2524 SRSO_MITIGATION_IBPB,
2525 SRSO_MITIGATION_IBPB_ON_VMEXIT,
2526 };
2527
2528 enum srso_mitigation_cmd {
2529 SRSO_CMD_OFF,
2530 SRSO_CMD_MICROCODE,
2531 SRSO_CMD_SAFE_RET,
2532 SRSO_CMD_IBPB,
2533 SRSO_CMD_IBPB_ON_VMEXIT,
2534 };
2535
2536 static const char * const srso_strings[] = {
2537 [SRSO_MITIGATION_NONE]
2538 [SRSO_MITIGATION_UCODE_NEEDED]
2539 [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDE
2540 [SRSO_MITIGATION_MICROCODE]
2541 [SRSO_MITIGATION_SAFE_RET]
2542 [SRSO_MITIGATION_IBPB]
2543 [SRSO_MITIGATION_IBPB_ON_VMEXIT]
2544 };
2545
2546 static enum srso_mitigation srso_mitigation _
2547 static enum srso_mitigation_cmd srso_cmd __ro
2548
2549 static int __init srso_parse_cmdline(char *st
2550 {
2551 if (!str)
2552 return -EINVAL;
2553
2554 if (!strcmp(str, "off"))
2555 srso_cmd = SRSO_CMD_OFF;
2556 else if (!strcmp(str, "microcode"))
2557 srso_cmd = SRSO_CMD_MICROCODE
2558 else if (!strcmp(str, "safe-ret"))
2559 srso_cmd = SRSO_CMD_SAFE_RET;
2560 else if (!strcmp(str, "ibpb"))
2561 srso_cmd = SRSO_CMD_IBPB;
2562 else if (!strcmp(str, "ibpb-vmexit"))
2563 srso_cmd = SRSO_CMD_IBPB_ON_V
2564 else
2565 pr_err("Ignoring unknown SRSO
2566
2567 return 0;
2568 }
2569 early_param("spec_rstack_overflow", srso_pars
2570
2571 #define SRSO_NOTICE "WARNING: See https://ker
2572
2573 static void __init srso_select_mitigation(voi
2574 {
2575 bool has_microcode = boot_cpu_has(X86
2576
2577 if (!boot_cpu_has_bug(X86_BUG_SRSO) |
2578 cpu_mitigations_off() ||
2579 srso_cmd == SRSO_CMD_OFF) {
2580 if (boot_cpu_has(X86_FEATURE_
2581 x86_pred_cmd = PRED_C
2582 return;
2583 }
2584
2585 if (has_microcode) {
2586 /*
2587 * Zen1/2 with SMT off aren't
2588 * IBPB microcode has been ap
2589 *
2590 * Zen1/2 don't have SBPB, no
2591 */
2592 if (boot_cpu_data.x86 < 0x19
2593 setup_force_cpu_cap(X
2594 return;
2595 }
2596
2597 if (retbleed_mitigation == RE
2598 srso_mitigation = SRS
2599 goto out;
2600 }
2601 } else {
2602 pr_warn("IBPB-extending micro
2603 pr_warn(SRSO_NOTICE);
2604
2605 /* may be overwritten by SRSO
2606 srso_mitigation = SRSO_MITIGA
2607 }
2608
2609 switch (srso_cmd) {
2610 case SRSO_CMD_MICROCODE:
2611 if (has_microcode) {
2612 srso_mitigation = SRS
2613 pr_warn(SRSO_NOTICE);
2614 }
2615 break;
2616
2617 case SRSO_CMD_SAFE_RET:
2618 if (IS_ENABLED(CONFIG_MITIGAT
2619 /*
2620 * Enable the return
2621 * like ftrace, stati
2622 */
2623 setup_force_cpu_cap(X
2624 setup_force_cpu_cap(X
2625
2626 if (boot_cpu_data.x86
2627 setup_force_c
2628 x86_return_th
2629 } else {
2630 setup_force_c
2631 x86_return_th
2632 }
2633 if (has_microcode)
2634 srso_mitigati
2635 else
2636 srso_mitigati
2637 } else {
2638 pr_err("WARNING: kern
2639 }
2640 break;
2641
2642 case SRSO_CMD_IBPB:
2643 if (IS_ENABLED(CONFIG_MITIGAT
2644 if (has_microcode) {
2645 setup_force_c
2646 srso_mitigati
2647
2648 /*
2649 * IBPB on en
2650 * software-b
2651 * other miti
2652 */
2653 setup_clear_c
2654 setup_clear_c
2655 }
2656 } else {
2657 pr_err("WARNING: kern
2658 }
2659 break;
2660
2661 case SRSO_CMD_IBPB_ON_VMEXIT:
2662 if (IS_ENABLED(CONFIG_MITIGAT
2663 if (!boot_cpu_has(X86
2664 setup_force_c
2665 srso_mitigati
2666
2667 /*
2668 * There is n
2669 * all predic
2670 * regardless
2671 */
2672 setup_clear_c
2673 }
2674 } else {
2675 pr_err("WARNING: kern
2676 }
2677 break;
2678 default:
2679 break;
2680 }
2681
2682 out:
2683 pr_info("%s\n", srso_strings[srso_mit
2684 }
2685
2686 #undef pr_fmt
2687 #define pr_fmt(fmt) fmt
2688
2689 #ifdef CONFIG_SYSFS
2690
2691 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inv
2692
2693 #if IS_ENABLED(CONFIG_KVM_INTEL)
2694 static const char * const l1tf_vmx_states[] =
2695 [VMENTER_L1D_FLUSH_AUTO]
2696 [VMENTER_L1D_FLUSH_NEVER]
2697 [VMENTER_L1D_FLUSH_COND]
2698 [VMENTER_L1D_FLUSH_ALWAYS]
2699 [VMENTER_L1D_FLUSH_EPT_DISABLED]
2700 [VMENTER_L1D_FLUSH_NOT_REQUIRED]
2701 };
2702
2703 static ssize_t l1tf_show_state(char *buf)
2704 {
2705 if (l1tf_vmx_mitigation == VMENTER_L1
2706 return sysfs_emit(buf, "%s\n"
2707
2708 if (l1tf_vmx_mitigation == VMENTER_L1
2709 (l1tf_vmx_mitigation == VMENTER_L
2710 sched_smt_active())) {
2711 return sysfs_emit(buf, "%s; V
2712 l1tf_vmx_st
2713 }
2714
2715 return sysfs_emit(buf, "%s; VMX: %s,
2716 l1tf_vmx_states[l1t
2717 sched_smt_active()
2718 }
2719
2720 static ssize_t itlb_multihit_show_state(char
2721 {
2722 if (!boot_cpu_has(X86_FEATURE_MSR_IA3
2723 !boot_cpu_has(X86_FEATURE_VMX))
2724 return sysfs_emit(buf, "KVM:
2725 else if (!(cr4_read_shadow() & X86_CR
2726 return sysfs_emit(buf, "KVM:
2727 else if (itlb_multihit_kvm_mitigation
2728 return sysfs_emit(buf, "KVM:
2729 else
2730 return sysfs_emit(buf, "KVM:
2731 }
2732 #else
2733 static ssize_t l1tf_show_state(char *buf)
2734 {
2735 return sysfs_emit(buf, "%s\n", L1TF_D
2736 }
2737
2738 static ssize_t itlb_multihit_show_state(char
2739 {
2740 return sysfs_emit(buf, "Processor vul
2741 }
2742 #endif
2743
2744 static ssize_t mds_show_state(char *buf)
2745 {
2746 if (boot_cpu_has(X86_FEATURE_HYPERVIS
2747 return sysfs_emit(buf, "%s; S
2748 mds_strings
2749 }
2750
2751 if (boot_cpu_has(X86_BUG_MSBDS_ONLY))
2752 return sysfs_emit(buf, "%s; S
2753 (mds_mitiga
2754 sched_smt_
2755 }
2756
2757 return sysfs_emit(buf, "%s; SMT %s\n"
2758 sched_smt_active()
2759 }
2760
2761 static ssize_t tsx_async_abort_show_state(cha
2762 {
2763 if ((taa_mitigation == TAA_MITIGATION
2764 (taa_mitigation == TAA_MITIGATION
2765 return sysfs_emit(buf, "%s\n"
2766
2767 if (boot_cpu_has(X86_FEATURE_HYPERVIS
2768 return sysfs_emit(buf, "%s; S
2769 taa_strings
2770 }
2771
2772 return sysfs_emit(buf, "%s; SMT %s\n"
2773 sched_smt_active()
2774 }
2775
2776 static ssize_t mmio_stale_data_show_state(cha
2777 {
2778 if (boot_cpu_has_bug(X86_BUG_MMIO_UNK
2779 return sysfs_emit(buf, "Unkno
2780
2781 if (mmio_mitigation == MMIO_MITIGATIO
2782 return sysfs_emit(buf, "%s\n"
2783
2784 if (boot_cpu_has(X86_FEATURE_HYPERVIS
2785 return sysfs_emit(buf, "%s; S
2786 mmio_string
2787 }
2788
2789 return sysfs_emit(buf, "%s; SMT %s\n"
2790 sched_smt_active()
2791 }
2792
2793 static ssize_t rfds_show_state(char *buf)
2794 {
2795 return sysfs_emit(buf, "%s\n", rfds_s
2796 }
2797
2798 static char *stibp_state(void)
2799 {
2800 if (spectre_v2_in_eibrs_mode(spectre_
2801 !boot_cpu_has(X86_FEATURE_AUTOIBR
2802 return "";
2803
2804 switch (spectre_v2_user_stibp) {
2805 case SPECTRE_V2_USER_NONE:
2806 return "; STIBP: disabled";
2807 case SPECTRE_V2_USER_STRICT:
2808 return "; STIBP: forced";
2809 case SPECTRE_V2_USER_STRICT_PREFERRED
2810 return "; STIBP: always-on";
2811 case SPECTRE_V2_USER_PRCTL:
2812 case SPECTRE_V2_USER_SECCOMP:
2813 if (static_key_enabled(&switc
2814 return "; STIBP: cond
2815 }
2816 return "";
2817 }
2818
2819 static char *ibpb_state(void)
2820 {
2821 if (boot_cpu_has(X86_FEATURE_IBPB)) {
2822 if (static_key_enabled(&switc
2823 return "; IBPB: alway
2824 if (static_key_enabled(&switc
2825 return "; IBPB: condi
2826 return "; IBPB: disabled";
2827 }
2828 return "";
2829 }
2830
2831 static char *pbrsb_eibrs_state(void)
2832 {
2833 if (boot_cpu_has_bug(X86_BUG_EIBRS_PB
2834 if (boot_cpu_has(X86_FEATURE_
2835 boot_cpu_has(X86_FEATURE_
2836 return "; PBRSB-eIBRS
2837 else
2838 return "; PBRSB-eIBRS
2839 } else {
2840 return "; PBRSB-eIBRS: Not af
2841 }
2842 }
2843
2844 static const char *spectre_bhi_state(void)
2845 {
2846 if (!boot_cpu_has_bug(X86_BUG_BHI))
2847 return "; BHI: Not affected";
2848 else if (boot_cpu_has(X86_FEATURE_CLE
2849 return "; BHI: BHI_DIS_S";
2850 else if (boot_cpu_has(X86_FEATURE_CLE
2851 return "; BHI: SW loop, KVM:
2852 else if (boot_cpu_has(X86_FEATURE_RET
2853 !boot_cpu_has(X86_FEATURE_RE
2854 rrsba_disabled)
2855 return "; BHI: Retpoline";
2856 else if (boot_cpu_has(X86_FEATURE_CLE
2857 return "; BHI: Vulnerable, KV
2858
2859 return "; BHI: Vulnerable";
2860 }
2861
2862 static ssize_t spectre_v2_show_state(char *bu
2863 {
2864 if (spectre_v2_enabled == SPECTRE_V2_
2865 return sysfs_emit(buf, "Vulne
2866
2867 if (spectre_v2_enabled == SPECTRE_V2_
2868 return sysfs_emit(buf, "Vulne
2869
2870 if (sched_smt_active() && unprivilege
2871 spectre_v2_enabled == SPECTRE_V2_
2872 return sysfs_emit(buf, "Vulne
2873
2874 return sysfs_emit(buf, "%s%s%s%s%s%s%
2875 spectre_v2_strings[
2876 ibpb_state(),
2877 boot_cpu_has(X86_FE
2878 stibp_state(),
2879 boot_cpu_has(X86_FE
2880 pbrsb_eibrs_state()
2881 spectre_bhi_state()
2882 /* this should alwa
2883 spectre_v2_module_s
2884 }
2885
2886 static ssize_t srbds_show_state(char *buf)
2887 {
2888 return sysfs_emit(buf, "%s\n", srbds_
2889 }
2890
2891 static ssize_t retbleed_show_state(char *buf)
2892 {
2893 if (retbleed_mitigation == RETBLEED_M
2894 retbleed_mitigation == RETBLEED_M
2895 if (boot_cpu_data.x86_vendor
2896 boot_cpu_data.x86_vendor
2897 return sysfs_emit(buf
2898
2899 return sysfs_emit(buf, "%s; S
2900 !sched_smt_
2901 spectre_v2_
2902 spectre_v2_
2903 "enabled wi
2904 }
2905
2906 return sysfs_emit(buf, "%s\n", retble
2907 }
2908
2909 static ssize_t srso_show_state(char *buf)
2910 {
2911 if (boot_cpu_has(X86_FEATURE_SRSO_NO)
2912 return sysfs_emit(buf, "Mitig
2913
2914 return sysfs_emit(buf, "%s\n", srso_s
2915 }
2916
2917 static ssize_t gds_show_state(char *buf)
2918 {
2919 return sysfs_emit(buf, "%s\n", gds_st
2920 }
2921
2922 static ssize_t cpu_show_common(struct device
2923 char *buf, uns
2924 {
2925 if (!boot_cpu_has_bug(bug))
2926 return sysfs_emit(buf, "Not a
2927
2928 switch (bug) {
2929 case X86_BUG_CPU_MELTDOWN:
2930 if (boot_cpu_has(X86_FEATURE_
2931 return sysfs_emit(buf
2932
2933 if (hypervisor_is_type(X86_HY
2934 return sysfs_emit(buf
2935
2936 break;
2937
2938 case X86_BUG_SPECTRE_V1:
2939 return sysfs_emit(buf, "%s\n"
2940
2941 case X86_BUG_SPECTRE_V2:
2942 return spectre_v2_show_state(
2943
2944 case X86_BUG_SPEC_STORE_BYPASS:
2945 return sysfs_emit(buf, "%s\n"
2946
2947 case X86_BUG_L1TF:
2948 if (boot_cpu_has(X86_FEATURE_
2949 return l1tf_show_stat
2950 break;
2951
2952 case X86_BUG_MDS:
2953 return mds_show_state(buf);
2954
2955 case X86_BUG_TAA:
2956 return tsx_async_abort_show_s
2957
2958 case X86_BUG_ITLB_MULTIHIT:
2959 return itlb_multihit_show_sta
2960
2961 case X86_BUG_SRBDS:
2962 return srbds_show_state(buf);
2963
2964 case X86_BUG_MMIO_STALE_DATA:
2965 case X86_BUG_MMIO_UNKNOWN:
2966 return mmio_stale_data_show_s
2967
2968 case X86_BUG_RETBLEED:
2969 return retbleed_show_state(bu
2970
2971 case X86_BUG_SRSO:
2972 return srso_show_state(buf);
2973
2974 case X86_BUG_GDS:
2975 return gds_show_state(buf);
2976
2977 case X86_BUG_RFDS:
2978 return rfds_show_state(buf);
2979
2980 default:
2981 break;
2982 }
2983
2984 return sysfs_emit(buf, "Vulnerable\n"
2985 }
2986
2987 ssize_t cpu_show_meltdown(struct device *dev,
2988 {
2989 return cpu_show_common(dev, attr, buf
2990 }
2991
2992 ssize_t cpu_show_spectre_v1(struct device *de
2993 {
2994 return cpu_show_common(dev, attr, buf
2995 }
2996
2997 ssize_t cpu_show_spectre_v2(struct device *de
2998 {
2999 return cpu_show_common(dev, attr, buf
3000 }
3001
3002 ssize_t cpu_show_spec_store_bypass(struct dev
3003 {
3004 return cpu_show_common(dev, attr, buf
3005 }
3006
3007 ssize_t cpu_show_l1tf(struct device *dev, str
3008 {
3009 return cpu_show_common(dev, attr, buf
3010 }
3011
3012 ssize_t cpu_show_mds(struct device *dev, stru
3013 {
3014 return cpu_show_common(dev, attr, buf
3015 }
3016
3017 ssize_t cpu_show_tsx_async_abort(struct devic
3018 {
3019 return cpu_show_common(dev, attr, buf
3020 }
3021
3022 ssize_t cpu_show_itlb_multihit(struct device
3023 {
3024 return cpu_show_common(dev, attr, buf
3025 }
3026
3027 ssize_t cpu_show_srbds(struct device *dev, st
3028 {
3029 return cpu_show_common(dev, attr, buf
3030 }
3031
3032 ssize_t cpu_show_mmio_stale_data(struct devic
3033 {
3034 if (boot_cpu_has_bug(X86_BUG_MMIO_UNK
3035 return cpu_show_common(dev, a
3036 else
3037 return cpu_show_common(dev, a
3038 }
3039
3040 ssize_t cpu_show_retbleed(struct device *dev,
3041 {
3042 return cpu_show_common(dev, attr, buf
3043 }
3044
3045 ssize_t cpu_show_spec_rstack_overflow(struct
3046 {
3047 return cpu_show_common(dev, attr, buf
3048 }
3049
3050 ssize_t cpu_show_gds(struct device *dev, stru
3051 {
3052 return cpu_show_common(dev, attr, buf
3053 }
3054
3055 ssize_t cpu_show_reg_file_data_sampling(struc
3056 {
3057 return cpu_show_common(dev, attr, buf
3058 }
3059 #endif
3060
3061 void __warn_thunk(void)
3062 {
3063 WARN_ONCE(1, "Unpatched return thunk
3064 }
3065
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.