1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023 Ventana Micro Systems Inc.
4 */
5
6 #define pr_fmt(fmt) "riscv-pv: " fmt
7
8 #include <linux/cpuhotplug.h>
9 #include <linux/compiler.h>
10 #include <linux/errno.h>
11 #include <linux/init.h>
12 #include <linux/jump_label.h>
13 #include <linux/kconfig.h>
14 #include <linux/kernel.h>
15 #include <linux/percpu-defs.h>
16 #include <linux/printk.h>
17 #include <linux/static_call.h>
18 #include <linux/types.h>
19
20 #include <asm/barrier.h>
21 #include <asm/page.h>
22 #include <asm/paravirt.h>
23 #include <asm/sbi.h>
24
25 struct static_key paravirt_steal_enabled;
26 struct static_key paravirt_steal_rq_enabled;
27
28 static u64 native_steal_clock(int cpu)
29 {
30 return 0;
31 }
32
33 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
34
35 static bool steal_acc = true;
36 static int __init parse_no_stealacc(char *arg)
37 {
38 steal_acc = false;
39 return 0;
40 }
41
42 early_param("no-steal-acc", parse_no_stealacc);
43
44 static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64);
45
46 static bool __init has_pv_steal_clock(void)
47 {
48 if (sbi_spec_version >= sbi_mk_version(2, 0) &&
49 sbi_probe_extension(SBI_EXT_STA) > 0) {
50 pr_info("SBI STA extension detected\n");
51 return true;
52 }
53
54 return false;
55 }
56
57 static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi,
58 unsigned long flags)
59 {
60 struct sbiret ret;
61
62 ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM,
63 lo, hi, flags, 0, 0, 0);
64 if (ret.error) {
65 if (lo == SBI_SHMEM_DISABLE && hi == SBI_SHMEM_DISABLE)
66 pr_warn("Failed to disable steal-time shmem");
67 else
68 pr_warn("Failed to set steal-time shmem");
69 return sbi_err_map_linux_errno(ret.error);
70 }
71
72 return 0;
73 }
74
75 static int pv_time_cpu_online(unsigned int cpu)
76 {
77 struct sbi_sta_struct *st = this_cpu_ptr(&steal_time);
78 phys_addr_t pa = __pa(st);
79 unsigned long lo = (unsigned long)pa;
80 unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0;
81
82 return sbi_sta_steal_time_set_shmem(lo, hi, 0);
83 }
84
85 static int pv_time_cpu_down_prepare(unsigned int cpu)
86 {
87 return sbi_sta_steal_time_set_shmem(SBI_SHMEM_DISABLE,
88 SBI_SHMEM_DISABLE, 0);
89 }
90
91 static u64 pv_time_steal_clock(int cpu)
92 {
93 struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu);
94 __le32 sequence;
95 __le64 steal;
96
97 /*
98 * Check the sequence field before and after reading the steal
99 * field. Repeat the read if it is different or odd.
100 */
101 do {
102 sequence = READ_ONCE(st->sequence);
103 virt_rmb();
104 steal = READ_ONCE(st->steal);
105 virt_rmb();
106 } while ((le32_to_cpu(sequence) & 1) ||
107 sequence != READ_ONCE(st->sequence));
108
109 return le64_to_cpu(steal);
110 }
111
112 int __init pv_time_init(void)
113 {
114 int ret;
115
116 if (!has_pv_steal_clock())
117 return 0;
118
119 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
120 "riscv/pv_time:online",
121 pv_time_cpu_online,
122 pv_time_cpu_down_prepare);
123 if (ret < 0)
124 return ret;
125
126 static_call_update(pv_steal_clock, pv_time_steal_clock);
127
128 static_key_slow_inc(¶virt_steal_enabled);
129 if (steal_acc)
130 static_key_slow_inc(¶virt_steal_rq_enabled);
131
132 pr_info("Computing paravirt steal-time\n");
133
134 return 0;
135 }
136
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.