1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 /* 2 /*
3 * Shadow Call Stack support. 3 * Shadow Call Stack support.
4 * 4 *
5 * Copyright (C) 2019 Google LLC 5 * Copyright (C) 2019 Google LLC
6 */ 6 */
7 7
8 #include <linux/cpuhotplug.h> 8 #include <linux/cpuhotplug.h>
9 #include <linux/kasan.h> 9 #include <linux/kasan.h>
10 #include <linux/mm.h> 10 #include <linux/mm.h>
11 #include <linux/scs.h> 11 #include <linux/scs.h>
12 #include <linux/vmalloc.h> 12 #include <linux/vmalloc.h>
13 #include <linux/vmstat.h> 13 #include <linux/vmstat.h>
14 14
15 #ifdef CONFIG_DYNAMIC_SCS <<
16 DEFINE_STATIC_KEY_FALSE(dynamic_scs_enabled); <<
17 #endif <<
18 <<
19 static void __scs_account(void *s, int account 15 static void __scs_account(void *s, int account)
20 { 16 {
21 struct page *scs_page = vmalloc_to_pag 17 struct page *scs_page = vmalloc_to_page(s);
22 18
23 mod_node_page_state(page_pgdat(scs_pag 19 mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
24 account * (SCS_SIZ 20 account * (SCS_SIZE / SZ_1K));
25 } 21 }
26 22
27 /* Matches NR_CACHED_STACKS for VMAP_STACK */ 23 /* Matches NR_CACHED_STACKS for VMAP_STACK */
28 #define NR_CACHED_SCS 2 24 #define NR_CACHED_SCS 2
29 static DEFINE_PER_CPU(void *, scs_cache[NR_CAC 25 static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
30 26
31 static void *__scs_alloc(int node) 27 static void *__scs_alloc(int node)
32 { 28 {
33 int i; 29 int i;
34 void *s; 30 void *s;
35 31
36 for (i = 0; i < NR_CACHED_SCS; i++) { 32 for (i = 0; i < NR_CACHED_SCS; i++) {
37 s = this_cpu_xchg(scs_cache[i] 33 s = this_cpu_xchg(scs_cache[i], NULL);
38 if (s) { 34 if (s) {
39 s = kasan_unpoison_vma !! 35 kasan_unpoison_vmalloc(s, SCS_SIZE);
40 <<
41 memset(s, 0, SCS_SIZE) 36 memset(s, 0, SCS_SIZE);
42 goto out; !! 37 return s;
43 } 38 }
44 } 39 }
45 40
46 s = __vmalloc_node_range(SCS_SIZE, 1, !! 41 return __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END,
47 GFP_SCS, P 42 GFP_SCS, PAGE_KERNEL, 0, node,
48 __builtin_ 43 __builtin_return_address(0));
49 <<
50 out: <<
51 return kasan_reset_tag(s); <<
52 } 44 }
53 45
54 void *scs_alloc(int node) 46 void *scs_alloc(int node)
55 { 47 {
56 void *s; 48 void *s;
57 49
58 s = __scs_alloc(node); 50 s = __scs_alloc(node);
59 if (!s) 51 if (!s)
60 return NULL; 52 return NULL;
61 53
62 *__scs_magic(s) = SCS_END_MAGIC; 54 *__scs_magic(s) = SCS_END_MAGIC;
63 55
64 /* 56 /*
65 * Poison the allocation to catch unin 57 * Poison the allocation to catch unintentional accesses to
66 * the shadow stack when KASAN is enab 58 * the shadow stack when KASAN is enabled.
67 */ 59 */
68 kasan_poison_vmalloc(s, SCS_SIZE); 60 kasan_poison_vmalloc(s, SCS_SIZE);
69 __scs_account(s, 1); 61 __scs_account(s, 1);
70 return s; 62 return s;
71 } 63 }
72 64
73 void scs_free(void *s) 65 void scs_free(void *s)
74 { 66 {
75 int i; 67 int i;
76 68
77 __scs_account(s, -1); 69 __scs_account(s, -1);
78 70
79 /* 71 /*
80 * We cannot sleep as this can be call 72 * We cannot sleep as this can be called in interrupt context,
81 * so use this_cpu_cmpxchg to update t 73 * so use this_cpu_cmpxchg to update the cache, and vfree_atomic
82 * to free the stack. 74 * to free the stack.
83 */ 75 */
84 76
85 for (i = 0; i < NR_CACHED_SCS; i++) 77 for (i = 0; i < NR_CACHED_SCS; i++)
86 if (this_cpu_cmpxchg(scs_cache 78 if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
87 return; 79 return;
88 80
89 kasan_unpoison_vmalloc(s, SCS_SIZE, KA !! 81 kasan_unpoison_vmalloc(s, SCS_SIZE);
90 vfree_atomic(s); 82 vfree_atomic(s);
91 } 83 }
92 84
93 static int scs_cleanup(unsigned int cpu) 85 static int scs_cleanup(unsigned int cpu)
94 { 86 {
95 int i; 87 int i;
96 void **cache = per_cpu_ptr(scs_cache, 88 void **cache = per_cpu_ptr(scs_cache, cpu);
97 89
98 for (i = 0; i < NR_CACHED_SCS; i++) { 90 for (i = 0; i < NR_CACHED_SCS; i++) {
99 vfree(cache[i]); 91 vfree(cache[i]);
100 cache[i] = NULL; 92 cache[i] = NULL;
101 } 93 }
102 94
103 return 0; 95 return 0;
104 } 96 }
105 97
106 void __init scs_init(void) 98 void __init scs_init(void)
107 { 99 {
108 if (!scs_is_enabled()) <<
109 return; <<
110 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN 100 cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
111 scs_cleanup); 101 scs_cleanup);
112 } 102 }
113 103
114 int scs_prepare(struct task_struct *tsk, int n 104 int scs_prepare(struct task_struct *tsk, int node)
115 { 105 {
116 void *s; !! 106 void *s = scs_alloc(node);
117 <<
118 if (!scs_is_enabled()) <<
119 return 0; <<
120 107
121 s = scs_alloc(node); <<
122 if (!s) 108 if (!s)
123 return -ENOMEM; 109 return -ENOMEM;
124 110
125 task_scs(tsk) = task_scs_sp(tsk) = s; 111 task_scs(tsk) = task_scs_sp(tsk) = s;
126 return 0; 112 return 0;
127 } 113 }
128 114
129 static void scs_check_usage(struct task_struct 115 static void scs_check_usage(struct task_struct *tsk)
130 { 116 {
131 static unsigned long highest; 117 static unsigned long highest;
132 118
133 unsigned long *p, prev, curr = highest 119 unsigned long *p, prev, curr = highest, used = 0;
134 120
135 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USA 121 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
136 return; 122 return;
137 123
138 for (p = task_scs(tsk); p < __scs_magi 124 for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
139 if (!READ_ONCE_NOCHECK(*p)) 125 if (!READ_ONCE_NOCHECK(*p))
140 break; 126 break;
141 used += sizeof(*p); 127 used += sizeof(*p);
142 } 128 }
143 129
144 while (used > curr) { 130 while (used > curr) {
145 prev = cmpxchg_relaxed(&highes 131 prev = cmpxchg_relaxed(&highest, curr, used);
146 132
147 if (prev == curr) { 133 if (prev == curr) {
148 pr_info("%s (%d): high 134 pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
149 tsk->comm, tas 135 tsk->comm, task_pid_nr(tsk), used);
150 break; 136 break;
151 } 137 }
152 138
153 curr = prev; 139 curr = prev;
154 } 140 }
155 } 141 }
156 142
157 void scs_release(struct task_struct *tsk) 143 void scs_release(struct task_struct *tsk)
158 { 144 {
159 void *s = task_scs(tsk); 145 void *s = task_scs(tsk);
160 146
161 if (!scs_is_enabled() || !s) !! 147 if (!s)
162 return; 148 return;
163 149
164 WARN(task_scs_end_corrupted(tsk), 150 WARN(task_scs_end_corrupted(tsk),
165 "corrupted shadow stack detected 151 "corrupted shadow stack detected when freeing task\n");
166 scs_check_usage(tsk); 152 scs_check_usage(tsk);
167 scs_free(s); 153 scs_free(s);
168 } 154 }
169 155
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.