1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
3
4 #include <linux/jump_label.h>
5 #include <linux/uaccess.h>
6 #include <linux/export.h>
7 #include <linux/instrumented.h>
8 #include <linux/string.h>
9 #include <linux/types.h>
10
11 #include <asm/mce.h>
12
13 #ifdef CONFIG_X86_MCE
14 static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
15
16 void enable_copy_mc_fragile(void)
17 {
18 static_branch_inc(©_mc_fragile_key);
19 }
20 #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key))
21
22 /*
23 * Similar to copy_user_handle_tail, probe for the write fault point, or
24 * source exception point.
25 */
26 __visible notrace unsigned long
27 copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
28 {
29 for (; len; --len, to++, from++)
30 if (copy_mc_fragile(to, from, 1))
31 break;
32 return len;
33 }
34 #else
35 /*
36 * No point in doing careful copying, or consulting a static key when
37 * there is no #MC handler in the CONFIG_X86_MCE=n case.
38 */
39 void enable_copy_mc_fragile(void)
40 {
41 }
42 #define copy_mc_fragile_enabled (0)
43 #endif
44
45 unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
46
47 /**
48 * copy_mc_to_kernel - memory copy that handles source exceptions
49 *
50 * @dst: destination address
51 * @src: source address
52 * @len: number of bytes to copy
53 *
54 * Call into the 'fragile' version on systems that benefit from avoiding
55 * corner case poison consumption scenarios, For example, accessing
56 * poison across 2 cachelines with a single instruction. Almost all
57 * other uses case can use copy_mc_enhanced_fast_string() for a fast
58 * recoverable copy, or fallback to plain memcpy.
59 *
60 * Return 0 for success, or number of bytes not copied if there was an
61 * exception.
62 */
63 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
64 {
65 unsigned long ret;
66
67 if (copy_mc_fragile_enabled) {
68 instrument_memcpy_before(dst, src, len);
69 ret = copy_mc_fragile(dst, src, len);
70 instrument_memcpy_after(dst, src, len, ret);
71 return ret;
72 }
73 if (static_cpu_has(X86_FEATURE_ERMS)) {
74 instrument_memcpy_before(dst, src, len);
75 ret = copy_mc_enhanced_fast_string(dst, src, len);
76 instrument_memcpy_after(dst, src, len, ret);
77 return ret;
78 }
79 memcpy(dst, src, len);
80 return 0;
81 }
82 EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
83
84 unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len)
85 {
86 unsigned long ret;
87
88 if (copy_mc_fragile_enabled) {
89 instrument_copy_to_user(dst, src, len);
90 __uaccess_begin();
91 ret = copy_mc_fragile((__force void *)dst, src, len);
92 __uaccess_end();
93 return ret;
94 }
95
96 if (static_cpu_has(X86_FEATURE_ERMS)) {
97 instrument_copy_to_user(dst, src, len);
98 __uaccess_begin();
99 ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
100 __uaccess_end();
101 return ret;
102 }
103
104 return copy_user_generic((__force void *)dst, src, len);
105 }
106
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.