1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright(c) 2016-2020 Intel Corporation. A
3
4 #include <linux/linkage.h>
5 #include <asm/asm.h>
6
7 #ifndef CONFIG_UML
8
9 #ifdef CONFIG_X86_MCE
10
11 /*
12 * copy_mc_fragile - copy memory with indicati
13 *
14 * The 'fragile' version is opted into by plat
15 * pains to avoid unrecoverable corner cases l
16 * instruction sequences, and consuming poison
17 * boundary. The non-fragile version is equiva
18 * regardless of CPU machine-check-recovery ca
19 */
20 SYM_FUNC_START(copy_mc_fragile)
21 cmpl $8, %edx
22 /* Less than 8 bytes? Go to byte copy
23 jb .L_no_whole_words
24
25 /* Check for bad alignment of source *
26 testl $7, %esi
27 /* Already aligned */
28 jz .L_8byte_aligned
29
30 /* Copy one byte at a time until sourc
31 movl %esi, %ecx
32 andl $7, %ecx
33 subl $8, %ecx
34 negl %ecx
35 subl %ecx, %edx
36 .L_read_leading_bytes:
37 movb (%rsi), %al
38 .L_write_leading_bytes:
39 movb %al, (%rdi)
40 incq %rsi
41 incq %rdi
42 decl %ecx
43 jnz .L_read_leading_bytes
44
45 .L_8byte_aligned:
46 movl %edx, %ecx
47 andl $7, %edx
48 shrl $3, %ecx
49 jz .L_no_whole_words
50
51 .L_read_words:
52 movq (%rsi), %r8
53 .L_write_words:
54 movq %r8, (%rdi)
55 addq $8, %rsi
56 addq $8, %rdi
57 decl %ecx
58 jnz .L_read_words
59
60 /* Any trailing bytes? */
61 .L_no_whole_words:
62 andl %edx, %edx
63 jz .L_done_memcpy_trap
64
65 /* Copy trailing bytes */
66 movl %edx, %ecx
67 .L_read_trailing_bytes:
68 movb (%rsi), %al
69 .L_write_trailing_bytes:
70 movb %al, (%rdi)
71 incq %rsi
72 incq %rdi
73 decl %ecx
74 jnz .L_read_trailing_bytes
75
76 /* Copy successful. Return zero */
77 .L_done_memcpy_trap:
78 xorl %eax, %eax
79 .L_done:
80 RET
81
82 /*
83 * Return number of bytes not copied f
84 * there is no "tail" handling since t
85 * aligned and poison is cacheline ali
86 */
87 .E_read_words:
88 shll $3, %ecx
89 .E_leading_bytes:
90 addl %edx, %ecx
91 .E_trailing_bytes:
92 mov %ecx, %eax
93 jmp .L_done
94
95 /*
96 * For write fault handling, given the
97 * we handle faults on multi-byte writ
98 * copy up to the write-protected page
99 */
100 .E_write_words:
101 shll $3, %ecx
102 addl %edx, %ecx
103 movl %ecx, %edx
104 jmp copy_mc_fragile_handle_tail
105
106 _ASM_EXTABLE_TYPE(.L_read_leading_byte
107 _ASM_EXTABLE_TYPE(.L_read_words, .E_re
108 _ASM_EXTABLE_TYPE(.L_read_trailing_byt
109 _ASM_EXTABLE(.L_write_leading_bytes, .
110 _ASM_EXTABLE(.L_write_words, .E_write_
111 _ASM_EXTABLE(.L_write_trailing_bytes,
112
113 SYM_FUNC_END(copy_mc_fragile)
114 #endif /* CONFIG_X86_MCE */
115
116 /*
117 * copy_mc_enhanced_fast_string - memory copy
118 *
119 * Fast string copy + fault / exception handli
120 * support machine check exception recovery, b
121 * recovering from fast-string exceptions then
122 * added to the copy_mc_fragile_key set of qui
123 * machine check recovery support this version
124 * standard memcpy.
125 */
126 SYM_FUNC_START(copy_mc_enhanced_fast_string)
127 movq %rdi, %rax
128 movq %rdx, %rcx
129 .L_copy:
130 rep movsb
131 /* Copy successful. Return zero */
132 xorl %eax, %eax
133 RET
134
135 .E_copy:
136 /*
137 * On fault %rcx is updated such that
138 * optionally be restarted at the faul
139 * contains 'bytes remaining'. A non-z
140 * to copy_mc_generic() users, or indi
141 * user-copy routines.
142 */
143 movq %rcx, %rax
144 RET
145
146 _ASM_EXTABLE_TYPE(.L_copy, .E_copy, EX
147
148 SYM_FUNC_END(copy_mc_enhanced_fast_string)
149 #endif /* !CONFIG_UML */
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.