1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@r
4 * Copyright 2002 Andi Kleen, SuSE Labs.
5 *
6 * Functions to copy from and to user space.
7 */
8
9 #include <linux/export.h>
10 #include <linux/linkage.h>
11 #include <asm/cpufeatures.h>
12 #include <asm/alternative.h>
13 #include <asm/asm.h>
14
15 /*
16 * rep_movs_alternative - memory copy with exc
17 * This version is for CPUs that don't have FS
18 *
19 * Input:
20 * rdi destination
21 * rsi source
22 * rcx count
23 *
24 * Output:
25 * rcx uncopied bytes or 0 if successful.
26 *
27 * NOTE! The calling convention is very intent
28 * for 'rep movs', so that we can rewrite the
29 * just a plain 'rep movs' on machines that ha
30 * it simpler for us, we can clobber rsi/rdi a
31 */
32 SYM_FUNC_START(rep_movs_alternative)
33 cmpq $64,%rcx
34 jae .Llarge
35
36 cmp $8,%ecx
37 jae .Lword
38
39 testl %ecx,%ecx
40 je .Lexit
41
42 .Lcopy_user_tail:
43 0: movb (%rsi),%al
44 1: movb %al,(%rdi)
45 inc %rdi
46 inc %rsi
47 dec %rcx
48 jne .Lcopy_user_tail
49 .Lexit:
50 RET
51
52 _ASM_EXTABLE_UA( 0b, .Lexit)
53 _ASM_EXTABLE_UA( 1b, .Lexit)
54
55 .p2align 4
56 .Lword:
57 2: movq (%rsi),%rax
58 3: movq %rax,(%rdi)
59 addq $8,%rsi
60 addq $8,%rdi
61 sub $8,%ecx
62 je .Lexit
63 cmp $8,%ecx
64 jae .Lword
65 jmp .Lcopy_user_tail
66
67 _ASM_EXTABLE_UA( 2b, .Lcopy_user_tail)
68 _ASM_EXTABLE_UA( 3b, .Lcopy_user_tail)
69
70 .Llarge:
71 0: ALTERNATIVE "jmp .Llarge_movsq", "rep
72 1: RET
73
74 _ASM_EXTABLE_UA( 0b, 1b)
75
76 .Llarge_movsq:
77 movq %rcx,%rax
78 shrq $3,%rcx
79 andl $7,%eax
80 0: rep movsq
81 movl %eax,%ecx
82 testl %ecx,%ecx
83 jne .Lcopy_user_tail
84 RET
85
86 1: leaq (%rax,%rcx,8),%rcx
87 jmp .Lcopy_user_tail
88
89 _ASM_EXTABLE_UA( 0b, 1b)
90 SYM_FUNC_END(rep_movs_alternative)
91 EXPORT_SYMBOL(rep_movs_alternative)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.