1 /* SPDX-License-Identifier: GPL-2.0-or-later * 1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* 2 /*
3 * String handling functions for PowerPC. 3 * String handling functions for PowerPC.
4 * 4 *
5 * Copyright (C) 1996 Paul Mackerras. 5 * Copyright (C) 1996 Paul Mackerras.
6 */ 6 */
7 #include <linux/export.h> 7 #include <linux/export.h>
8 #include <asm/processor.h> 8 #include <asm/processor.h>
9 #include <asm/errno.h> 9 #include <asm/errno.h>
10 #include <asm/ppc_asm.h> 10 #include <asm/ppc_asm.h>
11 #include <asm/kasan.h> 11 #include <asm/kasan.h>
12 12
13 #ifndef CONFIG_KASAN 13 #ifndef CONFIG_KASAN
14 _GLOBAL(__memset16) 14 _GLOBAL(__memset16)
15 rlwimi r4,r4,16,0,15 15 rlwimi r4,r4,16,0,15
16 /* fall through */ 16 /* fall through */
17 17
18 _GLOBAL(__memset32) 18 _GLOBAL(__memset32)
19 rldimi r4,r4,32,0 19 rldimi r4,r4,32,0
20 /* fall through */ 20 /* fall through */
21 21
22 _GLOBAL(__memset64) 22 _GLOBAL(__memset64)
23 neg r0,r3 23 neg r0,r3
24 andi. r0,r0,7 24 andi. r0,r0,7
25 cmplw cr1,r5,r0 25 cmplw cr1,r5,r0
26 b .Lms 26 b .Lms
27 EXPORT_SYMBOL(__memset16) 27 EXPORT_SYMBOL(__memset16)
28 EXPORT_SYMBOL(__memset32) 28 EXPORT_SYMBOL(__memset32)
29 EXPORT_SYMBOL(__memset64) 29 EXPORT_SYMBOL(__memset64)
30 #endif 30 #endif
31 31
32 _GLOBAL_KASAN(memset) 32 _GLOBAL_KASAN(memset)
33 neg r0,r3 33 neg r0,r3
34 rlwimi r4,r4,8,16,23 34 rlwimi r4,r4,8,16,23
35 andi. r0,r0,7 /* # b 35 andi. r0,r0,7 /* # bytes to be 8-byte aligned */
36 rlwimi r4,r4,16,0,15 36 rlwimi r4,r4,16,0,15
37 cmplw cr1,r5,r0 /* do 37 cmplw cr1,r5,r0 /* do we get that far? */
38 rldimi r4,r4,32,0 38 rldimi r4,r4,32,0
39 .Lms: PPC_MTOCRF(1,r0) 39 .Lms: PPC_MTOCRF(1,r0)
40 mr r6,r3 40 mr r6,r3
41 blt cr1,8f 41 blt cr1,8f
42 beq 3f /* if 42 beq 3f /* if already 8-byte aligned */
43 subf r5,r0,r5 43 subf r5,r0,r5
44 bf 31,1f 44 bf 31,1f
45 stb r4,0(r6) 45 stb r4,0(r6)
46 addi r6,r6,1 46 addi r6,r6,1
47 1: bf 30,2f 47 1: bf 30,2f
48 sth r4,0(r6) 48 sth r4,0(r6)
49 addi r6,r6,2 49 addi r6,r6,2
50 2: bf 29,3f 50 2: bf 29,3f
51 stw r4,0(r6) 51 stw r4,0(r6)
52 addi r6,r6,4 52 addi r6,r6,4
53 3: srdi. r0,r5,6 53 3: srdi. r0,r5,6
54 clrldi r5,r5,58 54 clrldi r5,r5,58
55 mtctr r0 55 mtctr r0
56 beq 5f 56 beq 5f
57 .balign 16 57 .balign 16
58 4: std r4,0(r6) 58 4: std r4,0(r6)
59 std r4,8(r6) 59 std r4,8(r6)
60 std r4,16(r6) 60 std r4,16(r6)
61 std r4,24(r6) 61 std r4,24(r6)
62 std r4,32(r6) 62 std r4,32(r6)
63 std r4,40(r6) 63 std r4,40(r6)
64 std r4,48(r6) 64 std r4,48(r6)
65 std r4,56(r6) 65 std r4,56(r6)
66 addi r6,r6,64 66 addi r6,r6,64
67 bdnz 4b 67 bdnz 4b
68 5: srwi. r0,r5,3 68 5: srwi. r0,r5,3
69 clrlwi r5,r5,29 69 clrlwi r5,r5,29
70 PPC_MTOCRF(1,r0) 70 PPC_MTOCRF(1,r0)
71 beq 8f 71 beq 8f
72 bf 29,6f 72 bf 29,6f
73 std r4,0(r6) 73 std r4,0(r6)
74 std r4,8(r6) 74 std r4,8(r6)
75 std r4,16(r6) 75 std r4,16(r6)
76 std r4,24(r6) 76 std r4,24(r6)
77 addi r6,r6,32 77 addi r6,r6,32
78 6: bf 30,7f 78 6: bf 30,7f
79 std r4,0(r6) 79 std r4,0(r6)
80 std r4,8(r6) 80 std r4,8(r6)
81 addi r6,r6,16 81 addi r6,r6,16
82 7: bf 31,8f 82 7: bf 31,8f
83 std r4,0(r6) 83 std r4,0(r6)
84 addi r6,r6,8 84 addi r6,r6,8
85 8: cmpwi r5,0 85 8: cmpwi r5,0
86 PPC_MTOCRF(1,r5) 86 PPC_MTOCRF(1,r5)
87 beqlr 87 beqlr
88 bf 29,9f 88 bf 29,9f
89 stw r4,0(r6) 89 stw r4,0(r6)
90 addi r6,r6,4 90 addi r6,r6,4
91 9: bf 30,10f 91 9: bf 30,10f
92 sth r4,0(r6) 92 sth r4,0(r6)
93 addi r6,r6,2 93 addi r6,r6,2
94 10: bflr 31 94 10: bflr 31
95 stb r4,0(r6) 95 stb r4,0(r6)
96 blr 96 blr
97 EXPORT_SYMBOL(memset) 97 EXPORT_SYMBOL(memset)
98 EXPORT_SYMBOL_KASAN(memset) 98 EXPORT_SYMBOL_KASAN(memset)
99 99
100 _GLOBAL_TOC_KASAN(memmove) 100 _GLOBAL_TOC_KASAN(memmove)
101 cmplw 0,r3,r4 101 cmplw 0,r3,r4
102 bgt backwards_memcpy 102 bgt backwards_memcpy
103 b memcpy 103 b memcpy
104 104
105 _GLOBAL(backwards_memcpy) 105 _GLOBAL(backwards_memcpy)
106 rlwinm. r7,r5,32-3,3,31 /* r0 106 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
107 add r6,r3,r5 107 add r6,r3,r5
108 add r4,r4,r5 108 add r4,r4,r5
109 beq 2f 109 beq 2f
110 andi. r0,r6,3 110 andi. r0,r6,3
111 mtctr r7 111 mtctr r7
112 bne 5f 112 bne 5f
113 .balign 16 113 .balign 16
114 1: lwz r7,-4(r4) 114 1: lwz r7,-4(r4)
115 lwzu r8,-8(r4) 115 lwzu r8,-8(r4)
116 stw r7,-4(r6) 116 stw r7,-4(r6)
117 stwu r8,-8(r6) 117 stwu r8,-8(r6)
118 bdnz 1b 118 bdnz 1b
119 andi. r5,r5,7 119 andi. r5,r5,7
120 2: cmplwi 0,r5,4 120 2: cmplwi 0,r5,4
121 blt 3f 121 blt 3f
122 lwzu r0,-4(r4) 122 lwzu r0,-4(r4)
123 subi r5,r5,4 123 subi r5,r5,4
124 stwu r0,-4(r6) 124 stwu r0,-4(r6)
125 3: cmpwi 0,r5,0 125 3: cmpwi 0,r5,0
126 beqlr 126 beqlr
127 mtctr r5 127 mtctr r5
128 4: lbzu r0,-1(r4) 128 4: lbzu r0,-1(r4)
129 stbu r0,-1(r6) 129 stbu r0,-1(r6)
130 bdnz 4b 130 bdnz 4b
131 blr 131 blr
132 5: mtctr r0 132 5: mtctr r0
133 6: lbzu r7,-1(r4) 133 6: lbzu r7,-1(r4)
134 stbu r7,-1(r6) 134 stbu r7,-1(r6)
135 bdnz 6b 135 bdnz 6b
136 subf r5,r0,r5 136 subf r5,r0,r5
137 rlwinm. r7,r5,32-3,3,31 137 rlwinm. r7,r5,32-3,3,31
138 beq 2b 138 beq 2b
139 mtctr r7 139 mtctr r7
140 b 1b 140 b 1b
141 EXPORT_SYMBOL(memmove) 141 EXPORT_SYMBOL(memmove)
142 EXPORT_SYMBOL_KASAN(memmove) 142 EXPORT_SYMBOL_KASAN(memmove)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.