1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2014-15 Synopsys, Inc. (www.s
4 */
5
6 #include <linux/linkage.h>
7
8 #ifdef __LITTLE_ENDIAN__
9 # define SHIFT_1(RX,RY,IMM) asl RX, RY
10 # define SHIFT_2(RX,RY,IMM) lsr RX, RY
11 # define MERGE_1(RX,RY,IMM) asl RX, RY
12 # define MERGE_2(RX,RY,IMM)
13 # define EXTRACT_1(RX,RY,IMM) and RX, RY
14 # define EXTRACT_2(RX,RY,IMM) lsr RX, RY
15 #else
16 # define SHIFT_1(RX,RY,IMM) lsr RX, RY
17 # define SHIFT_2(RX,RY,IMM) asl RX, RY
18 # define MERGE_1(RX,RY,IMM) asl RX, RY
19 # define MERGE_2(RX,RY,IMM) asl RX, RY
20 # define EXTRACT_1(RX,RY,IMM) lsr RX, RY
21 # define EXTRACT_2(RX,RY,IMM) lsr RX, RY
22 #endif
23
24 #ifdef CONFIG_ARC_HAS_LL64
25 # define LOADX(DST,RX) ldd.ab DST, [
26 # define STOREX(SRC,RX) std.ab SRC, [
27 # define ZOLSHFT 5
28 # define ZOLAND 0x1F
29 #else
30 # define LOADX(DST,RX) ld.ab DST, [
31 # define STOREX(SRC,RX) st.ab SRC, [
32 # define ZOLSHFT 4
33 # define ZOLAND 0xF
34 #endif
35
36 ENTRY_CFI(memcpy)
37 mov.f 0, r2
38 ;;; if size is zero
39 jz.d [blink]
40 mov r3, r0 ; don;t clobbe
41
42 ;;; if size <= 8
43 cmp r2, 8
44 bls.d @.Lsmallchunk
45 mov.f lp_count, r2
46
47 and.f r4, r0, 0x03
48 rsub lp_count, r4, 4
49 lpnz @.Laligndestination
50 ;; LOOP BEGIN
51 ldb.ab r5, [r1,1]
52 sub r2, r2, 1
53 stb.ab r5, [r3,1]
54 .Laligndestination:
55
56 ;;; Check the alignment of the source
57 and.f r4, r1, 0x03
58 bnz.d @.Lsourceunaligned
59
60 ;;; CASE 0: Both source and destination are 32
61 ;;; Convert len to Dwords, unfold x4
62 lsr.f lp_count, r2, ZOLSHFT
63 lpnz @.Lcopy32_64bytes
64 ;; LOOP START
65 LOADX (r6, r1)
66 LOADX (r8, r1)
67 LOADX (r10, r1)
68 LOADX (r4, r1)
69 STOREX (r6, r3)
70 STOREX (r8, r3)
71 STOREX (r10, r3)
72 STOREX (r4, r3)
73 .Lcopy32_64bytes:
74
75 and.f lp_count, r2, ZOLAND ;Last rem
76 .Lsmallchunk:
77 lpnz @.Lcopyremainingbytes
78 ;; LOOP START
79 ldb.ab r5, [r1,1]
80 stb.ab r5, [r3,1]
81 .Lcopyremainingbytes:
82
83 j [blink]
84 ;;; END CASE 0
85
86 .Lsourceunaligned:
87 cmp r4, 2
88 beq.d @.LunalignedOffby2
89 sub r2, r2, 1
90
91 bhi.d @.LunalignedOffby3
92 ldb.ab r5, [r1, 1]
93
94 ;;; CASE 1: The source is unaligned, off by 1
95 ;; Hence I need to read 1 byte for a 1
96 ;; and 2bytes to reach 32bit alignment
97 ldh.ab r6, [r1, 2]
98 sub r2, r2, 2
99 ;; Convert to words, unfold x2
100 lsr.f lp_count, r2, 3
101 MERGE_1 (r6, r6, 8)
102 MERGE_2 (r5, r5, 24)
103 or r5, r5, r6
104
105 ;; Both src and dst are aligned
106 lpnz @.Lcopy8bytes_1
107 ;; LOOP START
108 ld.ab r6, [r1, 4]
109 ld.ab r8, [r1,4]
110
111 SHIFT_1 (r7, r6, 24)
112 or r7, r7, r5
113 SHIFT_2 (r5, r6, 8)
114
115 SHIFT_1 (r9, r8, 24)
116 or r9, r9, r5
117 SHIFT_2 (r5, r8, 8)
118
119 st.ab r7, [r3, 4]
120 st.ab r9, [r3, 4]
121 .Lcopy8bytes_1:
122
123 ;; Write back the remaining 16bits
124 EXTRACT_1 (r6, r5, 16)
125 sth.ab r6, [r3, 2]
126 ;; Write back the remaining 8bits
127 EXTRACT_2 (r5, r5, 16)
128 stb.ab r5, [r3, 1]
129
130 and.f lp_count, r2, 0x07 ;Last 8byte
131 lpnz @.Lcopybytewise_1
132 ;; LOOP START
133 ldb.ab r6, [r1,1]
134 stb.ab r6, [r3,1]
135 .Lcopybytewise_1:
136 j [blink]
137
138 .LunalignedOffby2:
139 ;;; CASE 2: The source is unaligned, off by 2
140 ldh.ab r5, [r1, 2]
141 sub r2, r2, 1
142
143 ;; Both src and dst are aligned
144 ;; Convert to words, unfold x2
145 lsr.f lp_count, r2, 3
146 #ifdef __BIG_ENDIAN__
147 asl.nz r5, r5, 16
148 #endif
149 lpnz @.Lcopy8bytes_2
150 ;; LOOP START
151 ld.ab r6, [r1, 4]
152 ld.ab r8, [r1,4]
153
154 SHIFT_1 (r7, r6, 16)
155 or r7, r7, r5
156 SHIFT_2 (r5, r6, 16)
157
158 SHIFT_1 (r9, r8, 16)
159 or r9, r9, r5
160 SHIFT_2 (r5, r8, 16)
161
162 st.ab r7, [r3, 4]
163 st.ab r9, [r3, 4]
164 .Lcopy8bytes_2:
165
166 #ifdef __BIG_ENDIAN__
167 lsr.nz r5, r5, 16
168 #endif
169 sth.ab r5, [r3, 2]
170
171 and.f lp_count, r2, 0x07 ;Last 8byte
172 lpnz @.Lcopybytewise_2
173 ;; LOOP START
174 ldb.ab r6, [r1,1]
175 stb.ab r6, [r3,1]
176 .Lcopybytewise_2:
177 j [blink]
178
179 .LunalignedOffby3:
180 ;;; CASE 3: The source is unaligned, off by 3
181 ;;; Hence, I need to read 1byte for achieve th
182
183 ;; Both src and dst are aligned
184 ;; Convert to words, unfold x2
185 lsr.f lp_count, r2, 3
186 #ifdef __BIG_ENDIAN__
187 asl.ne r5, r5, 24
188 #endif
189 lpnz @.Lcopy8bytes_3
190 ;; LOOP START
191 ld.ab r6, [r1, 4]
192 ld.ab r8, [r1,4]
193
194 SHIFT_1 (r7, r6, 8)
195 or r7, r7, r5
196 SHIFT_2 (r5, r6, 24)
197
198 SHIFT_1 (r9, r8, 8)
199 or r9, r9, r5
200 SHIFT_2 (r5, r8, 24)
201
202 st.ab r7, [r3, 4]
203 st.ab r9, [r3, 4]
204 .Lcopy8bytes_3:
205
206 #ifdef __BIG_ENDIAN__
207 lsr.nz r5, r5, 24
208 #endif
209 stb.ab r5, [r3, 1]
210
211 and.f lp_count, r2, 0x07 ;Last 8byte
212 lpnz @.Lcopybytewise_3
213 ;; LOOP START
214 ldb.ab r6, [r1,1]
215 stb.ab r6, [r3,1]
216 .Lcopybytewise_3:
217 j [blink]
218
219 END_CFI(memcpy)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.