1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2013 ARM Ltd.
4 * Copyright (C) 2013 Linaro.
5 *
6 * This code is based on glibc cortex strings
7 * be found @
8 *
9 * http://bazaar.launchpad.net/~linaro-toolcha
10 * files/head:/src/aarch64/
11 */
12
13
14 /*
15 * Copy a buffer from src to dest (alignment h
16 *
17 * Parameters:
18 * x0 - dest
19 * x1 - src
20 * x2 - n
21 * Returns:
22 * x0 - dest
23 */
24 dstin .req x0
25 src .req x1
26 count .req x2
27 tmp1 .req x3
28 tmp1w .req w3
29 tmp2 .req x4
30 tmp2w .req w4
31 dst .req x6
32
33 A_l .req x7
34 A_h .req x8
35 B_l .req x9
36 B_h .req x10
37 C_l .req x11
38 C_h .req x12
39 D_l .req x13
40 D_h .req x14
41
42 mov dst, dstin
43 cmp count, #16
44 /*When memory length is less than 16,
45 b.lo .Ltiny15
46
47 neg tmp2, src
48 ands tmp2, tmp2, #15/* Bytes to rea
49 b.eq .LSrcAligned
50 sub count, count, tmp2
51 /*
52 * Copy the leading memory data from sr
53 * address order.By this way,the risk o
54 * memory data is eliminated when the d
55 * dst is less than 16. The memory acce
56 */
57 tbz tmp2, #0, 1f
58 ldrb1 tmp1w, src, #1
59 strb1 tmp1w, dst, #1
60 1:
61 tbz tmp2, #1, 2f
62 ldrh1 tmp1w, src, #2
63 strh1 tmp1w, dst, #2
64 2:
65 tbz tmp2, #2, 3f
66 ldr1 tmp1w, src, #4
67 str1 tmp1w, dst, #4
68 3:
69 tbz tmp2, #3, .LSrcAligned
70 ldr1 tmp1, src, #8
71 str1 tmp1, dst, #8
72
73 .LSrcAligned:
74 cmp count, #64
75 b.ge .Lcpy_over64
76 /*
77 * Deal with small copies quickly by dr
78 * exit block.
79 */
80 .Ltail63:
81 /*
82 * Copy up to 48 bytes of data. At this
83 * bottom 6 bits of count to be accurat
84 */
85 ands tmp1, count, #0x30
86 b.eq .Ltiny15
87 cmp tmp1w, #0x20
88 b.eq 1f
89 b.lt 2f
90 ldp1 A_l, A_h, src, #16
91 stp1 A_l, A_h, dst, #16
92 1:
93 ldp1 A_l, A_h, src, #16
94 stp1 A_l, A_h, dst, #16
95 2:
96 ldp1 A_l, A_h, src, #16
97 stp1 A_l, A_h, dst, #16
98 .Ltiny15:
99 /*
100 * Prefer to break one ldp/stp into sev
101 * memory in an increasing address orde
102 * bytes from (src-16) to (dst-16) and
103 * address,which way is used in origina
104 * the original memcpy process here, me
105 * precondition that src address is at
106 * address,otherwise some source data w
107 * call memcpy directly. To make memmov
108 * memcpy's dependency on memmove, with
109 */
110 tbz count, #3, 1f
111 ldr1 tmp1, src, #8
112 str1 tmp1, dst, #8
113 1:
114 tbz count, #2, 2f
115 ldr1 tmp1w, src, #4
116 str1 tmp1w, dst, #4
117 2:
118 tbz count, #1, 3f
119 ldrh1 tmp1w, src, #2
120 strh1 tmp1w, dst, #2
121 3:
122 tbz count, #0, .Lexitfunc
123 ldrb1 tmp1w, src, #1
124 strb1 tmp1w, dst, #1
125
126 b .Lexitfunc
127
128 .Lcpy_over64:
129 subs count, count, #128
130 b.ge .Lcpy_body_large
131 /*
132 * Less than 128 bytes to copy, so hand
133 * to the tail.
134 */
135 ldp1 A_l, A_h, src, #16
136 stp1 A_l, A_h, dst, #16
137 ldp1 B_l, B_h, src, #16
138 ldp1 C_l, C_h, src, #16
139 stp1 B_l, B_h, dst, #16
140 stp1 C_l, C_h, dst, #16
141 ldp1 D_l, D_h, src, #16
142 stp1 D_l, D_h, dst, #16
143
144 tst count, #0x3f
145 b.ne .Ltail63
146 b .Lexitfunc
147
148 /*
149 * Critical loop. Start at a new cache
150 * 64 bytes per line this ensures the e
151 */
152 .p2align L1_CACHE_SHIFT
153 .Lcpy_body_large:
154 /* pre-get 64 bytes data. */
155 ldp1 A_l, A_h, src, #16
156 ldp1 B_l, B_h, src, #16
157 ldp1 C_l, C_h, src, #16
158 ldp1 D_l, D_h, src, #16
159 1:
160 /*
161 * interlace the load of next 64 bytes
162 * loaded 64 bytes data.
163 */
164 stp1 A_l, A_h, dst, #16
165 ldp1 A_l, A_h, src, #16
166 stp1 B_l, B_h, dst, #16
167 ldp1 B_l, B_h, src, #16
168 stp1 C_l, C_h, dst, #16
169 ldp1 C_l, C_h, src, #16
170 stp1 D_l, D_h, dst, #16
171 ldp1 D_l, D_h, src, #16
172 subs count, count, #64
173 b.ge 1b
174 stp1 A_l, A_h, dst, #16
175 stp1 B_l, B_h, dst, #16
176 stp1 C_l, C_h, dst, #16
177 stp1 D_l, D_h, dst, #16
178
179 tst count, #0x3f
180 b.ne .Ltail63
181 .Lexitfunc:
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.