1 /*
2 * This file is subject to the terms and condi
3 * License. See the file "COPYING" in the mai
4 * for more details.
5 *
6 * Unified implementation of memcpy, memmove a
7 *
8 * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf
9 * Copyright (C) 1999, 2000, 01, 2002 Silicon
10 * Copyright (C) 2002 Broadcom, Inc.
11 * memcpy/copy_user author: Mark Vandevoorde
12 *
13 * Mnemonic names for arguments to memcpy/__co
14 */
15
16 #include <linux/export.h>
17 #include <asm/asm.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/regdef.h>
20
21 #define dst a0
22 #define src a1
23 #define len a2
24
25 /*
26 * Spec
27 *
28 * memcpy copies len bytes from src to dst and
29 * It assumes that
30 * - src and dst don't overlap
31 * - src is readable
32 * - dst is writable
33 * memcpy uses the standard calling convention
34 *
35 * __copy_user copies up to len bytes from src
36 * the number of uncopied bytes due to an exce
37 * __copy_user assumes that src and dst don't
38 * implementing one of the following:
39 * copy_to_user
40 * - src is readable (no exceptions when
41 * copy_from_user
42 * - dst is writable (no exceptions when
43 * __copy_user uses a non-standard calling con
44 * arch/mips/include/asm/uaccess.h
45 *
46 * When an exception happens on a load, the ha
47 # ensure that all of the destination buffer i
48 * leaking information to user mode programs.
49 */
50
51 /*
52 * Implementation
53 */
54
55 /*
56 * The exception handler for loads requires th
57 * 1- AT contain the address of the byte just
58 * of the copy,
59 * 2- src_entry <= src < AT, and
60 * 3- (dst - src) == (dst_entry - src_entry),
61 * The _entry suffix denotes values when __cop
62 *
63 * (1) is set up up by uaccess.h and maintaine
64 * (2) is met by incrementing src by the numbe
65 * (3) is met by not doing loads between a pai
66 *
67 * The exception handlers for stores adjust le
68 * These handlers do not need to overwrite any
69 *
70 * For __rmemcpy and memmove an exception is a
71 * they're not protected.
72 */
73
74 #define EXC(inst_reg,addr,handler)
75 9: inst_reg, addr;
76 .section __ex_table,"a";
77 PTR_WD 9b, handler;
78 .previous
79
80 /*
81 * Only on the 64-bit kernel we can made use o
82 */
83
84 #define LOAD ld
85 #define LOADL ldl
86 #define LOADR ldr
87 #define STOREL sdl
88 #define STORER sdr
89 #define STORE sd
90 #define ADD daddu
91 #define SUB dsubu
92 #define SRL dsrl
93 #define SRA dsra
94 #define SLL dsll
95 #define SLLV dsllv
96 #define SRLV dsrlv
97 #define NBYTES 8
98 #define LOG_NBYTES 3
99
100 /*
101 * As we are sharing code base with the mips32
102 * register definitions). We need to redefine
103 * the n64 ABI register naming to the o32 ABI
104 */
105 #undef t0
106 #undef t1
107 #undef t2
108 #undef t3
109 #define t0 $8
110 #define t1 $9
111 #define t2 $10
112 #define t3 $11
113 #define t4 $12
114 #define t5 $13
115 #define t6 $14
116 #define t7 $15
117
118 #ifdef CONFIG_CPU_LITTLE_ENDIAN
119 #define LDFIRST LOADR
120 #define LDREST LOADL
121 #define STFIRST STORER
122 #define STREST STOREL
123 #define SHIFT_DISCARD SLLV
124 #else
125 #define LDFIRST LOADL
126 #define LDREST LOADR
127 #define STFIRST STOREL
128 #define STREST STORER
129 #define SHIFT_DISCARD SRLV
130 #endif
131
132 #define FIRST(unit) ((unit)*NBYTES)
133 #define REST(unit) (FIRST(unit)+NBYTES-1)
134 #define UNIT(unit) FIRST(unit)
135
136 #define ADDRMASK (NBYTES-1)
137
138 .text
139 .set noreorder
140 .set noat
141
142 /*
143 * A combined memcpy/__copy_user
144 * __copy_user sets len to 0 for success; else
145 * the number of uncopied bytes.
146 * memcpy sets v0 to dst.
147 */
148 .align 5
149 LEAF(memcpy)
150 EXPORT_SYMBOL(memcpy)
151 move v0, dst
152 __memcpy:
153 FEXPORT(__raw_copy_from_user)
154 EXPORT_SYMBOL(__raw_copy_from_user)
155 FEXPORT(__raw_copy_to_user)
156 EXPORT_SYMBOL(__raw_copy_to_user)
157 /*
158 * Note: dst & src may be unaligned, l
159 * Temps
160 */
161 #
162 # Octeon doesn't care if the destinati
163 # can fix it faster than we can specia
164 #
165 pref 0, 0(src)
166 sltu t0, len, NBYTES # Chec
167 bnez t0, copy_bytes_checklen
168 and t0, src, ADDRMASK # Chec
169 bnez t0, src_unaligned
170 sltu t0, len, 4*NBYTES # Chec
171 bnez t0, less_than_4units
172 sltu t0, len, 8*NBYTES # Chec
173 bnez t0, less_than_8units
174 sltu t0, len, 16*NBYTES # Chec
175 bnez t0, cleanup_both_aligned
176 sltu t0, len, 128+1 # Chec
177 bnez t0, 1f # Skip
178 sltu t0, len, 256+1 # Chec
179 bnez t0, 1f # Skip
180 pref 0, 128(src) # We m
181 #
182 # This is where we loop if there is mo
183 2: pref 0, 256(src) # We m
184 #
185 # This is where we loop if we can't pr
186 1:
187 EXC( LOAD t0, UNIT(0)(src), l_exc)
188 EXC( LOAD t1, UNIT(1)(src), l_exc_
189 EXC( LOAD t2, UNIT(2)(src), l_exc_
190 EXC( LOAD t3, UNIT(3)(src), l_exc_
191 SUB len, len, 16*NBYTES
192 EXC( STORE t0, UNIT(0)(dst), s_exc_
193 EXC( STORE t1, UNIT(1)(dst), s_exc_
194 EXC( STORE t2, UNIT(2)(dst), s_exc_
195 EXC( STORE t3, UNIT(3)(dst), s_exc_
196 EXC( LOAD t0, UNIT(4)(src), l_exc_
197 EXC( LOAD t1, UNIT(5)(src), l_exc_
198 EXC( LOAD t2, UNIT(6)(src), l_exc_
199 EXC( LOAD t3, UNIT(7)(src), l_exc_
200 EXC( STORE t0, UNIT(4)(dst), s_exc_
201 EXC( STORE t1, UNIT(5)(dst), s_exc_
202 EXC( STORE t2, UNIT(6)(dst), s_exc_
203 ADD src, src, 16*NBYTES
204 EXC( STORE t3, UNIT(7)(dst), s_exc_
205 ADD dst, dst, 16*NBYTES
206 EXC( LOAD t0, UNIT(-8)(src), l_exc_
207 EXC( LOAD t1, UNIT(-7)(src), l_exc_
208 EXC( LOAD t2, UNIT(-6)(src), l_exc_
209 EXC( LOAD t3, UNIT(-5)(src), l_exc_
210 EXC( STORE t0, UNIT(-8)(dst), s_exc_
211 EXC( STORE t1, UNIT(-7)(dst), s_exc_
212 EXC( STORE t2, UNIT(-6)(dst), s_exc_
213 EXC( STORE t3, UNIT(-5)(dst), s_exc_
214 EXC( LOAD t0, UNIT(-4)(src), l_exc_
215 EXC( LOAD t1, UNIT(-3)(src), l_exc_
216 EXC( LOAD t2, UNIT(-2)(src), l_exc_
217 EXC( LOAD t3, UNIT(-1)(src), l_exc_
218 EXC( STORE t0, UNIT(-4)(dst), s_exc_
219 EXC( STORE t1, UNIT(-3)(dst), s_exc_
220 EXC( STORE t2, UNIT(-2)(dst), s_exc_
221 EXC( STORE t3, UNIT(-1)(dst), s_exc_
222 sltu t0, len, 256+1 # See
223 beqz t0, 2b
224 sltu t0, len, 128 # See
225 beqz t0, 1b
226 nop
227 #
228 # Jump here if there are less than 16*
229 #
230 cleanup_both_aligned:
231 beqz len, done
232 sltu t0, len, 8*NBYTES
233 bnez t0, less_than_8units
234 nop
235 EXC( LOAD t0, UNIT(0)(src), l_exc)
236 EXC( LOAD t1, UNIT(1)(src), l_exc_
237 EXC( LOAD t2, UNIT(2)(src), l_exc_
238 EXC( LOAD t3, UNIT(3)(src), l_exc_
239 SUB len, len, 8*NBYTES
240 EXC( STORE t0, UNIT(0)(dst), s_exc_
241 EXC( STORE t1, UNIT(1)(dst), s_exc_
242 EXC( STORE t2, UNIT(2)(dst), s_exc_
243 EXC( STORE t3, UNIT(3)(dst), s_exc_
244 EXC( LOAD t0, UNIT(4)(src), l_exc_
245 EXC( LOAD t1, UNIT(5)(src), l_exc_
246 EXC( LOAD t2, UNIT(6)(src), l_exc_
247 EXC( LOAD t3, UNIT(7)(src), l_exc_
248 EXC( STORE t0, UNIT(4)(dst), s_exc_
249 EXC( STORE t1, UNIT(5)(dst), s_exc_
250 EXC( STORE t2, UNIT(6)(dst), s_exc_
251 EXC( STORE t3, UNIT(7)(dst), s_exc_
252 ADD src, src, 8*NBYTES
253 beqz len, done
254 ADD dst, dst, 8*NBYTES
255 #
256 # Jump here if there are less than 8*N
257 #
258 less_than_8units:
259 sltu t0, len, 4*NBYTES
260 bnez t0, less_than_4units
261 nop
262 EXC( LOAD t0, UNIT(0)(src), l_exc)
263 EXC( LOAD t1, UNIT(1)(src), l_exc_
264 EXC( LOAD t2, UNIT(2)(src), l_exc_
265 EXC( LOAD t3, UNIT(3)(src), l_exc_
266 SUB len, len, 4*NBYTES
267 EXC( STORE t0, UNIT(0)(dst), s_exc_
268 EXC( STORE t1, UNIT(1)(dst), s_exc_
269 EXC( STORE t2, UNIT(2)(dst), s_exc_
270 EXC( STORE t3, UNIT(3)(dst), s_exc_
271 ADD src, src, 4*NBYTES
272 beqz len, done
273 ADD dst, dst, 4*NBYTES
274 #
275 # Jump here if there are less than 4*N
276 # we may need to copy up to 3 NBYTES w
277 #
278 less_than_4units:
279 sltu t0, len, 1*NBYTES
280 bnez t0, copy_bytes_checklen
281 nop
282 #
283 # 1) Copy NBYTES, then check length ag
284 #
285 EXC( LOAD t0, 0(src), l_exc)
286 SUB len, len, NBYTES
287 sltu t1, len, 8
288 EXC( STORE t0, 0(dst), s_exc_
289 ADD src, src, NBYTES
290 bnez t1, copy_bytes_checklen
291 ADD dst, dst, NBYTES
292 #
293 # 2) Copy NBYTES, then check length ag
294 #
295 EXC( LOAD t0, 0(src), l_exc)
296 SUB len, len, NBYTES
297 sltu t1, len, 8
298 EXC( STORE t0, 0(dst), s_exc_
299 ADD src, src, NBYTES
300 bnez t1, copy_bytes_checklen
301 ADD dst, dst, NBYTES
302 #
303 # 3) Copy NBYTES, then check length ag
304 #
305 EXC( LOAD t0, 0(src), l_exc)
306 SUB len, len, NBYTES
307 ADD src, src, NBYTES
308 ADD dst, dst, NBYTES
309 b copy_bytes_checklen
310 EXC( STORE t0, -8(dst), s_exc_
311
312 src_unaligned:
313 #define rem t8
314 SRL t0, len, LOG_NBYTES+2 # +2
315 beqz t0, cleanup_src_unaligned
316 and rem, len, (4*NBYTES-1) # rem
317 1:
318 /*
319 * Avoid consecutive LD*'s to the same registe
320 * implementations can't issue them in the sam
321 * It's OK to load FIRST(N+1) before REST(N) b
322 * are to the same unit (unless src is aligned
323 */
324 EXC( LDFIRST t0, FIRST(0)(src), l_exc)
325 EXC( LDFIRST t1, FIRST(1)(src), l_exc_
326 SUB len, len, 4*NBYTES
327 EXC( LDREST t0, REST(0)(src), l_exc_
328 EXC( LDREST t1, REST(1)(src), l_exc_
329 EXC( LDFIRST t2, FIRST(2)(src), l_exc_
330 EXC( LDFIRST t3, FIRST(3)(src), l_exc_
331 EXC( LDREST t2, REST(2)(src), l_exc_
332 EXC( LDREST t3, REST(3)(src), l_exc_
333 ADD src, src, 4*NBYTES
334 EXC( STORE t0, UNIT(0)(dst), s_exc_
335 EXC( STORE t1, UNIT(1)(dst), s_exc_
336 EXC( STORE t2, UNIT(2)(dst), s_exc_
337 EXC( STORE t3, UNIT(3)(dst), s_exc_
338 bne len, rem, 1b
339 ADD dst, dst, 4*NBYTES
340
341 cleanup_src_unaligned:
342 beqz len, done
343 and rem, len, NBYTES-1 # rem = le
344 beq rem, len, copy_bytes
345 nop
346 1:
347 EXC( LDFIRST t0, FIRST(0)(src), l_exc)
348 EXC( LDREST t0, REST(0)(src), l_exc_
349 SUB len, len, NBYTES
350 EXC( STORE t0, 0(dst), s_exc_
351 ADD src, src, NBYTES
352 bne len, rem, 1b
353 ADD dst, dst, NBYTES
354
355 copy_bytes_checklen:
356 beqz len, done
357 nop
358 copy_bytes:
359 /* 0 < len < NBYTES */
360 #define COPY_BYTE(N) \
361 EXC( lb t0, N(src), l_exc); \
362 SUB len, len, 1; \
363 beqz len, done; \
364 EXC( sb t0, N(dst), s_exc_p1)
365
366 COPY_BYTE(0)
367 COPY_BYTE(1)
368 COPY_BYTE(2)
369 COPY_BYTE(3)
370 COPY_BYTE(4)
371 COPY_BYTE(5)
372 EXC( lb t0, NBYTES-2(src), l_exc)
373 SUB len, len, 1
374 jr ra
375 EXC( sb t0, NBYTES-2(dst), s_exc_p1)
376 done:
377 jr ra
378 nop
379 END(memcpy)
380
381 l_exc_copy_rewind16:
382 /* Rewind src and dst by 16*NBYTES for
383 SUB src, src, 16*NBYTES
384 SUB dst, dst, 16*NBYTES
385 l_exc_copy:
386 /*
387 * Copy bytes from src until faulting
388 * lb faults)
389 *
390 * When reached by a faulting LDFIRST/
391 * may be more than a byte beyond the
392 * Hence, the lb below may get an exce
393 *
394 * Assumes src < THREAD_BUADDR($28)
395 */
396 LOAD t0, TI_TASK($28)
397 LOAD t0, THREAD_BUADDR(t0)
398 1:
399 EXC( lb t1, 0(src), l_exc)
400 ADD src, src, 1
401 sb t1, 0(dst) # can't fault
402 bne src, t0, 1b
403 ADD dst, dst, 1
404 l_exc:
405 LOAD t0, TI_TASK($28)
406 LOAD t0, THREAD_BUADDR(t0) # t0 i
407 SUB len, AT, t0 # len
408 jr ra
409 nop
410
411
412 #define SEXC(n) \
413 s_exc_p ## n ## u: \
414 jr ra; \
415 ADD len, len, n*NBYTES
416
417 SEXC(16)
418 SEXC(15)
419 SEXC(14)
420 SEXC(13)
421 SEXC(12)
422 SEXC(11)
423 SEXC(10)
424 SEXC(9)
425 SEXC(8)
426 SEXC(7)
427 SEXC(6)
428 SEXC(5)
429 SEXC(4)
430 SEXC(3)
431 SEXC(2)
432 SEXC(1)
433
434 s_exc_p1:
435 jr ra
436 ADD len, len, 1
437 s_exc:
438 jr ra
439 nop
440
441 .align 5
442 LEAF(memmove)
443 EXPORT_SYMBOL(memmove)
444 ADD t0, a0, a2
445 ADD t1, a1, a2
446 sltu t0, a1, t0
447 sltu t1, a0, t1
448 and t0, t1
449 beqz t0, __memcpy
450 move v0, a0
451 beqz a2, r_out
452 END(memmove)
453
454 /* fall through to __rmemcpy */
455 LEAF(__rmemcpy)
456 sltu t0, a1, a0
457 beqz t0, r_end_bytes_up
458 nop
459 ADD a0, a2
460 ADD a1, a2
461
462 r_end_bytes:
463 lb t0, -1(a1)
464 SUB a2, a2, 0x1
465 sb t0, -1(a0)
466 SUB a1, a1, 0x1
467 bnez a2, r_end_bytes
468 SUB a0, a0, 0x1
469
470 r_out:
471 jr ra
472 move a2, zero
473
474 r_end_bytes_up:
475 lb t0, (a1)
476 SUB a2, a2, 0x1
477 sb t0, (a0)
478 ADD a1, a1, 0x1
479 bnez a2, r_end_bytes_up
480 ADD a0, a0, 0x1
481
482 jr ra
483 move a2, zero
484 END(__rmemcpy)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.