Linux/arch/hexagon/lib/memset.S

Version: ~ [ linux-6.12-rc7 ] ~ [ linux-6.11.7 ] ~ [ linux-6.10.14 ] ~ [ linux-6.9.12 ] ~ [ linux-6.8.12 ] ~ [ linux-6.7.12 ] ~ [ linux-6.6.60 ] ~ [ linux-6.5.13 ] ~ [ linux-6.4.16 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.116 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.171 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.229 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.285 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.323 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.336 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.12 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

Diff markup

1 /* SPDX-License-Identifier: GPL-2.0-only */ << 2 /* 1 /* 3 * Copyright (c) 2011, The Linux Foundation. A !! 2 * This file is subject to the terms and conditions of the GNU General Public >> 3 * License. See the file "COPYING" in the main directory of this archive >> 4 * for more details. >> 5 * >> 6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle >> 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. >> 8 * Copyright (C) 2007 by Maciej W. Rozycki >> 9 * Copyright (C) 2011, 2012 MIPS Technologies, Inc. 4 */ 10 */ >> 11 #include <asm/asm.h> >> 12 #include <asm/asm-offsets.h> >> 13 #include <asm/regdef.h> >> 14 >> 15 #if LONGSIZE == 4 >> 16 #define LONG_S_L swl >> 17 #define LONG_S_R swr >> 18 #else >> 19 #define LONG_S_L sdl >> 20 #define LONG_S_R sdr >> 21 #endif 5 22 >> 23 #ifdef CONFIG_CPU_MICROMIPS >> 24 #define STORSIZE (LONGSIZE * 2) >> 25 #define STORMASK (STORSIZE - 1) >> 26 #define FILL64RG t8 >> 27 #define FILLPTRG t7 >> 28 #undef LONG_S >> 29 #define LONG_S LONG_SP >> 30 #else >> 31 #define STORSIZE LONGSIZE >> 32 #define STORMASK LONGMASK >> 33 #define FILL64RG a1 >> 34 #define FILLPTRG t0 >> 35 #endif 6 36 7 /* HEXAGON assembly optimized memset */ !! 37 #define EX(insn,reg,addr,handler) \ 8 /* Replaces the standard library function mems !! 38 9: insn reg, addr; \ >> 39 .section __ex_table,"a"; \ >> 40 PTR 9b, handler; \ >> 41 .previous >> 42 >> 43 .macro f_fill64 dst, offset, val, fixup >> 44 EX(LONG_S, \val, (\offset + 0 * STORSIZE)(\dst), \fixup) >> 45 EX(LONG_S, \val, (\offset + 1 * STORSIZE)(\dst), \fixup) >> 46 EX(LONG_S, \val, (\offset + 2 * STORSIZE)(\dst), \fixup) >> 47 EX(LONG_S, \val, (\offset + 3 * STORSIZE)(\dst), \fixup) >> 48 #if ((defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) || !defined(CONFIG_CPU_MICROMIPS)) >> 49 EX(LONG_S, \val, (\offset + 4 * STORSIZE)(\dst), \fixup) >> 50 EX(LONG_S, \val, (\offset + 5 * STORSIZE)(\dst), \fixup) >> 51 EX(LONG_S, \val, (\offset + 6 * STORSIZE)(\dst), \fixup) >> 52 EX(LONG_S, \val, (\offset + 7 * STORSIZE)(\dst), \fixup) >> 53 #endif >> 54 #if (!defined(CONFIG_CPU_MICROMIPS) && (LONGSIZE == 4)) >> 55 EX(LONG_S, \val, (\offset + 8 * STORSIZE)(\dst), \fixup) >> 56 EX(LONG_S, \val, (\offset + 9 * STORSIZE)(\dst), \fixup) >> 57 EX(LONG_S, \val, (\offset + 10 * STORSIZE)(\dst), \fixup) >> 58 EX(LONG_S, \val, (\offset + 11 * STORSIZE)(\dst), \fixup) >> 59 EX(LONG_S, \val, (\offset + 12 * STORSIZE)(\dst), \fixup) >> 60 EX(LONG_S, \val, (\offset + 13 * STORSIZE)(\dst), \fixup) >> 61 EX(LONG_S, \val, (\offset + 14 * STORSIZE)(\dst), \fixup) >> 62 EX(LONG_S, \val, (\offset + 15 * STORSIZE)(\dst), \fixup) >> 63 #endif >> 64 .endm 9 65 >> 66 /* >> 67 * memset(void *s, int c, size_t n) >> 68 * >> 69 * a0: start of area to clear >> 70 * a1: char to fill with >> 71 * a2: size of area to clear >> 72 */ >> 73 .set noreorder >> 74 .align 5 >> 75 LEAF(memset) >> 76 beqz a1, 1f >> 77 move v0, a0 /* result */ >> 78 >> 79 andi a1, 0xff /* spread fillword */ >> 80 LONG_SLL t1, a1, 8 >> 81 or a1, t1 >> 82 LONG_SLL t1, a1, 16 >> 83 #if LONGSIZE == 8 >> 84 or a1, t1 >> 85 LONG_SLL t1, a1, 32 >> 86 #endif >> 87 or a1, t1 >> 88 1: 10 89 11 .macro HEXAGON_OPT_FUNC_BEGIN name !! 90 FEXPORT(__bzero) 12 .text !! 91 sltiu t0, a2, STORSIZE /* very small region? */ 13 .p2align 4 !! 92 bnez t0, .Lsmall_memset 14 .globl \name !! 93 andi t0, a0, STORMASK /* aligned? */ 15 .type \name, @function !! 94 16 \name: !! 95 #ifdef CONFIG_CPU_MICROMIPS 17 .endm !! 96 move t8, a1 /* used by 'swp' instruction */ >> 97 move t9, a1 >> 98 #endif >> 99 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS >> 100 beqz t0, 1f >> 101 PTR_SUBU t0, STORSIZE /* alignment in bytes */ >> 102 #else >> 103 .set noat >> 104 li AT, STORSIZE >> 105 beqz t0, 1f >> 106 PTR_SUBU t0, AT /* alignment in bytes */ >> 107 .set at >> 108 #endif 18 109 19 .macro HEXAGON_OPT_FUNC_FINISH name !! 110 R10KCBARRIER(0(ra)) 20 .size \name, . - \name !! 111 #ifdef __MIPSEB__ 21 .endm !! 112 EX(LONG_S_L, a1, (a0), .Lfirst_fixup) /* make word/dword aligned */ >> 113 #endif >> 114 #ifdef __MIPSEL__ >> 115 EX(LONG_S_R, a1, (a0), .Lfirst_fixup) /* make word/dword aligned */ >> 116 #endif >> 117 PTR_SUBU a0, t0 /* long align ptr */ >> 118 PTR_ADDU a2, t0 /* correct size */ 22 119 23 /* FUNCTION: memset (v2 version) */ !! 120 1: ori t1, a2, 0x3f /* # of full blocks */ 24 #if __HEXAGON_ARCH__ < 3 !! 121 xori t1, 0x3f 25 HEXAGON_OPT_FUNC_BEGIN memset !! 122 beqz t1, .Lmemset_partial /* no block to fill */ 26 { !! 123 andi t0, a2, 0x40-STORSIZE 27 r6 = #8 !! 124 28 r7 = extractu(r0, #3 , #0) !! 125 PTR_ADDU t1, a0 /* end address */ 29 p0 = cmp.eq(r2, #0) !! 126 .set reorder 30 p1 = cmp.gtu(r2, #7) !! 127 1: PTR_ADDIU a0, 64 31 } !! 128 R10KCBARRIER(0(ra)) 32 { !! 129 f_fill64 a0, -64, FILL64RG, .Lfwd_fixup 33 r4 = vsplatb(r1) !! 130 bne t1, a0, 1b 34 r8 = r0 /* leave r0 !! 131 .set noreorder 35 r9 = sub(r6, r7) /* bytes unt !! 132 36 if p0 jumpr r31 /* count == !! 133 .Lmemset_partial: 37 } !! 134 R10KCBARRIER(0(ra)) 38 { !! 135 PTR_LA t1, 2f /* where to start */ 39 r3 = #0 !! 136 #ifdef CONFIG_CPU_MICROMIPS 40 r7 = #0 !! 137 LONG_SRL t7, t0, 1 41 p0 = tstbit(r9, #0) !! 138 #endif 42 if p1 jump 2f /* skip byte loo !! 139 #if LONGSIZE == 4 43 } !! 140 PTR_SUBU t1, FILLPTRG 44 !! 141 #else 45 /* less than 8 bytes to set, so just set a byt !! 142 .set noat 46 !! 143 LONG_SRL AT, FILLPTRG, 1 47 loop0(1f, r2) /* byte loop */ !! 144 PTR_SUBU t1, AT 48 .falign !! 145 .set at 49 1: /* byte loop */ << 50 { << 51 memb(r8++#1) = r4 << 52 }:endloop0 << 53 jumpr r31 << 54 .falign << 55 2: /* skip byte loop */ << 56 { << 57 r6 = #1 << 58 p0 = tstbit(r9, #1) << 59 p1 = cmp.eq(r2, #1) << 60 if !p0 jump 3f /* skip initial << 61 } << 62 { << 63 memb(r8++#1) = r4 << 64 r3:2 = sub(r3:2, r7:6) << 65 if p1 jumpr r31 << 66 } << 67 .falign << 68 3: /* skip initial byte store */ << 69 { << 70 r6 = #2 << 71 p0 = tstbit(r9, #2) << 72 p1 = cmp.eq(r2, #2) << 73 if !p0 jump 4f /* skip initial << 74 } << 75 { << 76 memh(r8++#2) = r4 << 77 r3:2 = sub(r3:2, r7:6) << 78 if p1 jumpr r31 << 79 } << 80 .falign << 81 4: /* skip initial half store */ << 82 { << 83 r6 = #4 << 84 p0 = cmp.gtu(r2, #7) << 85 p1 = cmp.eq(r2, #4) << 86 if !p0 jump 5f /* skip initial << 87 } << 88 { << 89 memw(r8++#4) = r4 << 90 r3:2 = sub(r3:2, r7:6) << 91 p0 = cmp.gtu(r2, #11) << 92 if p1 jumpr r31 << 93 } << 94 .falign << 95 5: /* skip initial word store */ << 96 { << 97 r10 = lsr(r2, #3) << 98 p1 = cmp.eq(r3, #1) << 99 if !p0 jump 7f /* skip double << 100 } << 101 { << 102 r5 = r4 << 103 r6 = #8 << 104 loop0(6f, r10) /* double loop << 105 } << 106 << 107 /* set bytes a double word at a time */ << 108 << 109 .falign << 110 6: /* double loop */ << 111 { << 112 memd(r8++#8) = r5:4 << 113 r3:2 = sub(r3:2, r7:6) << 114 p1 = cmp.eq(r2, #8) << 115 }:endloop0 << 116 .falign << 117 7: /* skip double loop */ << 118 { << 119 p0 = tstbit(r2, #2) << 120 if p1 jumpr r31 << 121 } << 122 { << 123 r6 = #4 << 124 p0 = tstbit(r2, #1) << 125 p1 = cmp.eq(r2, #4) << 126 if !p0 jump 8f /* skip final w << 127 } << 128 { << 129 memw(r8++#4) = r4 << 130 r3:2 = sub(r3:2, r7:6) << 131 if p1 jumpr r31 << 132 } << 133 .falign << 134 8: /* skip final word store */ << 135 { << 136 p1 = cmp.eq(r2, #2) << 137 if !p0 jump 9f /* skip final h << 138 } << 139 { << 140 memh(r8++#2) = r4 << 141 if p1 jumpr r31 << 142 } << 143 .falign << 144 9: /* skip final half store */ << 145 { << 146 memb(r8++#1) = r4 << 147 jumpr r31 << 148 } << 149 HEXAGON_OPT_FUNC_FINISH memset << 150 #endif << 151 << 152 << 153 /* FUNCTION: memset (v3 and higher version) << 154 #if __HEXAGON_ARCH__ >= 3 << 155 HEXAGON_OPT_FUNC_BEGIN memset << 156 { << 157 r7=vsplatb(r1) << 158 r6 = r0 << 159 if (r2==#0) jump:nt .L1 << 160 } << 161 { << 162 r5:4=combine(r7,r7) << 163 p0 = cmp.gtu(r2,#8) << 164 if (p0.new) jump:nt .L3 << 165 } << 166 { << 167 r3 = r0 << 168 loop0(.L47,r2) << 169 } << 170 .falign << 171 .L47: << 172 { << 173 memb(r3++#1) = r1 << 174 }:endloop0 /* start=.L47 */ << 175 jumpr r31 << 176 .L3: << 177 { << 178 p0 = tstbit(r0,#0) << 179 if (!p0.new) jump:nt .L8 << 180 p1 = cmp.eq(r2, #1) << 181 } << 182 { << 183 r6 = add(r0, #1) << 184 r2 = add(r2,#-1) << 185 memb(r0) = r1 << 186 if (p1) jump .L1 << 187 } << 188 .L8: << 189 { << 190 p0 = tstbit(r6,#1) << 191 if (!p0.new) jump:nt .L10 << 192 } << 193 { << 194 r2 = add(r2,#-2) << 195 memh(r6++#2) = r7 << 196 p0 = cmp.eq(r2, #2) << 197 if (p0.new) jump:nt .L1 << 198 } << 199 .L10: << 200 { << 201 p0 = tstbit(r6,#2) << 202 if (!p0.new) jump:nt .L12 << 203 } << 204 { << 205 r2 = add(r2,#-4) << 206 memw(r6++#4) = r7 << 207 p0 = cmp.eq(r2, #4) << 208 if (p0.new) jump:nt .L1 << 209 } << 210 .L12: << 211 { << 212 p0 = cmp.gtu(r2,#127) << 213 if (!p0.new) jump:nt .L14 << 214 } << 215 r3 = and(r6,#31) << 216 if (r3==#0) jump:nt .L17 << 217 { << 218 memd(r6++#8) = r5:4 << 219 r2 = add(r2,#-8) << 220 } << 221 r3 = and(r6,#31) << 222 if (r3==#0) jump:nt .L17 << 223 { << 224 memd(r6++#8) = r5:4 << 225 r2 = add(r2,#-8) << 226 } << 227 r3 = and(r6,#31) << 228 if (r3==#0) jump:nt .L17 << 229 { << 230 memd(r6++#8) = r5:4 << 231 r2 = add(r2,#-8) << 232 } << 233 .L17: << 234 { << 235 r3 = lsr(r2,#5) << 236 if (r1!=#0) jump:nt .L18 << 237 } << 238 { << 239 r8 = r3 << 240 r3 = r6 << 241 loop0(.L46,r3) << 242 } << 243 .falign << 244 .L46: << 245 { << 246 dczeroa(r6) << 247 r6 = add(r6,#32) << 248 r2 = add(r2,#-32) << 249 }:endloop0 /* start=.L46 */ << 250 .L14: << 251 { << 252 p0 = cmp.gtu(r2,#7) << 253 if (!p0.new) jump:nt .L28 << 254 r8 = lsr(r2,#3) << 255 } << 256 loop0(.L44,r8) << 257 .falign << 258 .L44: << 259 { << 260 memd(r6++#8) = r5:4 << 261 r2 = add(r2,#-8) << 262 }:endloop0 /* start=.L44 */ << 263 .L28: << 264 { << 265 p0 = tstbit(r2,#2) << 266 if (!p0.new) jump:nt .L33 << 267 } << 268 { << 269 r2 = add(r2,#-4) << 270 memw(r6++#4) = r7 << 271 } << 272 .L33: << 273 { << 274 p0 = tstbit(r2,#1) << 275 if (!p0.new) jump:nt .L35 << 276 } << 277 { << 278 r2 = add(r2,#-2) << 279 memh(r6++#2) = r7 << 280 } << 281 .L35: << 282 p0 = cmp.eq(r2,#1) << 283 if (p0) memb(r6) = r1 << 284 .L1: << 285 jumpr r31 << 286 .L18: << 287 loop0(.L45,r3) << 288 .falign << 289 .L45: << 290 dczeroa(r6) << 291 { << 292 memd(r6++#8) = r5:4 << 293 r2 = add(r2,#-32) << 294 } << 295 memd(r6++#8) = r5:4 << 296 memd(r6++#8) = r5:4 << 297 { << 298 memd(r6++#8) = r5:4 << 299 }:endloop0 /* start=.L45 */ << 300 jump .L14 << 301 HEXAGON_OPT_FUNC_FINISH memset << 302 #endif 146 #endif >> 147 jr t1 >> 148 PTR_ADDU a0, t0 /* dest ptr */ >> 149 >> 150 .set push >> 151 .set noreorder >> 152 .set nomacro >> 153 f_fill64 a0, -64, FILL64RG, .Lpartial_fixup /* ... but first do longs ... */ >> 154 2: .set pop >> 155 andi a2, STORMASK /* At most one long to go */ >> 156 >> 157 beqz a2, 1f >> 158 PTR_ADDU a0, a2 /* What's left */ >> 159 R10KCBARRIER(0(ra)) >> 160 #ifdef __MIPSEB__ >> 161 EX(LONG_S_R, a1, -1(a0), .Llast_fixup) >> 162 #endif >> 163 #ifdef __MIPSEL__ >> 164 EX(LONG_S_L, a1, -1(a0), .Llast_fixup) >> 165 #endif >> 166 1: jr ra >> 167 move a2, zero >> 168 >> 169 .Lsmall_memset: >> 170 beqz a2, 2f >> 171 PTR_ADDU t1, a0, a2 >> 172 >> 173 1: PTR_ADDIU a0, 1 /* fill bytewise */ >> 174 R10KCBARRIER(0(ra)) >> 175 bne t1, a0, 1b >> 176 sb a1, -1(a0) >> 177 >> 178 2: jr ra /* done */ >> 179 move a2, zero >> 180 END(memset) >> 181 >> 182 .Lfirst_fixup: >> 183 jr ra >> 184 nop >> 185 >> 186 .Lfwd_fixup: >> 187 PTR_L t0, TI_TASK($28) >> 188 andi a2, 0x3f >> 189 LONG_L t0, THREAD_BUADDR(t0) >> 190 LONG_ADDU a2, t1 >> 191 jr ra >> 192 LONG_SUBU a2, t0 >> 193 >> 194 .Lpartial_fixup: >> 195 PTR_L t0, TI_TASK($28) >> 196 andi a2, STORMASK >> 197 LONG_L t0, THREAD_BUADDR(t0) >> 198 LONG_ADDU a2, t1 >> 199 jr ra >> 200 LONG_SUBU a2, t0 >> 201 >> 202 .Llast_fixup: >> 203 jr ra >> 204 andi v1, a2, STORMASK

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

TOMOYO Linux Cross Reference
Linux/arch/hexagon/lib/memset.S

Diff markup

Differences between /arch/hexagon/lib/memset.S (Version linux-6.12-rc7) and /arch/mips/lib/memset.S (Version linux-3.10.108)

TOMOYO Linux Cross Reference Linux/arch/hexagon/lib/memset.S

Diff markup

Differences between /arch/hexagon/lib/memset.S (Version linux-6.12-rc7) and /arch/mips/lib/memset.S (Version linux-3.10.108)

TOMOYO Linux Cross Reference
Linux/arch/hexagon/lib/memset.S