1 /* SPDX-License-Identifier: GPL-2.0-only */ !! 1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* !! 2 /* clear_page.S: UltraSparc optimized copy page.
3 * Copyright (C) 2012 ARM Ltd. !! 3 *
>> 4 * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
>> 5 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
4 */ 6 */
5 7
6 #include <linux/linkage.h> !! 8 #include <linux/export.h>
7 #include <linux/const.h> !! 9 #include <asm/visasm.h>
8 #include <asm/assembler.h> !! 10 #include <asm/thread_info.h>
9 #include <asm/page.h> 11 #include <asm/page.h>
10 #include <asm/cpufeature.h> !! 12 #include <linux/pgtable.h>
11 #include <asm/alternative.h> !! 13 #include <asm/spitfire.h>
>> 14 #include <asm/head.h>
>> 15
>> 16 /* What we used to do was lock a TLB entry into a specific
>> 17 * TLB slot, clear the page with interrupts disabled, then
>> 18 * restore the original TLB entry. This was great for
>> 19 * disturbing the TLB as little as possible, but it meant
>> 20 * we had to keep interrupts disabled for a long time.
>> 21 *
>> 22 * Now, we simply use the normal TLB loading mechanism,
>> 23 * and this makes the cpu choose a slot all by itself.
>> 24 * Then we do a normal TLB flush on exit. We need only
>> 25 * disable preemption during the clear.
>> 26 */
>> 27
>> 28 #define DCACHE_SIZE (PAGE_SIZE * 2)
>> 29
>> 30 #if (PAGE_SHIFT == 13)
>> 31 #define PAGE_SIZE_REM 0x80
>> 32 #elif (PAGE_SHIFT == 16)
>> 33 #define PAGE_SIZE_REM 0x100
>> 34 #else
>> 35 #error Wrong PAGE_SHIFT specified
>> 36 #endif
>> 37
>> 38 #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
>> 39 fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
>> 40 fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
>> 41 fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
>> 42 fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
>> 43
>> 44 .text
>> 45
>> 46 .align 32
>> 47 .globl copy_user_page
>> 48 .type copy_user_page,#function
>> 49 EXPORT_SYMBOL(copy_user_page)
>> 50 copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
>> 51 lduw [%g6 + TI_PRE_COUNT], %o4
>> 52 sethi %hi(PAGE_OFFSET), %g2
>> 53 sethi %hi(PAGE_SIZE), %o3
>> 54
>> 55 ldx [%g2 + %lo(PAGE_OFFSET)], %g2
>> 56 sethi %hi(PAGE_KERNEL_LOCKED), %g3
>> 57
>> 58 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
>> 59 sub %o0, %g2, %g1 ! dest paddr
>> 60
>> 61 sub %o1, %g2, %g2 ! src paddr
>> 62
>> 63 and %o2, %o3, %o0 ! vaddr D-cache alias bit
>> 64 or %g1, %g3, %g1 ! dest TTE data
>> 65
>> 66 or %g2, %g3, %g2 ! src TTE data
>> 67 sethi %hi(TLBTEMP_BASE), %o3
>> 68
>> 69 sethi %hi(DCACHE_SIZE), %o1
>> 70 add %o0, %o3, %o0 ! dest TTE vaddr
>> 71
>> 72 add %o4, 1, %o2
>> 73 add %o0, %o1, %o1 ! src TTE vaddr
>> 74
>> 75 /* Disable preemption. */
>> 76 mov TLB_TAG_ACCESS, %g3
>> 77 stw %o2, [%g6 + TI_PRE_COUNT]
>> 78
>> 79 /* Load TLB entries. */
>> 80 rdpr %pstate, %o2
>> 81 wrpr %o2, PSTATE_IE, %pstate
>> 82 stxa %o0, [%g3] ASI_DMMU
>> 83 stxa %g1, [%g0] ASI_DTLB_DATA_IN
>> 84 membar #Sync
>> 85 stxa %o1, [%g3] ASI_DMMU
>> 86 stxa %g2, [%g0] ASI_DTLB_DATA_IN
>> 87 membar #Sync
>> 88 wrpr %o2, 0x0, %pstate
>> 89
>> 90 cheetah_copy_page_insn:
>> 91 ba,pt %xcc, 9f
>> 92 nop
12 93
13 /* <<
14 * Copy a page from src to dest (both are page <<
15 * <<
16 * Parameters: <<
17 * x0 - dest <<
18 * x1 - src <<
19 */ <<
20 SYM_FUNC_START(__pi_copy_page) <<
21 ldp x2, x3, [x1] <<
22 ldp x4, x5, [x1, #16] <<
23 ldp x6, x7, [x1, #32] <<
24 ldp x8, x9, [x1, #48] <<
25 ldp x10, x11, [x1, #64] <<
26 ldp x12, x13, [x1, #80] <<
27 ldp x14, x15, [x1, #96] <<
28 ldp x16, x17, [x1, #112] <<
29 <<
30 add x0, x0, #256 <<
31 add x1, x1, #128 <<
32 1: 94 1:
33 tst x0, #(PAGE_SIZE - 1) !! 95 VISEntryHalf
34 !! 96 membar #StoreLoad | #StoreStore | #LoadStore
35 stnp x2, x3, [x0, #-256] !! 97 sethi %hi((PAGE_SIZE/64)-2), %o2
36 ldp x2, x3, [x1] !! 98 mov %o0, %g1
37 stnp x4, x5, [x0, #16 - 256] !! 99 prefetch [%o1 + 0x000], #one_read
38 ldp x4, x5, [x1, #16] !! 100 or %o2, %lo((PAGE_SIZE/64)-2), %o2
39 stnp x6, x7, [x0, #32 - 256] !! 101 prefetch [%o1 + 0x040], #one_read
40 ldp x6, x7, [x1, #32] !! 102 prefetch [%o1 + 0x080], #one_read
41 stnp x8, x9, [x0, #48 - 256] !! 103 prefetch [%o1 + 0x0c0], #one_read
42 ldp x8, x9, [x1, #48] !! 104 ldd [%o1 + 0x000], %f0
43 stnp x10, x11, [x0, #64 - 256] !! 105 prefetch [%o1 + 0x100], #one_read
44 ldp x10, x11, [x1, #64] !! 106 ldd [%o1 + 0x008], %f2
45 stnp x12, x13, [x0, #80 - 256] !! 107 prefetch [%o1 + 0x140], #one_read
46 ldp x12, x13, [x1, #80] !! 108 ldd [%o1 + 0x010], %f4
47 stnp x14, x15, [x0, #96 - 256] !! 109 prefetch [%o1 + 0x180], #one_read
48 ldp x14, x15, [x1, #96] !! 110 fsrc2 %f0, %f16
49 stnp x16, x17, [x0, #112 - 256] !! 111 ldd [%o1 + 0x018], %f6
50 ldp x16, x17, [x1, #112] !! 112 fsrc2 %f2, %f18
51 !! 113 ldd [%o1 + 0x020], %f8
52 add x0, x0, #128 !! 114 fsrc2 %f4, %f20
53 add x1, x1, #128 !! 115 ldd [%o1 + 0x028], %f10
54 !! 116 fsrc2 %f6, %f22
55 b.ne 1b !! 117 ldd [%o1 + 0x030], %f12
56 !! 118 fsrc2 %f8, %f24
57 stnp x2, x3, [x0, #-256] !! 119 ldd [%o1 + 0x038], %f14
58 stnp x4, x5, [x0, #16 - 256] !! 120 fsrc2 %f10, %f26
59 stnp x6, x7, [x0, #32 - 256] !! 121 ldd [%o1 + 0x040], %f0
60 stnp x8, x9, [x0, #48 - 256] !! 122 1: ldd [%o1 + 0x048], %f2
61 stnp x10, x11, [x0, #64 - 256] !! 123 fsrc2 %f12, %f28
62 stnp x12, x13, [x0, #80 - 256] !! 124 ldd [%o1 + 0x050], %f4
63 stnp x14, x15, [x0, #96 - 256] !! 125 fsrc2 %f14, %f30
64 stnp x16, x17, [x0, #112 - 256] !! 126 stda %f16, [%o0] ASI_BLK_P
65 !! 127 ldd [%o1 + 0x058], %f6
66 ret !! 128 fsrc2 %f0, %f16
67 SYM_FUNC_END(__pi_copy_page) !! 129 ldd [%o1 + 0x060], %f8
68 SYM_FUNC_ALIAS(copy_page, __pi_copy_page) !! 130 fsrc2 %f2, %f18
69 EXPORT_SYMBOL(copy_page) !! 131 ldd [%o1 + 0x068], %f10
>> 132 fsrc2 %f4, %f20
>> 133 ldd [%o1 + 0x070], %f12
>> 134 fsrc2 %f6, %f22
>> 135 ldd [%o1 + 0x078], %f14
>> 136 fsrc2 %f8, %f24
>> 137 ldd [%o1 + 0x080], %f0
>> 138 prefetch [%o1 + 0x180], #one_read
>> 139 fsrc2 %f10, %f26
>> 140 subcc %o2, 1, %o2
>> 141 add %o0, 0x40, %o0
>> 142 bne,pt %xcc, 1b
>> 143 add %o1, 0x40, %o1
>> 144
>> 145 ldd [%o1 + 0x048], %f2
>> 146 fsrc2 %f12, %f28
>> 147 ldd [%o1 + 0x050], %f4
>> 148 fsrc2 %f14, %f30
>> 149 stda %f16, [%o0] ASI_BLK_P
>> 150 ldd [%o1 + 0x058], %f6
>> 151 fsrc2 %f0, %f16
>> 152 ldd [%o1 + 0x060], %f8
>> 153 fsrc2 %f2, %f18
>> 154 ldd [%o1 + 0x068], %f10
>> 155 fsrc2 %f4, %f20
>> 156 ldd [%o1 + 0x070], %f12
>> 157 fsrc2 %f6, %f22
>> 158 add %o0, 0x40, %o0
>> 159 ldd [%o1 + 0x078], %f14
>> 160 fsrc2 %f8, %f24
>> 161 fsrc2 %f10, %f26
>> 162 fsrc2 %f12, %f28
>> 163 fsrc2 %f14, %f30
>> 164 stda %f16, [%o0] ASI_BLK_P
>> 165 membar #Sync
>> 166 VISExitHalf
>> 167 ba,pt %xcc, 5f
>> 168 nop
>> 169
>> 170 9:
>> 171 VISEntry
>> 172 ldub [%g6 + TI_FAULT_CODE], %g3
>> 173 mov %o0, %g1
>> 174 cmp %g3, 0
>> 175 rd %asi, %g3
>> 176 be,a,pt %icc, 1f
>> 177 wr %g0, ASI_BLK_P, %asi
>> 178 wr %g0, ASI_BLK_COMMIT_P, %asi
>> 179 1: ldda [%o1] ASI_BLK_P, %f0
>> 180 add %o1, 0x40, %o1
>> 181 ldda [%o1] ASI_BLK_P, %f16
>> 182 add %o1, 0x40, %o1
>> 183 sethi %hi(PAGE_SIZE), %o2
>> 184 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
>> 185 ldda [%o1] ASI_BLK_P, %f32
>> 186 stda %f48, [%o0] %asi
>> 187 add %o1, 0x40, %o1
>> 188 sub %o2, 0x40, %o2
>> 189 add %o0, 0x40, %o0
>> 190 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
>> 191 ldda [%o1] ASI_BLK_P, %f0
>> 192 stda %f48, [%o0] %asi
>> 193 add %o1, 0x40, %o1
>> 194 sub %o2, 0x40, %o2
>> 195 add %o0, 0x40, %o0
>> 196 TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
>> 197 ldda [%o1] ASI_BLK_P, %f16
>> 198 stda %f48, [%o0] %asi
>> 199 sub %o2, 0x40, %o2
>> 200 add %o1, 0x40, %o1
>> 201 cmp %o2, PAGE_SIZE_REM
>> 202 bne,pt %xcc, 1b
>> 203 add %o0, 0x40, %o0
>> 204 #if (PAGE_SHIFT == 16)
>> 205 TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
>> 206 ldda [%o1] ASI_BLK_P, %f32
>> 207 stda %f48, [%o0] %asi
>> 208 add %o1, 0x40, %o1
>> 209 sub %o2, 0x40, %o2
>> 210 add %o0, 0x40, %o0
>> 211 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
>> 212 ldda [%o1] ASI_BLK_P, %f0
>> 213 stda %f48, [%o0] %asi
>> 214 add %o1, 0x40, %o1
>> 215 sub %o2, 0x40, %o2
>> 216 add %o0, 0x40, %o0
>> 217 membar #Sync
>> 218 stda %f32, [%o0] %asi
>> 219 add %o0, 0x40, %o0
>> 220 stda %f0, [%o0] %asi
>> 221 #else
>> 222 membar #Sync
>> 223 stda %f0, [%o0] %asi
>> 224 add %o0, 0x40, %o0
>> 225 stda %f16, [%o0] %asi
>> 226 #endif
>> 227 membar #Sync
>> 228 wr %g3, 0x0, %asi
>> 229 VISExit
>> 230
>> 231 5:
>> 232 stxa %g0, [%g1] ASI_DMMU_DEMAP
>> 233 membar #Sync
>> 234
>> 235 sethi %hi(DCACHE_SIZE), %g2
>> 236 stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP
>> 237 membar #Sync
>> 238
>> 239 retl
>> 240 stw %o4, [%g6 + TI_PRE_COUNT]
>> 241
>> 242 .size copy_user_page, .-copy_user_page
>> 243
>> 244 .globl cheetah_patch_copy_page
>> 245 cheetah_patch_copy_page:
>> 246 sethi %hi(0x01000000), %o1 ! NOP
>> 247 sethi %hi(cheetah_copy_page_insn), %o0
>> 248 or %o0, %lo(cheetah_copy_page_insn), %o0
>> 249 stw %o1, [%o0]
>> 250 membar #StoreStore
>> 251 flush %o0
>> 252 retl
>> 253 nop
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.