1 // SPDX-License-Identifier: GPL-2.0-or-later 1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* -*- linux-c -*- --------------------------- 2 /* -*- linux-c -*- ------------------------------------------------------- *
3 * 3 *
4 * Copyright 2002 H. Peter Anvin - All Right 4 * Copyright 2002 H. Peter Anvin - All Rights Reserved
5 * 5 *
6 * ------------------------------------------- 6 * ----------------------------------------------------------------------- */
7 7
8 /* 8 /*
9 * raid6/sse2.c 9 * raid6/sse2.c
10 * 10 *
11 * SSE-2 implementation of RAID-6 syndrome fun 11 * SSE-2 implementation of RAID-6 syndrome functions
12 * 12 *
13 */ 13 */
14 14
15 #include <linux/raid/pq.h> 15 #include <linux/raid/pq.h>
16 #include "x86.h" 16 #include "x86.h"
17 17
18 static const struct raid6_sse_constants { 18 static const struct raid6_sse_constants {
19 u64 x1d[2]; 19 u64 x1d[2];
20 } raid6_sse_constants __attribute__((aligned( 20 } raid6_sse_constants __attribute__((aligned(16))) = {
21 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1 21 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
22 }; 22 };
23 23
24 static int raid6_have_sse2(void) 24 static int raid6_have_sse2(void)
25 { 25 {
26 /* Not really boot_cpu but "all_cpus" 26 /* Not really boot_cpu but "all_cpus" */
27 return boot_cpu_has(X86_FEATURE_MMX) & 27 return boot_cpu_has(X86_FEATURE_MMX) &&
28 boot_cpu_has(X86_FEATURE_FXSR) 28 boot_cpu_has(X86_FEATURE_FXSR) &&
29 boot_cpu_has(X86_FEATURE_XMM) 29 boot_cpu_has(X86_FEATURE_XMM) &&
30 boot_cpu_has(X86_FEATURE_XMM2) 30 boot_cpu_has(X86_FEATURE_XMM2);
31 } 31 }
32 32
33 /* 33 /*
34 * Plain SSE2 implementation 34 * Plain SSE2 implementation
35 */ 35 */
36 static void raid6_sse21_gen_syndrome(int disks 36 static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
37 { 37 {
38 u8 **dptr = (u8 **)ptrs; 38 u8 **dptr = (u8 **)ptrs;
39 u8 *p, *q; 39 u8 *p, *q;
40 int d, z, z0; 40 int d, z, z0;
41 41
42 z0 = disks - 3; /* Highest dat 42 z0 = disks - 3; /* Highest data disk */
43 p = dptr[z0+1]; /* XOR parity 43 p = dptr[z0+1]; /* XOR parity */
44 q = dptr[z0+2]; /* RS syndrome 44 q = dptr[z0+2]; /* RS syndrome */
45 45
46 kernel_fpu_begin(); 46 kernel_fpu_begin();
47 47
48 asm volatile("movdqa %0,%%xmm0" : : "m 48 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
49 asm volatile("pxor %xmm5,%xmm5"); 49 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
50 50
51 for ( d = 0 ; d < bytes ; d += 16 ) { 51 for ( d = 0 ; d < bytes ; d += 16 ) {
52 asm volatile("prefetchnta %0" 52 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
53 asm volatile("movdqa %0,%%xmm2 53 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
54 asm volatile("prefetchnta %0" 54 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
55 asm volatile("movdqa %xmm2,%xm 55 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
56 asm volatile("movdqa %0,%%xmm6 56 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
57 for ( z = z0-2 ; z >= 0 ; z-- 57 for ( z = z0-2 ; z >= 0 ; z-- ) {
58 asm volatile("prefetch 58 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
59 asm volatile("pcmpgtb 59 asm volatile("pcmpgtb %xmm4,%xmm5");
60 asm volatile("paddb %x 60 asm volatile("paddb %xmm4,%xmm4");
61 asm volatile("pand %xm 61 asm volatile("pand %xmm0,%xmm5");
62 asm volatile("pxor %xm 62 asm volatile("pxor %xmm5,%xmm4");
63 asm volatile("pxor %xm 63 asm volatile("pxor %xmm5,%xmm5");
64 asm volatile("pxor %xm 64 asm volatile("pxor %xmm6,%xmm2");
65 asm volatile("pxor %xm 65 asm volatile("pxor %xmm6,%xmm4");
66 asm volatile("movdqa % 66 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
67 } 67 }
68 asm volatile("pcmpgtb %xmm4,%x 68 asm volatile("pcmpgtb %xmm4,%xmm5");
69 asm volatile("paddb %xmm4,%xmm 69 asm volatile("paddb %xmm4,%xmm4");
70 asm volatile("pand %xmm0,%xmm5 70 asm volatile("pand %xmm0,%xmm5");
71 asm volatile("pxor %xmm5,%xmm4 71 asm volatile("pxor %xmm5,%xmm4");
72 asm volatile("pxor %xmm5,%xmm5 72 asm volatile("pxor %xmm5,%xmm5");
73 asm volatile("pxor %xmm6,%xmm2 73 asm volatile("pxor %xmm6,%xmm2");
74 asm volatile("pxor %xmm6,%xmm4 74 asm volatile("pxor %xmm6,%xmm4");
75 75
76 asm volatile("movntdq %%xmm2,% 76 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
77 asm volatile("pxor %xmm2,%xmm2 77 asm volatile("pxor %xmm2,%xmm2");
78 asm volatile("movntdq %%xmm4,% 78 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
79 asm volatile("pxor %xmm4,%xmm4 79 asm volatile("pxor %xmm4,%xmm4");
80 } 80 }
81 81
82 asm volatile("sfence" : : : "memory"); 82 asm volatile("sfence" : : : "memory");
83 kernel_fpu_end(); 83 kernel_fpu_end();
84 } 84 }
85 85
86 86
87 static void raid6_sse21_xor_syndrome(int disks 87 static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
88 size_t by 88 size_t bytes, void **ptrs)
89 { 89 {
90 u8 **dptr = (u8 **)ptrs; 90 u8 **dptr = (u8 **)ptrs;
91 u8 *p, *q; 91 u8 *p, *q;
92 int d, z, z0; 92 int d, z, z0;
93 93
94 z0 = stop; /* P/Q right s 94 z0 = stop; /* P/Q right side optimization */
95 p = dptr[disks-2]; /* XOR parity 95 p = dptr[disks-2]; /* XOR parity */
96 q = dptr[disks-1]; /* RS syndrome 96 q = dptr[disks-1]; /* RS syndrome */
97 97
98 kernel_fpu_begin(); 98 kernel_fpu_begin();
99 99
100 asm volatile("movdqa %0,%%xmm0" : : "m 100 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
101 101
102 for ( d = 0 ; d < bytes ; d += 16 ) { 102 for ( d = 0 ; d < bytes ; d += 16 ) {
103 asm volatile("movdqa %0,%%xmm4 103 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
104 asm volatile("movdqa %0,%%xmm2 104 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
105 asm volatile("pxor %xmm4,%xmm2 105 asm volatile("pxor %xmm4,%xmm2");
106 /* P/Q data pages */ 106 /* P/Q data pages */
107 for ( z = z0-1 ; z >= start ; 107 for ( z = z0-1 ; z >= start ; z-- ) {
108 asm volatile("pxor %xm 108 asm volatile("pxor %xmm5,%xmm5");
109 asm volatile("pcmpgtb 109 asm volatile("pcmpgtb %xmm4,%xmm5");
110 asm volatile("paddb %x 110 asm volatile("paddb %xmm4,%xmm4");
111 asm volatile("pand %xm 111 asm volatile("pand %xmm0,%xmm5");
112 asm volatile("pxor %xm 112 asm volatile("pxor %xmm5,%xmm4");
113 asm volatile("movdqa % 113 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
114 asm volatile("pxor %xm 114 asm volatile("pxor %xmm5,%xmm2");
115 asm volatile("pxor %xm 115 asm volatile("pxor %xmm5,%xmm4");
116 } 116 }
117 /* P/Q left side optimization 117 /* P/Q left side optimization */
118 for ( z = start-1 ; z >= 0 ; z 118 for ( z = start-1 ; z >= 0 ; z-- ) {
119 asm volatile("pxor %xm 119 asm volatile("pxor %xmm5,%xmm5");
120 asm volatile("pcmpgtb 120 asm volatile("pcmpgtb %xmm4,%xmm5");
121 asm volatile("paddb %x 121 asm volatile("paddb %xmm4,%xmm4");
122 asm volatile("pand %xm 122 asm volatile("pand %xmm0,%xmm5");
123 asm volatile("pxor %xm 123 asm volatile("pxor %xmm5,%xmm4");
124 } 124 }
125 asm volatile("pxor %0,%%xmm4" 125 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
126 /* Don't use movntdq for r/w m 126 /* Don't use movntdq for r/w memory area < cache line */
127 asm volatile("movdqa %%xmm4,%0 127 asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
128 asm volatile("movdqa %%xmm2,%0 128 asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
129 } 129 }
130 130
131 asm volatile("sfence" : : : "memory"); 131 asm volatile("sfence" : : : "memory");
132 kernel_fpu_end(); 132 kernel_fpu_end();
133 } 133 }
134 134
135 const struct raid6_calls raid6_sse2x1 = { 135 const struct raid6_calls raid6_sse2x1 = {
136 raid6_sse21_gen_syndrome, 136 raid6_sse21_gen_syndrome,
137 raid6_sse21_xor_syndrome, 137 raid6_sse21_xor_syndrome,
138 raid6_have_sse2, 138 raid6_have_sse2,
139 "sse2x1", 139 "sse2x1",
140 1 /* Has cache h 140 1 /* Has cache hints */
141 }; 141 };
142 142
143 /* 143 /*
144 * Unrolled-by-2 SSE2 implementation 144 * Unrolled-by-2 SSE2 implementation
145 */ 145 */
146 static void raid6_sse22_gen_syndrome(int disks 146 static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
147 { 147 {
148 u8 **dptr = (u8 **)ptrs; 148 u8 **dptr = (u8 **)ptrs;
149 u8 *p, *q; 149 u8 *p, *q;
150 int d, z, z0; 150 int d, z, z0;
151 151
152 z0 = disks - 3; /* Highest dat 152 z0 = disks - 3; /* Highest data disk */
153 p = dptr[z0+1]; /* XOR parity 153 p = dptr[z0+1]; /* XOR parity */
154 q = dptr[z0+2]; /* RS syndrome 154 q = dptr[z0+2]; /* RS syndrome */
155 155
156 kernel_fpu_begin(); 156 kernel_fpu_begin();
157 157
158 asm volatile("movdqa %0,%%xmm0" : : "m 158 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
159 asm volatile("pxor %xmm5,%xmm5"); /* Z 159 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
160 asm volatile("pxor %xmm7,%xmm7"); /* Z 160 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
161 161
162 /* We uniformly assume a single prefet 162 /* We uniformly assume a single prefetch covers at least 32 bytes */
163 for ( d = 0 ; d < bytes ; d += 32 ) { 163 for ( d = 0 ; d < bytes ; d += 32 ) {
164 asm volatile("prefetchnta %0" 164 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
165 asm volatile("movdqa %0,%%xmm2 165 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
166 asm volatile("movdqa %0,%%xmm3 166 asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
167 asm volatile("movdqa %xmm2,%xm 167 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
168 asm volatile("movdqa %xmm3,%xm 168 asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
169 for ( z = z0-1 ; z >= 0 ; z-- 169 for ( z = z0-1 ; z >= 0 ; z-- ) {
170 asm volatile("prefetch 170 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
171 asm volatile("pcmpgtb 171 asm volatile("pcmpgtb %xmm4,%xmm5");
172 asm volatile("pcmpgtb 172 asm volatile("pcmpgtb %xmm6,%xmm7");
173 asm volatile("paddb %x 173 asm volatile("paddb %xmm4,%xmm4");
174 asm volatile("paddb %x 174 asm volatile("paddb %xmm6,%xmm6");
175 asm volatile("pand %xm 175 asm volatile("pand %xmm0,%xmm5");
176 asm volatile("pand %xm 176 asm volatile("pand %xmm0,%xmm7");
177 asm volatile("pxor %xm 177 asm volatile("pxor %xmm5,%xmm4");
178 asm volatile("pxor %xm 178 asm volatile("pxor %xmm7,%xmm6");
179 asm volatile("movdqa % 179 asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
180 asm volatile("movdqa % 180 asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
181 asm volatile("pxor %xm 181 asm volatile("pxor %xmm5,%xmm2");
182 asm volatile("pxor %xm 182 asm volatile("pxor %xmm7,%xmm3");
183 asm volatile("pxor %xm 183 asm volatile("pxor %xmm5,%xmm4");
184 asm volatile("pxor %xm 184 asm volatile("pxor %xmm7,%xmm6");
185 asm volatile("pxor %xm 185 asm volatile("pxor %xmm5,%xmm5");
186 asm volatile("pxor %xm 186 asm volatile("pxor %xmm7,%xmm7");
187 } 187 }
188 asm volatile("movntdq %%xmm2,% 188 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
189 asm volatile("movntdq %%xmm3,% 189 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
190 asm volatile("movntdq %%xmm4,% 190 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
191 asm volatile("movntdq %%xmm6,% 191 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
192 } 192 }
193 193
194 asm volatile("sfence" : : : "memory"); 194 asm volatile("sfence" : : : "memory");
195 kernel_fpu_end(); 195 kernel_fpu_end();
196 } 196 }
197 197
198 static void raid6_sse22_xor_syndrome(int disks 198 static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
199 size_t by 199 size_t bytes, void **ptrs)
200 { 200 {
201 u8 **dptr = (u8 **)ptrs; 201 u8 **dptr = (u8 **)ptrs;
202 u8 *p, *q; 202 u8 *p, *q;
203 int d, z, z0; 203 int d, z, z0;
204 204
205 z0 = stop; /* P/Q right s 205 z0 = stop; /* P/Q right side optimization */
206 p = dptr[disks-2]; /* XOR parity 206 p = dptr[disks-2]; /* XOR parity */
207 q = dptr[disks-1]; /* RS syndrome 207 q = dptr[disks-1]; /* RS syndrome */
208 208
209 kernel_fpu_begin(); 209 kernel_fpu_begin();
210 210
211 asm volatile("movdqa %0,%%xmm0" : : "m 211 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
212 212
213 for ( d = 0 ; d < bytes ; d += 32 ) { 213 for ( d = 0 ; d < bytes ; d += 32 ) {
214 asm volatile("movdqa %0,%%xmm4 214 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
215 asm volatile("movdqa %0,%%xmm6 215 asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
216 asm volatile("movdqa %0,%%xmm2 216 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
217 asm volatile("movdqa %0,%%xmm3 217 asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
218 asm volatile("pxor %xmm4,%xmm2 218 asm volatile("pxor %xmm4,%xmm2");
219 asm volatile("pxor %xmm6,%xmm3 219 asm volatile("pxor %xmm6,%xmm3");
220 /* P/Q data pages */ 220 /* P/Q data pages */
221 for ( z = z0-1 ; z >= start ; 221 for ( z = z0-1 ; z >= start ; z-- ) {
222 asm volatile("pxor %xm 222 asm volatile("pxor %xmm5,%xmm5");
223 asm volatile("pxor %xm 223 asm volatile("pxor %xmm7,%xmm7");
224 asm volatile("pcmpgtb 224 asm volatile("pcmpgtb %xmm4,%xmm5");
225 asm volatile("pcmpgtb 225 asm volatile("pcmpgtb %xmm6,%xmm7");
226 asm volatile("paddb %x 226 asm volatile("paddb %xmm4,%xmm4");
227 asm volatile("paddb %x 227 asm volatile("paddb %xmm6,%xmm6");
228 asm volatile("pand %xm 228 asm volatile("pand %xmm0,%xmm5");
229 asm volatile("pand %xm 229 asm volatile("pand %xmm0,%xmm7");
230 asm volatile("pxor %xm 230 asm volatile("pxor %xmm5,%xmm4");
231 asm volatile("pxor %xm 231 asm volatile("pxor %xmm7,%xmm6");
232 asm volatile("movdqa % 232 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
233 asm volatile("movdqa % 233 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
234 asm volatile("pxor %xm 234 asm volatile("pxor %xmm5,%xmm2");
235 asm volatile("pxor %xm 235 asm volatile("pxor %xmm7,%xmm3");
236 asm volatile("pxor %xm 236 asm volatile("pxor %xmm5,%xmm4");
237 asm volatile("pxor %xm 237 asm volatile("pxor %xmm7,%xmm6");
238 } 238 }
239 /* P/Q left side optimization 239 /* P/Q left side optimization */
240 for ( z = start-1 ; z >= 0 ; z 240 for ( z = start-1 ; z >= 0 ; z-- ) {
241 asm volatile("pxor %xm 241 asm volatile("pxor %xmm5,%xmm5");
242 asm volatile("pxor %xm 242 asm volatile("pxor %xmm7,%xmm7");
243 asm volatile("pcmpgtb 243 asm volatile("pcmpgtb %xmm4,%xmm5");
244 asm volatile("pcmpgtb 244 asm volatile("pcmpgtb %xmm6,%xmm7");
245 asm volatile("paddb %x 245 asm volatile("paddb %xmm4,%xmm4");
246 asm volatile("paddb %x 246 asm volatile("paddb %xmm6,%xmm6");
247 asm volatile("pand %xm 247 asm volatile("pand %xmm0,%xmm5");
248 asm volatile("pand %xm 248 asm volatile("pand %xmm0,%xmm7");
249 asm volatile("pxor %xm 249 asm volatile("pxor %xmm5,%xmm4");
250 asm volatile("pxor %xm 250 asm volatile("pxor %xmm7,%xmm6");
251 } 251 }
252 asm volatile("pxor %0,%%xmm4" 252 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
253 asm volatile("pxor %0,%%xmm6" 253 asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
254 /* Don't use movntdq for r/w m 254 /* Don't use movntdq for r/w memory area < cache line */
255 asm volatile("movdqa %%xmm4,%0 255 asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
256 asm volatile("movdqa %%xmm6,%0 256 asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
257 asm volatile("movdqa %%xmm2,%0 257 asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
258 asm volatile("movdqa %%xmm3,%0 258 asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
259 } 259 }
260 260
261 asm volatile("sfence" : : : "memory"); 261 asm volatile("sfence" : : : "memory");
262 kernel_fpu_end(); 262 kernel_fpu_end();
263 } 263 }
264 264
265 const struct raid6_calls raid6_sse2x2 = { 265 const struct raid6_calls raid6_sse2x2 = {
266 raid6_sse22_gen_syndrome, 266 raid6_sse22_gen_syndrome,
267 raid6_sse22_xor_syndrome, 267 raid6_sse22_xor_syndrome,
268 raid6_have_sse2, 268 raid6_have_sse2,
269 "sse2x2", 269 "sse2x2",
270 1 /* Has cache h 270 1 /* Has cache hints */
271 }; 271 };
272 272
273 #ifdef CONFIG_X86_64 273 #ifdef CONFIG_X86_64
274 274
275 /* 275 /*
276 * Unrolled-by-4 SSE2 implementation 276 * Unrolled-by-4 SSE2 implementation
277 */ 277 */
278 static void raid6_sse24_gen_syndrome(int disks 278 static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
279 { 279 {
280 u8 **dptr = (u8 **)ptrs; 280 u8 **dptr = (u8 **)ptrs;
281 u8 *p, *q; 281 u8 *p, *q;
282 int d, z, z0; 282 int d, z, z0;
283 283
284 z0 = disks - 3; /* Highest dat 284 z0 = disks - 3; /* Highest data disk */
285 p = dptr[z0+1]; /* XOR parity 285 p = dptr[z0+1]; /* XOR parity */
286 q = dptr[z0+2]; /* RS syndrome 286 q = dptr[z0+2]; /* RS syndrome */
287 287
288 kernel_fpu_begin(); 288 kernel_fpu_begin();
289 289
290 asm volatile("movdqa %0,%%xmm0" :: "m" 290 asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
291 asm volatile("pxor %xmm2,%xmm2"); 291 asm volatile("pxor %xmm2,%xmm2"); /* P[0] */
292 asm volatile("pxor %xmm3,%xmm3"); 292 asm volatile("pxor %xmm3,%xmm3"); /* P[1] */
293 asm volatile("pxor %xmm4,%xmm4"); 293 asm volatile("pxor %xmm4,%xmm4"); /* Q[0] */
294 asm volatile("pxor %xmm5,%xmm5"); 294 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
295 asm volatile("pxor %xmm6,%xmm6"); 295 asm volatile("pxor %xmm6,%xmm6"); /* Q[1] */
296 asm volatile("pxor %xmm7,%xmm7"); 296 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
297 asm volatile("pxor %xmm10,%xmm10"); 297 asm volatile("pxor %xmm10,%xmm10"); /* P[2] */
298 asm volatile("pxor %xmm11,%xmm11"); 298 asm volatile("pxor %xmm11,%xmm11"); /* P[3] */
299 asm volatile("pxor %xmm12,%xmm12"); 299 asm volatile("pxor %xmm12,%xmm12"); /* Q[2] */
300 asm volatile("pxor %xmm13,%xmm13"); 300 asm volatile("pxor %xmm13,%xmm13"); /* Zero temp */
301 asm volatile("pxor %xmm14,%xmm14"); 301 asm volatile("pxor %xmm14,%xmm14"); /* Q[3] */
302 asm volatile("pxor %xmm15,%xmm15"); 302 asm volatile("pxor %xmm15,%xmm15"); /* Zero temp */
303 303
304 for ( d = 0 ; d < bytes ; d += 64 ) { 304 for ( d = 0 ; d < bytes ; d += 64 ) {
305 for ( z = z0 ; z >= 0 ; z-- ) 305 for ( z = z0 ; z >= 0 ; z-- ) {
306 /* The second prefetch 306 /* The second prefetch seems to improve performance... */
307 asm volatile("prefetch 307 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
308 asm volatile("prefetch 308 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
309 asm volatile("pcmpgtb 309 asm volatile("pcmpgtb %xmm4,%xmm5");
310 asm volatile("pcmpgtb 310 asm volatile("pcmpgtb %xmm6,%xmm7");
311 asm volatile("pcmpgtb 311 asm volatile("pcmpgtb %xmm12,%xmm13");
312 asm volatile("pcmpgtb 312 asm volatile("pcmpgtb %xmm14,%xmm15");
313 asm volatile("paddb %x 313 asm volatile("paddb %xmm4,%xmm4");
314 asm volatile("paddb %x 314 asm volatile("paddb %xmm6,%xmm6");
315 asm volatile("paddb %x 315 asm volatile("paddb %xmm12,%xmm12");
316 asm volatile("paddb %x 316 asm volatile("paddb %xmm14,%xmm14");
317 asm volatile("pand %xm 317 asm volatile("pand %xmm0,%xmm5");
318 asm volatile("pand %xm 318 asm volatile("pand %xmm0,%xmm7");
319 asm volatile("pand %xm 319 asm volatile("pand %xmm0,%xmm13");
320 asm volatile("pand %xm 320 asm volatile("pand %xmm0,%xmm15");
321 asm volatile("pxor %xm 321 asm volatile("pxor %xmm5,%xmm4");
322 asm volatile("pxor %xm 322 asm volatile("pxor %xmm7,%xmm6");
323 asm volatile("pxor %xm 323 asm volatile("pxor %xmm13,%xmm12");
324 asm volatile("pxor %xm 324 asm volatile("pxor %xmm15,%xmm14");
325 asm volatile("movdqa % 325 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
326 asm volatile("movdqa % 326 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
327 asm volatile("movdqa % 327 asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
328 asm volatile("movdqa % 328 asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
329 asm volatile("pxor %xm 329 asm volatile("pxor %xmm5,%xmm2");
330 asm volatile("pxor %xm 330 asm volatile("pxor %xmm7,%xmm3");
331 asm volatile("pxor %xm 331 asm volatile("pxor %xmm13,%xmm10");
332 asm volatile("pxor %xm 332 asm volatile("pxor %xmm15,%xmm11");
333 asm volatile("pxor %xm 333 asm volatile("pxor %xmm5,%xmm4");
334 asm volatile("pxor %xm 334 asm volatile("pxor %xmm7,%xmm6");
335 asm volatile("pxor %xm 335 asm volatile("pxor %xmm13,%xmm12");
336 asm volatile("pxor %xm 336 asm volatile("pxor %xmm15,%xmm14");
337 asm volatile("pxor %xm 337 asm volatile("pxor %xmm5,%xmm5");
338 asm volatile("pxor %xm 338 asm volatile("pxor %xmm7,%xmm7");
339 asm volatile("pxor %xm 339 asm volatile("pxor %xmm13,%xmm13");
340 asm volatile("pxor %xm 340 asm volatile("pxor %xmm15,%xmm15");
341 } 341 }
342 asm volatile("movntdq %%xmm2,% 342 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
343 asm volatile("pxor %xmm2,%xmm2 343 asm volatile("pxor %xmm2,%xmm2");
344 asm volatile("movntdq %%xmm3,% 344 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
345 asm volatile("pxor %xmm3,%xmm3 345 asm volatile("pxor %xmm3,%xmm3");
346 asm volatile("movntdq %%xmm10, 346 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
347 asm volatile("pxor %xmm10,%xmm 347 asm volatile("pxor %xmm10,%xmm10");
348 asm volatile("movntdq %%xmm11, 348 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
349 asm volatile("pxor %xmm11,%xmm 349 asm volatile("pxor %xmm11,%xmm11");
350 asm volatile("movntdq %%xmm4,% 350 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
351 asm volatile("pxor %xmm4,%xmm4 351 asm volatile("pxor %xmm4,%xmm4");
352 asm volatile("movntdq %%xmm6,% 352 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
353 asm volatile("pxor %xmm6,%xmm6 353 asm volatile("pxor %xmm6,%xmm6");
354 asm volatile("movntdq %%xmm12, 354 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
355 asm volatile("pxor %xmm12,%xmm 355 asm volatile("pxor %xmm12,%xmm12");
356 asm volatile("movntdq %%xmm14, 356 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
357 asm volatile("pxor %xmm14,%xmm 357 asm volatile("pxor %xmm14,%xmm14");
358 } 358 }
359 359
360 asm volatile("sfence" : : : "memory"); 360 asm volatile("sfence" : : : "memory");
361 kernel_fpu_end(); 361 kernel_fpu_end();
362 } 362 }
363 363
364 static void raid6_sse24_xor_syndrome(int disks 364 static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
365 size_t by 365 size_t bytes, void **ptrs)
366 { 366 {
367 u8 **dptr = (u8 **)ptrs; 367 u8 **dptr = (u8 **)ptrs;
368 u8 *p, *q; 368 u8 *p, *q;
369 int d, z, z0; 369 int d, z, z0;
370 370
371 z0 = stop; /* P/Q right s 371 z0 = stop; /* P/Q right side optimization */
372 p = dptr[disks-2]; /* XOR parity 372 p = dptr[disks-2]; /* XOR parity */
373 q = dptr[disks-1]; /* RS syndrome 373 q = dptr[disks-1]; /* RS syndrome */
374 374
375 kernel_fpu_begin(); 375 kernel_fpu_begin();
376 376
377 asm volatile("movdqa %0,%%xmm0" :: "m" 377 asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
378 378
379 for ( d = 0 ; d < bytes ; d += 64 ) { 379 for ( d = 0 ; d < bytes ; d += 64 ) {
380 asm volatile("movdqa %0,%%xmm4 380 asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
381 asm volatile("movdqa %0,%%xmm6 381 asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
382 asm volatile("movdqa %0,%%xmm1 382 asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
383 asm volatile("movdqa %0,%%xmm1 383 asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
384 asm volatile("movdqa %0,%%xmm2 384 asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
385 asm volatile("movdqa %0,%%xmm3 385 asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
386 asm volatile("movdqa %0,%%xmm1 386 asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
387 asm volatile("movdqa %0,%%xmm1 387 asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
388 asm volatile("pxor %xmm4,%xmm2 388 asm volatile("pxor %xmm4,%xmm2");
389 asm volatile("pxor %xmm6,%xmm3 389 asm volatile("pxor %xmm6,%xmm3");
390 asm volatile("pxor %xmm12,%xmm 390 asm volatile("pxor %xmm12,%xmm10");
391 asm volatile("pxor %xmm14,%xmm 391 asm volatile("pxor %xmm14,%xmm11");
392 /* P/Q data pages */ 392 /* P/Q data pages */
393 for ( z = z0-1 ; z >= start ; 393 for ( z = z0-1 ; z >= start ; z-- ) {
394 asm volatile("prefetch 394 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
395 asm volatile("prefetch 395 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
396 asm volatile("pxor %xm 396 asm volatile("pxor %xmm5,%xmm5");
397 asm volatile("pxor %xm 397 asm volatile("pxor %xmm7,%xmm7");
398 asm volatile("pxor %xm 398 asm volatile("pxor %xmm13,%xmm13");
399 asm volatile("pxor %xm 399 asm volatile("pxor %xmm15,%xmm15");
400 asm volatile("pcmpgtb 400 asm volatile("pcmpgtb %xmm4,%xmm5");
401 asm volatile("pcmpgtb 401 asm volatile("pcmpgtb %xmm6,%xmm7");
402 asm volatile("pcmpgtb 402 asm volatile("pcmpgtb %xmm12,%xmm13");
403 asm volatile("pcmpgtb 403 asm volatile("pcmpgtb %xmm14,%xmm15");
404 asm volatile("paddb %x 404 asm volatile("paddb %xmm4,%xmm4");
405 asm volatile("paddb %x 405 asm volatile("paddb %xmm6,%xmm6");
406 asm volatile("paddb %x 406 asm volatile("paddb %xmm12,%xmm12");
407 asm volatile("paddb %x 407 asm volatile("paddb %xmm14,%xmm14");
408 asm volatile("pand %xm 408 asm volatile("pand %xmm0,%xmm5");
409 asm volatile("pand %xm 409 asm volatile("pand %xmm0,%xmm7");
410 asm volatile("pand %xm 410 asm volatile("pand %xmm0,%xmm13");
411 asm volatile("pand %xm 411 asm volatile("pand %xmm0,%xmm15");
412 asm volatile("pxor %xm 412 asm volatile("pxor %xmm5,%xmm4");
413 asm volatile("pxor %xm 413 asm volatile("pxor %xmm7,%xmm6");
414 asm volatile("pxor %xm 414 asm volatile("pxor %xmm13,%xmm12");
415 asm volatile("pxor %xm 415 asm volatile("pxor %xmm15,%xmm14");
416 asm volatile("movdqa % 416 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
417 asm volatile("movdqa % 417 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
418 asm volatile("movdqa % 418 asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
419 asm volatile("movdqa % 419 asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
420 asm volatile("pxor %xm 420 asm volatile("pxor %xmm5,%xmm2");
421 asm volatile("pxor %xm 421 asm volatile("pxor %xmm7,%xmm3");
422 asm volatile("pxor %xm 422 asm volatile("pxor %xmm13,%xmm10");
423 asm volatile("pxor %xm 423 asm volatile("pxor %xmm15,%xmm11");
424 asm volatile("pxor %xm 424 asm volatile("pxor %xmm5,%xmm4");
425 asm volatile("pxor %xm 425 asm volatile("pxor %xmm7,%xmm6");
426 asm volatile("pxor %xm 426 asm volatile("pxor %xmm13,%xmm12");
427 asm volatile("pxor %xm 427 asm volatile("pxor %xmm15,%xmm14");
428 } 428 }
429 asm volatile("prefetchnta %0" 429 asm volatile("prefetchnta %0" :: "m" (q[d]));
430 asm volatile("prefetchnta %0" 430 asm volatile("prefetchnta %0" :: "m" (q[d+32]));
431 /* P/Q left side optimization 431 /* P/Q left side optimization */
432 for ( z = start-1 ; z >= 0 ; z 432 for ( z = start-1 ; z >= 0 ; z-- ) {
433 asm volatile("pxor %xm 433 asm volatile("pxor %xmm5,%xmm5");
434 asm volatile("pxor %xm 434 asm volatile("pxor %xmm7,%xmm7");
435 asm volatile("pxor %xm 435 asm volatile("pxor %xmm13,%xmm13");
436 asm volatile("pxor %xm 436 asm volatile("pxor %xmm15,%xmm15");
437 asm volatile("pcmpgtb 437 asm volatile("pcmpgtb %xmm4,%xmm5");
438 asm volatile("pcmpgtb 438 asm volatile("pcmpgtb %xmm6,%xmm7");
439 asm volatile("pcmpgtb 439 asm volatile("pcmpgtb %xmm12,%xmm13");
440 asm volatile("pcmpgtb 440 asm volatile("pcmpgtb %xmm14,%xmm15");
441 asm volatile("paddb %x 441 asm volatile("paddb %xmm4,%xmm4");
442 asm volatile("paddb %x 442 asm volatile("paddb %xmm6,%xmm6");
443 asm volatile("paddb %x 443 asm volatile("paddb %xmm12,%xmm12");
444 asm volatile("paddb %x 444 asm volatile("paddb %xmm14,%xmm14");
445 asm volatile("pand %xm 445 asm volatile("pand %xmm0,%xmm5");
446 asm volatile("pand %xm 446 asm volatile("pand %xmm0,%xmm7");
447 asm volatile("pand %xm 447 asm volatile("pand %xmm0,%xmm13");
448 asm volatile("pand %xm 448 asm volatile("pand %xmm0,%xmm15");
449 asm volatile("pxor %xm 449 asm volatile("pxor %xmm5,%xmm4");
450 asm volatile("pxor %xm 450 asm volatile("pxor %xmm7,%xmm6");
451 asm volatile("pxor %xm 451 asm volatile("pxor %xmm13,%xmm12");
452 asm volatile("pxor %xm 452 asm volatile("pxor %xmm15,%xmm14");
453 } 453 }
454 asm volatile("movntdq %%xmm2,% 454 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
455 asm volatile("movntdq %%xmm3,% 455 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
456 asm volatile("movntdq %%xmm10, 456 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
457 asm volatile("movntdq %%xmm11, 457 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
458 asm volatile("pxor %0,%%xmm4" 458 asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
459 asm volatile("pxor %0,%%xmm6" 459 asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
460 asm volatile("pxor %0,%%xmm12" 460 asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
461 asm volatile("pxor %0,%%xmm14" 461 asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
462 asm volatile("movntdq %%xmm4,% 462 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
463 asm volatile("movntdq %%xmm6,% 463 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
464 asm volatile("movntdq %%xmm12, 464 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
465 asm volatile("movntdq %%xmm14, 465 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
466 } 466 }
467 asm volatile("sfence" : : : "memory"); 467 asm volatile("sfence" : : : "memory");
468 kernel_fpu_end(); 468 kernel_fpu_end();
469 } 469 }
470 470
471 471
472 const struct raid6_calls raid6_sse2x4 = { 472 const struct raid6_calls raid6_sse2x4 = {
473 raid6_sse24_gen_syndrome, 473 raid6_sse24_gen_syndrome,
474 raid6_sse24_xor_syndrome, 474 raid6_sse24_xor_syndrome,
475 raid6_have_sse2, 475 raid6_have_sse2,
476 "sse2x4", 476 "sse2x4",
477 1 /* Has cache h 477 1 /* Has cache hints */
478 }; 478 };
479 479
480 #endif /* CONFIG_X86_64 */ 480 #endif /* CONFIG_X86_64 */
481 481
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.