1 // SPDX-License-Identifier: GPL-2.0-only 1 // SPDX-License-Identifier: GPL-2.0-only
2 /* 2 /*
3 * linux/lib/raid6/neon.c - RAID6 syndrome cal 3 * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
4 * 4 *
5 * Copyright (C) 2013 Linaro Ltd <ard.biesheuv 5 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */ 6 */
7 7
8 #include <linux/raid/pq.h> 8 #include <linux/raid/pq.h>
9 9
10 #ifdef __KERNEL__ 10 #ifdef __KERNEL__
11 #include <asm/neon.h> 11 #include <asm/neon.h>
12 #else 12 #else
13 #define kernel_neon_begin() 13 #define kernel_neon_begin()
14 #define kernel_neon_end() 14 #define kernel_neon_end()
15 #define cpu_has_neon() (1) 15 #define cpu_has_neon() (1)
16 #endif 16 #endif
17 17
18 /* 18 /*
19 * There are 2 reasons these wrappers are kept 19 * There are 2 reasons these wrappers are kept in a separate compilation unit
20 * from the actual implementations in neonN.c 20 * from the actual implementations in neonN.c (generated from neon.uc by
21 * unroll.awk): 21 * unroll.awk):
22 * - the actual implementations use NEON intri 22 * - the actual implementations use NEON intrinsics, and the GCC support header
23 * (arm_neon.h) is not fully compatible (typ 23 * (arm_neon.h) is not fully compatible (type wise) with the kernel;
24 * - the neonN.c files are compiled with -mfpu 24 * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
25 * and we have to make sure that we never us 25 * and we have to make sure that we never use *any* NEON/VFP instructions
26 * outside a kernel_neon_begin()/kernel_neon 26 * outside a kernel_neon_begin()/kernel_neon_end() pair.
27 */ 27 */
28 28
29 #define RAID6_NEON_WRAPPER(_n) 29 #define RAID6_NEON_WRAPPER(_n) \
30 static void raid6_neon ## _n ## _gen_s 30 static void raid6_neon ## _n ## _gen_syndrome(int disks, \
31 size_t 31 size_t bytes, void **ptrs) \
32 { 32 { \
33 void raid6_neon ## _n ## _gen 33 void raid6_neon ## _n ## _gen_syndrome_real(int, \
34 34 unsigned long, void**); \
35 kernel_neon_begin(); 35 kernel_neon_begin(); \
36 raid6_neon ## _n ## _gen_syndr 36 raid6_neon ## _n ## _gen_syndrome_real(disks, \
37 (unsig 37 (unsigned long)bytes, ptrs); \
38 kernel_neon_end(); 38 kernel_neon_end(); \
39 } 39 } \
40 static void raid6_neon ## _n ## _xor_s 40 static void raid6_neon ## _n ## _xor_syndrome(int disks, \
41 int st 41 int start, int stop, \
42 size_t 42 size_t bytes, void **ptrs) \
43 { 43 { \
44 void raid6_neon ## _n ## _xor 44 void raid6_neon ## _n ## _xor_syndrome_real(int, \
45 int, int, unsi 45 int, int, unsigned long, void**); \
46 kernel_neon_begin(); 46 kernel_neon_begin(); \
47 raid6_neon ## _n ## _xor_syndr 47 raid6_neon ## _n ## _xor_syndrome_real(disks, \
48 start, stop, (unsigned 48 start, stop, (unsigned long)bytes, ptrs); \
49 kernel_neon_end(); 49 kernel_neon_end(); \
50 } 50 } \
51 struct raid6_calls const raid6_neonx # 51 struct raid6_calls const raid6_neonx ## _n = { \
52 raid6_neon ## _n ## _gen_syndr 52 raid6_neon ## _n ## _gen_syndrome, \
53 raid6_neon ## _n ## _xor_syndr 53 raid6_neon ## _n ## _xor_syndrome, \
54 raid6_have_neon, 54 raid6_have_neon, \
55 "neonx" #_n, 55 "neonx" #_n, \
56 0 56 0 \
57 } 57 }
58 58
59 static int raid6_have_neon(void) 59 static int raid6_have_neon(void)
60 { 60 {
61 return cpu_has_neon(); 61 return cpu_has_neon();
62 } 62 }
63 63
64 RAID6_NEON_WRAPPER(1); 64 RAID6_NEON_WRAPPER(1);
65 RAID6_NEON_WRAPPER(2); 65 RAID6_NEON_WRAPPER(2);
66 RAID6_NEON_WRAPPER(4); 66 RAID6_NEON_WRAPPER(4);
67 RAID6_NEON_WRAPPER(8); 67 RAID6_NEON_WRAPPER(8);
68 68
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.