1 // SPDX-License-Identifier: GPL-2.0-only <<
2 /* 1 /*
3 * linux/lib/raid6/neon.c - RAID6 syndrome cal 2 * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
4 * 3 *
5 * Copyright (C) 2013 Linaro Ltd <ard.biesheuv 4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
>> 5 *
>> 6 * This program is free software; you can redistribute it and/or modify
>> 7 * it under the terms of the GNU General Public License version 2 as
>> 8 * published by the Free Software Foundation.
6 */ 9 */
7 10
8 #include <linux/raid/pq.h> 11 #include <linux/raid/pq.h>
9 12
10 #ifdef __KERNEL__ 13 #ifdef __KERNEL__
11 #include <asm/neon.h> 14 #include <asm/neon.h>
12 #else 15 #else
13 #define kernel_neon_begin() 16 #define kernel_neon_begin()
14 #define kernel_neon_end() 17 #define kernel_neon_end()
15 #define cpu_has_neon() (1) 18 #define cpu_has_neon() (1)
16 #endif 19 #endif
17 20
18 /* 21 /*
19 * There are 2 reasons these wrappers are kept 22 * There are 2 reasons these wrappers are kept in a separate compilation unit
20 * from the actual implementations in neonN.c 23 * from the actual implementations in neonN.c (generated from neon.uc by
21 * unroll.awk): 24 * unroll.awk):
22 * - the actual implementations use NEON intri 25 * - the actual implementations use NEON intrinsics, and the GCC support header
23 * (arm_neon.h) is not fully compatible (typ 26 * (arm_neon.h) is not fully compatible (type wise) with the kernel;
24 * - the neonN.c files are compiled with -mfpu 27 * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
25 * and we have to make sure that we never us 28 * and we have to make sure that we never use *any* NEON/VFP instructions
26 * outside a kernel_neon_begin()/kernel_neon 29 * outside a kernel_neon_begin()/kernel_neon_end() pair.
27 */ 30 */
28 31
29 #define RAID6_NEON_WRAPPER(_n) 32 #define RAID6_NEON_WRAPPER(_n) \
30 static void raid6_neon ## _n ## _gen_s 33 static void raid6_neon ## _n ## _gen_syndrome(int disks, \
31 size_t 34 size_t bytes, void **ptrs) \
32 { 35 { \
33 void raid6_neon ## _n ## _gen 36 void raid6_neon ## _n ## _gen_syndrome_real(int, \
34 37 unsigned long, void**); \
35 kernel_neon_begin(); 38 kernel_neon_begin(); \
36 raid6_neon ## _n ## _gen_syndr 39 raid6_neon ## _n ## _gen_syndrome_real(disks, \
37 (unsig 40 (unsigned long)bytes, ptrs); \
38 kernel_neon_end(); 41 kernel_neon_end(); \
39 } 42 } \
40 static void raid6_neon ## _n ## _xor_s 43 static void raid6_neon ## _n ## _xor_syndrome(int disks, \
41 int st 44 int start, int stop, \
42 size_t 45 size_t bytes, void **ptrs) \
43 { 46 { \
44 void raid6_neon ## _n ## _xor 47 void raid6_neon ## _n ## _xor_syndrome_real(int, \
45 int, int, unsi 48 int, int, unsigned long, void**); \
46 kernel_neon_begin(); 49 kernel_neon_begin(); \
47 raid6_neon ## _n ## _xor_syndr 50 raid6_neon ## _n ## _xor_syndrome_real(disks, \
48 start, stop, (unsigned 51 start, stop, (unsigned long)bytes, ptrs); \
49 kernel_neon_end(); 52 kernel_neon_end(); \
50 } 53 } \
51 struct raid6_calls const raid6_neonx # 54 struct raid6_calls const raid6_neonx ## _n = { \
52 raid6_neon ## _n ## _gen_syndr 55 raid6_neon ## _n ## _gen_syndrome, \
53 raid6_neon ## _n ## _xor_syndr 56 raid6_neon ## _n ## _xor_syndrome, \
54 raid6_have_neon, 57 raid6_have_neon, \
55 "neonx" #_n, 58 "neonx" #_n, \
56 0 59 0 \
57 } 60 }
58 61
59 static int raid6_have_neon(void) 62 static int raid6_have_neon(void)
60 { 63 {
61 return cpu_has_neon(); 64 return cpu_has_neon();
62 } 65 }
63 66
64 RAID6_NEON_WRAPPER(1); 67 RAID6_NEON_WRAPPER(1);
65 RAID6_NEON_WRAPPER(2); 68 RAID6_NEON_WRAPPER(2);
66 RAID6_NEON_WRAPPER(4); 69 RAID6_NEON_WRAPPER(4);
67 RAID6_NEON_WRAPPER(8); 70 RAID6_NEON_WRAPPER(8);
68 71
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.