1 // SPDX-License-Identifier: GPL-2.0-only <<
2 /* 1 /*
3 * Copyright (C) 2012 Intel Corporation 2 * Copyright (C) 2012 Intel Corporation
4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheu 3 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
>> 4 *
>> 5 * This program is free software; you can redistribute it and/or
>> 6 * modify it under the terms of the GNU General Public License
>> 7 * as published by the Free Software Foundation; version 2
>> 8 * of the License.
5 */ 9 */
6 10
7 #include <linux/raid/pq.h> 11 #include <linux/raid/pq.h>
8 12
9 #ifdef __KERNEL__ 13 #ifdef __KERNEL__
10 #include <asm/neon.h> 14 #include <asm/neon.h>
11 #include "neon.h" <<
12 #else 15 #else
13 #define kernel_neon_begin() 16 #define kernel_neon_begin()
14 #define kernel_neon_end() 17 #define kernel_neon_end()
15 #define cpu_has_neon() (1) 18 #define cpu_has_neon() (1)
16 #endif 19 #endif
17 20
18 static int raid6_has_neon(void) 21 static int raid6_has_neon(void)
19 { 22 {
20 return cpu_has_neon(); 23 return cpu_has_neon();
21 } 24 }
>> 25
>> 26 void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
>> 27 uint8_t *dq, const uint8_t *pbmul,
>> 28 const uint8_t *qmul);
>> 29
>> 30 void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
>> 31 const uint8_t *qmul);
22 32
23 static void raid6_2data_recov_neon(int disks, 33 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
24 int failb, void **ptrs) 34 int failb, void **ptrs)
25 { 35 {
26 u8 *p, *q, *dp, *dq; 36 u8 *p, *q, *dp, *dq;
27 const u8 *pbmul; /* P multiplie 37 const u8 *pbmul; /* P multiplier table for B data */
28 const u8 *qmul; /* Q multiplie 38 const u8 *qmul; /* Q multiplier table (for both) */
29 39
30 p = (u8 *)ptrs[disks - 2]; 40 p = (u8 *)ptrs[disks - 2];
31 q = (u8 *)ptrs[disks - 1]; 41 q = (u8 *)ptrs[disks - 1];
32 42
33 /* 43 /*
34 * Compute syndrome with zero for the 44 * Compute syndrome with zero for the missing data pages
35 * Use the dead data pages as temporar 45 * Use the dead data pages as temporary storage for
36 * delta p and delta q 46 * delta p and delta q
37 */ 47 */
38 dp = (u8 *)ptrs[faila]; 48 dp = (u8 *)ptrs[faila];
39 ptrs[faila] = (void *)raid6_empty_zero 49 ptrs[faila] = (void *)raid6_empty_zero_page;
40 ptrs[disks - 2] = dp; 50 ptrs[disks - 2] = dp;
41 dq = (u8 *)ptrs[failb]; 51 dq = (u8 *)ptrs[failb];
42 ptrs[failb] = (void *)raid6_empty_zero 52 ptrs[failb] = (void *)raid6_empty_zero_page;
43 ptrs[disks - 1] = dq; 53 ptrs[disks - 1] = dq;
44 54
45 raid6_call.gen_syndrome(disks, bytes, 55 raid6_call.gen_syndrome(disks, bytes, ptrs);
46 56
47 /* Restore pointer table */ 57 /* Restore pointer table */
48 ptrs[faila] = dp; 58 ptrs[faila] = dp;
49 ptrs[failb] = dq; 59 ptrs[failb] = dq;
50 ptrs[disks - 2] = p; 60 ptrs[disks - 2] = p;
51 ptrs[disks - 1] = q; 61 ptrs[disks - 1] = q;
52 62
53 /* Now, pick the proper data tables */ 63 /* Now, pick the proper data tables */
54 pbmul = raid6_vgfmul[raid6_gfexi[failb 64 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
55 qmul = raid6_vgfmul[raid6_gfinv[raid6 65 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
56 raid6 66 raid6_gfexp[failb]]];
57 67
58 kernel_neon_begin(); 68 kernel_neon_begin();
59 __raid6_2data_recov_neon(bytes, p, q, 69 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
60 kernel_neon_end(); 70 kernel_neon_end();
61 } 71 }
62 72
63 static void raid6_datap_recov_neon(int disks, 73 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
64 void **ptrs) 74 void **ptrs)
65 { 75 {
66 u8 *p, *q, *dq; 76 u8 *p, *q, *dq;
67 const u8 *qmul; /* Q multiplie 77 const u8 *qmul; /* Q multiplier table */
68 78
69 p = (u8 *)ptrs[disks - 2]; 79 p = (u8 *)ptrs[disks - 2];
70 q = (u8 *)ptrs[disks - 1]; 80 q = (u8 *)ptrs[disks - 1];
71 81
72 /* 82 /*
73 * Compute syndrome with zero for the 83 * Compute syndrome with zero for the missing data page
74 * Use the dead data page as temporary 84 * Use the dead data page as temporary storage for delta q
75 */ 85 */
76 dq = (u8 *)ptrs[faila]; 86 dq = (u8 *)ptrs[faila];
77 ptrs[faila] = (void *)raid6_empty_zero 87 ptrs[faila] = (void *)raid6_empty_zero_page;
78 ptrs[disks - 1] = dq; 88 ptrs[disks - 1] = dq;
79 89
80 raid6_call.gen_syndrome(disks, bytes, 90 raid6_call.gen_syndrome(disks, bytes, ptrs);
81 91
82 /* Restore pointer table */ 92 /* Restore pointer table */
83 ptrs[faila] = dq; 93 ptrs[faila] = dq;
84 ptrs[disks - 1] = q; 94 ptrs[disks - 1] = q;
85 95
86 /* Now, pick the proper data tables */ 96 /* Now, pick the proper data tables */
87 qmul = raid6_vgfmul[raid6_gfinv[raid6_ 97 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
88 98
89 kernel_neon_begin(); 99 kernel_neon_begin();
90 __raid6_datap_recov_neon(bytes, p, q, 100 __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
91 kernel_neon_end(); 101 kernel_neon_end();
92 } 102 }
93 103
94 const struct raid6_recov_calls raid6_recov_neo 104 const struct raid6_recov_calls raid6_recov_neon = {
95 .data2 = raid6_2data_recov_ne 105 .data2 = raid6_2data_recov_neon,
96 .datap = raid6_datap_recov_ne 106 .datap = raid6_datap_recov_neon,
97 .valid = raid6_has_neon, 107 .valid = raid6_has_neon,
98 .name = "neon", 108 .name = "neon",
99 .priority = 10, 109 .priority = 10,
100 }; 110 };
101 111
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.