1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/export.h>
4 #include <asm/checksum.h>
5 #include <asm/fpu.h>
6
7 /*
8 * Computes the checksum of a memory block at src, length len,
9 * and adds in "sum" (32-bit). If copy is true copies to dst.
10 *
11 * Returns a 32-bit number suitable for feeding into itself
12 * or csum_tcpudp_magic.
13 *
14 * This function must be called with even lengths, except
15 * for the last fragment, which may be odd.
16 *
17 * It's best to have src and dst aligned on a 64-bit boundary.
18 */
19 static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __wsum sum, bool copy)
20 {
21 DECLARE_KERNEL_FPU_ONSTACK8(vxstate);
22
23 if (!cpu_has_vx()) {
24 if (copy)
25 memcpy(dst, src, len);
26 return cksm(dst, len, sum);
27 }
28 kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23);
29 fpu_vlvgf(16, (__force u32)sum, 1);
30 fpu_vzero(17);
31 fpu_vzero(18);
32 fpu_vzero(19);
33 while (len >= 64) {
34 fpu_vlm(20, 23, src);
35 if (copy) {
36 fpu_vstm(20, 23, dst);
37 dst += 64;
38 }
39 fpu_vcksm(16, 20, 16);
40 fpu_vcksm(17, 21, 17);
41 fpu_vcksm(18, 22, 18);
42 fpu_vcksm(19, 23, 19);
43 src += 64;
44 len -= 64;
45 }
46 while (len >= 32) {
47 fpu_vlm(20, 21, src);
48 if (copy) {
49 fpu_vstm(20, 21, dst);
50 dst += 32;
51 }
52 fpu_vcksm(16, 20, 16);
53 fpu_vcksm(17, 21, 17);
54 src += 32;
55 len -= 32;
56 }
57 while (len >= 16) {
58 fpu_vl(20, src);
59 if (copy) {
60 fpu_vst(20, dst);
61 dst += 16;
62 }
63 fpu_vcksm(16, 20, 16);
64 src += 16;
65 len -= 16;
66 }
67 if (len) {
68 fpu_vll(20, len - 1, src);
69 if (copy)
70 fpu_vstl(20, len - 1, dst);
71 fpu_vcksm(16, 20, 16);
72 }
73 fpu_vcksm(18, 19, 18);
74 fpu_vcksm(16, 17, 16);
75 fpu_vcksm(16, 18, 16);
76 sum = (__force __wsum)fpu_vlgvf(16, 1);
77 kernel_fpu_end(&vxstate, KERNEL_VXR_V16V23);
78 return sum;
79 }
80
81 __wsum csum_partial(const void *buff, int len, __wsum sum)
82 {
83 return csum_copy(NULL, buff, len, sum, false);
84 }
85 EXPORT_SYMBOL(csum_partial);
86
87 __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
88 {
89 return csum_copy(dst, src, len, 0, true);
90 }
91 EXPORT_SYMBOL(csum_partial_copy_nocheck);
92
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.