1 // SPDX-License-Identifier: GPL-2.0-only
2 // Copyright (C) 2019-2020 Arm Ltd.
3
4 #include <linux/compiler.h>
5 #include <linux/kasan-checks.h>
6 #include <linux/kernel.h>
7
8 #include <net/checksum.h>
9
10 /* Looks dumb, but generates nice-ish code */
11 static u64 accumulate(u64 sum, u64 data)
12 {
13 __uint128_t tmp = (__uint128_t)sum + data;
14 return tmp + (tmp >> 64);
15 }
16
17 /*
18 * We over-read the buffer and this makes KASAN unhappy. Instead, disable
19 * instrumentation and call kasan explicitly.
20 */
21 unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
22 {
23 unsigned int offset, shift, sum;
24 const u64 *ptr;
25 u64 data, sum64 = 0;
26
27 if (unlikely(len <= 0))
28 return 0;
29
30 offset = (unsigned long)buff & 7;
31 /*
32 * This is to all intents and purposes safe, since rounding down cannot
33 * result in a different page or cache line being accessed, and @buff
34 * should absolutely not be pointing to anything read-sensitive. We do,
35 * however, have to be careful not to piss off KASAN, which means using
36 * unchecked reads to accommodate the head and tail, for which we'll
37 * compensate with an explicit check up-front.
38 */
39 kasan_check_read(buff, len);
40 ptr = (u64 *)(buff - offset);
41 len = len + offset - 8;
42
43 /*
44 * Head: zero out any excess leading bytes. Shifting back by the same
45 * amount should be at least as fast as any other way of handling the
46 * odd/even alignment, and means we can ignore it until the very end.
47 */
48 shift = offset * 8;
49 data = *ptr++;
50 #ifdef __LITTLE_ENDIAN
51 data = (data >> shift) << shift;
52 #else
53 data = (data << shift) >> shift;
54 #endif
55
56 /*
57 * Body: straightforward aligned loads from here on (the paired loads
58 * underlying the quadword type still only need dword alignment). The
59 * main loop strictly excludes the tail, so the second loop will always
60 * run at least once.
61 */
62 while (unlikely(len > 64)) {
63 __uint128_t tmp1, tmp2, tmp3, tmp4;
64
65 tmp1 = *(__uint128_t *)ptr;
66 tmp2 = *(__uint128_t *)(ptr + 2);
67 tmp3 = *(__uint128_t *)(ptr + 4);
68 tmp4 = *(__uint128_t *)(ptr + 6);
69
70 len -= 64;
71 ptr += 8;
72
73 /* This is the "don't dump the carry flag into a GPR" idiom */
74 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
75 tmp2 += (tmp2 >> 64) | (tmp2 << 64);
76 tmp3 += (tmp3 >> 64) | (tmp3 << 64);
77 tmp4 += (tmp4 >> 64) | (tmp4 << 64);
78 tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
79 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
80 tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
81 tmp3 += (tmp3 >> 64) | (tmp3 << 64);
82 tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
83 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
84 tmp1 = ((tmp1 >> 64) << 64) | sum64;
85 tmp1 += (tmp1 >> 64) | (tmp1 << 64);
86 sum64 = tmp1 >> 64;
87 }
88 while (len > 8) {
89 __uint128_t tmp;
90
91 sum64 = accumulate(sum64, data);
92 tmp = *(__uint128_t *)ptr;
93
94 len -= 16;
95 ptr += 2;
96
97 #ifdef __LITTLE_ENDIAN
98 data = tmp >> 64;
99 sum64 = accumulate(sum64, tmp);
100 #else
101 data = tmp;
102 sum64 = accumulate(sum64, tmp >> 64);
103 #endif
104 }
105 if (len > 0) {
106 sum64 = accumulate(sum64, data);
107 data = *ptr;
108 len -= 8;
109 }
110 /*
111 * Tail: zero any over-read bytes similarly to the head, again
112 * preserving odd/even alignment.
113 */
114 shift = len * -8;
115 #ifdef __LITTLE_ENDIAN
116 data = (data << shift) >> shift;
117 #else
118 data = (data >> shift) << shift;
119 #endif
120 sum64 = accumulate(sum64, data);
121
122 /* Finally, folding */
123 sum64 += (sum64 >> 32) | (sum64 << 32);
124 sum = sum64 >> 32;
125 sum += (sum >> 16) | (sum << 16);
126 if (offset & 1)
127 return (u16)swab32(sum);
128
129 return sum >> 16;
130 }
131
132 __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
133 const struct in6_addr *daddr,
134 __u32 len, __u8 proto, __wsum csum)
135 {
136 __uint128_t src, dst;
137 u64 sum = (__force u64)csum;
138
139 src = *(const __uint128_t *)saddr->s6_addr;
140 dst = *(const __uint128_t *)daddr->s6_addr;
141
142 sum += (__force u32)htonl(len);
143 #ifdef __LITTLE_ENDIAN
144 sum += (u32)proto << 24;
145 #else
146 sum += proto;
147 #endif
148 src += (src >> 64) | (src << 64);
149 dst += (dst >> 64) | (dst << 64);
150
151 sum = accumulate(sum, src >> 64);
152 sum = accumulate(sum, dst >> 64);
153
154 sum += ((sum >> 32) | (sum << 32));
155 return csum_fold((__force __wsum)(sum >> 32));
156 }
157 EXPORT_SYMBOL(csum_ipv6_magic);
158
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.