1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * SM4-CCM AEAD Algorithm using ARMv8 Crypto Extensions
4 * as specified in rfc8998
5 * https://datatracker.ietf.org/doc/html/rfc8998
6 *
7 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
8 */
9
10 #include <linux/linkage.h>
11 #include <linux/cfi_types.h>
12 #include <asm/assembler.h>
13 #include "sm4-ce-asm.h"
14
15 .arch armv8-a+crypto
16
17 .irp b, 0, 1, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 25, 26, 27, 28, 29, 30, 31
18 .set .Lv\b\().4s, \b
19 .endr
20
21 .macro sm4e, vd, vn
22 .inst 0xcec08400 | (.L\vn << 5) | .L\vd
23 .endm
24
25 /* Register macros */
26
27 #define RMAC v16
28
29 /* Helper macros. */
30
31 #define inc_le128(vctr) \
32 mov vctr.d[1], x8; \
33 mov vctr.d[0], x7; \
34 adds x8, x8, #1; \
35 rev64 vctr.16b, vctr.16b; \
36 adc x7, x7, xzr;
37
38
39 .align 3
40 SYM_FUNC_START(sm4_ce_cbcmac_update)
41 /* input:
42 * x0: round key array, CTX
43 * x1: mac
44 * x2: src
45 * w3: nblocks
46 */
47 SM4_PREPARE(x0)
48
49 ld1 {RMAC.16b}, [x1]
50
51 .Lcbcmac_loop_4x:
52 cmp w3, #4
53 blt .Lcbcmac_loop_1x
54
55 sub w3, w3, #4
56
57 ld1 {v0.16b-v3.16b}, [x2], #64
58
59 SM4_CRYPT_BLK(RMAC)
60 eor RMAC.16b, RMAC.16b, v0.16b
61 SM4_CRYPT_BLK(RMAC)
62 eor RMAC.16b, RMAC.16b, v1.16b
63 SM4_CRYPT_BLK(RMAC)
64 eor RMAC.16b, RMAC.16b, v2.16b
65 SM4_CRYPT_BLK(RMAC)
66 eor RMAC.16b, RMAC.16b, v3.16b
67
68 cbz w3, .Lcbcmac_end
69 b .Lcbcmac_loop_4x
70
71 .Lcbcmac_loop_1x:
72 sub w3, w3, #1
73
74 ld1 {v0.16b}, [x2], #16
75
76 SM4_CRYPT_BLK(RMAC)
77 eor RMAC.16b, RMAC.16b, v0.16b
78
79 cbnz w3, .Lcbcmac_loop_1x
80
81 .Lcbcmac_end:
82 st1 {RMAC.16b}, [x1]
83 ret
84 SYM_FUNC_END(sm4_ce_cbcmac_update)
85
86 .align 3
87 SYM_FUNC_START(sm4_ce_ccm_final)
88 /* input:
89 * x0: round key array, CTX
90 * x1: ctr0 (big endian, 128 bit)
91 * x2: mac
92 */
93 SM4_PREPARE(x0)
94
95 ld1 {RMAC.16b}, [x2]
96 ld1 {v0.16b}, [x1]
97
98 SM4_CRYPT_BLK2(RMAC, v0)
99
100 /* en-/decrypt the mac with ctr0 */
101 eor RMAC.16b, RMAC.16b, v0.16b
102 st1 {RMAC.16b}, [x2]
103
104 ret
105 SYM_FUNC_END(sm4_ce_ccm_final)
106
107 .align 3
108 SYM_TYPED_FUNC_START(sm4_ce_ccm_enc)
109 /* input:
110 * x0: round key array, CTX
111 * x1: dst
112 * x2: src
113 * x3: ctr (big endian, 128 bit)
114 * w4: nbytes
115 * x5: mac
116 */
117 SM4_PREPARE(x0)
118
119 ldp x7, x8, [x3]
120 rev x7, x7
121 rev x8, x8
122
123 ld1 {RMAC.16b}, [x5]
124
125 .Lccm_enc_loop_4x:
126 cmp w4, #(4 * 16)
127 blt .Lccm_enc_loop_1x
128
129 sub w4, w4, #(4 * 16)
130
131 /* construct CTRs */
132 inc_le128(v8) /* +0 */
133 inc_le128(v9) /* +1 */
134 inc_le128(v10) /* +2 */
135 inc_le128(v11) /* +3 */
136
137 ld1 {v0.16b-v3.16b}, [x2], #64
138
139 SM4_CRYPT_BLK2(v8, RMAC)
140 eor v8.16b, v8.16b, v0.16b
141 eor RMAC.16b, RMAC.16b, v0.16b
142 SM4_CRYPT_BLK2(v9, RMAC)
143 eor v9.16b, v9.16b, v1.16b
144 eor RMAC.16b, RMAC.16b, v1.16b
145 SM4_CRYPT_BLK2(v10, RMAC)
146 eor v10.16b, v10.16b, v2.16b
147 eor RMAC.16b, RMAC.16b, v2.16b
148 SM4_CRYPT_BLK2(v11, RMAC)
149 eor v11.16b, v11.16b, v3.16b
150 eor RMAC.16b, RMAC.16b, v3.16b
151
152 st1 {v8.16b-v11.16b}, [x1], #64
153
154 cbz w4, .Lccm_enc_end
155 b .Lccm_enc_loop_4x
156
157 .Lccm_enc_loop_1x:
158 cmp w4, #16
159 blt .Lccm_enc_tail
160
161 sub w4, w4, #16
162
163 /* construct CTRs */
164 inc_le128(v8)
165
166 ld1 {v0.16b}, [x2], #16
167
168 SM4_CRYPT_BLK2(v8, RMAC)
169 eor v8.16b, v8.16b, v0.16b
170 eor RMAC.16b, RMAC.16b, v0.16b
171
172 st1 {v8.16b}, [x1], #16
173
174 cbz w4, .Lccm_enc_end
175 b .Lccm_enc_loop_1x
176
177 .Lccm_enc_tail:
178 /* construct CTRs */
179 inc_le128(v8)
180
181 SM4_CRYPT_BLK2(RMAC, v8)
182
183 /* store new MAC */
184 st1 {RMAC.16b}, [x5]
185
186 .Lccm_enc_tail_loop:
187 ldrb w0, [x2], #1 /* get 1 byte from input */
188 umov w9, v8.b[0] /* get top crypted CTR byte */
189 umov w6, RMAC.b[0] /* get top MAC byte */
190
191 eor w9, w9, w0 /* w9 = CTR ^ input */
192 eor w6, w6, w0 /* w6 = MAC ^ input */
193
194 strb w9, [x1], #1 /* store out byte */
195 strb w6, [x5], #1 /* store MAC byte */
196
197 subs w4, w4, #1
198 beq .Lccm_enc_ret
199
200 /* shift out one byte */
201 ext RMAC.16b, RMAC.16b, RMAC.16b, #1
202 ext v8.16b, v8.16b, v8.16b, #1
203
204 b .Lccm_enc_tail_loop
205
206 .Lccm_enc_end:
207 /* store new MAC */
208 st1 {RMAC.16b}, [x5]
209
210 /* store new CTR */
211 rev x7, x7
212 rev x8, x8
213 stp x7, x8, [x3]
214
215 .Lccm_enc_ret:
216 ret
217 SYM_FUNC_END(sm4_ce_ccm_enc)
218
219 .align 3
220 SYM_TYPED_FUNC_START(sm4_ce_ccm_dec)
221 /* input:
222 * x0: round key array, CTX
223 * x1: dst
224 * x2: src
225 * x3: ctr (big endian, 128 bit)
226 * w4: nbytes
227 * x5: mac
228 */
229 SM4_PREPARE(x0)
230
231 ldp x7, x8, [x3]
232 rev x7, x7
233 rev x8, x8
234
235 ld1 {RMAC.16b}, [x5]
236
237 .Lccm_dec_loop_4x:
238 cmp w4, #(4 * 16)
239 blt .Lccm_dec_loop_1x
240
241 sub w4, w4, #(4 * 16)
242
243 /* construct CTRs */
244 inc_le128(v8) /* +0 */
245 inc_le128(v9) /* +1 */
246 inc_le128(v10) /* +2 */
247 inc_le128(v11) /* +3 */
248
249 ld1 {v0.16b-v3.16b}, [x2], #64
250
251 SM4_CRYPT_BLK2(v8, RMAC)
252 eor v8.16b, v8.16b, v0.16b
253 eor RMAC.16b, RMAC.16b, v8.16b
254 SM4_CRYPT_BLK2(v9, RMAC)
255 eor v9.16b, v9.16b, v1.16b
256 eor RMAC.16b, RMAC.16b, v9.16b
257 SM4_CRYPT_BLK2(v10, RMAC)
258 eor v10.16b, v10.16b, v2.16b
259 eor RMAC.16b, RMAC.16b, v10.16b
260 SM4_CRYPT_BLK2(v11, RMAC)
261 eor v11.16b, v11.16b, v3.16b
262 eor RMAC.16b, RMAC.16b, v11.16b
263
264 st1 {v8.16b-v11.16b}, [x1], #64
265
266 cbz w4, .Lccm_dec_end
267 b .Lccm_dec_loop_4x
268
269 .Lccm_dec_loop_1x:
270 cmp w4, #16
271 blt .Lccm_dec_tail
272
273 sub w4, w4, #16
274
275 /* construct CTRs */
276 inc_le128(v8)
277
278 ld1 {v0.16b}, [x2], #16
279
280 SM4_CRYPT_BLK2(v8, RMAC)
281 eor v8.16b, v8.16b, v0.16b
282 eor RMAC.16b, RMAC.16b, v8.16b
283
284 st1 {v8.16b}, [x1], #16
285
286 cbz w4, .Lccm_dec_end
287 b .Lccm_dec_loop_1x
288
289 .Lccm_dec_tail:
290 /* construct CTRs */
291 inc_le128(v8)
292
293 SM4_CRYPT_BLK2(RMAC, v8)
294
295 /* store new MAC */
296 st1 {RMAC.16b}, [x5]
297
298 .Lccm_dec_tail_loop:
299 ldrb w0, [x2], #1 /* get 1 byte from input */
300 umov w9, v8.b[0] /* get top crypted CTR byte */
301 umov w6, RMAC.b[0] /* get top MAC byte */
302
303 eor w9, w9, w0 /* w9 = CTR ^ input */
304 eor w6, w6, w9 /* w6 = MAC ^ output */
305
306 strb w9, [x1], #1 /* store out byte */
307 strb w6, [x5], #1 /* store MAC byte */
308
309 subs w4, w4, #1
310 beq .Lccm_dec_ret
311
312 /* shift out one byte */
313 ext RMAC.16b, RMAC.16b, RMAC.16b, #1
314 ext v8.16b, v8.16b, v8.16b, #1
315
316 b .Lccm_dec_tail_loop
317
318 .Lccm_dec_end:
319 /* store new MAC */
320 st1 {RMAC.16b}, [x5]
321
322 /* store new CTR */
323 rev x7, x7
324 rev x8, x8
325 stp x7, x8, [x3]
326
327 .Lccm_dec_ret:
328 ret
329 SYM_FUNC_END(sm4_ce_ccm_dec)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.