1 /* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
2 //
3 // This file is dual-licensed, meaning that you can use it under your
4 // choice of either of the following two licenses:
5 //
6 // Copyright 2023 The OpenSSL Project Authors. All Rights Reserved.
7 //
8 // Licensed under the Apache License 2.0 (the "License"). You can obtain
9 // a copy in the file LICENSE in the source distribution or at
10 // https://www.openssl.org/source/license.html
11 //
12 // or
13 //
14 // Copyright (c) 2023, Jerry Shih <jerry.shih@sifive.com>
15 // Copyright 2024 Google LLC
16 // All rights reserved.
17 //
18 // Redistribution and use in source and binary forms, with or without
19 // modification, are permitted provided that the following conditions
20 // are met:
21 // 1. Redistributions of source code must retain the above copyright
22 // notice, this list of conditions and the following disclaimer.
23 // 2. Redistributions in binary form must reproduce the above copyright
24 // notice, this list of conditions and the following disclaimer in the
25 // documentation and/or other materials provided with the distribution.
26 //
27 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
39 // The generated code of this file depends on the following RISC-V extensions:
40 // - RV64I
41 // - RISC-V Vector ('V') with VLEN >= 128
42 // - RISC-V Vector AES block cipher extension ('Zvkned')
43 // - RISC-V Vector Cryptography Bit-manipulation extension ('Zvkb')
44
45 #include <linux/linkage.h>
46
47 .text
48 .option arch, +zvkned, +zvkb
49
50 #include "aes-macros.S"
51
52 #define KEYP a0
53 #define INP a1
54 #define OUTP a2
55 #define LEN a3
56 #define IVP a4
57
58 #define LEN32 a5
59 #define VL_E32 a6
60 #define VL_BLOCKS a7
61
62 .macro aes_ctr32_crypt keylen
63 // LEN32 = number of blocks, rounded up, in 32-bit words.
64 addi t0, LEN, 15
65 srli t0, t0, 4
66 slli LEN32, t0, 2
67
68 // Create a mask that selects the last 32-bit word of each 128-bit
69 // block. This is the word that contains the (big-endian) counter.
70 li t0, 0x88
71 vsetvli t1, zero, e8, m1, ta, ma
72 vmv.v.x v0, t0
73
74 // Load the IV into v31. The last 32-bit word contains the counter.
75 vsetivli zero, 4, e32, m1, ta, ma
76 vle32.v v31, (IVP)
77
78 // Convert the big-endian counter into little-endian.
79 vsetivli zero, 4, e32, m1, ta, mu
80 vrev8.v v31, v31, v0.t
81
82 // Splat the IV to v16 (with LMUL=4). The number of copies is the
83 // maximum number of blocks that will be processed per iteration.
84 vsetvli zero, LEN32, e32, m4, ta, ma
85 vmv.v.i v16, 0
86 vaesz.vs v16, v31
87
88 // v20 = [x, x, x, 0, x, x, x, 1, ...]
89 viota.m v20, v0, v0.t
90 // v16 = [IV0, IV1, IV2, counter+0, IV0, IV1, IV2, counter+1, ...]
91 vsetvli VL_E32, LEN32, e32, m4, ta, mu
92 vadd.vv v16, v16, v20, v0.t
93
94 j 2f
95 1:
96 // Set the number of blocks to process in this iteration. vl=VL_E32 is
97 // the length in 32-bit words, i.e. 4 times the number of blocks.
98 vsetvli VL_E32, LEN32, e32, m4, ta, mu
99
100 // Increment the counters by the number of blocks processed in the
101 // previous iteration.
102 vadd.vx v16, v16, VL_BLOCKS, v0.t
103 2:
104 // Prepare the AES inputs into v24.
105 vmv.v.v v24, v16
106 vrev8.v v24, v24, v0.t // Convert counters back to big-endian.
107
108 // Encrypt the AES inputs to create the next portion of the keystream.
109 aes_encrypt v24, \keylen
110
111 // XOR the data with the keystream.
112 vsetvli t0, LEN, e8, m4, ta, ma
113 vle8.v v20, (INP)
114 vxor.vv v20, v20, v24
115 vse8.v v20, (OUTP)
116
117 // Advance the pointers and update the remaining length.
118 add INP, INP, t0
119 add OUTP, OUTP, t0
120 sub LEN, LEN, t0
121 sub LEN32, LEN32, VL_E32
122 srli VL_BLOCKS, VL_E32, 2
123
124 // Repeat if more data remains.
125 bnez LEN, 1b
126
127 // Update *IVP to contain the next counter.
128 vsetivli zero, 4, e32, m1, ta, mu
129 vadd.vx v16, v16, VL_BLOCKS, v0.t
130 vrev8.v v16, v16, v0.t // Convert counters back to big-endian.
131 vse32.v v16, (IVP)
132
133 ret
134 .endm
135
136 // void aes_ctr32_crypt_zvkned_zvkb(const struct crypto_aes_ctx *key,
137 // const u8 *in, u8 *out, size_t len,
138 // u8 iv[16]);
139 SYM_FUNC_START(aes_ctr32_crypt_zvkned_zvkb)
140 aes_begin KEYP, 128f, 192f
141 aes_ctr32_crypt 256
142 128:
143 aes_ctr32_crypt 128
144 192:
145 aes_ctr32_crypt 192
146 SYM_FUNC_END(aes_ctr32_crypt_zvkned_zvkb)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.