1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11 #include <linux/module.h>
12 #include <linux/crypto.h>
13 #include <linux/kernel.h>
14 #include <asm/simd.h>
15 #include <crypto/internal/simd.h>
16 #include <crypto/internal/skcipher.h>
17 #include <crypto/sm4.h>
18 #include "sm4-avx.h"
19
20 #define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22 asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24 asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26
27 static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
28 unsigned int key_len)
29 {
30 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
31
32 return sm4_expandkey(ctx, key, key_len);
33 }
34
35 static int cbc_decrypt(struct skcipher_request *req)
36 {
37 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
38 sm4_aesni_avx2_cbc_dec_blk16);
39 }
40
41
42 static int ctr_crypt(struct skcipher_request *req)
43 {
44 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
45 sm4_aesni_avx2_ctr_enc_blk16);
46 }
47
48 static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
49 {
50 .base = {
51 .cra_name = "__ecb(sm4)",
52 .cra_driver_name = "__ecb-sm4-aesni-avx2",
53 .cra_priority = 500,
54 .cra_flags = CRYPTO_ALG_INTERNAL,
55 .cra_blocksize = SM4_BLOCK_SIZE,
56 .cra_ctxsize = sizeof(struct sm4_ctx),
57 .cra_module = THIS_MODULE,
58 },
59 .min_keysize = SM4_KEY_SIZE,
60 .max_keysize = SM4_KEY_SIZE,
61 .walksize = 16 * SM4_BLOCK_SIZE,
62 .setkey = sm4_skcipher_setkey,
63 .encrypt = sm4_avx_ecb_encrypt,
64 .decrypt = sm4_avx_ecb_decrypt,
65 }, {
66 .base = {
67 .cra_name = "__cbc(sm4)",
68 .cra_driver_name = "__cbc-sm4-aesni-avx2",
69 .cra_priority = 500,
70 .cra_flags = CRYPTO_ALG_INTERNAL,
71 .cra_blocksize = SM4_BLOCK_SIZE,
72 .cra_ctxsize = sizeof(struct sm4_ctx),
73 .cra_module = THIS_MODULE,
74 },
75 .min_keysize = SM4_KEY_SIZE,
76 .max_keysize = SM4_KEY_SIZE,
77 .ivsize = SM4_BLOCK_SIZE,
78 .walksize = 16 * SM4_BLOCK_SIZE,
79 .setkey = sm4_skcipher_setkey,
80 .encrypt = sm4_cbc_encrypt,
81 .decrypt = cbc_decrypt,
82 }, {
83 .base = {
84 .cra_name = "__ctr(sm4)",
85 .cra_driver_name = "__ctr-sm4-aesni-avx2",
86 .cra_priority = 500,
87 .cra_flags = CRYPTO_ALG_INTERNAL,
88 .cra_blocksize = 1,
89 .cra_ctxsize = sizeof(struct sm4_ctx),
90 .cra_module = THIS_MODULE,
91 },
92 .min_keysize = SM4_KEY_SIZE,
93 .max_keysize = SM4_KEY_SIZE,
94 .ivsize = SM4_BLOCK_SIZE,
95 .chunksize = SM4_BLOCK_SIZE,
96 .walksize = 16 * SM4_BLOCK_SIZE,
97 .setkey = sm4_skcipher_setkey,
98 .encrypt = ctr_crypt,
99 .decrypt = ctr_crypt,
100 }
101 };
102
103 static struct simd_skcipher_alg *
104 simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
105
106 static int __init sm4_init(void)
107 {
108 const char *feature_name;
109
110 if (!boot_cpu_has(X86_FEATURE_AVX) ||
111 !boot_cpu_has(X86_FEATURE_AVX2) ||
112 !boot_cpu_has(X86_FEATURE_AES) ||
113 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
114 pr_info("AVX2 or AES-NI instructions are not detected.\n");
115 return -ENODEV;
116 }
117
118 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
119 &feature_name)) {
120 pr_info("CPU feature '%s' is not supported.\n", feature_name);
121 return -ENODEV;
122 }
123
124 return simd_register_skciphers_compat(sm4_aesni_avx2_skciphers,
125 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
126 simd_sm4_aesni_avx2_skciphers);
127 }
128
129 static void __exit sm4_exit(void)
130 {
131 simd_unregister_skciphers(sm4_aesni_avx2_skciphers,
132 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
133 simd_sm4_aesni_avx2_skciphers);
134 }
135
136 module_init(sm4_init);
137 module_exit(sm4_exit);
138
139 MODULE_LICENSE("GPL v2");
140 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
141 MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
142 MODULE_ALIAS_CRYPTO("sm4");
143 MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
144
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.