1 // SPDX-License-Identifier: GPL-2.0-only 1 2 /* 3 * Using hardware provided CRC32 instruction t 4 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78 5 * CRC32 is a new instruction in Intel SSE4.2, 6 * http://www.intel.com/products/processor/man 7 * Intel(R) 64 and IA-32 Architectures Softwar 8 * Volume 2A: Instruction Set Reference, A-M 9 * 10 * Copyright (C) 2008 Intel Corporation 11 * Authors: Austin Zhang <austin_zhang@linux.i 12 * Kent Liu <kent.liu@intel.com> 13 */ 14 #include <linux/init.h> 15 #include <linux/module.h> 16 #include <linux/string.h> 17 #include <linux/kernel.h> 18 #include <crypto/internal/hash.h> 19 #include <crypto/internal/simd.h> 20 21 #include <asm/cpufeatures.h> 22 #include <asm/cpu_device_id.h> 23 #include <asm/simd.h> 24 25 #define CHKSUM_BLOCK_SIZE 1 26 #define CHKSUM_DIGEST_SIZE 4 27 28 #define SCALE_F sizeof(unsigned long) 29 30 #ifdef CONFIG_X86_64 31 #define CRC32_INST "crc32q %1, %q0" 32 #else 33 #define CRC32_INST "crc32l %1, %0" 34 #endif 35 36 #ifdef CONFIG_X86_64 37 /* 38 * use carryless multiply version of crc32c wh 39 * size is >= 512 to account 40 * for fpu state save/restore overhead. 41 */ 42 #define CRC32C_PCL_BREAKEVEN 512 43 44 asmlinkage unsigned int crc_pcl(const u8 *buff 45 unsigned int c 46 #endif /* CONFIG_X86_64 */ 47 48 static u32 crc32c_intel_le_hw_byte(u32 crc, un 49 { 50 while (length--) { 51 asm("crc32b %1, %0" 52 : "+r" (crc) : "rm" (*data 53 data++; 54 } 55 56 return crc; 57 } 58 59 static u32 __pure crc32c_intel_le_hw(u32 crc, 60 { 61 unsigned int iquotient = len / SCALE_F 62 unsigned int iremainder = len % SCALE_ 63 unsigned long *ptmp = (unsigned long * 64 65 while (iquotient--) { 66 asm(CRC32_INST 67 : "+r" (crc) : "rm" (*ptmp 68 ptmp++; 69 } 70 71 if (iremainder) 72 crc = crc32c_intel_le_hw_byte( 73 iremainder); 74 75 return crc; 76 } 77 78 /* 79 * Setting the seed allows arbitrary accumulat 80 * If your algorithm starts with ~0, then XOR 81 * the seed. 82 */ 83 static int crc32c_intel_setkey(struct crypto_s 84 unsigned int keylen) 85 { 86 u32 *mctx = crypto_shash_ctx(hash); 87 88 if (keylen != sizeof(u32)) 89 return -EINVAL; 90 *mctx = le32_to_cpup((__le32 *)key); 91 return 0; 92 } 93 94 static int crc32c_intel_init(struct shash_desc 95 { 96 u32 *mctx = crypto_shash_ctx(desc->tfm 97 u32 *crcp = shash_desc_ctx(desc); 98 99 *crcp = *mctx; 100 101 return 0; 102 } 103 104 static int crc32c_intel_update(struct shash_de 105 unsigned int le 106 { 107 u32 *crcp = shash_desc_ctx(desc); 108 109 *crcp = crc32c_intel_le_hw(*crcp, data 110 return 0; 111 } 112 113 static int __crc32c_intel_finup(u32 *crcp, con 114 u8 *out) 115 { 116 *(__le32 *)out = ~cpu_to_le32(crc32c_i 117 return 0; 118 } 119 120 static int crc32c_intel_finup(struct shash_des 121 unsigned int len 122 { 123 return __crc32c_intel_finup(shash_desc 124 } 125 126 static int crc32c_intel_final(struct shash_des 127 { 128 u32 *crcp = shash_desc_ctx(desc); 129 130 *(__le32 *)out = ~cpu_to_le32p(crcp); 131 return 0; 132 } 133 134 static int crc32c_intel_digest(struct shash_de 135 unsigned int le 136 { 137 return __crc32c_intel_finup(crypto_sha 138 out); 139 } 140 141 static int crc32c_intel_cra_init(struct crypto 142 { 143 u32 *key = crypto_tfm_ctx(tfm); 144 145 *key = ~0; 146 147 return 0; 148 } 149 150 #ifdef CONFIG_X86_64 151 static int crc32c_pcl_intel_update(struct shas 152 unsigned int le 153 { 154 u32 *crcp = shash_desc_ctx(desc); 155 156 /* 157 * use faster PCL version if datasize 158 * overcome kernel fpu state save/rest 159 */ 160 if (len >= CRC32C_PCL_BREAKEVEN && cry 161 kernel_fpu_begin(); 162 *crcp = crc_pcl(data, len, *cr 163 kernel_fpu_end(); 164 } else 165 *crcp = crc32c_intel_le_hw(*cr 166 return 0; 167 } 168 169 static int __crc32c_pcl_intel_finup(u32 *crcp, 170 u8 *out) 171 { 172 if (len >= CRC32C_PCL_BREAKEVEN && cry 173 kernel_fpu_begin(); 174 *(__le32 *)out = ~cpu_to_le32( 175 kernel_fpu_end(); 176 } else 177 *(__le32 *)out = 178 ~cpu_to_le32(crc32c_in 179 return 0; 180 } 181 182 static int crc32c_pcl_intel_finup(struct shash 183 unsigned int len 184 { 185 return __crc32c_pcl_intel_finup(shash_ 186 } 187 188 static int crc32c_pcl_intel_digest(struct shas 189 unsigned int le 190 { 191 return __crc32c_pcl_intel_finup(crypto 192 out); 193 } 194 #endif /* CONFIG_X86_64 */ 195 196 static struct shash_alg alg = { 197 .setkey = crc32c 198 .init = crc32c 199 .update = crc32c 200 .final = crc32c 201 .finup = crc32c 202 .digest = crc32c 203 .descsize = sizeof 204 .digestsize = CHKSUM 205 .base = { 206 .cra_name = 207 .cra_driver_name = 208 .cra_priority = 209 .cra_flags = 210 .cra_blocksize = 211 .cra_ctxsize = 212 .cra_module = 213 .cra_init = 214 } 215 }; 216 217 static const struct x86_cpu_id crc32c_cpu_id[] 218 X86_MATCH_FEATURE(X86_FEATURE_XMM4_2, 219 {} 220 }; 221 MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id); 222 223 static int __init crc32c_intel_mod_init(void) 224 { 225 if (!x86_match_cpu(crc32c_cpu_id)) 226 return -ENODEV; 227 #ifdef CONFIG_X86_64 228 if (boot_cpu_has(X86_FEATURE_PCLMULQDQ 229 alg.update = crc32c_pcl_intel_ 230 alg.finup = crc32c_pcl_intel_f 231 alg.digest = crc32c_pcl_intel_ 232 } 233 #endif 234 return crypto_register_shash(&alg); 235 } 236 237 static void __exit crc32c_intel_mod_fini(void) 238 { 239 crypto_unregister_shash(&alg); 240 } 241 242 module_init(crc32c_intel_mod_init); 243 module_exit(crc32c_intel_mod_fini); 244 245 MODULE_AUTHOR("Austin Zhang <austin.zhang@inte 246 MODULE_DESCRIPTION("CRC32c (Castagnoli) optimi 247 MODULE_LICENSE("GPL"); 248 249 MODULE_ALIAS_CRYPTO("crc32c"); 250 MODULE_ALIAS_CRYPTO("crc32c-intel"); 251
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.