1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * x86_64/AVX2 assembler optimized version of Serpent 4 * 5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6 * 7 * Based on AVX assembler implementation of Serpent by: 8 * Copyright © 2012 Johannes Goetzfried 9 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 10 */ 11 12 #include <linux/linkage.h> 13 #include <asm/frame.h> 14 #include "glue_helper-asm-avx2.S" 15 16 .file "serpent-avx2-asm_64.S" 17 18 .section .rodata.cst16.bswap128_mask, "aM", @progbits, 16 19 .align 16 20 .Lbswap128_mask: 21 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 22 23 .text 24 25 #define CTX %rdi 26 27 #define RNOT %ymm0 28 #define tp %ymm1 29 30 #define RA1 %ymm2 31 #define RA2 %ymm3 32 #define RB1 %ymm4 33 #define RB2 %ymm5 34 #define RC1 %ymm6 35 #define RC2 %ymm7 36 #define RD1 %ymm8 37 #define RD2 %ymm9 38 #define RE1 %ymm10 39 #define RE2 %ymm11 40 41 #define RK0 %ymm12 42 #define RK1 %ymm13 43 #define RK2 %ymm14 44 #define RK3 %ymm15 45 46 #define RK0x %xmm12 47 #define RK1x %xmm13 48 #define RK2x %xmm14 49 #define RK3x %xmm15 50 51 #define S0_1(x0, x1, x2, x3, x4) \ 52 vpor x0, x3, tp; \ 53 vpxor x3, x0, x0; \ 54 vpxor x2, x3, x4; \ 55 vpxor RNOT, x4, x4; \ 56 vpxor x1, tp, x3; \ 57 vpand x0, x1, x1; \ 58 vpxor x4, x1, x1; \ 59 vpxor x0, x2, x2; 60 #define S0_2(x0, x1, x2, x3, x4) \ 61 vpxor x3, x0, x0; \ 62 vpor x0, x4, x4; \ 63 vpxor x2, x0, x0; \ 64 vpand x1, x2, x2; \ 65 vpxor x2, x3, x3; \ 66 vpxor RNOT, x1, x1; \ 67 vpxor x4, x2, x2; \ 68 vpxor x2, x1, x1; 69 70 #define S1_1(x0, x1, x2, x3, x4) \ 71 vpxor x0, x1, tp; \ 72 vpxor x3, x0, x0; \ 73 vpxor RNOT, x3, x3; \ 74 vpand tp, x1, x4; \ 75 vpor tp, x0, x0; \ 76 vpxor x2, x3, x3; \ 77 vpxor x3, x0, x0; \ 78 vpxor x3, tp, x1; 79 #define S1_2(x0, x1, x2, x3, x4) \ 80 vpxor x4, x3, x3; \ 81 vpor x4, x1, x1; \ 82 vpxor x2, x4, x4; \ 83 vpand x0, x2, x2; \ 84 vpxor x1, x2, x2; \ 85 vpor x0, x1, x1; \ 86 vpxor RNOT, x0, x0; \ 87 vpxor x2, x0, x0; \ 88 vpxor x1, x4, x4; 89 90 #define S2_1(x0, x1, x2, x3, x4) \ 91 vpxor RNOT, x3, x3; \ 92 vpxor x0, x1, x1; \ 93 vpand x2, x0, tp; \ 94 vpxor x3, tp, tp; \ 95 vpor x0, x3, x3; \ 96 vpxor x1, x2, x2; \ 97 vpxor x1, x3, x3; \ 98 vpand tp, x1, x1; 99 #define S2_2(x0, x1, x2, x3, x4) \ 100 vpxor x2, tp, tp; \ 101 vpand x3, x2, x2; \ 102 vpor x1, x3, x3; \ 103 vpxor RNOT, tp, tp; \ 104 vpxor tp, x3, x3; \ 105 vpxor tp, x0, x4; \ 106 vpxor x2, tp, x0; \ 107 vpor x2, x1, x1; 108 109 #define S3_1(x0, x1, x2, x3, x4) \ 110 vpxor x3, x1, tp; \ 111 vpor x0, x3, x3; \ 112 vpand x0, x1, x4; \ 113 vpxor x2, x0, x0; \ 114 vpxor tp, x2, x2; \ 115 vpand x3, tp, x1; \ 116 vpxor x3, x2, x2; \ 117 vpor x4, x0, x0; \ 118 vpxor x3, x4, x4; 119 #define S3_2(x0, x1, x2, x3, x4) \ 120 vpxor x0, x1, x1; \ 121 vpand x3, x0, x0; \ 122 vpand x4, x3, x3; \ 123 vpxor x2, x3, x3; \ 124 vpor x1, x4, x4; \ 125 vpand x1, x2, x2; \ 126 vpxor x3, x4, x4; \ 127 vpxor x3, x0, x0; \ 128 vpxor x2, x3, x3; 129 130 #define S4_1(x0, x1, x2, x3, x4) \ 131 vpand x0, x3, tp; \ 132 vpxor x3, x0, x0; \ 133 vpxor x2, tp, tp; \ 134 vpor x3, x2, x2; \ 135 vpxor x1, x0, x0; \ 136 vpxor tp, x3, x4; \ 137 vpor x0, x2, x2; \ 138 vpxor x1, x2, x2; 139 #define S4_2(x0, x1, x2, x3, x4) \ 140 vpand x0, x1, x1; \ 141 vpxor x4, x1, x1; \ 142 vpand x2, x4, x4; \ 143 vpxor tp, x2, x2; \ 144 vpxor x0, x4, x4; \ 145 vpor x1, tp, x3; \ 146 vpxor RNOT, x1, x1; \ 147 vpxor x0, x3, x3; 148 149 #define S5_1(x0, x1, x2, x3, x4) \ 150 vpor x0, x1, tp; \ 151 vpxor tp, x2, x2; \ 152 vpxor RNOT, x3, x3; \ 153 vpxor x0, x1, x4; \ 154 vpxor x2, x0, x0; \ 155 vpand x4, tp, x1; \ 156 vpor x3, x4, x4; \ 157 vpxor x0, x4, x4; 158 #define S5_2(x0, x1, x2, x3, x4) \ 159 vpand x3, x0, x0; \ 160 vpxor x3, x1, x1; \ 161 vpxor x2, x3, x3; \ 162 vpxor x1, x0, x0; \ 163 vpand x4, x2, x2; \ 164 vpxor x2, x1, x1; \ 165 vpand x0, x2, x2; \ 166 vpxor x2, x3, x3; 167 168 #define S6_1(x0, x1, x2, x3, x4) \ 169 vpxor x0, x3, x3; \ 170 vpxor x2, x1, tp; \ 171 vpxor x0, x2, x2; \ 172 vpand x3, x0, x0; \ 173 vpor x3, tp, tp; \ 174 vpxor RNOT, x1, x4; \ 175 vpxor tp, x0, x0; \ 176 vpxor x2, tp, x1; 177 #define S6_2(x0, x1, x2, x3, x4) \ 178 vpxor x4, x3, x3; \ 179 vpxor x0, x4, x4; \ 180 vpand x0, x2, x2; \ 181 vpxor x1, x4, x4; \ 182 vpxor x3, x2, x2; \ 183 vpand x1, x3, x3; \ 184 vpxor x0, x3, x3; \ 185 vpxor x2, x1, x1; 186 187 #define S7_1(x0, x1, x2, x3, x4) \ 188 vpxor RNOT, x1, tp; \ 189 vpxor RNOT, x0, x0; \ 190 vpand x2, tp, x1; \ 191 vpxor x3, x1, x1; \ 192 vpor tp, x3, x3; \ 193 vpxor x2, tp, x4; \ 194 vpxor x3, x2, x2; \ 195 vpxor x0, x3, x3; \ 196 vpor x1, x0, x0; 197 #define S7_2(x0, x1, x2, x3, x4) \ 198 vpand x0, x2, x2; \ 199 vpxor x4, x0, x0; \ 200 vpxor x3, x4, x4; \ 201 vpand x0, x3, x3; \ 202 vpxor x1, x4, x4; \ 203 vpxor x4, x2, x2; \ 204 vpxor x1, x3, x3; \ 205 vpor x0, x4, x4; \ 206 vpxor x1, x4, x4; 207 208 #define SI0_1(x0, x1, x2, x3, x4) \ 209 vpxor x0, x1, x1; \ 210 vpor x1, x3, tp; \ 211 vpxor x1, x3, x4; \ 212 vpxor RNOT, x0, x0; \ 213 vpxor tp, x2, x2; \ 214 vpxor x0, tp, x3; \ 215 vpand x1, x0, x0; \ 216 vpxor x2, x0, x0; 217 #define SI0_2(x0, x1, x2, x3, x4) \ 218 vpand x3, x2, x2; \ 219 vpxor x4, x3, x3; \ 220 vpxor x3, x2, x2; \ 221 vpxor x3, x1, x1; \ 222 vpand x0, x3, x3; \ 223 vpxor x0, x1, x1; \ 224 vpxor x2, x0, x0; \ 225 vpxor x3, x4, x4; 226 227 #define SI1_1(x0, x1, x2, x3, x4) \ 228 vpxor x3, x1, x1; \ 229 vpxor x2, x0, tp; \ 230 vpxor RNOT, x2, x2; \ 231 vpor x1, x0, x4; \ 232 vpxor x3, x4, x4; \ 233 vpand x1, x3, x3; \ 234 vpxor x2, x1, x1; \ 235 vpand x4, x2, x2; 236 #define SI1_2(x0, x1, x2, x3, x4) \ 237 vpxor x1, x4, x4; \ 238 vpor x3, x1, x1; \ 239 vpxor tp, x3, x3; \ 240 vpxor tp, x2, x2; \ 241 vpor x4, tp, x0; \ 242 vpxor x4, x2, x2; \ 243 vpxor x0, x1, x1; \ 244 vpxor x1, x4, x4; 245 246 #define SI2_1(x0, x1, x2, x3, x4) \ 247 vpxor x1, x2, x2; \ 248 vpxor RNOT, x3, tp; \ 249 vpor x2, tp, tp; \ 250 vpxor x3, x2, x2; \ 251 vpxor x0, x3, x4; \ 252 vpxor x1, tp, x3; \ 253 vpor x2, x1, x1; \ 254 vpxor x0, x2, x2; 255 #define SI2_2(x0, x1, x2, x3, x4) \ 256 vpxor x4, x1, x1; \ 257 vpor x3, x4, x4; \ 258 vpxor x3, x2, x2; \ 259 vpxor x2, x4, x4; \ 260 vpand x1, x2, x2; \ 261 vpxor x3, x2, x2; \ 262 vpxor x4, x3, x3; \ 263 vpxor x0, x4, x4; 264 265 #define SI3_1(x0, x1, x2, x3, x4) \ 266 vpxor x1, x2, x2; \ 267 vpand x2, x1, tp; \ 268 vpxor x0, tp, tp; \ 269 vpor x1, x0, x0; \ 270 vpxor x3, x1, x4; \ 271 vpxor x3, x0, x0; \ 272 vpor tp, x3, x3; \ 273 vpxor x2, tp, x1; 274 #define SI3_2(x0, x1, x2, x3, x4) \ 275 vpxor x3, x1, x1; \ 276 vpxor x2, x0, x0; \ 277 vpxor x3, x2, x2; \ 278 vpand x1, x3, x3; \ 279 vpxor x0, x1, x1; \ 280 vpand x2, x0, x0; \ 281 vpxor x3, x4, x4; \ 282 vpxor x0, x3, x3; \ 283 vpxor x1, x0, x0; 284 285 #define SI4_1(x0, x1, x2, x3, x4) \ 286 vpxor x3, x2, x2; \ 287 vpand x1, x0, tp; \ 288 vpxor x2, tp, tp; \ 289 vpor x3, x2, x2; \ 290 vpxor RNOT, x0, x4; \ 291 vpxor tp, x1, x1; \ 292 vpxor x2, tp, x0; \ 293 vpand x4, x2, x2; 294 #define SI4_2(x0, x1, x2, x3, x4) \ 295 vpxor x0, x2, x2; \ 296 vpor x4, x0, x0; \ 297 vpxor x3, x0, x0; \ 298 vpand x2, x3, x3; \ 299 vpxor x3, x4, x4; \ 300 vpxor x1, x3, x3; \ 301 vpand x0, x1, x1; \ 302 vpxor x1, x4, x4; \ 303 vpxor x3, x0, x0; 304 305 #define SI5_1(x0, x1, x2, x3, x4) \ 306 vpor x2, x1, tp; \ 307 vpxor x1, x2, x2; \ 308 vpxor x3, tp, tp; \ 309 vpand x1, x3, x3; \ 310 vpxor x3, x2, x2; \ 311 vpor x0, x3, x3; \ 312 vpxor RNOT, x0, x0; \ 313 vpxor x2, x3, x3; \ 314 vpor x0, x2, x2; 315 #define SI5_2(x0, x1, x2, x3, x4) \ 316 vpxor tp, x1, x4; \ 317 vpxor x4, x2, x2; \ 318 vpand x0, x4, x4; \ 319 vpxor tp, x0, x0; \ 320 vpxor x3, tp, x1; \ 321 vpand x2, x0, x0; \ 322 vpxor x3, x2, x2; \ 323 vpxor x2, x0, x0; \ 324 vpxor x4, x2, x2; \ 325 vpxor x3, x4, x4; 326 327 #define SI6_1(x0, x1, x2, x3, x4) \ 328 vpxor x2, x0, x0; \ 329 vpand x3, x0, tp; \ 330 vpxor x3, x2, x2; \ 331 vpxor x2, tp, tp; \ 332 vpxor x1, x3, x3; \ 333 vpor x0, x2, x2; \ 334 vpxor x3, x2, x2; \ 335 vpand tp, x3, x3; 336 #define SI6_2(x0, x1, x2, x3, x4) \ 337 vpxor RNOT, tp, tp; \ 338 vpxor x1, x3, x3; \ 339 vpand x2, x1, x1; \ 340 vpxor tp, x0, x4; \ 341 vpxor x4, x3, x3; \ 342 vpxor x2, x4, x4; \ 343 vpxor x1, tp, x0; \ 344 vpxor x0, x2, x2; 345 346 #define SI7_1(x0, x1, x2, x3, x4) \ 347 vpand x0, x3, tp; \ 348 vpxor x2, x0, x0; \ 349 vpor x3, x2, x2; \ 350 vpxor x1, x3, x4; \ 351 vpxor RNOT, x0, x0; \ 352 vpor tp, x1, x1; \ 353 vpxor x0, x4, x4; \ 354 vpand x2, x0, x0; \ 355 vpxor x1, x0, x0; 356 #define SI7_2(x0, x1, x2, x3, x4) \ 357 vpand x2, x1, x1; \ 358 vpxor x2, tp, x3; \ 359 vpxor x3, x4, x4; \ 360 vpand x3, x2, x2; \ 361 vpor x0, x3, x3; \ 362 vpxor x4, x1, x1; \ 363 vpxor x4, x3, x3; \ 364 vpand x0, x4, x4; \ 365 vpxor x2, x4, x4; 366 367 #define get_key(i,j,t) \ 368 vpbroadcastd (4*(i)+(j))*4(CTX), t; 369 370 #define K2(x0, x1, x2, x3, x4, i) \ 371 get_key(i, 0, RK0); \ 372 get_key(i, 1, RK1); \ 373 get_key(i, 2, RK2); \ 374 get_key(i, 3, RK3); \ 375 vpxor RK0, x0 ## 1, x0 ## 1; \ 376 vpxor RK1, x1 ## 1, x1 ## 1; \ 377 vpxor RK2, x2 ## 1, x2 ## 1; \ 378 vpxor RK3, x3 ## 1, x3 ## 1; \ 379 vpxor RK0, x0 ## 2, x0 ## 2; \ 380 vpxor RK1, x1 ## 2, x1 ## 2; \ 381 vpxor RK2, x2 ## 2, x2 ## 2; \ 382 vpxor RK3, x3 ## 2, x3 ## 2; 383 384 #define LK2(x0, x1, x2, x3, x4, i) \ 385 vpslld $13, x0 ## 1, x4 ## 1; \ 386 vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \ 387 vpor x4 ## 1, x0 ## 1, x0 ## 1; \ 388 vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ 389 vpslld $3, x2 ## 1, x4 ## 1; \ 390 vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \ 391 vpor x4 ## 1, x2 ## 1, x2 ## 1; \ 392 vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ 393 vpslld $13, x0 ## 2, x4 ## 2; \ 394 vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \ 395 vpor x4 ## 2, x0 ## 2, x0 ## 2; \ 396 vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ 397 vpslld $3, x2 ## 2, x4 ## 2; \ 398 vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \ 399 vpor x4 ## 2, x2 ## 2, x2 ## 2; \ 400 vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ 401 vpslld $1, x1 ## 1, x4 ## 1; \ 402 vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \ 403 vpor x4 ## 1, x1 ## 1, x1 ## 1; \ 404 vpslld $3, x0 ## 1, x4 ## 1; \ 405 vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ 406 vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ 407 get_key(i, 1, RK1); \ 408 vpslld $1, x1 ## 2, x4 ## 2; \ 409 vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \ 410 vpor x4 ## 2, x1 ## 2, x1 ## 2; \ 411 vpslld $3, x0 ## 2, x4 ## 2; \ 412 vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ 413 vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ 414 get_key(i, 3, RK3); \ 415 vpslld $7, x3 ## 1, x4 ## 1; \ 416 vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \ 417 vpor x4 ## 1, x3 ## 1, x3 ## 1; \ 418 vpslld $7, x1 ## 1, x4 ## 1; \ 419 vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ 420 vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ 421 vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ 422 vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ 423 get_key(i, 0, RK0); \ 424 vpslld $7, x3 ## 2, x4 ## 2; \ 425 vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \ 426 vpor x4 ## 2, x3 ## 2, x3 ## 2; \ 427 vpslld $7, x1 ## 2, x4 ## 2; \ 428 vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ 429 vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ 430 vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ 431 vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ 432 get_key(i, 2, RK2); \ 433 vpxor RK1, x1 ## 1, x1 ## 1; \ 434 vpxor RK3, x3 ## 1, x3 ## 1; \ 435 vpslld $5, x0 ## 1, x4 ## 1; \ 436 vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \ 437 vpor x4 ## 1, x0 ## 1, x0 ## 1; \ 438 vpslld $22, x2 ## 1, x4 ## 1; \ 439 vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \ 440 vpor x4 ## 1, x2 ## 1, x2 ## 1; \ 441 vpxor RK0, x0 ## 1, x0 ## 1; \ 442 vpxor RK2, x2 ## 1, x2 ## 1; \ 443 vpxor RK1, x1 ## 2, x1 ## 2; \ 444 vpxor RK3, x3 ## 2, x3 ## 2; \ 445 vpslld $5, x0 ## 2, x4 ## 2; \ 446 vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \ 447 vpor x4 ## 2, x0 ## 2, x0 ## 2; \ 448 vpslld $22, x2 ## 2, x4 ## 2; \ 449 vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \ 450 vpor x4 ## 2, x2 ## 2, x2 ## 2; \ 451 vpxor RK0, x0 ## 2, x0 ## 2; \ 452 vpxor RK2, x2 ## 2, x2 ## 2; 453 454 #define KL2(x0, x1, x2, x3, x4, i) \ 455 vpxor RK0, x0 ## 1, x0 ## 1; \ 456 vpxor RK2, x2 ## 1, x2 ## 1; \ 457 vpsrld $5, x0 ## 1, x4 ## 1; \ 458 vpslld $(32 - 5), x0 ## 1, x0 ## 1; \ 459 vpor x4 ## 1, x0 ## 1, x0 ## 1; \ 460 vpxor RK3, x3 ## 1, x3 ## 1; \ 461 vpxor RK1, x1 ## 1, x1 ## 1; \ 462 vpsrld $22, x2 ## 1, x4 ## 1; \ 463 vpslld $(32 - 22), x2 ## 1, x2 ## 1; \ 464 vpor x4 ## 1, x2 ## 1, x2 ## 1; \ 465 vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ 466 vpxor RK0, x0 ## 2, x0 ## 2; \ 467 vpxor RK2, x2 ## 2, x2 ## 2; \ 468 vpsrld $5, x0 ## 2, x4 ## 2; \ 469 vpslld $(32 - 5), x0 ## 2, x0 ## 2; \ 470 vpor x4 ## 2, x0 ## 2, x0 ## 2; \ 471 vpxor RK3, x3 ## 2, x3 ## 2; \ 472 vpxor RK1, x1 ## 2, x1 ## 2; \ 473 vpsrld $22, x2 ## 2, x4 ## 2; \ 474 vpslld $(32 - 22), x2 ## 2, x2 ## 2; \ 475 vpor x4 ## 2, x2 ## 2, x2 ## 2; \ 476 vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ 477 vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ 478 vpslld $7, x1 ## 1, x4 ## 1; \ 479 vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ 480 vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ 481 vpsrld $1, x1 ## 1, x4 ## 1; \ 482 vpslld $(32 - 1), x1 ## 1, x1 ## 1; \ 483 vpor x4 ## 1, x1 ## 1, x1 ## 1; \ 484 vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ 485 vpslld $7, x1 ## 2, x4 ## 2; \ 486 vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ 487 vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ 488 vpsrld $1, x1 ## 2, x4 ## 2; \ 489 vpslld $(32 - 1), x1 ## 2, x1 ## 2; \ 490 vpor x4 ## 2, x1 ## 2, x1 ## 2; \ 491 vpsrld $7, x3 ## 1, x4 ## 1; \ 492 vpslld $(32 - 7), x3 ## 1, x3 ## 1; \ 493 vpor x4 ## 1, x3 ## 1, x3 ## 1; \ 494 vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ 495 vpslld $3, x0 ## 1, x4 ## 1; \ 496 vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ 497 vpsrld $7, x3 ## 2, x4 ## 2; \ 498 vpslld $(32 - 7), x3 ## 2, x3 ## 2; \ 499 vpor x4 ## 2, x3 ## 2, x3 ## 2; \ 500 vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ 501 vpslld $3, x0 ## 2, x4 ## 2; \ 502 vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ 503 vpsrld $13, x0 ## 1, x4 ## 1; \ 504 vpslld $(32 - 13), x0 ## 1, x0 ## 1; \ 505 vpor x4 ## 1, x0 ## 1, x0 ## 1; \ 506 vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ 507 vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ 508 vpsrld $3, x2 ## 1, x4 ## 1; \ 509 vpslld $(32 - 3), x2 ## 1, x2 ## 1; \ 510 vpor x4 ## 1, x2 ## 1, x2 ## 1; \ 511 vpsrld $13, x0 ## 2, x4 ## 2; \ 512 vpslld $(32 - 13), x0 ## 2, x0 ## 2; \ 513 vpor x4 ## 2, x0 ## 2, x0 ## 2; \ 514 vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ 515 vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ 516 vpsrld $3, x2 ## 2, x4 ## 2; \ 517 vpslld $(32 - 3), x2 ## 2, x2 ## 2; \ 518 vpor x4 ## 2, x2 ## 2, x2 ## 2; 519 520 #define S(SBOX, x0, x1, x2, x3, x4) \ 521 SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ 522 SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ 523 SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ 524 SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); 525 526 #define SP(SBOX, x0, x1, x2, x3, x4, i) \ 527 get_key(i, 0, RK0); \ 528 SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ 529 get_key(i, 2, RK2); \ 530 SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ 531 get_key(i, 3, RK3); \ 532 SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ 533 get_key(i, 1, RK1); \ 534 SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ 535 536 #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ 537 vpunpckldq x1, x0, t0; \ 538 vpunpckhdq x1, x0, t2; \ 539 vpunpckldq x3, x2, t1; \ 540 vpunpckhdq x3, x2, x3; \ 541 \ 542 vpunpcklqdq t1, t0, x0; \ 543 vpunpckhqdq t1, t0, x1; \ 544 vpunpcklqdq x3, t2, x2; \ 545 vpunpckhqdq x3, t2, x3; 546 547 #define read_blocks(x0, x1, x2, x3, t0, t1, t2) \ 548 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) 549 550 #define write_blocks(x0, x1, x2, x3, t0, t1, t2) \ 551 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) 552 553 SYM_FUNC_START_LOCAL(__serpent_enc_blk16) 554 /* input: 555 * %rdi: ctx, CTX 556 * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: plaintext 557 * output: 558 * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: ciphertext 559 */ 560 561 vpcmpeqd RNOT, RNOT, RNOT; 562 563 read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); 564 read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 565 566 K2(RA, RB, RC, RD, RE, 0); 567 S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1); 568 S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2); 569 S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3); 570 S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4); 571 S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5); 572 S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6); 573 S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7); 574 S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8); 575 S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9); 576 S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10); 577 S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11); 578 S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12); 579 S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13); 580 S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14); 581 S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15); 582 S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16); 583 S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17); 584 S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18); 585 S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19); 586 S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20); 587 S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21); 588 S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22); 589 S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23); 590 S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24); 591 S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25); 592 S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26); 593 S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27); 594 S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28); 595 S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29); 596 S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30); 597 S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31); 598 S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32); 599 600 write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); 601 write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 602 603 RET; 604 SYM_FUNC_END(__serpent_enc_blk16) 605 606 SYM_FUNC_START_LOCAL(__serpent_dec_blk16) 607 /* input: 608 * %rdi: ctx, CTX 609 * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: ciphertext 610 * output: 611 * RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2: plaintext 612 */ 613 614 vpcmpeqd RNOT, RNOT, RNOT; 615 616 read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2); 617 read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 618 619 K2(RA, RB, RC, RD, RE, 32); 620 SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31); 621 SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30); 622 SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29); 623 SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28); 624 SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27); 625 SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26); 626 SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25); 627 SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24); 628 SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23); 629 SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22); 630 SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21); 631 SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20); 632 SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19); 633 SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18); 634 SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17); 635 SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16); 636 SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15); 637 SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14); 638 SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13); 639 SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12); 640 SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11); 641 SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10); 642 SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9); 643 SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8); 644 SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7); 645 SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6); 646 SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5); 647 SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4); 648 SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3); 649 SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2); 650 SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1); 651 S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0); 652 653 write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2); 654 write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); 655 656 RET; 657 SYM_FUNC_END(__serpent_dec_blk16) 658 659 SYM_FUNC_START(serpent_ecb_enc_16way) 660 /* input: 661 * %rdi: ctx, CTX 662 * %rsi: dst 663 * %rdx: src 664 */ 665 FRAME_BEGIN 666 667 vzeroupper; 668 669 load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 670 671 call __serpent_enc_blk16; 672 673 store_16way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 674 675 vzeroupper; 676 677 FRAME_END 678 RET; 679 SYM_FUNC_END(serpent_ecb_enc_16way) 680 681 SYM_FUNC_START(serpent_ecb_dec_16way) 682 /* input: 683 * %rdi: ctx, CTX 684 * %rsi: dst 685 * %rdx: src 686 */ 687 FRAME_BEGIN 688 689 vzeroupper; 690 691 load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 692 693 call __serpent_dec_blk16; 694 695 store_16way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); 696 697 vzeroupper; 698 699 FRAME_END 700 RET; 701 SYM_FUNC_END(serpent_ecb_dec_16way) 702 703 SYM_FUNC_START(serpent_cbc_dec_16way) 704 /* input: 705 * %rdi: ctx, CTX 706 * %rsi: dst 707 * %rdx: src 708 */ 709 FRAME_BEGIN 710 711 vzeroupper; 712 713 load_16way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 714 715 call __serpent_dec_blk16; 716 717 store_cbc_16way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2, 718 RK0); 719 720 vzeroupper; 721 722 FRAME_END 723 RET; 724 SYM_FUNC_END(serpent_cbc_dec_16way)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.