1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 # 3 # Accelerated poly1305 implementation for ppc64le. 4 # 5 # Copyright 2023- IBM Corp. All rights reserved 6 # 7 #=================================================================================== 8 # Written by Danny Tsen <dtsen@us.ibm.com> 9 # 10 # Poly1305 - this version mainly using vector/VSX/Scalar 11 # - 26 bits limbs 12 # - Handle multiple 64 byte blcok. 13 # 14 # Block size 16 bytes 15 # key = (r, s) 16 # clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF 17 # p = 2^130 - 5 18 # a += m 19 # a = (r + a) % p 20 # a += s 21 # 22 # Improve performance by breaking down polynominal to the sum of products with 23 # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r 24 # 25 # 07/22/21 - this revison based on the above sum of products. Setup r^4, r^3, r^2, r and s3, s2, s1, s0 26 # to 9 vectors for multiplications. 27 # 28 # setup r^4, r^3, r^2, r vectors 29 # vs [r^1, r^3, r^2, r^4] 30 # vs0 = [r0,.....] 31 # vs1 = [r1,.....] 32 # vs2 = [r2,.....] 33 # vs3 = [r3,.....] 34 # vs4 = [r4,.....] 35 # vs5 = [r1*5,...] 36 # vs6 = [r2*5,...] 37 # vs7 = [r2*5,...] 38 # vs8 = [r4*5,...] 39 # 40 # Each word in a vector consists a member of a "r/s" in [a * r/s]. 41 # 42 # r0, r4*5, r3*5, r2*5, r1*5; 43 # r1, r0, r4*5, r3*5, r2*5; 44 # r2, r1, r0, r4*5, r3*5; 45 # r3, r2, r1, r0, r4*5; 46 # r4, r3, r2, r1, r0 ; 47 # 48 # 49 # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) 50 # k = 32 bytes key 51 # r3 = k (r, s) 52 # r4 = mlen 53 # r5 = m 54 # 55 #include <asm/ppc_asm.h> 56 #include <asm/asm-offsets.h> 57 #include <asm/asm-compat.h> 58 #include <linux/linkage.h> 59 60 .machine "any" 61 62 .text 63 64 .macro SAVE_GPR GPR OFFSET FRAME 65 std \GPR,\OFFSET(\FRAME) 66 .endm 67 68 .macro SAVE_VRS VRS OFFSET FRAME 69 li 16, \OFFSET 70 stvx \VRS, 16, \FRAME 71 .endm 72 73 .macro SAVE_VSX VSX OFFSET FRAME 74 li 16, \OFFSET 75 stxvx \VSX, 16, \FRAME 76 .endm 77 78 .macro RESTORE_GPR GPR OFFSET FRAME 79 ld \GPR,\OFFSET(\FRAME) 80 .endm 81 82 .macro RESTORE_VRS VRS OFFSET FRAME 83 li 16, \OFFSET 84 lvx \VRS, 16, \FRAME 85 .endm 86 87 .macro RESTORE_VSX VSX OFFSET FRAME 88 li 16, \OFFSET 89 lxvx \VSX, 16, \FRAME 90 .endm 91 92 .macro SAVE_REGS 93 mflr 0 94 std 0, 16(1) 95 stdu 1,-752(1) 96 97 SAVE_GPR 14, 112, 1 98 SAVE_GPR 15, 120, 1 99 SAVE_GPR 16, 128, 1 100 SAVE_GPR 17, 136, 1 101 SAVE_GPR 18, 144, 1 102 SAVE_GPR 19, 152, 1 103 SAVE_GPR 20, 160, 1 104 SAVE_GPR 21, 168, 1 105 SAVE_GPR 22, 176, 1 106 SAVE_GPR 23, 184, 1 107 SAVE_GPR 24, 192, 1 108 SAVE_GPR 25, 200, 1 109 SAVE_GPR 26, 208, 1 110 SAVE_GPR 27, 216, 1 111 SAVE_GPR 28, 224, 1 112 SAVE_GPR 29, 232, 1 113 SAVE_GPR 30, 240, 1 114 SAVE_GPR 31, 248, 1 115 116 addi 9, 1, 256 117 SAVE_VRS 20, 0, 9 118 SAVE_VRS 21, 16, 9 119 SAVE_VRS 22, 32, 9 120 SAVE_VRS 23, 48, 9 121 SAVE_VRS 24, 64, 9 122 SAVE_VRS 25, 80, 9 123 SAVE_VRS 26, 96, 9 124 SAVE_VRS 27, 112, 9 125 SAVE_VRS 28, 128, 9 126 SAVE_VRS 29, 144, 9 127 SAVE_VRS 30, 160, 9 128 SAVE_VRS 31, 176, 9 129 130 SAVE_VSX 14, 192, 9 131 SAVE_VSX 15, 208, 9 132 SAVE_VSX 16, 224, 9 133 SAVE_VSX 17, 240, 9 134 SAVE_VSX 18, 256, 9 135 SAVE_VSX 19, 272, 9 136 SAVE_VSX 20, 288, 9 137 SAVE_VSX 21, 304, 9 138 SAVE_VSX 22, 320, 9 139 SAVE_VSX 23, 336, 9 140 SAVE_VSX 24, 352, 9 141 SAVE_VSX 25, 368, 9 142 SAVE_VSX 26, 384, 9 143 SAVE_VSX 27, 400, 9 144 SAVE_VSX 28, 416, 9 145 SAVE_VSX 29, 432, 9 146 SAVE_VSX 30, 448, 9 147 SAVE_VSX 31, 464, 9 148 .endm # SAVE_REGS 149 150 .macro RESTORE_REGS 151 addi 9, 1, 256 152 RESTORE_VRS 20, 0, 9 153 RESTORE_VRS 21, 16, 9 154 RESTORE_VRS 22, 32, 9 155 RESTORE_VRS 23, 48, 9 156 RESTORE_VRS 24, 64, 9 157 RESTORE_VRS 25, 80, 9 158 RESTORE_VRS 26, 96, 9 159 RESTORE_VRS 27, 112, 9 160 RESTORE_VRS 28, 128, 9 161 RESTORE_VRS 29, 144, 9 162 RESTORE_VRS 30, 160, 9 163 RESTORE_VRS 31, 176, 9 164 165 RESTORE_VSX 14, 192, 9 166 RESTORE_VSX 15, 208, 9 167 RESTORE_VSX 16, 224, 9 168 RESTORE_VSX 17, 240, 9 169 RESTORE_VSX 18, 256, 9 170 RESTORE_VSX 19, 272, 9 171 RESTORE_VSX 20, 288, 9 172 RESTORE_VSX 21, 304, 9 173 RESTORE_VSX 22, 320, 9 174 RESTORE_VSX 23, 336, 9 175 RESTORE_VSX 24, 352, 9 176 RESTORE_VSX 25, 368, 9 177 RESTORE_VSX 26, 384, 9 178 RESTORE_VSX 27, 400, 9 179 RESTORE_VSX 28, 416, 9 180 RESTORE_VSX 29, 432, 9 181 RESTORE_VSX 30, 448, 9 182 RESTORE_VSX 31, 464, 9 183 184 RESTORE_GPR 14, 112, 1 185 RESTORE_GPR 15, 120, 1 186 RESTORE_GPR 16, 128, 1 187 RESTORE_GPR 17, 136, 1 188 RESTORE_GPR 18, 144, 1 189 RESTORE_GPR 19, 152, 1 190 RESTORE_GPR 20, 160, 1 191 RESTORE_GPR 21, 168, 1 192 RESTORE_GPR 22, 176, 1 193 RESTORE_GPR 23, 184, 1 194 RESTORE_GPR 24, 192, 1 195 RESTORE_GPR 25, 200, 1 196 RESTORE_GPR 26, 208, 1 197 RESTORE_GPR 27, 216, 1 198 RESTORE_GPR 28, 224, 1 199 RESTORE_GPR 29, 232, 1 200 RESTORE_GPR 30, 240, 1 201 RESTORE_GPR 31, 248, 1 202 203 addi 1, 1, 752 204 ld 0, 16(1) 205 mtlr 0 206 .endm # RESTORE_REGS 207 208 # 209 # p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5; 210 # p[1] = a0*r1 + a1*r0 + a2*r4*5 + a3*r3*5 + a4*r2*5; 211 # p[2] = a0*r2 + a1*r1 + a2*r0 + a3*r4*5 + a4*r3*5; 212 # p[3] = a0*r3 + a1*r2 + a2*r1 + a3*r0 + a4*r4*5; 213 # p[4] = a0*r4 + a1*r3 + a2*r2 + a3*r1 + a4*r0 ; 214 # 215 # [r^2, r^3, r^1, r^4] 216 # [m3, m2, m4, m1] 217 # 218 # multiply odd and even words 219 .macro mul_odd 220 vmulouw 14, 4, 26 221 vmulouw 10, 5, 3 222 vmulouw 11, 6, 2 223 vmulouw 12, 7, 1 224 vmulouw 13, 8, 0 225 vmulouw 15, 4, 27 226 vaddudm 14, 14, 10 227 vaddudm 14, 14, 11 228 vmulouw 10, 5, 26 229 vmulouw 11, 6, 3 230 vaddudm 14, 14, 12 231 vaddudm 14, 14, 13 # x0 232 vaddudm 15, 15, 10 233 vaddudm 15, 15, 11 234 vmulouw 12, 7, 2 235 vmulouw 13, 8, 1 236 vaddudm 15, 15, 12 237 vaddudm 15, 15, 13 # x1 238 vmulouw 16, 4, 28 239 vmulouw 10, 5, 27 240 vmulouw 11, 6, 26 241 vaddudm 16, 16, 10 242 vaddudm 16, 16, 11 243 vmulouw 12, 7, 3 244 vmulouw 13, 8, 2 245 vaddudm 16, 16, 12 246 vaddudm 16, 16, 13 # x2 247 vmulouw 17, 4, 29 248 vmulouw 10, 5, 28 249 vmulouw 11, 6, 27 250 vaddudm 17, 17, 10 251 vaddudm 17, 17, 11 252 vmulouw 12, 7, 26 253 vmulouw 13, 8, 3 254 vaddudm 17, 17, 12 255 vaddudm 17, 17, 13 # x3 256 vmulouw 18, 4, 30 257 vmulouw 10, 5, 29 258 vmulouw 11, 6, 28 259 vaddudm 18, 18, 10 260 vaddudm 18, 18, 11 261 vmulouw 12, 7, 27 262 vmulouw 13, 8, 26 263 vaddudm 18, 18, 12 264 vaddudm 18, 18, 13 # x4 265 .endm 266 267 .macro mul_even 268 vmuleuw 9, 4, 26 269 vmuleuw 10, 5, 3 270 vmuleuw 11, 6, 2 271 vmuleuw 12, 7, 1 272 vmuleuw 13, 8, 0 273 vaddudm 14, 14, 9 274 vaddudm 14, 14, 10 275 vaddudm 14, 14, 11 276 vaddudm 14, 14, 12 277 vaddudm 14, 14, 13 # x0 278 279 vmuleuw 9, 4, 27 280 vmuleuw 10, 5, 26 281 vmuleuw 11, 6, 3 282 vmuleuw 12, 7, 2 283 vmuleuw 13, 8, 1 284 vaddudm 15, 15, 9 285 vaddudm 15, 15, 10 286 vaddudm 15, 15, 11 287 vaddudm 15, 15, 12 288 vaddudm 15, 15, 13 # x1 289 290 vmuleuw 9, 4, 28 291 vmuleuw 10, 5, 27 292 vmuleuw 11, 6, 26 293 vmuleuw 12, 7, 3 294 vmuleuw 13, 8, 2 295 vaddudm 16, 16, 9 296 vaddudm 16, 16, 10 297 vaddudm 16, 16, 11 298 vaddudm 16, 16, 12 299 vaddudm 16, 16, 13 # x2 300 301 vmuleuw 9, 4, 29 302 vmuleuw 10, 5, 28 303 vmuleuw 11, 6, 27 304 vmuleuw 12, 7, 26 305 vmuleuw 13, 8, 3 306 vaddudm 17, 17, 9 307 vaddudm 17, 17, 10 308 vaddudm 17, 17, 11 309 vaddudm 17, 17, 12 310 vaddudm 17, 17, 13 # x3 311 312 vmuleuw 9, 4, 30 313 vmuleuw 10, 5, 29 314 vmuleuw 11, 6, 28 315 vmuleuw 12, 7, 27 316 vmuleuw 13, 8, 26 317 vaddudm 18, 18, 9 318 vaddudm 18, 18, 10 319 vaddudm 18, 18, 11 320 vaddudm 18, 18, 12 321 vaddudm 18, 18, 13 # x4 322 .endm 323 324 # 325 # poly1305_setup_r 326 # 327 # setup r^4, r^3, r^2, r vectors 328 # [r, r^3, r^2, r^4] 329 # vs0 = [r0,...] 330 # vs1 = [r1,...] 331 # vs2 = [r2,...] 332 # vs3 = [r3,...] 333 # vs4 = [r4,...] 334 # vs5 = [r4*5,...] 335 # vs6 = [r3*5,...] 336 # vs7 = [r2*5,...] 337 # vs8 = [r1*5,...] 338 # 339 # r0, r4*5, r3*5, r2*5, r1*5; 340 # r1, r0, r4*5, r3*5, r2*5; 341 # r2, r1, r0, r4*5, r3*5; 342 # r3, r2, r1, r0, r4*5; 343 # r4, r3, r2, r1, r0 ; 344 # 345 .macro poly1305_setup_r 346 347 # save r 348 xxlor 26, 58, 58 349 xxlor 27, 59, 59 350 xxlor 28, 60, 60 351 xxlor 29, 61, 61 352 xxlor 30, 62, 62 353 354 xxlxor 31, 31, 31 355 356 # [r, r^3, r^2, r^4] 357 # compute r^2 358 vmr 4, 26 359 vmr 5, 27 360 vmr 6, 28 361 vmr 7, 29 362 vmr 8, 30 363 bl do_mul # r^2 r^1 364 xxpermdi 58, 58, 36, 0x3 # r0 365 xxpermdi 59, 59, 37, 0x3 # r1 366 xxpermdi 60, 60, 38, 0x3 # r2 367 xxpermdi 61, 61, 39, 0x3 # r3 368 xxpermdi 62, 62, 40, 0x3 # r4 369 xxpermdi 36, 36, 36, 0x3 370 xxpermdi 37, 37, 37, 0x3 371 xxpermdi 38, 38, 38, 0x3 372 xxpermdi 39, 39, 39, 0x3 373 xxpermdi 40, 40, 40, 0x3 374 vspltisb 13, 2 375 vsld 9, 27, 13 376 vsld 10, 28, 13 377 vsld 11, 29, 13 378 vsld 12, 30, 13 379 vaddudm 0, 9, 27 380 vaddudm 1, 10, 28 381 vaddudm 2, 11, 29 382 vaddudm 3, 12, 30 383 384 bl do_mul # r^4 r^3 385 vmrgow 26, 26, 4 386 vmrgow 27, 27, 5 387 vmrgow 28, 28, 6 388 vmrgow 29, 29, 7 389 vmrgow 30, 30, 8 390 vspltisb 13, 2 391 vsld 9, 27, 13 392 vsld 10, 28, 13 393 vsld 11, 29, 13 394 vsld 12, 30, 13 395 vaddudm 0, 9, 27 396 vaddudm 1, 10, 28 397 vaddudm 2, 11, 29 398 vaddudm 3, 12, 30 399 400 # r^2 r^4 401 xxlor 0, 58, 58 402 xxlor 1, 59, 59 403 xxlor 2, 60, 60 404 xxlor 3, 61, 61 405 xxlor 4, 62, 62 406 xxlor 5, 32, 32 407 xxlor 6, 33, 33 408 xxlor 7, 34, 34 409 xxlor 8, 35, 35 410 411 vspltw 9, 26, 3 412 vspltw 10, 26, 2 413 vmrgow 26, 10, 9 414 vspltw 9, 27, 3 415 vspltw 10, 27, 2 416 vmrgow 27, 10, 9 417 vspltw 9, 28, 3 418 vspltw 10, 28, 2 419 vmrgow 28, 10, 9 420 vspltw 9, 29, 3 421 vspltw 10, 29, 2 422 vmrgow 29, 10, 9 423 vspltw 9, 30, 3 424 vspltw 10, 30, 2 425 vmrgow 30, 10, 9 426 427 vsld 9, 27, 13 428 vsld 10, 28, 13 429 vsld 11, 29, 13 430 vsld 12, 30, 13 431 vaddudm 0, 9, 27 432 vaddudm 1, 10, 28 433 vaddudm 2, 11, 29 434 vaddudm 3, 12, 30 435 .endm 436 437 SYM_FUNC_START_LOCAL(do_mul) 438 mul_odd 439 440 # do reduction ( h %= p ) 441 # carry reduction 442 vspltisb 9, 2 443 vsrd 10, 14, 31 444 vsrd 11, 17, 31 445 vand 7, 17, 25 446 vand 4, 14, 25 447 vaddudm 18, 18, 11 448 vsrd 12, 18, 31 449 vaddudm 15, 15, 10 450 451 vsrd 11, 15, 31 452 vand 8, 18, 25 453 vand 5, 15, 25 454 vaddudm 4, 4, 12 455 vsld 10, 12, 9 456 vaddudm 6, 16, 11 457 458 vsrd 13, 6, 31 459 vand 6, 6, 25 460 vaddudm 4, 4, 10 461 vsrd 10, 4, 31 462 vaddudm 7, 7, 13 463 464 vsrd 11, 7, 31 465 vand 7, 7, 25 466 vand 4, 4, 25 467 vaddudm 5, 5, 10 468 vaddudm 8, 8, 11 469 blr 470 SYM_FUNC_END(do_mul) 471 472 # 473 # init key 474 # 475 .macro do_poly1305_init 476 addis 10, 2, rmask@toc@ha 477 addi 10, 10, rmask@toc@l 478 479 ld 11, 0(10) 480 ld 12, 8(10) 481 482 li 14, 16 483 li 15, 32 484 addis 10, 2, cnum@toc@ha 485 addi 10, 10, cnum@toc@l 486 lvx 25, 0, 10 # v25 - mask 487 lvx 31, 14, 10 # v31 = 1a 488 lvx 19, 15, 10 # v19 = 1 << 24 489 lxv 24, 48(10) # vs24 490 lxv 25, 64(10) # vs25 491 492 # initialize 493 # load key from r3 to vectors 494 ld 9, 24(3) 495 ld 10, 32(3) 496 and. 9, 9, 11 497 and. 10, 10, 12 498 499 # break 26 bits 500 extrdi 14, 9, 26, 38 501 extrdi 15, 9, 26, 12 502 extrdi 16, 9, 12, 0 503 mtvsrdd 58, 0, 14 504 insrdi 16, 10, 14, 38 505 mtvsrdd 59, 0, 15 506 extrdi 17, 10, 26, 24 507 mtvsrdd 60, 0, 16 508 extrdi 18, 10, 24, 0 509 mtvsrdd 61, 0, 17 510 mtvsrdd 62, 0, 18 511 512 # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5 513 li 9, 5 514 mtvsrdd 36, 0, 9 515 vmulouw 0, 27, 4 # v0 = rr0 516 vmulouw 1, 28, 4 # v1 = rr1 517 vmulouw 2, 29, 4 # v2 = rr2 518 vmulouw 3, 30, 4 # v3 = rr3 519 .endm 520 521 # 522 # poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) 523 # k = 32 bytes key 524 # r3 = k (r, s) 525 # r4 = mlen 526 # r5 = m 527 # 528 SYM_FUNC_START(poly1305_p10le_4blocks) 529 .align 5 530 cmpdi 5, 64 531 blt Out_no_poly1305 532 533 SAVE_REGS 534 535 do_poly1305_init 536 537 li 21, 0 # counter to message 538 539 poly1305_setup_r 540 541 # load previous H state 542 # break/convert r6 to 26 bits 543 ld 9, 0(3) 544 ld 10, 8(3) 545 ld 19, 16(3) 546 sldi 19, 19, 24 547 mtvsrdd 41, 0, 19 548 extrdi 14, 9, 26, 38 549 extrdi 15, 9, 26, 12 550 extrdi 16, 9, 12, 0 551 mtvsrdd 36, 0, 14 552 insrdi 16, 10, 14, 38 553 mtvsrdd 37, 0, 15 554 extrdi 17, 10, 26, 24 555 mtvsrdd 38, 0, 16 556 extrdi 18, 10, 24, 0 557 mtvsrdd 39, 0, 17 558 mtvsrdd 40, 0, 18 559 vor 8, 8, 9 560 561 # input m1 m2 562 add 20, 4, 21 563 xxlor 49, 24, 24 564 xxlor 50, 25, 25 565 lxvw4x 43, 0, 20 566 addi 17, 20, 16 567 lxvw4x 44, 0, 17 568 vperm 14, 11, 12, 17 569 vperm 15, 11, 12, 18 570 vand 9, 14, 25 # a0 571 vsrd 10, 14, 31 # >> 26 572 vsrd 11, 10, 31 # 12 bits left 573 vand 10, 10, 25 # a1 574 vspltisb 13, 12 575 vand 16, 15, 25 576 vsld 12, 16, 13 577 vor 11, 11, 12 578 vand 11, 11, 25 # a2 579 vspltisb 13, 14 580 vsrd 12, 15, 13 # >> 14 581 vsrd 13, 12, 31 # >> 26, a4 582 vand 12, 12, 25 # a3 583 584 vaddudm 20, 4, 9 585 vaddudm 21, 5, 10 586 vaddudm 22, 6, 11 587 vaddudm 23, 7, 12 588 vaddudm 24, 8, 13 589 590 # m3 m4 591 addi 17, 17, 16 592 lxvw4x 43, 0, 17 593 addi 17, 17, 16 594 lxvw4x 44, 0, 17 595 vperm 14, 11, 12, 17 596 vperm 15, 11, 12, 18 597 vand 9, 14, 25 # a0 598 vsrd 10, 14, 31 # >> 26 599 vsrd 11, 10, 31 # 12 bits left 600 vand 10, 10, 25 # a1 601 vspltisb 13, 12 602 vand 16, 15, 25 603 vsld 12, 16, 13 604 vspltisb 13, 14 605 vor 11, 11, 12 606 vand 11, 11, 25 # a2 607 vsrd 12, 15, 13 # >> 14 608 vsrd 13, 12, 31 # >> 26, a4 609 vand 12, 12, 25 # a3 610 611 # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] 612 vmrgow 4, 9, 20 613 vmrgow 5, 10, 21 614 vmrgow 6, 11, 22 615 vmrgow 7, 12, 23 616 vmrgow 8, 13, 24 617 vaddudm 8, 8, 19 618 619 addi 5, 5, -64 # len -= 64 620 addi 21, 21, 64 # offset += 64 621 622 li 9, 64 623 divdu 31, 5, 9 624 625 cmpdi 31, 0 626 ble Skip_block_loop 627 628 mtctr 31 629 630 # h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r 631 # Rewrite the polynominal sum of product as follows, 632 # h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2 633 # h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2 --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2 634 # .... Repeat 635 # h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2 --> 636 # h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1 --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r 637 # 638 loop_4blocks: 639 640 # Multiply odd words and even words 641 mul_odd 642 mul_even 643 # carry reduction 644 vspltisb 9, 2 645 vsrd 10, 14, 31 646 vsrd 11, 17, 31 647 vand 7, 17, 25 648 vand 4, 14, 25 649 vaddudm 18, 18, 11 650 vsrd 12, 18, 31 651 vaddudm 15, 15, 10 652 653 vsrd 11, 15, 31 654 vand 8, 18, 25 655 vand 5, 15, 25 656 vaddudm 4, 4, 12 657 vsld 10, 12, 9 658 vaddudm 6, 16, 11 659 660 vsrd 13, 6, 31 661 vand 6, 6, 25 662 vaddudm 4, 4, 10 663 vsrd 10, 4, 31 664 vaddudm 7, 7, 13 665 666 vsrd 11, 7, 31 667 vand 7, 7, 25 668 vand 4, 4, 25 669 vaddudm 5, 5, 10 670 vaddudm 8, 8, 11 671 672 # input m1 m2 m3 m4 673 add 20, 4, 21 674 xxlor 49, 24, 24 675 xxlor 50, 25, 25 676 lxvw4x 43, 0, 20 677 addi 17, 20, 16 678 lxvw4x 44, 0, 17 679 vperm 14, 11, 12, 17 680 vperm 15, 11, 12, 18 681 addi 17, 17, 16 682 lxvw4x 43, 0, 17 683 addi 17, 17, 16 684 lxvw4x 44, 0, 17 685 vperm 17, 11, 12, 17 686 vperm 18, 11, 12, 18 687 688 vand 20, 14, 25 # a0 689 vand 9, 17, 25 # a0 690 vsrd 21, 14, 31 # >> 26 691 vsrd 22, 21, 31 # 12 bits left 692 vsrd 10, 17, 31 # >> 26 693 vsrd 11, 10, 31 # 12 bits left 694 695 vand 21, 21, 25 # a1 696 vand 10, 10, 25 # a1 697 698 vspltisb 13, 12 699 vand 16, 15, 25 700 vsld 23, 16, 13 701 vor 22, 22, 23 702 vand 22, 22, 25 # a2 703 vand 16, 18, 25 704 vsld 12, 16, 13 705 vor 11, 11, 12 706 vand 11, 11, 25 # a2 707 vspltisb 13, 14 708 vsrd 23, 15, 13 # >> 14 709 vsrd 24, 23, 31 # >> 26, a4 710 vand 23, 23, 25 # a3 711 vsrd 12, 18, 13 # >> 14 712 vsrd 13, 12, 31 # >> 26, a4 713 vand 12, 12, 25 # a3 714 715 vaddudm 4, 4, 20 716 vaddudm 5, 5, 21 717 vaddudm 6, 6, 22 718 vaddudm 7, 7, 23 719 vaddudm 8, 8, 24 720 721 # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] 722 vmrgow 4, 9, 4 723 vmrgow 5, 10, 5 724 vmrgow 6, 11, 6 725 vmrgow 7, 12, 7 726 vmrgow 8, 13, 8 727 vaddudm 8, 8, 19 728 729 addi 5, 5, -64 # len -= 64 730 addi 21, 21, 64 # offset += 64 731 732 bdnz loop_4blocks 733 734 Skip_block_loop: 735 xxlor 58, 0, 0 736 xxlor 59, 1, 1 737 xxlor 60, 2, 2 738 xxlor 61, 3, 3 739 xxlor 62, 4, 4 740 xxlor 32, 5, 5 741 xxlor 33, 6, 6 742 xxlor 34, 7, 7 743 xxlor 35, 8, 8 744 745 # Multiply odd words and even words 746 mul_odd 747 mul_even 748 749 # Sum the products. 750 xxpermdi 41, 31, 46, 0 751 xxpermdi 42, 31, 47, 0 752 vaddudm 4, 14, 9 753 xxpermdi 36, 31, 36, 3 754 vaddudm 5, 15, 10 755 xxpermdi 37, 31, 37, 3 756 xxpermdi 43, 31, 48, 0 757 vaddudm 6, 16, 11 758 xxpermdi 38, 31, 38, 3 759 xxpermdi 44, 31, 49, 0 760 vaddudm 7, 17, 12 761 xxpermdi 39, 31, 39, 3 762 xxpermdi 45, 31, 50, 0 763 vaddudm 8, 18, 13 764 xxpermdi 40, 31, 40, 3 765 766 # carry reduction 767 vspltisb 9, 2 768 vsrd 10, 4, 31 769 vsrd 11, 7, 31 770 vand 7, 7, 25 771 vand 4, 4, 25 772 vaddudm 8, 8, 11 773 vsrd 12, 8, 31 774 vaddudm 5, 5, 10 775 776 vsrd 11, 5, 31 777 vand 8, 8, 25 778 vand 5, 5, 25 779 vaddudm 4, 4, 12 780 vsld 10, 12, 9 781 vaddudm 6, 6, 11 782 783 vsrd 13, 6, 31 784 vand 6, 6, 25 785 vaddudm 4, 4, 10 786 vsrd 10, 4, 31 787 vaddudm 7, 7, 13 788 789 vsrd 11, 7, 31 790 vand 7, 7, 25 791 vand 4, 4, 25 792 vaddudm 5, 5, 10 793 vsrd 10, 5, 31 794 vand 5, 5, 25 795 vaddudm 6, 6, 10 796 vaddudm 8, 8, 11 797 798 b do_final_update 799 800 do_final_update: 801 # combine 26 bit limbs 802 # v4, v5, v6, v7 and v8 are 26 bit vectors 803 vsld 5, 5, 31 804 vor 20, 4, 5 805 vspltisb 11, 12 806 vsrd 12, 6, 11 807 vsld 6, 6, 31 808 vsld 6, 6, 31 809 vor 20, 20, 6 810 vspltisb 11, 14 811 vsld 7, 7, 11 812 vor 21, 7, 12 813 mfvsrld 16, 40 # save last 2 bytes 814 vsld 8, 8, 11 815 vsld 8, 8, 31 816 vor 21, 21, 8 817 mfvsrld 17, 52 818 mfvsrld 19, 53 819 srdi 16, 16, 24 820 821 std 17, 0(3) 822 std 19, 8(3) 823 stw 16, 16(3) 824 825 Out_loop: 826 li 3, 0 827 828 RESTORE_REGS 829 830 blr 831 832 Out_no_poly1305: 833 li 3, 0 834 blr 835 SYM_FUNC_END(poly1305_p10le_4blocks) 836 837 # 838 # ======================================================================= 839 # The following functions implement 64 x 64 bits multiplication poly1305. 840 # 841 SYM_FUNC_START_LOCAL(Poly1305_init_64) 842 # mask 0x0FFFFFFC0FFFFFFC 843 # mask 0x0FFFFFFC0FFFFFFF 844 addis 10, 2, rmask@toc@ha 845 addi 10, 10, rmask@toc@l 846 ld 11, 0(10) 847 ld 12, 8(10) 848 849 # initialize 850 # load key from r3 851 ld 9, 24(3) 852 ld 10, 32(3) 853 and. 9, 9, 11 # cramp mask r0 854 and. 10, 10, 12 # cramp mask r1 855 856 srdi 21, 10, 2 857 add 19, 21, 10 # s1: r19 - (r1 >> 2) *5 858 859 # setup r and s 860 li 25, 0 861 mtvsrdd 32+0, 9, 19 # r0, s1 862 mtvsrdd 32+1, 10, 9 # r1, r0 863 mtvsrdd 32+2, 19, 25 # s1 864 mtvsrdd 32+3, 9, 25 # r0 865 866 blr 867 SYM_FUNC_END(Poly1305_init_64) 868 869 # Poly1305_mult 870 # v6 = (h0, h1), v8 = h2 871 # v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0 872 # 873 # Output: v7, v10, v11 874 # 875 SYM_FUNC_START_LOCAL(Poly1305_mult) 876 # 877 # d0 = h0 * r0 + h1 * s1 878 vmsumudm 7, 6, 0, 9 # h0 * r0, h1 * s1 879 880 # d1 = h0 * r1 + h1 * r0 + h2 * s1 881 vmsumudm 11, 6, 1, 9 # h0 * r1, h1 * r0 882 vmsumudm 10, 8, 2, 11 # d1 += h2 * s1 883 884 # d2 = r0 885 vmsumudm 11, 8, 3, 9 # d2 = h2 * r0 886 blr 887 SYM_FUNC_END(Poly1305_mult) 888 889 # 890 # carry reduction 891 # h %=p 892 # 893 # Input: v7, v10, v11 894 # Output: r27, r28, r29 895 # 896 SYM_FUNC_START_LOCAL(Carry_reduction) 897 mfvsrld 27, 32+7 898 mfvsrld 28, 32+10 899 mfvsrld 29, 32+11 900 mfvsrd 20, 32+7 # h0.h 901 mfvsrd 21, 32+10 # h1.h 902 903 addc 28, 28, 20 904 adde 29, 29, 21 905 srdi 22, 29, 0x2 906 sldi 23, 22, 0x2 907 add 23, 23, 22 # (h2 & 3) * 5 908 addc 27, 27, 23 # h0 909 addze 28, 28 # h1 910 andi. 29, 29, 0x3 # h2 911 blr 912 SYM_FUNC_END(Carry_reduction) 913 914 # 915 # poly1305 multiplication 916 # h *= r, h %= p 917 # d0 = h0 * r0 + h1 * s1 918 # d1 = h0 * r1 + h1 * r0 + h2 * s1 919 # d2 = h0 * r0 920 # 921 # 922 # unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit) 923 # - no highbit if final leftover block (highbit = 0) 924 # 925 SYM_FUNC_START(poly1305_64s) 926 cmpdi 5, 0 927 ble Out_no_poly1305_64 928 929 mflr 0 930 std 0, 16(1) 931 stdu 1,-400(1) 932 933 SAVE_GPR 14, 112, 1 934 SAVE_GPR 15, 120, 1 935 SAVE_GPR 16, 128, 1 936 SAVE_GPR 17, 136, 1 937 SAVE_GPR 18, 144, 1 938 SAVE_GPR 19, 152, 1 939 SAVE_GPR 20, 160, 1 940 SAVE_GPR 21, 168, 1 941 SAVE_GPR 22, 176, 1 942 SAVE_GPR 23, 184, 1 943 SAVE_GPR 24, 192, 1 944 SAVE_GPR 25, 200, 1 945 SAVE_GPR 26, 208, 1 946 SAVE_GPR 27, 216, 1 947 SAVE_GPR 28, 224, 1 948 SAVE_GPR 29, 232, 1 949 SAVE_GPR 30, 240, 1 950 SAVE_GPR 31, 248, 1 951 952 # Init poly1305 953 bl Poly1305_init_64 954 955 li 25, 0 # offset to inp and outp 956 957 add 11, 25, 4 958 959 # load h 960 # h0, h1, h2? 961 ld 27, 0(3) 962 ld 28, 8(3) 963 lwz 29, 16(3) 964 965 li 30, 16 966 divdu 31, 5, 30 967 968 mtctr 31 969 970 mr 24, 6 # highbit 971 972 Loop_block_64: 973 vxor 9, 9, 9 974 975 ld 20, 0(11) 976 ld 21, 8(11) 977 addi 11, 11, 16 978 979 addc 27, 27, 20 980 adde 28, 28, 21 981 adde 29, 29, 24 982 983 li 22, 0 984 mtvsrdd 32+6, 27, 28 # h0, h1 985 mtvsrdd 32+8, 29, 22 # h2 986 987 bl Poly1305_mult 988 989 bl Carry_reduction 990 991 bdnz Loop_block_64 992 993 std 27, 0(3) 994 std 28, 8(3) 995 stw 29, 16(3) 996 997 li 3, 0 998 999 RESTORE_GPR 14, 112, 1 1000 RESTORE_GPR 15, 120, 1 1001 RESTORE_GPR 16, 128, 1 1002 RESTORE_GPR 17, 136, 1 1003 RESTORE_GPR 18, 144, 1 1004 RESTORE_GPR 19, 152, 1 1005 RESTORE_GPR 20, 160, 1 1006 RESTORE_GPR 21, 168, 1 1007 RESTORE_GPR 22, 176, 1 1008 RESTORE_GPR 23, 184, 1 1009 RESTORE_GPR 24, 192, 1 1010 RESTORE_GPR 25, 200, 1 1011 RESTORE_GPR 26, 208, 1 1012 RESTORE_GPR 27, 216, 1 1013 RESTORE_GPR 28, 224, 1 1014 RESTORE_GPR 29, 232, 1 1015 RESTORE_GPR 30, 240, 1 1016 RESTORE_GPR 31, 248, 1 1017 1018 addi 1, 1, 400 1019 ld 0, 16(1) 1020 mtlr 0 1021 1022 blr 1023 1024 Out_no_poly1305_64: 1025 li 3, 0 1026 blr 1027 SYM_FUNC_END(poly1305_64s) 1028 1029 # 1030 # Input: r3 = h, r4 = s, r5 = mac 1031 # mac = h + s 1032 # 1033 SYM_FUNC_START(poly1305_emit_64) 1034 ld 10, 0(3) 1035 ld 11, 8(3) 1036 ld 12, 16(3) 1037 1038 # compare modulus 1039 # h + 5 + (-p) 1040 mr 6, 10 1041 mr 7, 11 1042 mr 8, 12 1043 addic. 6, 6, 5 1044 addze 7, 7 1045 addze 8, 8 1046 srdi 9, 8, 2 # overflow? 1047 cmpdi 9, 0 1048 beq Skip_h64 1049 mr 10, 6 1050 mr 11, 7 1051 mr 12, 8 1052 1053 Skip_h64: 1054 ld 6, 0(4) 1055 ld 7, 8(4) 1056 addc 10, 10, 6 1057 adde 11, 11, 7 1058 addze 12, 12 1059 1060 std 10, 0(5) 1061 std 11, 8(5) 1062 blr 1063 SYM_FUNC_END(poly1305_emit_64) 1064 1065 SYM_DATA_START_LOCAL(RMASK) 1066 .align 5 1067 rmask: 1068 .byte 0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f 1069 cnum: 1070 .long 0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000 1071 .long 0x1a, 0x00, 0x1a, 0x00 1072 .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 1073 .long 0x00010203, 0x04050607, 0x10111213, 0x14151617 1074 .long 0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f 1075 SYM_DATA_END(RMASK)
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.