1 #! /usr/bin/env perl
2 # SPDX-License-Identifier: GPL-2.0
3
4 # This code is taken from CRYPTOGAMs[1] and is included here using the option
5 # in the license to distribute the code under the GPL. Therefore this program
6 # is free software; you can redistribute it and/or modify it under the terms of
7 # the GNU General Public License version 2 as published by the Free Software
8 # Foundation.
9 #
10 # [1] https://www.openssl.org/~appro/cryptogams/
11
12 # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
13 # All rights reserved.
14 #
15 # Redistribution and use in source and binary forms, with or without
16 # modification, are permitted provided that the following conditions
17 # are met:
18 #
19 # * Redistributions of source code must retain copyright notices,
20 # this list of conditions and the following disclaimer.
21 #
22 # * Redistributions in binary form must reproduce the above
23 # copyright notice, this list of conditions and the following
24 # disclaimer in the documentation and/or other materials
25 # provided with the distribution.
26 #
27 # * Neither the name of the CRYPTOGAMS nor the names of its
28 # copyright holder and contributors may be used to endorse or
29 # promote products derived from this software without specific
30 # prior written permission.
31 #
32 # ALTERNATIVELY, provided that this notice is retained in full, this
33 # product may be distributed under the terms of the GNU General Public
34 # License (GPL), in which case the provisions of the GPL apply INSTEAD OF
35 # those given above.
36 #
37 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
38 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48
49 # ====================================================================
50 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
51 # project. The module is, however, dual licensed under OpenSSL and
52 # CRYPTOGAMS licenses depending on where you obtain it. For further
53 # details see https://www.openssl.org/~appro/cryptogams/.
54 # ====================================================================
55 #
56 # This module implements support for AES instructions as per PowerISA
57 # specification version 2.07, first implemented by POWER8 processor.
58 # The module is endian-agnostic in sense that it supports both big-
59 # and little-endian cases. Data alignment in parallelizable modes is
60 # handled with VSX loads and stores, which implies MSR.VSX flag being
61 # set. It should also be noted that ISA specification doesn't prohibit
62 # alignment exceptions for these instructions on page boundaries.
63 # Initially alignment was handled in pure AltiVec/VMX way [when data
64 # is aligned programmatically, which in turn guarantees exception-
65 # free execution], but it turned to hamper performance when vcipher
66 # instructions are interleaved. It's reckoned that eventual
67 # misalignment penalties at page boundaries are in average lower
68 # than additional overhead in pure AltiVec approach.
69 #
70 # May 2016
71 #
72 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
73 # systems were measured.
74 #
75 ######################################################################
76 # Current large-block performance in cycles per byte processed with
77 # 128-bit key (less is better).
78 #
79 # CBC en-/decrypt CTR XTS
80 # POWER8[le] 3.96/0.72 0.74 1.1
81 # POWER8[be] 3.75/0.65 0.66 1.0
82
83 $flavour = shift;
84
85 if ($flavour =~ /64/) {
86 $SIZE_T =8;
87 $LRSAVE =2*$SIZE_T;
88 $STU ="stdu";
89 $POP ="ld";
90 $PUSH ="std";
91 $UCMP ="cmpld";
92 $SHL ="sldi";
93 } elsif ($flavour =~ /32/) {
94 $SIZE_T =4;
95 $LRSAVE =$SIZE_T;
96 $STU ="stwu";
97 $POP ="lwz";
98 $PUSH ="stw";
99 $UCMP ="cmplw";
100 $SHL ="slwi";
101 } else { die "nonsense $flavour"; }
102
103 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
104
105 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
106 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108 die "can't locate ppc-xlate.pl";
109
110 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
111
112 $FRAME=8*$SIZE_T;
113 $prefix="aes_p8";
114
115 $sp="r1";
116 $vrsave="r12";
117
118 #########################################################################
119 {{{ # Key setup procedures #
120 my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
121 my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
122 my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
123
124 $code.=<<___;
125 .machine "any"
126
127 .text
128
129 .align 7
130 rcon:
131 .long 0x01000000, 0x01000000, 0x01000000, 0x01000000 ?rev
132 .long 0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000 ?rev
133 .long 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c ?rev
134 .long 0,0,0,0 ?asis
135 .long 0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe
136 Lconsts:
137 mflr r0
138 bcl 20,31,\$+4
139 mflr $ptr #vvvvv "distance between . and rcon
140 addi $ptr,$ptr,-0x58
141 mtlr r0
142 blr
143 .long 0
144 .byte 0,12,0x14,0,0,0,0,0
145 .asciz "AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
146
147 .globl .${prefix}_set_encrypt_key
148 Lset_encrypt_key:
149 mflr r11
150 $PUSH r11,$LRSAVE($sp)
151
152 li $ptr,-1
153 ${UCMP}i $inp,0
154 beq- Lenc_key_abort # if ($inp==0) return -1;
155 ${UCMP}i $out,0
156 beq- Lenc_key_abort # if ($out==0) return -1;
157 li $ptr,-2
158 cmpwi $bits,128
159 blt- Lenc_key_abort
160 cmpwi $bits,256
161 bgt- Lenc_key_abort
162 andi. r0,$bits,0x3f
163 bne- Lenc_key_abort
164
165 lis r0,0xfff0
166 mfspr $vrsave,256
167 mtspr 256,r0
168
169 bl Lconsts
170 mtlr r11
171
172 neg r9,$inp
173 lvx $in0,0,$inp
174 addi $inp,$inp,15 # 15 is not typo
175 lvsr $key,0,r9 # borrow $key
176 li r8,0x20
177 cmpwi $bits,192
178 lvx $in1,0,$inp
179 le?vspltisb $mask,0x0f # borrow $mask
180 lvx $rcon,0,$ptr
181 le?vxor $key,$key,$mask # adjust for byte swap
182 lvx $mask,r8,$ptr
183 addi $ptr,$ptr,0x10
184 vperm $in0,$in0,$in1,$key # align [and byte swap in LE]
185 li $cnt,8
186 vxor $zero,$zero,$zero
187 mtctr $cnt
188
189 ?lvsr $outperm,0,$out
190 vspltisb $outmask,-1
191 lvx $outhead,0,$out
192 ?vperm $outmask,$zero,$outmask,$outperm
193
194 blt Loop128
195 addi $inp,$inp,8
196 beq L192
197 addi $inp,$inp,8
198 b L256
199
200 .align 4
201 Loop128:
202 vperm $key,$in0,$in0,$mask # rotate-n-splat
203 vsldoi $tmp,$zero,$in0,12 # >>32
204 vperm $outtail,$in0,$in0,$outperm # rotate
205 vsel $stage,$outhead,$outtail,$outmask
206 vmr $outhead,$outtail
207 vcipherlast $key,$key,$rcon
208 stvx $stage,0,$out
209 addi $out,$out,16
210
211 vxor $in0,$in0,$tmp
212 vsldoi $tmp,$zero,$tmp,12 # >>32
213 vxor $in0,$in0,$tmp
214 vsldoi $tmp,$zero,$tmp,12 # >>32
215 vxor $in0,$in0,$tmp
216 vadduwm $rcon,$rcon,$rcon
217 vxor $in0,$in0,$key
218 bdnz Loop128
219
220 lvx $rcon,0,$ptr # last two round keys
221
222 vperm $key,$in0,$in0,$mask # rotate-n-splat
223 vsldoi $tmp,$zero,$in0,12 # >>32
224 vperm $outtail,$in0,$in0,$outperm # rotate
225 vsel $stage,$outhead,$outtail,$outmask
226 vmr $outhead,$outtail
227 vcipherlast $key,$key,$rcon
228 stvx $stage,0,$out
229 addi $out,$out,16
230
231 vxor $in0,$in0,$tmp
232 vsldoi $tmp,$zero,$tmp,12 # >>32
233 vxor $in0,$in0,$tmp
234 vsldoi $tmp,$zero,$tmp,12 # >>32
235 vxor $in0,$in0,$tmp
236 vadduwm $rcon,$rcon,$rcon
237 vxor $in0,$in0,$key
238
239 vperm $key,$in0,$in0,$mask # rotate-n-splat
240 vsldoi $tmp,$zero,$in0,12 # >>32
241 vperm $outtail,$in0,$in0,$outperm # rotate
242 vsel $stage,$outhead,$outtail,$outmask
243 vmr $outhead,$outtail
244 vcipherlast $key,$key,$rcon
245 stvx $stage,0,$out
246 addi $out,$out,16
247
248 vxor $in0,$in0,$tmp
249 vsldoi $tmp,$zero,$tmp,12 # >>32
250 vxor $in0,$in0,$tmp
251 vsldoi $tmp,$zero,$tmp,12 # >>32
252 vxor $in0,$in0,$tmp
253 vxor $in0,$in0,$key
254 vperm $outtail,$in0,$in0,$outperm # rotate
255 vsel $stage,$outhead,$outtail,$outmask
256 vmr $outhead,$outtail
257 stvx $stage,0,$out
258
259 addi $inp,$out,15 # 15 is not typo
260 addi $out,$out,0x50
261
262 li $rounds,10
263 b Ldone
264
265 .align 4
266 L192:
267 lvx $tmp,0,$inp
268 li $cnt,4
269 vperm $outtail,$in0,$in0,$outperm # rotate
270 vsel $stage,$outhead,$outtail,$outmask
271 vmr $outhead,$outtail
272 stvx $stage,0,$out
273 addi $out,$out,16
274 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
275 vspltisb $key,8 # borrow $key
276 mtctr $cnt
277 vsububm $mask,$mask,$key # adjust the mask
278
279 Loop192:
280 vperm $key,$in1,$in1,$mask # roate-n-splat
281 vsldoi $tmp,$zero,$in0,12 # >>32
282 vcipherlast $key,$key,$rcon
283
284 vxor $in0,$in0,$tmp
285 vsldoi $tmp,$zero,$tmp,12 # >>32
286 vxor $in0,$in0,$tmp
287 vsldoi $tmp,$zero,$tmp,12 # >>32
288 vxor $in0,$in0,$tmp
289
290 vsldoi $stage,$zero,$in1,8
291 vspltw $tmp,$in0,3
292 vxor $tmp,$tmp,$in1
293 vsldoi $in1,$zero,$in1,12 # >>32
294 vadduwm $rcon,$rcon,$rcon
295 vxor $in1,$in1,$tmp
296 vxor $in0,$in0,$key
297 vxor $in1,$in1,$key
298 vsldoi $stage,$stage,$in0,8
299
300 vperm $key,$in1,$in1,$mask # rotate-n-splat
301 vsldoi $tmp,$zero,$in0,12 # >>32
302 vperm $outtail,$stage,$stage,$outperm # rotate
303 vsel $stage,$outhead,$outtail,$outmask
304 vmr $outhead,$outtail
305 vcipherlast $key,$key,$rcon
306 stvx $stage,0,$out
307 addi $out,$out,16
308
309 vsldoi $stage,$in0,$in1,8
310 vxor $in0,$in0,$tmp
311 vsldoi $tmp,$zero,$tmp,12 # >>32
312 vperm $outtail,$stage,$stage,$outperm # rotate
313 vsel $stage,$outhead,$outtail,$outmask
314 vmr $outhead,$outtail
315 vxor $in0,$in0,$tmp
316 vsldoi $tmp,$zero,$tmp,12 # >>32
317 vxor $in0,$in0,$tmp
318 stvx $stage,0,$out
319 addi $out,$out,16
320
321 vspltw $tmp,$in0,3
322 vxor $tmp,$tmp,$in1
323 vsldoi $in1,$zero,$in1,12 # >>32
324 vadduwm $rcon,$rcon,$rcon
325 vxor $in1,$in1,$tmp
326 vxor $in0,$in0,$key
327 vxor $in1,$in1,$key
328 vperm $outtail,$in0,$in0,$outperm # rotate
329 vsel $stage,$outhead,$outtail,$outmask
330 vmr $outhead,$outtail
331 stvx $stage,0,$out
332 addi $inp,$out,15 # 15 is not typo
333 addi $out,$out,16
334 bdnz Loop192
335
336 li $rounds,12
337 addi $out,$out,0x20
338 b Ldone
339
340 .align 4
341 L256:
342 lvx $tmp,0,$inp
343 li $cnt,7
344 li $rounds,14
345 vperm $outtail,$in0,$in0,$outperm # rotate
346 vsel $stage,$outhead,$outtail,$outmask
347 vmr $outhead,$outtail
348 stvx $stage,0,$out
349 addi $out,$out,16
350 vperm $in1,$in1,$tmp,$key # align [and byte swap in LE]
351 mtctr $cnt
352
353 Loop256:
354 vperm $key,$in1,$in1,$mask # rotate-n-splat
355 vsldoi $tmp,$zero,$in0,12 # >>32
356 vperm $outtail,$in1,$in1,$outperm # rotate
357 vsel $stage,$outhead,$outtail,$outmask
358 vmr $outhead,$outtail
359 vcipherlast $key,$key,$rcon
360 stvx $stage,0,$out
361 addi $out,$out,16
362
363 vxor $in0,$in0,$tmp
364 vsldoi $tmp,$zero,$tmp,12 # >>32
365 vxor $in0,$in0,$tmp
366 vsldoi $tmp,$zero,$tmp,12 # >>32
367 vxor $in0,$in0,$tmp
368 vadduwm $rcon,$rcon,$rcon
369 vxor $in0,$in0,$key
370 vperm $outtail,$in0,$in0,$outperm # rotate
371 vsel $stage,$outhead,$outtail,$outmask
372 vmr $outhead,$outtail
373 stvx $stage,0,$out
374 addi $inp,$out,15 # 15 is not typo
375 addi $out,$out,16
376 bdz Ldone
377
378 vspltw $key,$in0,3 # just splat
379 vsldoi $tmp,$zero,$in1,12 # >>32
380 vsbox $key,$key
381
382 vxor $in1,$in1,$tmp
383 vsldoi $tmp,$zero,$tmp,12 # >>32
384 vxor $in1,$in1,$tmp
385 vsldoi $tmp,$zero,$tmp,12 # >>32
386 vxor $in1,$in1,$tmp
387
388 vxor $in1,$in1,$key
389 b Loop256
390
391 .align 4
392 Ldone:
393 lvx $in1,0,$inp # redundant in aligned case
394 vsel $in1,$outhead,$in1,$outmask
395 stvx $in1,0,$inp
396 li $ptr,0
397 mtspr 256,$vrsave
398 stw $rounds,0($out)
399
400 Lenc_key_abort:
401 mr r3,$ptr
402 blr
403 .long 0
404 .byte 0,12,0x14,1,0,0,3,0
405 .long 0
406 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
407
408 .globl .${prefix}_set_decrypt_key
409 $STU $sp,-$FRAME($sp)
410 mflr r10
411 $PUSH r10,$FRAME+$LRSAVE($sp)
412 bl Lset_encrypt_key
413 mtlr r10
414
415 cmpwi r3,0
416 bne- Ldec_key_abort
417
418 slwi $cnt,$rounds,4
419 subi $inp,$out,240 # first round key
420 srwi $rounds,$rounds,1
421 add $out,$inp,$cnt # last round key
422 mtctr $rounds
423
424 Ldeckey:
425 lwz r0, 0($inp)
426 lwz r6, 4($inp)
427 lwz r7, 8($inp)
428 lwz r8, 12($inp)
429 addi $inp,$inp,16
430 lwz r9, 0($out)
431 lwz r10,4($out)
432 lwz r11,8($out)
433 lwz r12,12($out)
434 stw r0, 0($out)
435 stw r6, 4($out)
436 stw r7, 8($out)
437 stw r8, 12($out)
438 subi $out,$out,16
439 stw r9, -16($inp)
440 stw r10,-12($inp)
441 stw r11,-8($inp)
442 stw r12,-4($inp)
443 bdnz Ldeckey
444
445 xor r3,r3,r3 # return value
446 Ldec_key_abort:
447 addi $sp,$sp,$FRAME
448 blr
449 .long 0
450 .byte 0,12,4,1,0x80,0,3,0
451 .long 0
452 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
453 ___
454 }}}
455 #########################################################################
456 {{{ # Single block en- and decrypt procedures #
457 sub gen_block () {
458 my $dir = shift;
459 my $n = $dir eq "de" ? "n" : "";
460 my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
461
462 $code.=<<___;
463 .globl .${prefix}_${dir}crypt
464 lwz $rounds,240($key)
465 lis r0,0xfc00
466 mfspr $vrsave,256
467 li $idx,15 # 15 is not typo
468 mtspr 256,r0
469
470 lvx v0,0,$inp
471 neg r11,$out
472 lvx v1,$idx,$inp
473 lvsl v2,0,$inp # inpperm
474 le?vspltisb v4,0x0f
475 ?lvsl v3,0,r11 # outperm
476 le?vxor v2,v2,v4
477 li $idx,16
478 vperm v0,v0,v1,v2 # align [and byte swap in LE]
479 lvx v1,0,$key
480 ?lvsl v5,0,$key # keyperm
481 srwi $rounds,$rounds,1
482 lvx v2,$idx,$key
483 addi $idx,$idx,16
484 subi $rounds,$rounds,1
485 ?vperm v1,v1,v2,v5 # align round key
486
487 vxor v0,v0,v1
488 lvx v1,$idx,$key
489 addi $idx,$idx,16
490 mtctr $rounds
491
492 Loop_${dir}c:
493 ?vperm v2,v2,v1,v5
494 v${n}cipher v0,v0,v2
495 lvx v2,$idx,$key
496 addi $idx,$idx,16
497 ?vperm v1,v1,v2,v5
498 v${n}cipher v0,v0,v1
499 lvx v1,$idx,$key
500 addi $idx,$idx,16
501 bdnz Loop_${dir}c
502
503 ?vperm v2,v2,v1,v5
504 v${n}cipher v0,v0,v2
505 lvx v2,$idx,$key
506 ?vperm v1,v1,v2,v5
507 v${n}cipherlast v0,v0,v1
508
509 vspltisb v2,-1
510 vxor v1,v1,v1
511 li $idx,15 # 15 is not typo
512 ?vperm v2,v1,v2,v3 # outmask
513 le?vxor v3,v3,v4
514 lvx v1,0,$out # outhead
515 vperm v0,v0,v0,v3 # rotate [and byte swap in LE]
516 vsel v1,v1,v0,v2
517 lvx v4,$idx,$out
518 stvx v1,0,$out
519 vsel v0,v0,v4,v2
520 stvx v0,$idx,$out
521
522 mtspr 256,$vrsave
523 blr
524 .long 0
525 .byte 0,12,0x14,0,0,0,3,0
526 .long 0
527 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
528 ___
529 }
530 &gen_block("en");
531 &gen_block("de");
532 }}}
533 #########################################################################
534 {{{ # CBC en- and decrypt procedures #
535 my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
536 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
537 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
538 map("v$_",(4..10));
539 $code.=<<___;
540 .globl .${prefix}_cbc_encrypt
541 ${UCMP}i $len,16
542 bltlr-
543
544 cmpwi $enc,0 # test direction
545 lis r0,0xffe0
546 mfspr $vrsave,256
547 mtspr 256,r0
548
549 li $idx,15
550 vxor $rndkey0,$rndkey0,$rndkey0
551 le?vspltisb $tmp,0x0f
552
553 lvx $ivec,0,$ivp # load [unaligned] iv
554 lvsl $inpperm,0,$ivp
555 lvx $inptail,$idx,$ivp
556 le?vxor $inpperm,$inpperm,$tmp
557 vperm $ivec,$ivec,$inptail,$inpperm
558
559 neg r11,$inp
560 ?lvsl $keyperm,0,$key # prepare for unaligned key
561 lwz $rounds,240($key)
562
563 lvsr $inpperm,0,r11 # prepare for unaligned load
564 lvx $inptail,0,$inp
565 addi $inp,$inp,15 # 15 is not typo
566 le?vxor $inpperm,$inpperm,$tmp
567
568 ?lvsr $outperm,0,$out # prepare for unaligned store
569 vspltisb $outmask,-1
570 lvx $outhead,0,$out
571 ?vperm $outmask,$rndkey0,$outmask,$outperm
572 le?vxor $outperm,$outperm,$tmp
573
574 srwi $rounds,$rounds,1
575 li $idx,16
576 subi $rounds,$rounds,1
577 beq Lcbc_dec
578
579 Lcbc_enc:
580 vmr $inout,$inptail
581 lvx $inptail,0,$inp
582 addi $inp,$inp,16
583 mtctr $rounds
584 subi $len,$len,16 # len-=16
585
586 lvx $rndkey0,0,$key
587 vperm $inout,$inout,$inptail,$inpperm
588 lvx $rndkey1,$idx,$key
589 addi $idx,$idx,16
590 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
591 vxor $inout,$inout,$rndkey0
592 lvx $rndkey0,$idx,$key
593 addi $idx,$idx,16
594 vxor $inout,$inout,$ivec
595
596 Loop_cbc_enc:
597 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
598 vcipher $inout,$inout,$rndkey1
599 lvx $rndkey1,$idx,$key
600 addi $idx,$idx,16
601 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
602 vcipher $inout,$inout,$rndkey0
603 lvx $rndkey0,$idx,$key
604 addi $idx,$idx,16
605 bdnz Loop_cbc_enc
606
607 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
608 vcipher $inout,$inout,$rndkey1
609 lvx $rndkey1,$idx,$key
610 li $idx,16
611 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
612 vcipherlast $ivec,$inout,$rndkey0
613 ${UCMP}i $len,16
614
615 vperm $tmp,$ivec,$ivec,$outperm
616 vsel $inout,$outhead,$tmp,$outmask
617 vmr $outhead,$tmp
618 stvx $inout,0,$out
619 addi $out,$out,16
620 bge Lcbc_enc
621
622 b Lcbc_done
623
624 .align 4
625 Lcbc_dec:
626 ${UCMP}i $len,128
627 bge _aesp8_cbc_decrypt8x
628 vmr $tmp,$inptail
629 lvx $inptail,0,$inp
630 addi $inp,$inp,16
631 mtctr $rounds
632 subi $len,$len,16 # len-=16
633
634 lvx $rndkey0,0,$key
635 vperm $tmp,$tmp,$inptail,$inpperm
636 lvx $rndkey1,$idx,$key
637 addi $idx,$idx,16
638 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
639 vxor $inout,$tmp,$rndkey0
640 lvx $rndkey0,$idx,$key
641 addi $idx,$idx,16
642
643 Loop_cbc_dec:
644 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
645 vncipher $inout,$inout,$rndkey1
646 lvx $rndkey1,$idx,$key
647 addi $idx,$idx,16
648 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
649 vncipher $inout,$inout,$rndkey0
650 lvx $rndkey0,$idx,$key
651 addi $idx,$idx,16
652 bdnz Loop_cbc_dec
653
654 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
655 vncipher $inout,$inout,$rndkey1
656 lvx $rndkey1,$idx,$key
657 li $idx,16
658 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
659 vncipherlast $inout,$inout,$rndkey0
660 ${UCMP}i $len,16
661
662 vxor $inout,$inout,$ivec
663 vmr $ivec,$tmp
664 vperm $tmp,$inout,$inout,$outperm
665 vsel $inout,$outhead,$tmp,$outmask
666 vmr $outhead,$tmp
667 stvx $inout,0,$out
668 addi $out,$out,16
669 bge Lcbc_dec
670
671 Lcbc_done:
672 addi $out,$out,-1
673 lvx $inout,0,$out # redundant in aligned case
674 vsel $inout,$outhead,$inout,$outmask
675 stvx $inout,0,$out
676
677 neg $enc,$ivp # write [unaligned] iv
678 li $idx,15 # 15 is not typo
679 vxor $rndkey0,$rndkey0,$rndkey0
680 vspltisb $outmask,-1
681 le?vspltisb $tmp,0x0f
682 ?lvsl $outperm,0,$enc
683 ?vperm $outmask,$rndkey0,$outmask,$outperm
684 le?vxor $outperm,$outperm,$tmp
685 lvx $outhead,0,$ivp
686 vperm $ivec,$ivec,$ivec,$outperm
687 vsel $inout,$outhead,$ivec,$outmask
688 lvx $inptail,$idx,$ivp
689 stvx $inout,0,$ivp
690 vsel $inout,$ivec,$inptail,$outmask
691 stvx $inout,$idx,$ivp
692
693 mtspr 256,$vrsave
694 blr
695 .long 0
696 .byte 0,12,0x14,0,0,0,6,0
697 .long 0
698 ___
699 #########################################################################
700 {{ # Optimized CBC decrypt procedure #
701 my $key_="r11";
702 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
703 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
704 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
705 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
706 # v26-v31 last 6 round keys
707 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
708
709 $code.=<<___;
710 .align 5
711 _aesp8_cbc_decrypt8x:
712 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
713 li r10,`$FRAME+8*16+15`
714 li r11,`$FRAME+8*16+31`
715 stvx v20,r10,$sp # ABI says so
716 addi r10,r10,32
717 stvx v21,r11,$sp
718 addi r11,r11,32
719 stvx v22,r10,$sp
720 addi r10,r10,32
721 stvx v23,r11,$sp
722 addi r11,r11,32
723 stvx v24,r10,$sp
724 addi r10,r10,32
725 stvx v25,r11,$sp
726 addi r11,r11,32
727 stvx v26,r10,$sp
728 addi r10,r10,32
729 stvx v27,r11,$sp
730 addi r11,r11,32
731 stvx v28,r10,$sp
732 addi r10,r10,32
733 stvx v29,r11,$sp
734 addi r11,r11,32
735 stvx v30,r10,$sp
736 stvx v31,r11,$sp
737 li r0,-1
738 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
739 li $x10,0x10
740 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
741 li $x20,0x20
742 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
743 li $x30,0x30
744 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
745 li $x40,0x40
746 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
747 li $x50,0x50
748 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
749 li $x60,0x60
750 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
751 li $x70,0x70
752 mtspr 256,r0
753
754 subi $rounds,$rounds,3 # -4 in total
755 subi $len,$len,128 # bias
756
757 lvx $rndkey0,$x00,$key # load key schedule
758 lvx v30,$x10,$key
759 addi $key,$key,0x20
760 lvx v31,$x00,$key
761 ?vperm $rndkey0,$rndkey0,v30,$keyperm
762 addi $key_,$sp,$FRAME+15
763 mtctr $rounds
764
765 Load_cbc_dec_key:
766 ?vperm v24,v30,v31,$keyperm
767 lvx v30,$x10,$key
768 addi $key,$key,0x20
769 stvx v24,$x00,$key_ # off-load round[1]
770 ?vperm v25,v31,v30,$keyperm
771 lvx v31,$x00,$key
772 stvx v25,$x10,$key_ # off-load round[2]
773 addi $key_,$key_,0x20
774 bdnz Load_cbc_dec_key
775
776 lvx v26,$x10,$key
777 ?vperm v24,v30,v31,$keyperm
778 lvx v27,$x20,$key
779 stvx v24,$x00,$key_ # off-load round[3]
780 ?vperm v25,v31,v26,$keyperm
781 lvx v28,$x30,$key
782 stvx v25,$x10,$key_ # off-load round[4]
783 addi $key_,$sp,$FRAME+15 # rewind $key_
784 ?vperm v26,v26,v27,$keyperm
785 lvx v29,$x40,$key
786 ?vperm v27,v27,v28,$keyperm
787 lvx v30,$x50,$key
788 ?vperm v28,v28,v29,$keyperm
789 lvx v31,$x60,$key
790 ?vperm v29,v29,v30,$keyperm
791 lvx $out0,$x70,$key # borrow $out0
792 ?vperm v30,v30,v31,$keyperm
793 lvx v24,$x00,$key_ # pre-load round[1]
794 ?vperm v31,v31,$out0,$keyperm
795 lvx v25,$x10,$key_ # pre-load round[2]
796
797 #lvx $inptail,0,$inp # "caller" already did this
798 #addi $inp,$inp,15 # 15 is not typo
799 subi $inp,$inp,15 # undo "caller"
800
801 le?li $idx,8
802 lvx_u $in0,$x00,$inp # load first 8 "words"
803 le?lvsl $inpperm,0,$idx
804 le?vspltisb $tmp,0x0f
805 lvx_u $in1,$x10,$inp
806 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
807 lvx_u $in2,$x20,$inp
808 le?vperm $in0,$in0,$in0,$inpperm
809 lvx_u $in3,$x30,$inp
810 le?vperm $in1,$in1,$in1,$inpperm
811 lvx_u $in4,$x40,$inp
812 le?vperm $in2,$in2,$in2,$inpperm
813 vxor $out0,$in0,$rndkey0
814 lvx_u $in5,$x50,$inp
815 le?vperm $in3,$in3,$in3,$inpperm
816 vxor $out1,$in1,$rndkey0
817 lvx_u $in6,$x60,$inp
818 le?vperm $in4,$in4,$in4,$inpperm
819 vxor $out2,$in2,$rndkey0
820 lvx_u $in7,$x70,$inp
821 addi $inp,$inp,0x80
822 le?vperm $in5,$in5,$in5,$inpperm
823 vxor $out3,$in3,$rndkey0
824 le?vperm $in6,$in6,$in6,$inpperm
825 vxor $out4,$in4,$rndkey0
826 le?vperm $in7,$in7,$in7,$inpperm
827 vxor $out5,$in5,$rndkey0
828 vxor $out6,$in6,$rndkey0
829 vxor $out7,$in7,$rndkey0
830
831 mtctr $rounds
832 b Loop_cbc_dec8x
833 .align 5
834 Loop_cbc_dec8x:
835 vncipher $out0,$out0,v24
836 vncipher $out1,$out1,v24
837 vncipher $out2,$out2,v24
838 vncipher $out3,$out3,v24
839 vncipher $out4,$out4,v24
840 vncipher $out5,$out5,v24
841 vncipher $out6,$out6,v24
842 vncipher $out7,$out7,v24
843 lvx v24,$x20,$key_ # round[3]
844 addi $key_,$key_,0x20
845
846 vncipher $out0,$out0,v25
847 vncipher $out1,$out1,v25
848 vncipher $out2,$out2,v25
849 vncipher $out3,$out3,v25
850 vncipher $out4,$out4,v25
851 vncipher $out5,$out5,v25
852 vncipher $out6,$out6,v25
853 vncipher $out7,$out7,v25
854 lvx v25,$x10,$key_ # round[4]
855 bdnz Loop_cbc_dec8x
856
857 subic $len,$len,128 # $len-=128
858 vncipher $out0,$out0,v24
859 vncipher $out1,$out1,v24
860 vncipher $out2,$out2,v24
861 vncipher $out3,$out3,v24
862 vncipher $out4,$out4,v24
863 vncipher $out5,$out5,v24
864 vncipher $out6,$out6,v24
865 vncipher $out7,$out7,v24
866
867 subfe. r0,r0,r0 # borrow?-1:0
868 vncipher $out0,$out0,v25
869 vncipher $out1,$out1,v25
870 vncipher $out2,$out2,v25
871 vncipher $out3,$out3,v25
872 vncipher $out4,$out4,v25
873 vncipher $out5,$out5,v25
874 vncipher $out6,$out6,v25
875 vncipher $out7,$out7,v25
876
877 and r0,r0,$len
878 vncipher $out0,$out0,v26
879 vncipher $out1,$out1,v26
880 vncipher $out2,$out2,v26
881 vncipher $out3,$out3,v26
882 vncipher $out4,$out4,v26
883 vncipher $out5,$out5,v26
884 vncipher $out6,$out6,v26
885 vncipher $out7,$out7,v26
886
887 add $inp,$inp,r0 # $inp is adjusted in such
888 # way that at exit from the
889 # loop inX-in7 are loaded
890 # with last "words"
891 vncipher $out0,$out0,v27
892 vncipher $out1,$out1,v27
893 vncipher $out2,$out2,v27
894 vncipher $out3,$out3,v27
895 vncipher $out4,$out4,v27
896 vncipher $out5,$out5,v27
897 vncipher $out6,$out6,v27
898 vncipher $out7,$out7,v27
899
900 addi $key_,$sp,$FRAME+15 # rewind $key_
901 vncipher $out0,$out0,v28
902 vncipher $out1,$out1,v28
903 vncipher $out2,$out2,v28
904 vncipher $out3,$out3,v28
905 vncipher $out4,$out4,v28
906 vncipher $out5,$out5,v28
907 vncipher $out6,$out6,v28
908 vncipher $out7,$out7,v28
909 lvx v24,$x00,$key_ # re-pre-load round[1]
910
911 vncipher $out0,$out0,v29
912 vncipher $out1,$out1,v29
913 vncipher $out2,$out2,v29
914 vncipher $out3,$out3,v29
915 vncipher $out4,$out4,v29
916 vncipher $out5,$out5,v29
917 vncipher $out6,$out6,v29
918 vncipher $out7,$out7,v29
919 lvx v25,$x10,$key_ # re-pre-load round[2]
920
921 vncipher $out0,$out0,v30
922 vxor $ivec,$ivec,v31 # xor with last round key
923 vncipher $out1,$out1,v30
924 vxor $in0,$in0,v31
925 vncipher $out2,$out2,v30
926 vxor $in1,$in1,v31
927 vncipher $out3,$out3,v30
928 vxor $in2,$in2,v31
929 vncipher $out4,$out4,v30
930 vxor $in3,$in3,v31
931 vncipher $out5,$out5,v30
932 vxor $in4,$in4,v31
933 vncipher $out6,$out6,v30
934 vxor $in5,$in5,v31
935 vncipher $out7,$out7,v30
936 vxor $in6,$in6,v31
937
938 vncipherlast $out0,$out0,$ivec
939 vncipherlast $out1,$out1,$in0
940 lvx_u $in0,$x00,$inp # load next input block
941 vncipherlast $out2,$out2,$in1
942 lvx_u $in1,$x10,$inp
943 vncipherlast $out3,$out3,$in2
944 le?vperm $in0,$in0,$in0,$inpperm
945 lvx_u $in2,$x20,$inp
946 vncipherlast $out4,$out4,$in3
947 le?vperm $in1,$in1,$in1,$inpperm
948 lvx_u $in3,$x30,$inp
949 vncipherlast $out5,$out5,$in4
950 le?vperm $in2,$in2,$in2,$inpperm
951 lvx_u $in4,$x40,$inp
952 vncipherlast $out6,$out6,$in5
953 le?vperm $in3,$in3,$in3,$inpperm
954 lvx_u $in5,$x50,$inp
955 vncipherlast $out7,$out7,$in6
956 le?vperm $in4,$in4,$in4,$inpperm
957 lvx_u $in6,$x60,$inp
958 vmr $ivec,$in7
959 le?vperm $in5,$in5,$in5,$inpperm
960 lvx_u $in7,$x70,$inp
961 addi $inp,$inp,0x80
962
963 le?vperm $out0,$out0,$out0,$inpperm
964 le?vperm $out1,$out1,$out1,$inpperm
965 stvx_u $out0,$x00,$out
966 le?vperm $in6,$in6,$in6,$inpperm
967 vxor $out0,$in0,$rndkey0
968 le?vperm $out2,$out2,$out2,$inpperm
969 stvx_u $out1,$x10,$out
970 le?vperm $in7,$in7,$in7,$inpperm
971 vxor $out1,$in1,$rndkey0
972 le?vperm $out3,$out3,$out3,$inpperm
973 stvx_u $out2,$x20,$out
974 vxor $out2,$in2,$rndkey0
975 le?vperm $out4,$out4,$out4,$inpperm
976 stvx_u $out3,$x30,$out
977 vxor $out3,$in3,$rndkey0
978 le?vperm $out5,$out5,$out5,$inpperm
979 stvx_u $out4,$x40,$out
980 vxor $out4,$in4,$rndkey0
981 le?vperm $out6,$out6,$out6,$inpperm
982 stvx_u $out5,$x50,$out
983 vxor $out5,$in5,$rndkey0
984 le?vperm $out7,$out7,$out7,$inpperm
985 stvx_u $out6,$x60,$out
986 vxor $out6,$in6,$rndkey0
987 stvx_u $out7,$x70,$out
988 addi $out,$out,0x80
989 vxor $out7,$in7,$rndkey0
990
991 mtctr $rounds
992 beq Loop_cbc_dec8x # did $len-=128 borrow?
993
994 addic. $len,$len,128
995 beq Lcbc_dec8x_done
996 nop
997 nop
998
999 Loop_cbc_dec8x_tail: # up to 7 "words" tail...
1000 vncipher $out1,$out1,v24
1001 vncipher $out2,$out2,v24
1002 vncipher $out3,$out3,v24
1003 vncipher $out4,$out4,v24
1004 vncipher $out5,$out5,v24
1005 vncipher $out6,$out6,v24
1006 vncipher $out7,$out7,v24
1007 lvx v24,$x20,$key_ # round[3]
1008 addi $key_,$key_,0x20
1009
1010 vncipher $out1,$out1,v25
1011 vncipher $out2,$out2,v25
1012 vncipher $out3,$out3,v25
1013 vncipher $out4,$out4,v25
1014 vncipher $out5,$out5,v25
1015 vncipher $out6,$out6,v25
1016 vncipher $out7,$out7,v25
1017 lvx v25,$x10,$key_ # round[4]
1018 bdnz Loop_cbc_dec8x_tail
1019
1020 vncipher $out1,$out1,v24
1021 vncipher $out2,$out2,v24
1022 vncipher $out3,$out3,v24
1023 vncipher $out4,$out4,v24
1024 vncipher $out5,$out5,v24
1025 vncipher $out6,$out6,v24
1026 vncipher $out7,$out7,v24
1027
1028 vncipher $out1,$out1,v25
1029 vncipher $out2,$out2,v25
1030 vncipher $out3,$out3,v25
1031 vncipher $out4,$out4,v25
1032 vncipher $out5,$out5,v25
1033 vncipher $out6,$out6,v25
1034 vncipher $out7,$out7,v25
1035
1036 vncipher $out1,$out1,v26
1037 vncipher $out2,$out2,v26
1038 vncipher $out3,$out3,v26
1039 vncipher $out4,$out4,v26
1040 vncipher $out5,$out5,v26
1041 vncipher $out6,$out6,v26
1042 vncipher $out7,$out7,v26
1043
1044 vncipher $out1,$out1,v27
1045 vncipher $out2,$out2,v27
1046 vncipher $out3,$out3,v27
1047 vncipher $out4,$out4,v27
1048 vncipher $out5,$out5,v27
1049 vncipher $out6,$out6,v27
1050 vncipher $out7,$out7,v27
1051
1052 vncipher $out1,$out1,v28
1053 vncipher $out2,$out2,v28
1054 vncipher $out3,$out3,v28
1055 vncipher $out4,$out4,v28
1056 vncipher $out5,$out5,v28
1057 vncipher $out6,$out6,v28
1058 vncipher $out7,$out7,v28
1059
1060 vncipher $out1,$out1,v29
1061 vncipher $out2,$out2,v29
1062 vncipher $out3,$out3,v29
1063 vncipher $out4,$out4,v29
1064 vncipher $out5,$out5,v29
1065 vncipher $out6,$out6,v29
1066 vncipher $out7,$out7,v29
1067
1068 vncipher $out1,$out1,v30
1069 vxor $ivec,$ivec,v31 # last round key
1070 vncipher $out2,$out2,v30
1071 vxor $in1,$in1,v31
1072 vncipher $out3,$out3,v30
1073 vxor $in2,$in2,v31
1074 vncipher $out4,$out4,v30
1075 vxor $in3,$in3,v31
1076 vncipher $out5,$out5,v30
1077 vxor $in4,$in4,v31
1078 vncipher $out6,$out6,v30
1079 vxor $in5,$in5,v31
1080 vncipher $out7,$out7,v30
1081 vxor $in6,$in6,v31
1082
1083 cmplwi $len,32 # switch($len)
1084 blt Lcbc_dec8x_one
1085 nop
1086 beq Lcbc_dec8x_two
1087 cmplwi $len,64
1088 blt Lcbc_dec8x_three
1089 nop
1090 beq Lcbc_dec8x_four
1091 cmplwi $len,96
1092 blt Lcbc_dec8x_five
1093 nop
1094 beq Lcbc_dec8x_six
1095
1096 Lcbc_dec8x_seven:
1097 vncipherlast $out1,$out1,$ivec
1098 vncipherlast $out2,$out2,$in1
1099 vncipherlast $out3,$out3,$in2
1100 vncipherlast $out4,$out4,$in3
1101 vncipherlast $out5,$out5,$in4
1102 vncipherlast $out6,$out6,$in5
1103 vncipherlast $out7,$out7,$in6
1104 vmr $ivec,$in7
1105
1106 le?vperm $out1,$out1,$out1,$inpperm
1107 le?vperm $out2,$out2,$out2,$inpperm
1108 stvx_u $out1,$x00,$out
1109 le?vperm $out3,$out3,$out3,$inpperm
1110 stvx_u $out2,$x10,$out
1111 le?vperm $out4,$out4,$out4,$inpperm
1112 stvx_u $out3,$x20,$out
1113 le?vperm $out5,$out5,$out5,$inpperm
1114 stvx_u $out4,$x30,$out
1115 le?vperm $out6,$out6,$out6,$inpperm
1116 stvx_u $out5,$x40,$out
1117 le?vperm $out7,$out7,$out7,$inpperm
1118 stvx_u $out6,$x50,$out
1119 stvx_u $out7,$x60,$out
1120 addi $out,$out,0x70
1121 b Lcbc_dec8x_done
1122
1123 .align 5
1124 Lcbc_dec8x_six:
1125 vncipherlast $out2,$out2,$ivec
1126 vncipherlast $out3,$out3,$in2
1127 vncipherlast $out4,$out4,$in3
1128 vncipherlast $out5,$out5,$in4
1129 vncipherlast $out6,$out6,$in5
1130 vncipherlast $out7,$out7,$in6
1131 vmr $ivec,$in7
1132
1133 le?vperm $out2,$out2,$out2,$inpperm
1134 le?vperm $out3,$out3,$out3,$inpperm
1135 stvx_u $out2,$x00,$out
1136 le?vperm $out4,$out4,$out4,$inpperm
1137 stvx_u $out3,$x10,$out
1138 le?vperm $out5,$out5,$out5,$inpperm
1139 stvx_u $out4,$x20,$out
1140 le?vperm $out6,$out6,$out6,$inpperm
1141 stvx_u $out5,$x30,$out
1142 le?vperm $out7,$out7,$out7,$inpperm
1143 stvx_u $out6,$x40,$out
1144 stvx_u $out7,$x50,$out
1145 addi $out,$out,0x60
1146 b Lcbc_dec8x_done
1147
1148 .align 5
1149 Lcbc_dec8x_five:
1150 vncipherlast $out3,$out3,$ivec
1151 vncipherlast $out4,$out4,$in3
1152 vncipherlast $out5,$out5,$in4
1153 vncipherlast $out6,$out6,$in5
1154 vncipherlast $out7,$out7,$in6
1155 vmr $ivec,$in7
1156
1157 le?vperm $out3,$out3,$out3,$inpperm
1158 le?vperm $out4,$out4,$out4,$inpperm
1159 stvx_u $out3,$x00,$out
1160 le?vperm $out5,$out5,$out5,$inpperm
1161 stvx_u $out4,$x10,$out
1162 le?vperm $out6,$out6,$out6,$inpperm
1163 stvx_u $out5,$x20,$out
1164 le?vperm $out7,$out7,$out7,$inpperm
1165 stvx_u $out6,$x30,$out
1166 stvx_u $out7,$x40,$out
1167 addi $out,$out,0x50
1168 b Lcbc_dec8x_done
1169
1170 .align 5
1171 Lcbc_dec8x_four:
1172 vncipherlast $out4,$out4,$ivec
1173 vncipherlast $out5,$out5,$in4
1174 vncipherlast $out6,$out6,$in5
1175 vncipherlast $out7,$out7,$in6
1176 vmr $ivec,$in7
1177
1178 le?vperm $out4,$out4,$out4,$inpperm
1179 le?vperm $out5,$out5,$out5,$inpperm
1180 stvx_u $out4,$x00,$out
1181 le?vperm $out6,$out6,$out6,$inpperm
1182 stvx_u $out5,$x10,$out
1183 le?vperm $out7,$out7,$out7,$inpperm
1184 stvx_u $out6,$x20,$out
1185 stvx_u $out7,$x30,$out
1186 addi $out,$out,0x40
1187 b Lcbc_dec8x_done
1188
1189 .align 5
1190 Lcbc_dec8x_three:
1191 vncipherlast $out5,$out5,$ivec
1192 vncipherlast $out6,$out6,$in5
1193 vncipherlast $out7,$out7,$in6
1194 vmr $ivec,$in7
1195
1196 le?vperm $out5,$out5,$out5,$inpperm
1197 le?vperm $out6,$out6,$out6,$inpperm
1198 stvx_u $out5,$x00,$out
1199 le?vperm $out7,$out7,$out7,$inpperm
1200 stvx_u $out6,$x10,$out
1201 stvx_u $out7,$x20,$out
1202 addi $out,$out,0x30
1203 b Lcbc_dec8x_done
1204
1205 .align 5
1206 Lcbc_dec8x_two:
1207 vncipherlast $out6,$out6,$ivec
1208 vncipherlast $out7,$out7,$in6
1209 vmr $ivec,$in7
1210
1211 le?vperm $out6,$out6,$out6,$inpperm
1212 le?vperm $out7,$out7,$out7,$inpperm
1213 stvx_u $out6,$x00,$out
1214 stvx_u $out7,$x10,$out
1215 addi $out,$out,0x20
1216 b Lcbc_dec8x_done
1217
1218 .align 5
1219 Lcbc_dec8x_one:
1220 vncipherlast $out7,$out7,$ivec
1221 vmr $ivec,$in7
1222
1223 le?vperm $out7,$out7,$out7,$inpperm
1224 stvx_u $out7,0,$out
1225 addi $out,$out,0x10
1226
1227 Lcbc_dec8x_done:
1228 le?vperm $ivec,$ivec,$ivec,$inpperm
1229 stvx_u $ivec,0,$ivp # write [unaligned] iv
1230
1231 li r10,`$FRAME+15`
1232 li r11,`$FRAME+31`
1233 stvx $inpperm,r10,$sp # wipe copies of round keys
1234 addi r10,r10,32
1235 stvx $inpperm,r11,$sp
1236 addi r11,r11,32
1237 stvx $inpperm,r10,$sp
1238 addi r10,r10,32
1239 stvx $inpperm,r11,$sp
1240 addi r11,r11,32
1241 stvx $inpperm,r10,$sp
1242 addi r10,r10,32
1243 stvx $inpperm,r11,$sp
1244 addi r11,r11,32
1245 stvx $inpperm,r10,$sp
1246 addi r10,r10,32
1247 stvx $inpperm,r11,$sp
1248 addi r11,r11,32
1249
1250 mtspr 256,$vrsave
1251 lvx v20,r10,$sp # ABI says so
1252 addi r10,r10,32
1253 lvx v21,r11,$sp
1254 addi r11,r11,32
1255 lvx v22,r10,$sp
1256 addi r10,r10,32
1257 lvx v23,r11,$sp
1258 addi r11,r11,32
1259 lvx v24,r10,$sp
1260 addi r10,r10,32
1261 lvx v25,r11,$sp
1262 addi r11,r11,32
1263 lvx v26,r10,$sp
1264 addi r10,r10,32
1265 lvx v27,r11,$sp
1266 addi r11,r11,32
1267 lvx v28,r10,$sp
1268 addi r10,r10,32
1269 lvx v29,r11,$sp
1270 addi r11,r11,32
1271 lvx v30,r10,$sp
1272 lvx v31,r11,$sp
1273 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1274 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1275 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1276 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1277 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1278 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1279 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1280 blr
1281 .long 0
1282 .byte 0,12,0x14,0,0x80,6,6,0
1283 .long 0
1284 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1285 ___
1286 }} }}}
1287
1288 #########################################################################
1289 {{{ # CTR procedure[s] #
1290
1291 ####################### WARNING: Here be dragons! #######################
1292 #
1293 # This code is written as 'ctr32', based on a 32-bit counter used
1294 # upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1295 # a 128-bit counter.
1296 #
1297 # This leads to subtle changes from the upstream code: the counter
1298 # is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
1299 # both the bulk (8 blocks at a time) path, and in the individual block
1300 # path. Be aware of this when doing updates.
1301 #
1302 # See:
1303 # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1304 # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1305 # https://github.com/openssl/openssl/pull/8942
1306 #
1307 #########################################################################
1308 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
1309 my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3));
1310 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
1311 map("v$_",(4..11));
1312 my $dat=$tmp;
1313
1314 $code.=<<___;
1315 .globl .${prefix}_ctr32_encrypt_blocks
1316 ${UCMP}i $len,1
1317 bltlr-
1318
1319 lis r0,0xfff0
1320 mfspr $vrsave,256
1321 mtspr 256,r0
1322
1323 li $idx,15
1324 vxor $rndkey0,$rndkey0,$rndkey0
1325 le?vspltisb $tmp,0x0f
1326
1327 lvx $ivec,0,$ivp # load [unaligned] iv
1328 lvsl $inpperm,0,$ivp
1329 lvx $inptail,$idx,$ivp
1330 vspltisb $one,1
1331 le?vxor $inpperm,$inpperm,$tmp
1332 vperm $ivec,$ivec,$inptail,$inpperm
1333 vsldoi $one,$rndkey0,$one,1
1334
1335 neg r11,$inp
1336 ?lvsl $keyperm,0,$key # prepare for unaligned key
1337 lwz $rounds,240($key)
1338
1339 lvsr $inpperm,0,r11 # prepare for unaligned load
1340 lvx $inptail,0,$inp
1341 addi $inp,$inp,15 # 15 is not typo
1342 le?vxor $inpperm,$inpperm,$tmp
1343
1344 srwi $rounds,$rounds,1
1345 li $idx,16
1346 subi $rounds,$rounds,1
1347
1348 ${UCMP}i $len,8
1349 bge _aesp8_ctr32_encrypt8x
1350
1351 ?lvsr $outperm,0,$out # prepare for unaligned store
1352 vspltisb $outmask,-1
1353 lvx $outhead,0,$out
1354 ?vperm $outmask,$rndkey0,$outmask,$outperm
1355 le?vxor $outperm,$outperm,$tmp
1356
1357 lvx $rndkey0,0,$key
1358 mtctr $rounds
1359 lvx $rndkey1,$idx,$key
1360 addi $idx,$idx,16
1361 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1362 vxor $inout,$ivec,$rndkey0
1363 lvx $rndkey0,$idx,$key
1364 addi $idx,$idx,16
1365 b Loop_ctr32_enc
1366
1367 .align 5
1368 Loop_ctr32_enc:
1369 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1370 vcipher $inout,$inout,$rndkey1
1371 lvx $rndkey1,$idx,$key
1372 addi $idx,$idx,16
1373 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1374 vcipher $inout,$inout,$rndkey0
1375 lvx $rndkey0,$idx,$key
1376 addi $idx,$idx,16
1377 bdnz Loop_ctr32_enc
1378
1379 vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
1380 vmr $dat,$inptail
1381 lvx $inptail,0,$inp
1382 addi $inp,$inp,16
1383 subic. $len,$len,1 # blocks--
1384
1385 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
1386 vcipher $inout,$inout,$rndkey1
1387 lvx $rndkey1,$idx,$key
1388 vperm $dat,$dat,$inptail,$inpperm
1389 li $idx,16
1390 ?vperm $rndkey1,$rndkey0,$rndkey1,$keyperm
1391 lvx $rndkey0,0,$key
1392 vxor $dat,$dat,$rndkey1 # last round key
1393 vcipherlast $inout,$inout,$dat
1394
1395 lvx $rndkey1,$idx,$key
1396 addi $idx,$idx,16
1397 vperm $inout,$inout,$inout,$outperm
1398 vsel $dat,$outhead,$inout,$outmask
1399 mtctr $rounds
1400 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
1401 vmr $outhead,$inout
1402 vxor $inout,$ivec,$rndkey0
1403 lvx $rndkey0,$idx,$key
1404 addi $idx,$idx,16
1405 stvx $dat,0,$out
1406 addi $out,$out,16
1407 bne Loop_ctr32_enc
1408
1409 addi $out,$out,-1
1410 lvx $inout,0,$out # redundant in aligned case
1411 vsel $inout,$outhead,$inout,$outmask
1412 stvx $inout,0,$out
1413
1414 mtspr 256,$vrsave
1415 blr
1416 .long 0
1417 .byte 0,12,0x14,0,0,0,6,0
1418 .long 0
1419 ___
1420 #########################################################################
1421 {{ # Optimized CTR procedure #
1422 my $key_="r11";
1423 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
1424 my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
1425 my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
1426 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1427 # v26-v31 last 6 round keys
1428 my ($tmp,$keyperm)=($in3,$in4); # aliases with "caller", redundant assignment
1429 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1430
1431 $code.=<<___;
1432 .align 5
1433 _aesp8_ctr32_encrypt8x:
1434 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1435 li r10,`$FRAME+8*16+15`
1436 li r11,`$FRAME+8*16+31`
1437 stvx v20,r10,$sp # ABI says so
1438 addi r10,r10,32
1439 stvx v21,r11,$sp
1440 addi r11,r11,32
1441 stvx v22,r10,$sp
1442 addi r10,r10,32
1443 stvx v23,r11,$sp
1444 addi r11,r11,32
1445 stvx v24,r10,$sp
1446 addi r10,r10,32
1447 stvx v25,r11,$sp
1448 addi r11,r11,32
1449 stvx v26,r10,$sp
1450 addi r10,r10,32
1451 stvx v27,r11,$sp
1452 addi r11,r11,32
1453 stvx v28,r10,$sp
1454 addi r10,r10,32
1455 stvx v29,r11,$sp
1456 addi r11,r11,32
1457 stvx v30,r10,$sp
1458 stvx v31,r11,$sp
1459 li r0,-1
1460 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1461 li $x10,0x10
1462 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1463 li $x20,0x20
1464 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1465 li $x30,0x30
1466 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1467 li $x40,0x40
1468 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1469 li $x50,0x50
1470 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1471 li $x60,0x60
1472 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1473 li $x70,0x70
1474 mtspr 256,r0
1475
1476 subi $rounds,$rounds,3 # -4 in total
1477
1478 lvx $rndkey0,$x00,$key # load key schedule
1479 lvx v30,$x10,$key
1480 addi $key,$key,0x20
1481 lvx v31,$x00,$key
1482 ?vperm $rndkey0,$rndkey0,v30,$keyperm
1483 addi $key_,$sp,$FRAME+15
1484 mtctr $rounds
1485
1486 Load_ctr32_enc_key:
1487 ?vperm v24,v30,v31,$keyperm
1488 lvx v30,$x10,$key
1489 addi $key,$key,0x20
1490 stvx v24,$x00,$key_ # off-load round[1]
1491 ?vperm v25,v31,v30,$keyperm
1492 lvx v31,$x00,$key
1493 stvx v25,$x10,$key_ # off-load round[2]
1494 addi $key_,$key_,0x20
1495 bdnz Load_ctr32_enc_key
1496
1497 lvx v26,$x10,$key
1498 ?vperm v24,v30,v31,$keyperm
1499 lvx v27,$x20,$key
1500 stvx v24,$x00,$key_ # off-load round[3]
1501 ?vperm v25,v31,v26,$keyperm
1502 lvx v28,$x30,$key
1503 stvx v25,$x10,$key_ # off-load round[4]
1504 addi $key_,$sp,$FRAME+15 # rewind $key_
1505 ?vperm v26,v26,v27,$keyperm
1506 lvx v29,$x40,$key
1507 ?vperm v27,v27,v28,$keyperm
1508 lvx v30,$x50,$key
1509 ?vperm v28,v28,v29,$keyperm
1510 lvx v31,$x60,$key
1511 ?vperm v29,v29,v30,$keyperm
1512 lvx $out0,$x70,$key # borrow $out0
1513 ?vperm v30,v30,v31,$keyperm
1514 lvx v24,$x00,$key_ # pre-load round[1]
1515 ?vperm v31,v31,$out0,$keyperm
1516 lvx v25,$x10,$key_ # pre-load round[2]
1517
1518 vadduqm $two,$one,$one
1519 subi $inp,$inp,15 # undo "caller"
1520 $SHL $len,$len,4
1521
1522 vadduqm $out1,$ivec,$one # counter values ...
1523 vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
1524 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1525 le?li $idx,8
1526 vadduqm $out3,$out1,$two
1527 vxor $out1,$out1,$rndkey0
1528 le?lvsl $inpperm,0,$idx
1529 vadduqm $out4,$out2,$two
1530 vxor $out2,$out2,$rndkey0
1531 le?vspltisb $tmp,0x0f
1532 vadduqm $out5,$out3,$two
1533 vxor $out3,$out3,$rndkey0
1534 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u
1535 vadduqm $out6,$out4,$two
1536 vxor $out4,$out4,$rndkey0
1537 vadduqm $out7,$out5,$two
1538 vxor $out5,$out5,$rndkey0
1539 vadduqm $ivec,$out6,$two # next counter value
1540 vxor $out6,$out6,$rndkey0
1541 vxor $out7,$out7,$rndkey0
1542
1543 mtctr $rounds
1544 b Loop_ctr32_enc8x
1545 .align 5
1546 Loop_ctr32_enc8x:
1547 vcipher $out0,$out0,v24
1548 vcipher $out1,$out1,v24
1549 vcipher $out2,$out2,v24
1550 vcipher $out3,$out3,v24
1551 vcipher $out4,$out4,v24
1552 vcipher $out5,$out5,v24
1553 vcipher $out6,$out6,v24
1554 vcipher $out7,$out7,v24
1555 Loop_ctr32_enc8x_middle:
1556 lvx v24,$x20,$key_ # round[3]
1557 addi $key_,$key_,0x20
1558
1559 vcipher $out0,$out0,v25
1560 vcipher $out1,$out1,v25
1561 vcipher $out2,$out2,v25
1562 vcipher $out3,$out3,v25
1563 vcipher $out4,$out4,v25
1564 vcipher $out5,$out5,v25
1565 vcipher $out6,$out6,v25
1566 vcipher $out7,$out7,v25
1567 lvx v25,$x10,$key_ # round[4]
1568 bdnz Loop_ctr32_enc8x
1569
1570 subic r11,$len,256 # $len-256, borrow $key_
1571 vcipher $out0,$out0,v24
1572 vcipher $out1,$out1,v24
1573 vcipher $out2,$out2,v24
1574 vcipher $out3,$out3,v24
1575 vcipher $out4,$out4,v24
1576 vcipher $out5,$out5,v24
1577 vcipher $out6,$out6,v24
1578 vcipher $out7,$out7,v24
1579
1580 subfe r0,r0,r0 # borrow?-1:0
1581 vcipher $out0,$out0,v25
1582 vcipher $out1,$out1,v25
1583 vcipher $out2,$out2,v25
1584 vcipher $out3,$out3,v25
1585 vcipher $out4,$out4,v25
1586 vcipher $out5,$out5,v25
1587 vcipher $out6,$out6,v25
1588 vcipher $out7,$out7,v25
1589
1590 and r0,r0,r11
1591 addi $key_,$sp,$FRAME+15 # rewind $key_
1592 vcipher $out0,$out0,v26
1593 vcipher $out1,$out1,v26
1594 vcipher $out2,$out2,v26
1595 vcipher $out3,$out3,v26
1596 vcipher $out4,$out4,v26
1597 vcipher $out5,$out5,v26
1598 vcipher $out6,$out6,v26
1599 vcipher $out7,$out7,v26
1600 lvx v24,$x00,$key_ # re-pre-load round[1]
1601
1602 subic $len,$len,129 # $len-=129
1603 vcipher $out0,$out0,v27
1604 addi $len,$len,1 # $len-=128 really
1605 vcipher $out1,$out1,v27
1606 vcipher $out2,$out2,v27
1607 vcipher $out3,$out3,v27
1608 vcipher $out4,$out4,v27
1609 vcipher $out5,$out5,v27
1610 vcipher $out6,$out6,v27
1611 vcipher $out7,$out7,v27
1612 lvx v25,$x10,$key_ # re-pre-load round[2]
1613
1614 vcipher $out0,$out0,v28
1615 lvx_u $in0,$x00,$inp # load input
1616 vcipher $out1,$out1,v28
1617 lvx_u $in1,$x10,$inp
1618 vcipher $out2,$out2,v28
1619 lvx_u $in2,$x20,$inp
1620 vcipher $out3,$out3,v28
1621 lvx_u $in3,$x30,$inp
1622 vcipher $out4,$out4,v28
1623 lvx_u $in4,$x40,$inp
1624 vcipher $out5,$out5,v28
1625 lvx_u $in5,$x50,$inp
1626 vcipher $out6,$out6,v28
1627 lvx_u $in6,$x60,$inp
1628 vcipher $out7,$out7,v28
1629 lvx_u $in7,$x70,$inp
1630 addi $inp,$inp,0x80
1631
1632 vcipher $out0,$out0,v29
1633 le?vperm $in0,$in0,$in0,$inpperm
1634 vcipher $out1,$out1,v29
1635 le?vperm $in1,$in1,$in1,$inpperm
1636 vcipher $out2,$out2,v29
1637 le?vperm $in2,$in2,$in2,$inpperm
1638 vcipher $out3,$out3,v29
1639 le?vperm $in3,$in3,$in3,$inpperm
1640 vcipher $out4,$out4,v29
1641 le?vperm $in4,$in4,$in4,$inpperm
1642 vcipher $out5,$out5,v29
1643 le?vperm $in5,$in5,$in5,$inpperm
1644 vcipher $out6,$out6,v29
1645 le?vperm $in6,$in6,$in6,$inpperm
1646 vcipher $out7,$out7,v29
1647 le?vperm $in7,$in7,$in7,$inpperm
1648
1649 add $inp,$inp,r0 # $inp is adjusted in such
1650 # way that at exit from the
1651 # loop inX-in7 are loaded
1652 # with last "words"
1653 subfe. r0,r0,r0 # borrow?-1:0
1654 vcipher $out0,$out0,v30
1655 vxor $in0,$in0,v31 # xor with last round key
1656 vcipher $out1,$out1,v30
1657 vxor $in1,$in1,v31
1658 vcipher $out2,$out2,v30
1659 vxor $in2,$in2,v31
1660 vcipher $out3,$out3,v30
1661 vxor $in3,$in3,v31
1662 vcipher $out4,$out4,v30
1663 vxor $in4,$in4,v31
1664 vcipher $out5,$out5,v30
1665 vxor $in5,$in5,v31
1666 vcipher $out6,$out6,v30
1667 vxor $in6,$in6,v31
1668 vcipher $out7,$out7,v30
1669 vxor $in7,$in7,v31
1670
1671 bne Lctr32_enc8x_break # did $len-129 borrow?
1672
1673 vcipherlast $in0,$out0,$in0
1674 vcipherlast $in1,$out1,$in1
1675 vadduqm $out1,$ivec,$one # counter values ...
1676 vcipherlast $in2,$out2,$in2
1677 vadduqm $out2,$ivec,$two
1678 vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0]
1679 vcipherlast $in3,$out3,$in3
1680 vadduqm $out3,$out1,$two
1681 vxor $out1,$out1,$rndkey0
1682 vcipherlast $in4,$out4,$in4
1683 vadduqm $out4,$out2,$two
1684 vxor $out2,$out2,$rndkey0
1685 vcipherlast $in5,$out5,$in5
1686 vadduqm $out5,$out3,$two
1687 vxor $out3,$out3,$rndkey0
1688 vcipherlast $in6,$out6,$in6
1689 vadduqm $out6,$out4,$two
1690 vxor $out4,$out4,$rndkey0
1691 vcipherlast $in7,$out7,$in7
1692 vadduqm $out7,$out5,$two
1693 vxor $out5,$out5,$rndkey0
1694 le?vperm $in0,$in0,$in0,$inpperm
1695 vadduqm $ivec,$out6,$two # next counter value
1696 vxor $out6,$out6,$rndkey0
1697 le?vperm $in1,$in1,$in1,$inpperm
1698 vxor $out7,$out7,$rndkey0
1699 mtctr $rounds
1700
1701 vcipher $out0,$out0,v24
1702 stvx_u $in0,$x00,$out
1703 le?vperm $in2,$in2,$in2,$inpperm
1704 vcipher $out1,$out1,v24
1705 stvx_u $in1,$x10,$out
1706 le?vperm $in3,$in3,$in3,$inpperm
1707 vcipher $out2,$out2,v24
1708 stvx_u $in2,$x20,$out
1709 le?vperm $in4,$in4,$in4,$inpperm
1710 vcipher $out3,$out3,v24
1711 stvx_u $in3,$x30,$out
1712 le?vperm $in5,$in5,$in5,$inpperm
1713 vcipher $out4,$out4,v24
1714 stvx_u $in4,$x40,$out
1715 le?vperm $in6,$in6,$in6,$inpperm
1716 vcipher $out5,$out5,v24
1717 stvx_u $in5,$x50,$out
1718 le?vperm $in7,$in7,$in7,$inpperm
1719 vcipher $out6,$out6,v24
1720 stvx_u $in6,$x60,$out
1721 vcipher $out7,$out7,v24
1722 stvx_u $in7,$x70,$out
1723 addi $out,$out,0x80
1724
1725 b Loop_ctr32_enc8x_middle
1726
1727 .align 5
1728 Lctr32_enc8x_break:
1729 cmpwi $len,-0x60
1730 blt Lctr32_enc8x_one
1731 nop
1732 beq Lctr32_enc8x_two
1733 cmpwi $len,-0x40
1734 blt Lctr32_enc8x_three
1735 nop
1736 beq Lctr32_enc8x_four
1737 cmpwi $len,-0x20
1738 blt Lctr32_enc8x_five
1739 nop
1740 beq Lctr32_enc8x_six
1741 cmpwi $len,0x00
1742 blt Lctr32_enc8x_seven
1743
1744 Lctr32_enc8x_eight:
1745 vcipherlast $out0,$out0,$in0
1746 vcipherlast $out1,$out1,$in1
1747 vcipherlast $out2,$out2,$in2
1748 vcipherlast $out3,$out3,$in3
1749 vcipherlast $out4,$out4,$in4
1750 vcipherlast $out5,$out5,$in5
1751 vcipherlast $out6,$out6,$in6
1752 vcipherlast $out7,$out7,$in7
1753
1754 le?vperm $out0,$out0,$out0,$inpperm
1755 le?vperm $out1,$out1,$out1,$inpperm
1756 stvx_u $out0,$x00,$out
1757 le?vperm $out2,$out2,$out2,$inpperm
1758 stvx_u $out1,$x10,$out
1759 le?vperm $out3,$out3,$out3,$inpperm
1760 stvx_u $out2,$x20,$out
1761 le?vperm $out4,$out4,$out4,$inpperm
1762 stvx_u $out3,$x30,$out
1763 le?vperm $out5,$out5,$out5,$inpperm
1764 stvx_u $out4,$x40,$out
1765 le?vperm $out6,$out6,$out6,$inpperm
1766 stvx_u $out5,$x50,$out
1767 le?vperm $out7,$out7,$out7,$inpperm
1768 stvx_u $out6,$x60,$out
1769 stvx_u $out7,$x70,$out
1770 addi $out,$out,0x80
1771 b Lctr32_enc8x_done
1772
1773 .align 5
1774 Lctr32_enc8x_seven:
1775 vcipherlast $out0,$out0,$in1
1776 vcipherlast $out1,$out1,$in2
1777 vcipherlast $out2,$out2,$in3
1778 vcipherlast $out3,$out3,$in4
1779 vcipherlast $out4,$out4,$in5
1780 vcipherlast $out5,$out5,$in6
1781 vcipherlast $out6,$out6,$in7
1782
1783 le?vperm $out0,$out0,$out0,$inpperm
1784 le?vperm $out1,$out1,$out1,$inpperm
1785 stvx_u $out0,$x00,$out
1786 le?vperm $out2,$out2,$out2,$inpperm
1787 stvx_u $out1,$x10,$out
1788 le?vperm $out3,$out3,$out3,$inpperm
1789 stvx_u $out2,$x20,$out
1790 le?vperm $out4,$out4,$out4,$inpperm
1791 stvx_u $out3,$x30,$out
1792 le?vperm $out5,$out5,$out5,$inpperm
1793 stvx_u $out4,$x40,$out
1794 le?vperm $out6,$out6,$out6,$inpperm
1795 stvx_u $out5,$x50,$out
1796 stvx_u $out6,$x60,$out
1797 addi $out,$out,0x70
1798 b Lctr32_enc8x_done
1799
1800 .align 5
1801 Lctr32_enc8x_six:
1802 vcipherlast $out0,$out0,$in2
1803 vcipherlast $out1,$out1,$in3
1804 vcipherlast $out2,$out2,$in4
1805 vcipherlast $out3,$out3,$in5
1806 vcipherlast $out4,$out4,$in6
1807 vcipherlast $out5,$out5,$in7
1808
1809 le?vperm $out0,$out0,$out0,$inpperm
1810 le?vperm $out1,$out1,$out1,$inpperm
1811 stvx_u $out0,$x00,$out
1812 le?vperm $out2,$out2,$out2,$inpperm
1813 stvx_u $out1,$x10,$out
1814 le?vperm $out3,$out3,$out3,$inpperm
1815 stvx_u $out2,$x20,$out
1816 le?vperm $out4,$out4,$out4,$inpperm
1817 stvx_u $out3,$x30,$out
1818 le?vperm $out5,$out5,$out5,$inpperm
1819 stvx_u $out4,$x40,$out
1820 stvx_u $out5,$x50,$out
1821 addi $out,$out,0x60
1822 b Lctr32_enc8x_done
1823
1824 .align 5
1825 Lctr32_enc8x_five:
1826 vcipherlast $out0,$out0,$in3
1827 vcipherlast $out1,$out1,$in4
1828 vcipherlast $out2,$out2,$in5
1829 vcipherlast $out3,$out3,$in6
1830 vcipherlast $out4,$out4,$in7
1831
1832 le?vperm $out0,$out0,$out0,$inpperm
1833 le?vperm $out1,$out1,$out1,$inpperm
1834 stvx_u $out0,$x00,$out
1835 le?vperm $out2,$out2,$out2,$inpperm
1836 stvx_u $out1,$x10,$out
1837 le?vperm $out3,$out3,$out3,$inpperm
1838 stvx_u $out2,$x20,$out
1839 le?vperm $out4,$out4,$out4,$inpperm
1840 stvx_u $out3,$x30,$out
1841 stvx_u $out4,$x40,$out
1842 addi $out,$out,0x50
1843 b Lctr32_enc8x_done
1844
1845 .align 5
1846 Lctr32_enc8x_four:
1847 vcipherlast $out0,$out0,$in4
1848 vcipherlast $out1,$out1,$in5
1849 vcipherlast $out2,$out2,$in6
1850 vcipherlast $out3,$out3,$in7
1851
1852 le?vperm $out0,$out0,$out0,$inpperm
1853 le?vperm $out1,$out1,$out1,$inpperm
1854 stvx_u $out0,$x00,$out
1855 le?vperm $out2,$out2,$out2,$inpperm
1856 stvx_u $out1,$x10,$out
1857 le?vperm $out3,$out3,$out3,$inpperm
1858 stvx_u $out2,$x20,$out
1859 stvx_u $out3,$x30,$out
1860 addi $out,$out,0x40
1861 b Lctr32_enc8x_done
1862
1863 .align 5
1864 Lctr32_enc8x_three:
1865 vcipherlast $out0,$out0,$in5
1866 vcipherlast $out1,$out1,$in6
1867 vcipherlast $out2,$out2,$in7
1868
1869 le?vperm $out0,$out0,$out0,$inpperm
1870 le?vperm $out1,$out1,$out1,$inpperm
1871 stvx_u $out0,$x00,$out
1872 le?vperm $out2,$out2,$out2,$inpperm
1873 stvx_u $out1,$x10,$out
1874 stvx_u $out2,$x20,$out
1875 addi $out,$out,0x30
1876 b Lctr32_enc8x_done
1877
1878 .align 5
1879 Lctr32_enc8x_two:
1880 vcipherlast $out0,$out0,$in6
1881 vcipherlast $out1,$out1,$in7
1882
1883 le?vperm $out0,$out0,$out0,$inpperm
1884 le?vperm $out1,$out1,$out1,$inpperm
1885 stvx_u $out0,$x00,$out
1886 stvx_u $out1,$x10,$out
1887 addi $out,$out,0x20
1888 b Lctr32_enc8x_done
1889
1890 .align 5
1891 Lctr32_enc8x_one:
1892 vcipherlast $out0,$out0,$in7
1893
1894 le?vperm $out0,$out0,$out0,$inpperm
1895 stvx_u $out0,0,$out
1896 addi $out,$out,0x10
1897
1898 Lctr32_enc8x_done:
1899 li r10,`$FRAME+15`
1900 li r11,`$FRAME+31`
1901 stvx $inpperm,r10,$sp # wipe copies of round keys
1902 addi r10,r10,32
1903 stvx $inpperm,r11,$sp
1904 addi r11,r11,32
1905 stvx $inpperm,r10,$sp
1906 addi r10,r10,32
1907 stvx $inpperm,r11,$sp
1908 addi r11,r11,32
1909 stvx $inpperm,r10,$sp
1910 addi r10,r10,32
1911 stvx $inpperm,r11,$sp
1912 addi r11,r11,32
1913 stvx $inpperm,r10,$sp
1914 addi r10,r10,32
1915 stvx $inpperm,r11,$sp
1916 addi r11,r11,32
1917
1918 mtspr 256,$vrsave
1919 lvx v20,r10,$sp # ABI says so
1920 addi r10,r10,32
1921 lvx v21,r11,$sp
1922 addi r11,r11,32
1923 lvx v22,r10,$sp
1924 addi r10,r10,32
1925 lvx v23,r11,$sp
1926 addi r11,r11,32
1927 lvx v24,r10,$sp
1928 addi r10,r10,32
1929 lvx v25,r11,$sp
1930 addi r11,r11,32
1931 lvx v26,r10,$sp
1932 addi r10,r10,32
1933 lvx v27,r11,$sp
1934 addi r11,r11,32
1935 lvx v28,r10,$sp
1936 addi r10,r10,32
1937 lvx v29,r11,$sp
1938 addi r11,r11,32
1939 lvx v30,r10,$sp
1940 lvx v31,r11,$sp
1941 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
1942 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
1943 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
1944 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
1945 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
1946 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
1947 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
1948 blr
1949 .long 0
1950 .byte 0,12,0x14,0,0x80,6,6,0
1951 .long 0
1952 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1953 ___
1954 }} }}}
1955
1956 #########################################################################
1957 {{{ # XTS procedures #
1958 # int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len, #
1959 # const AES_KEY *key1, const AES_KEY *key2, #
1960 # [const] unsigned char iv[16]); #
1961 # If $key2 is NULL, then a "tweak chaining" mode is engaged, in which #
1962 # input tweak value is assumed to be encrypted already, and last tweak #
1963 # value, one suitable for consecutive call on same chunk of data, is #
1964 # written back to original buffer. In addition, in "tweak chaining" #
1965 # mode only complete input blocks are processed. #
1966
1967 my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) = map("r$_",(3..10));
1968 my ($rndkey0,$rndkey1,$inout) = map("v$_",(0..2));
1969 my ($output,$inptail,$inpperm,$leperm,$keyperm) = map("v$_",(3..7));
1970 my ($tweak,$seven,$eighty7,$tmp,$tweak1) = map("v$_",(8..12));
1971 my $taillen = $key2;
1972
1973 ($inp,$idx) = ($idx,$inp); # reassign
1974
1975 $code.=<<___;
1976 .globl .${prefix}_xts_encrypt
1977 mr $inp,r3 # reassign
1978 li r3,-1
1979 ${UCMP}i $len,16
1980 bltlr-
1981
1982 lis r0,0xfff0
1983 mfspr r12,256 # save vrsave
1984 li r11,0
1985 mtspr 256,r0
1986
1987 vspltisb $seven,0x07 # 0x070707..07
1988 le?lvsl $leperm,r11,r11
1989 le?vspltisb $tmp,0x0f
1990 le?vxor $leperm,$leperm,$seven
1991
1992 li $idx,15
1993 lvx $tweak,0,$ivp # load [unaligned] iv
1994 lvsl $inpperm,0,$ivp
1995 lvx $inptail,$idx,$ivp
1996 le?vxor $inpperm,$inpperm,$tmp
1997 vperm $tweak,$tweak,$inptail,$inpperm
1998
1999 neg r11,$inp
2000 lvsr $inpperm,0,r11 # prepare for unaligned load
2001 lvx $inout,0,$inp
2002 addi $inp,$inp,15 # 15 is not typo
2003 le?vxor $inpperm,$inpperm,$tmp
2004
2005 ${UCMP}i $key2,0 # key2==NULL?
2006 beq Lxts_enc_no_key2
2007
2008 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
2009 lwz $rounds,240($key2)
2010 srwi $rounds,$rounds,1
2011 subi $rounds,$rounds,1
2012 li $idx,16
2013
2014 lvx $rndkey0,0,$key2
2015 lvx $rndkey1,$idx,$key2
2016 addi $idx,$idx,16
2017 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2018 vxor $tweak,$tweak,$rndkey0
2019 lvx $rndkey0,$idx,$key2
2020 addi $idx,$idx,16
2021 mtctr $rounds
2022
2023 Ltweak_xts_enc:
2024 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2025 vcipher $tweak,$tweak,$rndkey1
2026 lvx $rndkey1,$idx,$key2
2027 addi $idx,$idx,16
2028 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2029 vcipher $tweak,$tweak,$rndkey0
2030 lvx $rndkey0,$idx,$key2
2031 addi $idx,$idx,16
2032 bdnz Ltweak_xts_enc
2033
2034 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2035 vcipher $tweak,$tweak,$rndkey1
2036 lvx $rndkey1,$idx,$key2
2037 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2038 vcipherlast $tweak,$tweak,$rndkey0
2039
2040 li $ivp,0 # don't chain the tweak
2041 b Lxts_enc
2042
2043 Lxts_enc_no_key2:
2044 li $idx,-16
2045 and $len,$len,$idx # in "tweak chaining"
2046 # mode only complete
2047 # blocks are processed
2048 Lxts_enc:
2049 lvx $inptail,0,$inp
2050 addi $inp,$inp,16
2051
2052 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
2053 lwz $rounds,240($key1)
2054 srwi $rounds,$rounds,1
2055 subi $rounds,$rounds,1
2056 li $idx,16
2057
2058 vslb $eighty7,$seven,$seven # 0x808080..80
2059 vor $eighty7,$eighty7,$seven # 0x878787..87
2060 vspltisb $tmp,1 # 0x010101..01
2061 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2062
2063 ${UCMP}i $len,96
2064 bge _aesp8_xts_encrypt6x
2065
2066 andi. $taillen,$len,15
2067 subic r0,$len,32
2068 subi $taillen,$taillen,16
2069 subfe r0,r0,r0
2070 and r0,r0,$taillen
2071 add $inp,$inp,r0
2072
2073 lvx $rndkey0,0,$key1
2074 lvx $rndkey1,$idx,$key1
2075 addi $idx,$idx,16
2076 vperm $inout,$inout,$inptail,$inpperm
2077 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2078 vxor $inout,$inout,$tweak
2079 vxor $inout,$inout,$rndkey0
2080 lvx $rndkey0,$idx,$key1
2081 addi $idx,$idx,16
2082 mtctr $rounds
2083 b Loop_xts_enc
2084
2085 .align 5
2086 Loop_xts_enc:
2087 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2088 vcipher $inout,$inout,$rndkey1
2089 lvx $rndkey1,$idx,$key1
2090 addi $idx,$idx,16
2091 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2092 vcipher $inout,$inout,$rndkey0
2093 lvx $rndkey0,$idx,$key1
2094 addi $idx,$idx,16
2095 bdnz Loop_xts_enc
2096
2097 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2098 vcipher $inout,$inout,$rndkey1
2099 lvx $rndkey1,$idx,$key1
2100 li $idx,16
2101 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2102 vxor $rndkey0,$rndkey0,$tweak
2103 vcipherlast $output,$inout,$rndkey0
2104
2105 le?vperm $tmp,$output,$output,$leperm
2106 be?nop
2107 le?stvx_u $tmp,0,$out
2108 be?stvx_u $output,0,$out
2109 addi $out,$out,16
2110
2111 subic. $len,$len,16
2112 beq Lxts_enc_done
2113
2114 vmr $inout,$inptail
2115 lvx $inptail,0,$inp
2116 addi $inp,$inp,16
2117 lvx $rndkey0,0,$key1
2118 lvx $rndkey1,$idx,$key1
2119 addi $idx,$idx,16
2120
2121 subic r0,$len,32
2122 subfe r0,r0,r0
2123 and r0,r0,$taillen
2124 add $inp,$inp,r0
2125
2126 vsrab $tmp,$tweak,$seven # next tweak value
2127 vaddubm $tweak,$tweak,$tweak
2128 vsldoi $tmp,$tmp,$tmp,15
2129 vand $tmp,$tmp,$eighty7
2130 vxor $tweak,$tweak,$tmp
2131
2132 vperm $inout,$inout,$inptail,$inpperm
2133 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2134 vxor $inout,$inout,$tweak
2135 vxor $output,$output,$rndkey0 # just in case $len<16
2136 vxor $inout,$inout,$rndkey0
2137 lvx $rndkey0,$idx,$key1
2138 addi $idx,$idx,16
2139
2140 mtctr $rounds
2141 ${UCMP}i $len,16
2142 bge Loop_xts_enc
2143
2144 vxor $output,$output,$tweak
2145 lvsr $inpperm,0,$len # $inpperm is no longer needed
2146 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2147 vspltisb $tmp,-1
2148 vperm $inptail,$inptail,$tmp,$inpperm
2149 vsel $inout,$inout,$output,$inptail
2150
2151 subi r11,$out,17
2152 subi $out,$out,16
2153 mtctr $len
2154 li $len,16
2155 Loop_xts_enc_steal:
2156 lbzu r0,1(r11)
2157 stb r0,16(r11)
2158 bdnz Loop_xts_enc_steal
2159
2160 mtctr $rounds
2161 b Loop_xts_enc # one more time...
2162
2163 Lxts_enc_done:
2164 ${UCMP}i $ivp,0
2165 beq Lxts_enc_ret
2166
2167 vsrab $tmp,$tweak,$seven # next tweak value
2168 vaddubm $tweak,$tweak,$tweak
2169 vsldoi $tmp,$tmp,$tmp,15
2170 vand $tmp,$tmp,$eighty7
2171 vxor $tweak,$tweak,$tmp
2172
2173 le?vperm $tweak,$tweak,$tweak,$leperm
2174 stvx_u $tweak,0,$ivp
2175
2176 Lxts_enc_ret:
2177 mtspr 256,r12 # restore vrsave
2178 li r3,0
2179 blr
2180 .long 0
2181 .byte 0,12,0x04,0,0x80,6,6,0
2182 .long 0
2183 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2184
2185 .globl .${prefix}_xts_decrypt
2186 mr $inp,r3 # reassign
2187 li r3,-1
2188 ${UCMP}i $len,16
2189 bltlr-
2190
2191 lis r0,0xfff8
2192 mfspr r12,256 # save vrsave
2193 li r11,0
2194 mtspr 256,r0
2195
2196 andi. r0,$len,15
2197 neg r0,r0
2198 andi. r0,r0,16
2199 sub $len,$len,r0
2200
2201 vspltisb $seven,0x07 # 0x070707..07
2202 le?lvsl $leperm,r11,r11
2203 le?vspltisb $tmp,0x0f
2204 le?vxor $leperm,$leperm,$seven
2205
2206 li $idx,15
2207 lvx $tweak,0,$ivp # load [unaligned] iv
2208 lvsl $inpperm,0,$ivp
2209 lvx $inptail,$idx,$ivp
2210 le?vxor $inpperm,$inpperm,$tmp
2211 vperm $tweak,$tweak,$inptail,$inpperm
2212
2213 neg r11,$inp
2214 lvsr $inpperm,0,r11 # prepare for unaligned load
2215 lvx $inout,0,$inp
2216 addi $inp,$inp,15 # 15 is not typo
2217 le?vxor $inpperm,$inpperm,$tmp
2218
2219 ${UCMP}i $key2,0 # key2==NULL?
2220 beq Lxts_dec_no_key2
2221
2222 ?lvsl $keyperm,0,$key2 # prepare for unaligned key
2223 lwz $rounds,240($key2)
2224 srwi $rounds,$rounds,1
2225 subi $rounds,$rounds,1
2226 li $idx,16
2227
2228 lvx $rndkey0,0,$key2
2229 lvx $rndkey1,$idx,$key2
2230 addi $idx,$idx,16
2231 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2232 vxor $tweak,$tweak,$rndkey0
2233 lvx $rndkey0,$idx,$key2
2234 addi $idx,$idx,16
2235 mtctr $rounds
2236
2237 Ltweak_xts_dec:
2238 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2239 vcipher $tweak,$tweak,$rndkey1
2240 lvx $rndkey1,$idx,$key2
2241 addi $idx,$idx,16
2242 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2243 vcipher $tweak,$tweak,$rndkey0
2244 lvx $rndkey0,$idx,$key2
2245 addi $idx,$idx,16
2246 bdnz Ltweak_xts_dec
2247
2248 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2249 vcipher $tweak,$tweak,$rndkey1
2250 lvx $rndkey1,$idx,$key2
2251 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2252 vcipherlast $tweak,$tweak,$rndkey0
2253
2254 li $ivp,0 # don't chain the tweak
2255 b Lxts_dec
2256
2257 Lxts_dec_no_key2:
2258 neg $idx,$len
2259 andi. $idx,$idx,15
2260 add $len,$len,$idx # in "tweak chaining"
2261 # mode only complete
2262 # blocks are processed
2263 Lxts_dec:
2264 lvx $inptail,0,$inp
2265 addi $inp,$inp,16
2266
2267 ?lvsl $keyperm,0,$key1 # prepare for unaligned key
2268 lwz $rounds,240($key1)
2269 srwi $rounds,$rounds,1
2270 subi $rounds,$rounds,1
2271 li $idx,16
2272
2273 vslb $eighty7,$seven,$seven # 0x808080..80
2274 vor $eighty7,$eighty7,$seven # 0x878787..87
2275 vspltisb $tmp,1 # 0x010101..01
2276 vsldoi $eighty7,$eighty7,$tmp,15 # 0x870101..01
2277
2278 ${UCMP}i $len,96
2279 bge _aesp8_xts_decrypt6x
2280
2281 lvx $rndkey0,0,$key1
2282 lvx $rndkey1,$idx,$key1
2283 addi $idx,$idx,16
2284 vperm $inout,$inout,$inptail,$inpperm
2285 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2286 vxor $inout,$inout,$tweak
2287 vxor $inout,$inout,$rndkey0
2288 lvx $rndkey0,$idx,$key1
2289 addi $idx,$idx,16
2290 mtctr $rounds
2291
2292 ${UCMP}i $len,16
2293 blt Ltail_xts_dec
2294 be?b Loop_xts_dec
2295
2296 .align 5
2297 Loop_xts_dec:
2298 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2299 vncipher $inout,$inout,$rndkey1
2300 lvx $rndkey1,$idx,$key1
2301 addi $idx,$idx,16
2302 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2303 vncipher $inout,$inout,$rndkey0
2304 lvx $rndkey0,$idx,$key1
2305 addi $idx,$idx,16
2306 bdnz Loop_xts_dec
2307
2308 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2309 vncipher $inout,$inout,$rndkey1
2310 lvx $rndkey1,$idx,$key1
2311 li $idx,16
2312 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2313 vxor $rndkey0,$rndkey0,$tweak
2314 vncipherlast $output,$inout,$rndkey0
2315
2316 le?vperm $tmp,$output,$output,$leperm
2317 be?nop
2318 le?stvx_u $tmp,0,$out
2319 be?stvx_u $output,0,$out
2320 addi $out,$out,16
2321
2322 subic. $len,$len,16
2323 beq Lxts_dec_done
2324
2325 vmr $inout,$inptail
2326 lvx $inptail,0,$inp
2327 addi $inp,$inp,16
2328 lvx $rndkey0,0,$key1
2329 lvx $rndkey1,$idx,$key1
2330 addi $idx,$idx,16
2331
2332 vsrab $tmp,$tweak,$seven # next tweak value
2333 vaddubm $tweak,$tweak,$tweak
2334 vsldoi $tmp,$tmp,$tmp,15
2335 vand $tmp,$tmp,$eighty7
2336 vxor $tweak,$tweak,$tmp
2337
2338 vperm $inout,$inout,$inptail,$inpperm
2339 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2340 vxor $inout,$inout,$tweak
2341 vxor $inout,$inout,$rndkey0
2342 lvx $rndkey0,$idx,$key1
2343 addi $idx,$idx,16
2344
2345 mtctr $rounds
2346 ${UCMP}i $len,16
2347 bge Loop_xts_dec
2348
2349 Ltail_xts_dec:
2350 vsrab $tmp,$tweak,$seven # next tweak value
2351 vaddubm $tweak1,$tweak,$tweak
2352 vsldoi $tmp,$tmp,$tmp,15
2353 vand $tmp,$tmp,$eighty7
2354 vxor $tweak1,$tweak1,$tmp
2355
2356 subi $inp,$inp,16
2357 add $inp,$inp,$len
2358
2359 vxor $inout,$inout,$tweak # :-(
2360 vxor $inout,$inout,$tweak1 # :-)
2361
2362 Loop_xts_dec_short:
2363 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2364 vncipher $inout,$inout,$rndkey1
2365 lvx $rndkey1,$idx,$key1
2366 addi $idx,$idx,16
2367 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2368 vncipher $inout,$inout,$rndkey0
2369 lvx $rndkey0,$idx,$key1
2370 addi $idx,$idx,16
2371 bdnz Loop_xts_dec_short
2372
2373 ?vperm $rndkey1,$rndkey1,$rndkey0,$keyperm
2374 vncipher $inout,$inout,$rndkey1
2375 lvx $rndkey1,$idx,$key1
2376 li $idx,16
2377 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2378 vxor $rndkey0,$rndkey0,$tweak1
2379 vncipherlast $output,$inout,$rndkey0
2380
2381 le?vperm $tmp,$output,$output,$leperm
2382 be?nop
2383 le?stvx_u $tmp,0,$out
2384 be?stvx_u $output,0,$out
2385
2386 vmr $inout,$inptail
2387 lvx $inptail,0,$inp
2388 #addi $inp,$inp,16
2389 lvx $rndkey0,0,$key1
2390 lvx $rndkey1,$idx,$key1
2391 addi $idx,$idx,16
2392 vperm $inout,$inout,$inptail,$inpperm
2393 ?vperm $rndkey0,$rndkey0,$rndkey1,$keyperm
2394
2395 lvsr $inpperm,0,$len # $inpperm is no longer needed
2396 vxor $inptail,$inptail,$inptail # $inptail is no longer needed
2397 vspltisb $tmp,-1
2398 vperm $inptail,$inptail,$tmp,$inpperm
2399 vsel $inout,$inout,$output,$inptail
2400
2401 vxor $rndkey0,$rndkey0,$tweak
2402 vxor $inout,$inout,$rndkey0
2403 lvx $rndkey0,$idx,$key1
2404 addi $idx,$idx,16
2405
2406 subi r11,$out,1
2407 mtctr $len
2408 li $len,16
2409 Loop_xts_dec_steal:
2410 lbzu r0,1(r11)
2411 stb r0,16(r11)
2412 bdnz Loop_xts_dec_steal
2413
2414 mtctr $rounds
2415 b Loop_xts_dec # one more time...
2416
2417 Lxts_dec_done:
2418 ${UCMP}i $ivp,0
2419 beq Lxts_dec_ret
2420
2421 vsrab $tmp,$tweak,$seven # next tweak value
2422 vaddubm $tweak,$tweak,$tweak
2423 vsldoi $tmp,$tmp,$tmp,15
2424 vand $tmp,$tmp,$eighty7
2425 vxor $tweak,$tweak,$tmp
2426
2427 le?vperm $tweak,$tweak,$tweak,$leperm
2428 stvx_u $tweak,0,$ivp
2429
2430 Lxts_dec_ret:
2431 mtspr 256,r12 # restore vrsave
2432 li r3,0
2433 blr
2434 .long 0
2435 .byte 0,12,0x04,0,0x80,6,6,0
2436 .long 0
2437 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2438 ___
2439 #########################################################################
2440 {{ # Optimized XTS procedures #
2441 my $key_=$key2;
2442 my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
2443 $x00=0 if ($flavour =~ /osx/);
2444 my ($in0, $in1, $in2, $in3, $in4, $in5 )=map("v$_",(0..5));
2445 my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
2446 my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
2447 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2448 # v26-v31 last 6 round keys
2449 my ($keyperm)=($out0); # aliases with "caller", redundant assignment
2450 my $taillen=$x70;
2451
2452 $code.=<<___;
2453 .align 5
2454 _aesp8_xts_encrypt6x:
2455 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2456 mflr r11
2457 li r7,`$FRAME+8*16+15`
2458 li r3,`$FRAME+8*16+31`
2459 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
2460 stvx v20,r7,$sp # ABI says so
2461 addi r7,r7,32
2462 stvx v21,r3,$sp
2463 addi r3,r3,32
2464 stvx v22,r7,$sp
2465 addi r7,r7,32
2466 stvx v23,r3,$sp
2467 addi r3,r3,32
2468 stvx v24,r7,$sp
2469 addi r7,r7,32
2470 stvx v25,r3,$sp
2471 addi r3,r3,32
2472 stvx v26,r7,$sp
2473 addi r7,r7,32
2474 stvx v27,r3,$sp
2475 addi r3,r3,32
2476 stvx v28,r7,$sp
2477 addi r7,r7,32
2478 stvx v29,r3,$sp
2479 addi r3,r3,32
2480 stvx v30,r7,$sp
2481 stvx v31,r3,$sp
2482 li r0,-1
2483 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2484 li $x10,0x10
2485 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
2486 li $x20,0x20
2487 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
2488 li $x30,0x30
2489 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
2490 li $x40,0x40
2491 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
2492 li $x50,0x50
2493 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
2494 li $x60,0x60
2495 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
2496 li $x70,0x70
2497 mtspr 256,r0
2498
2499 xxlor 2, 32+$eighty7, 32+$eighty7
2500 vsldoi $eighty7,$tmp,$eighty7,1 # 0x010101..87
2501 xxlor 1, 32+$eighty7, 32+$eighty7
2502
2503 # Load XOR Lconsts.
2504 mr $x70, r6
2505 bl Lconsts
2506 lxvw4x 0, $x40, r6 # load XOR contents
2507 mr r6, $x70
2508 li $x70,0x70
2509
2510 subi $rounds,$rounds,3 # -4 in total
2511
2512 lvx $rndkey0,$x00,$key1 # load key schedule
2513 lvx v30,$x10,$key1
2514 addi $key1,$key1,0x20
2515 lvx v31,$x00,$key1
2516 ?vperm $rndkey0,$rndkey0,v30,$keyperm
2517 addi $key_,$sp,$FRAME+15
2518 mtctr $rounds
2519
2520 Load_xts_enc_key:
2521 ?vperm v24,v30,v31,$keyperm
2522 lvx v30,$x10,$key1
2523 addi $key1,$key1,0x20
2524 stvx v24,$x00,$key_ # off-load round[1]
2525 ?vperm v25,v31,v30,$keyperm
2526 lvx v31,$x00,$key1
2527 stvx v25,$x10,$key_ # off-load round[2]
2528 addi $key_,$key_,0x20
2529 bdnz Load_xts_enc_key
2530
2531 lvx v26,$x10,$key1
2532 ?vperm v24,v30,v31,$keyperm
2533 lvx v27,$x20,$key1
2534 stvx v24,$x00,$key_ # off-load round[3]
2535 ?vperm v25,v31,v26,$keyperm
2536 lvx v28,$x30,$key1
2537 stvx v25,$x10,$key_ # off-load round[4]
2538 addi $key_,$sp,$FRAME+15 # rewind $key_
2539 ?vperm v26,v26,v27,$keyperm
2540 lvx v29,$x40,$key1
2541 ?vperm v27,v27,v28,$keyperm
2542 lvx v30,$x50,$key1
2543 ?vperm v28,v28,v29,$keyperm
2544 lvx v31,$x60,$key1
2545 ?vperm v29,v29,v30,$keyperm
2546 lvx $twk5,$x70,$key1 # borrow $twk5
2547 ?vperm v30,v30,v31,$keyperm
2548 lvx v24,$x00,$key_ # pre-load round[1]
2549 ?vperm v31,v31,$twk5,$keyperm
2550 lvx v25,$x10,$key_ # pre-load round[2]
2551
2552 # Switch to use the following codes with 0x010101..87 to generate tweak.
2553 # eighty7 = 0x010101..87
2554 # vsrab tmp, tweak, seven # next tweak value, right shift 7 bits
2555 # vand tmp, tmp, eighty7 # last byte with carry
2556 # vaddubm tweak, tweak, tweak # left shift 1 bit (x2)
2557 # xxlor vsx, 0, 0
2558 # vpermxor tweak, tweak, tmp, vsx
2559
2560 vperm $in0,$inout,$inptail,$inpperm
2561 subi $inp,$inp,31 # undo "caller"
2562 vxor $twk0,$tweak,$rndkey0
2563 vsrab $tmp,$tweak,$seven # next tweak value
2564 vaddubm $tweak,$tweak,$tweak
2565 vand $tmp,$tmp,$eighty7
2566 vxor $out0,$in0,$twk0
2567 xxlor 32+$in1, 0, 0
2568 vpermxor $tweak, $tweak, $tmp, $in1
2569
2570 lvx_u $in1,$x10,$inp
2571 vxor $twk1,$tweak,$rndkey0
2572 vsrab $tmp,$tweak,$seven # next tweak value
2573 vaddubm $tweak,$tweak,$tweak
2574 le?vperm $in1,$in1,$in1,$leperm
2575 vand $tmp,$tmp,$eighty7
2576 vxor $out1,$in1,$twk1
2577 xxlor 32+$in2, 0, 0
2578 vpermxor $tweak, $tweak, $tmp, $in2
2579
2580 lvx_u $in2,$x20,$inp
2581 andi. $taillen,$len,15
2582 vxor $twk2,$tweak,$rndkey0
2583 vsrab $tmp,$tweak,$seven # next tweak value
2584 vaddubm $tweak,$tweak,$tweak
2585 le?vperm $in2,$in2,$in2,$leperm
2586 vand $tmp,$tmp,$eighty7
2587 vxor $out2,$in2,$twk2
2588 xxlor 32+$in3, 0, 0
2589 vpermxor $tweak, $tweak, $tmp, $in3
2590
2591 lvx_u $in3,$x30,$inp
2592 sub $len,$len,$taillen
2593 vxor $twk3,$tweak,$rndkey0
2594 vsrab $tmp,$tweak,$seven # next tweak value
2595 vaddubm $tweak,$tweak,$tweak
2596 le?vperm $in3,$in3,$in3,$leperm
2597 vand $tmp,$tmp,$eighty7
2598 vxor $out3,$in3,$twk3
2599 xxlor 32+$in4, 0, 0
2600 vpermxor $tweak, $tweak, $tmp, $in4
2601
2602 lvx_u $in4,$x40,$inp
2603 subi $len,$len,0x60
2604 vxor $twk4,$tweak,$rndkey0
2605 vsrab $tmp,$tweak,$seven # next tweak value
2606 vaddubm $tweak,$tweak,$tweak
2607 le?vperm $in4,$in4,$in4,$leperm
2608 vand $tmp,$tmp,$eighty7
2609 vxor $out4,$in4,$twk4
2610 xxlor 32+$in5, 0, 0
2611 vpermxor $tweak, $tweak, $tmp, $in5
2612
2613 lvx_u $in5,$x50,$inp
2614 addi $inp,$inp,0x60
2615 vxor $twk5,$tweak,$rndkey0
2616 vsrab $tmp,$tweak,$seven # next tweak value
2617 vaddubm $tweak,$tweak,$tweak
2618 le?vperm $in5,$in5,$in5,$leperm
2619 vand $tmp,$tmp,$eighty7
2620 vxor $out5,$in5,$twk5
2621 xxlor 32+$in0, 0, 0
2622 vpermxor $tweak, $tweak, $tmp, $in0
2623
2624 vxor v31,v31,$rndkey0
2625 mtctr $rounds
2626 b Loop_xts_enc6x
2627
2628 .align 5
2629 Loop_xts_enc6x:
2630 vcipher $out0,$out0,v24
2631 vcipher $out1,$out1,v24
2632 vcipher $out2,$out2,v24
2633 vcipher $out3,$out3,v24
2634 vcipher $out4,$out4,v24
2635 vcipher $out5,$out5,v24
2636 lvx v24,$x20,$key_ # round[3]
2637 addi $key_,$key_,0x20
2638
2639 vcipher $out0,$out0,v25
2640 vcipher $out1,$out1,v25
2641 vcipher $out2,$out2,v25
2642 vcipher $out3,$out3,v25
2643 vcipher $out4,$out4,v25
2644 vcipher $out5,$out5,v25
2645 lvx v25,$x10,$key_ # round[4]
2646 bdnz Loop_xts_enc6x
2647
2648 xxlor 32+$eighty7, 1, 1 # 0x010101..87
2649
2650 subic $len,$len,96 # $len-=96
2651 vxor $in0,$twk0,v31 # xor with last round key
2652 vcipher $out0,$out0,v24
2653 vcipher $out1,$out1,v24
2654 vsrab $tmp,$tweak,$seven # next tweak value
2655 vxor $twk0,$tweak,$rndkey0
2656 vaddubm $tweak,$tweak,$tweak
2657 vcipher $out2,$out2,v24
2658 vcipher $out3,$out3,v24
2659 vcipher $out4,$out4,v24
2660 vcipher $out5,$out5,v24
2661
2662 subfe. r0,r0,r0 # borrow?-1:0
2663 vand $tmp,$tmp,$eighty7
2664 vcipher $out0,$out0,v25
2665 vcipher $out1,$out1,v25
2666 xxlor 32+$in1, 0, 0
2667 vpermxor $tweak, $tweak, $tmp, $in1
2668 vcipher $out2,$out2,v25
2669 vcipher $out3,$out3,v25
2670 vxor $in1,$twk1,v31
2671 vsrab $tmp,$tweak,$seven # next tweak value
2672 vxor $twk1,$tweak,$rndkey0
2673 vcipher $out4,$out4,v25
2674 vcipher $out5,$out5,v25
2675
2676 and r0,r0,$len
2677 vaddubm $tweak,$tweak,$tweak
2678 vcipher $out0,$out0,v26
2679 vcipher $out1,$out1,v26
2680 vand $tmp,$tmp,$eighty7
2681 vcipher $out2,$out2,v26
2682 vcipher $out3,$out3,v26
2683 xxlor 32+$in2, 0, 0
2684 vpermxor $tweak, $tweak, $tmp, $in2
2685 vcipher $out4,$out4,v26
2686 vcipher $out5,$out5,v26
2687
2688 add $inp,$inp,r0 # $inp is adjusted in such
2689 # way that at exit from the
2690 # loop inX-in5 are loaded
2691 # with last "words"
2692 vxor $in2,$twk2,v31
2693 vsrab $tmp,$tweak,$seven # next tweak value
2694 vxor $twk2,$tweak,$rndkey0
2695 vaddubm $tweak,$tweak,$tweak
2696 vcipher $out0,$out0,v27
2697 vcipher $out1,$out1,v27
2698 vcipher $out2,$out2,v27
2699 vcipher $out3,$out3,v27
2700 vand $tmp,$tmp,$eighty7
2701 vcipher $out4,$out4,v27
2702 vcipher $out5,$out5,v27
2703
2704 addi $key_,$sp,$FRAME+15 # rewind $key_
2705 xxlor 32+$in3, 0, 0
2706 vpermxor $tweak, $tweak, $tmp, $in3
2707 vcipher $out0,$out0,v28
2708 vcipher $out1,$out1,v28
2709 vxor $in3,$twk3,v31
2710 vsrab $tmp,$tweak,$seven # next tweak value
2711 vxor $twk3,$tweak,$rndkey0
2712 vcipher $out2,$out2,v28
2713 vcipher $out3,$out3,v28
2714 vaddubm $tweak,$tweak,$tweak
2715 vcipher $out4,$out4,v28
2716 vcipher $out5,$out5,v28
2717 lvx v24,$x00,$key_ # re-pre-load round[1]
2718 vand $tmp,$tmp,$eighty7
2719
2720 vcipher $out0,$out0,v29
2721 vcipher $out1,$out1,v29
2722 xxlor 32+$in4, 0, 0
2723 vpermxor $tweak, $tweak, $tmp, $in4
2724 vcipher $out2,$out2,v29
2725 vcipher $out3,$out3,v29
2726 vxor $in4,$twk4,v31
2727 vsrab $tmp,$tweak,$seven # next tweak value
2728 vxor $twk4,$tweak,$rndkey0
2729 vcipher $out4,$out4,v29
2730 vcipher $out5,$out5,v29
2731 lvx v25,$x10,$key_ # re-pre-load round[2]
2732 vaddubm $tweak,$tweak,$tweak
2733
2734 vcipher $out0,$out0,v30
2735 vcipher $out1,$out1,v30
2736 vand $tmp,$tmp,$eighty7
2737 vcipher $out2,$out2,v30
2738 vcipher $out3,$out3,v30
2739 xxlor 32+$in5, 0, 0
2740 vpermxor $tweak, $tweak, $tmp, $in5
2741 vcipher $out4,$out4,v30
2742 vcipher $out5,$out5,v30
2743 vxor $in5,$twk5,v31
2744 vsrab $tmp,$tweak,$seven # next tweak value
2745 vxor $twk5,$tweak,$rndkey0
2746
2747 vcipherlast $out0,$out0,$in0
2748 lvx_u $in0,$x00,$inp # load next input block
2749 vaddubm $tweak,$tweak,$tweak
2750 vcipherlast $out1,$out1,$in1
2751 lvx_u $in1,$x10,$inp
2752 vcipherlast $out2,$out2,$in2
2753 le?vperm $in0,$in0,$in0,$leperm
2754 lvx_u $in2,$x20,$inp
2755 vand $tmp,$tmp,$eighty7
2756 vcipherlast $out3,$out3,$in3
2757 le?vperm $in1,$in1,$in1,$leperm
2758 lvx_u $in3,$x30,$inp
2759 vcipherlast $out4,$out4,$in4
2760 le?vperm $in2,$in2,$in2,$leperm
2761 lvx_u $in4,$x40,$inp
2762 xxlor 10, 32+$in0, 32+$in0
2763 xxlor 32+$in0, 0, 0
2764 vpermxor $tweak, $tweak, $tmp, $in0
2765 xxlor 32+$in0, 10, 10
2766 vcipherlast $tmp,$out5,$in5 # last block might be needed
2767 # in stealing mode
2768 le?vperm $in3,$in3,$in3,$leperm
2769 lvx_u $in5,$x50,$inp
2770 addi $inp,$inp,0x60
2771 le?vperm $in4,$in4,$in4,$leperm
2772 le?vperm $in5,$in5,$in5,$leperm
2773
2774 le?vperm $out0,$out0,$out0,$leperm
2775 le?vperm $out1,$out1,$out1,$leperm
2776 stvx_u $out0,$x00,$out # store output
2777 vxor $out0,$in0,$twk0
2778 le?vperm $out2,$out2,$out2,$leperm
2779 stvx_u $out1,$x10,$out
2780 vxor $out1,$in1,$twk1
2781 le?vperm $out3,$out3,$out3,$leperm
2782 stvx_u $out2,$x20,$out
2783 vxor $out2,$in2,$twk2
2784 le?vperm $out4,$out4,$out4,$leperm
2785 stvx_u $out3,$x30,$out
2786 vxor $out3,$in3,$twk3
2787 le?vperm $out5,$tmp,$tmp,$leperm
2788 stvx_u $out4,$x40,$out
2789 vxor $out4,$in4,$twk4
2790 le?stvx_u $out5,$x50,$out
2791 be?stvx_u $tmp, $x50,$out
2792 vxor $out5,$in5,$twk5
2793 addi $out,$out,0x60
2794
2795 mtctr $rounds
2796 beq Loop_xts_enc6x # did $len-=96 borrow?
2797
2798 xxlor 32+$eighty7, 2, 2 # 0x010101..87
2799
2800 addic. $len,$len,0x60
2801 beq Lxts_enc6x_zero
2802 cmpwi $len,0x20
2803 blt Lxts_enc6x_one
2804 nop
2805 beq Lxts_enc6x_two
2806 cmpwi $len,0x40
2807 blt Lxts_enc6x_three
2808 nop
2809 beq Lxts_enc6x_four
2810
2811 Lxts_enc6x_five:
2812 vxor $out0,$in1,$twk0
2813 vxor $out1,$in2,$twk1
2814 vxor $out2,$in3,$twk2
2815 vxor $out3,$in4,$twk3
2816 vxor $out4,$in5,$twk4
2817
2818 bl _aesp8_xts_enc5x
2819
2820 le?vperm $out0,$out0,$out0,$leperm
2821 vmr $twk0,$twk5 # unused tweak
2822 le?vperm $out1,$out1,$out1,$leperm
2823 stvx_u $out0,$x00,$out # store output
2824 le?vperm $out2,$out2,$out2,$leperm
2825 stvx_u $out1,$x10,$out
2826 le?vperm $out3,$out3,$out3,$leperm
2827 stvx_u $out2,$x20,$out
2828 vxor $tmp,$out4,$twk5 # last block prep for stealing
2829 le?vperm $out4,$out4,$out4,$leperm
2830 stvx_u $out3,$x30,$out
2831 stvx_u $out4,$x40,$out
2832 addi $out,$out,0x50
2833 bne Lxts_enc6x_steal
2834 b Lxts_enc6x_done
2835
2836 .align 4
2837 Lxts_enc6x_four:
2838 vxor $out0,$in2,$twk0
2839 vxor $out1,$in3,$twk1
2840 vxor $out2,$in4,$twk2
2841 vxor $out3,$in5,$twk3
2842 vxor $out4,$out4,$out4
2843
2844 bl _aesp8_xts_enc5x
2845
2846 le?vperm $out0,$out0,$out0,$leperm
2847 vmr $twk0,$twk4 # unused tweak
2848 le?vperm $out1,$out1,$out1,$leperm
2849 stvx_u $out0,$x00,$out # store output
2850 le?vperm $out2,$out2,$out2,$leperm
2851 stvx_u $out1,$x10,$out
2852 vxor $tmp,$out3,$twk4 # last block prep for stealing
2853 le?vperm $out3,$out3,$out3,$leperm
2854 stvx_u $out2,$x20,$out
2855 stvx_u $out3,$x30,$out
2856 addi $out,$out,0x40
2857 bne Lxts_enc6x_steal
2858 b Lxts_enc6x_done
2859
2860 .align 4
2861 Lxts_enc6x_three:
2862 vxor $out0,$in3,$twk0
2863 vxor $out1,$in4,$twk1
2864 vxor $out2,$in5,$twk2
2865 vxor $out3,$out3,$out3
2866 vxor $out4,$out4,$out4
2867
2868 bl _aesp8_xts_enc5x
2869
2870 le?vperm $out0,$out0,$out0,$leperm
2871 vmr $twk0,$twk3 # unused tweak
2872 le?vperm $out1,$out1,$out1,$leperm
2873 stvx_u $out0,$x00,$out # store output
2874 vxor $tmp,$out2,$twk3 # last block prep for stealing
2875 le?vperm $out2,$out2,$out2,$leperm
2876 stvx_u $out1,$x10,$out
2877 stvx_u $out2,$x20,$out
2878 addi $out,$out,0x30
2879 bne Lxts_enc6x_steal
2880 b Lxts_enc6x_done
2881
2882 .align 4
2883 Lxts_enc6x_two:
2884 vxor $out0,$in4,$twk0
2885 vxor $out1,$in5,$twk1
2886 vxor $out2,$out2,$out2
2887 vxor $out3,$out3,$out3
2888 vxor $out4,$out4,$out4
2889
2890 bl _aesp8_xts_enc5x
2891
2892 le?vperm $out0,$out0,$out0,$leperm
2893 vmr $twk0,$twk2 # unused tweak
2894 vxor $tmp,$out1,$twk2 # last block prep for stealing
2895 le?vperm $out1,$out1,$out1,$leperm
2896 stvx_u $out0,$x00,$out # store output
2897 stvx_u $out1,$x10,$out
2898 addi $out,$out,0x20
2899 bne Lxts_enc6x_steal
2900 b Lxts_enc6x_done
2901
2902 .align 4
2903 Lxts_enc6x_one:
2904 vxor $out0,$in5,$twk0
2905 nop
2906 Loop_xts_enc1x:
2907 vcipher $out0,$out0,v24
2908 lvx v24,$x20,$key_ # round[3]
2909 addi $key_,$key_,0x20
2910
2911 vcipher $out0,$out0,v25
2912 lvx v25,$x10,$key_ # round[4]
2913 bdnz Loop_xts_enc1x
2914
2915 add $inp,$inp,$taillen
2916 cmpwi $taillen,0
2917 vcipher $out0,$out0,v24
2918
2919 subi $inp,$inp,16
2920 vcipher $out0,$out0,v25
2921
2922 lvsr $inpperm,0,$taillen
2923 vcipher $out0,$out0,v26
2924
2925 lvx_u $in0,0,$inp
2926 vcipher $out0,$out0,v27
2927
2928 addi $key_,$sp,$FRAME+15 # rewind $key_
2929 vcipher $out0,$out0,v28
2930 lvx v24,$x00,$key_ # re-pre-load round[1]
2931
2932 vcipher $out0,$out0,v29
2933 lvx v25,$x10,$key_ # re-pre-load round[2]
2934 vxor $twk0,$twk0,v31
2935
2936 le?vperm $in0,$in0,$in0,$leperm
2937 vcipher $out0,$out0,v30
2938
2939 vperm $in0,$in0,$in0,$inpperm
2940 vcipherlast $out0,$out0,$twk0
2941
2942 vmr $twk0,$twk1 # unused tweak
2943 vxor $tmp,$out0,$twk1 # last block prep for stealing
2944 le?vperm $out0,$out0,$out0,$leperm
2945 stvx_u $out0,$x00,$out # store output
2946 addi $out,$out,0x10
2947 bne Lxts_enc6x_steal
2948 b Lxts_enc6x_done
2949
2950 .align 4
2951 Lxts_enc6x_zero:
2952 cmpwi $taillen,0
2953 beq Lxts_enc6x_done
2954
2955 add $inp,$inp,$taillen
2956 subi $inp,$inp,16
2957 lvx_u $in0,0,$inp
2958 lvsr $inpperm,0,$taillen # $in5 is no more
2959 le?vperm $in0,$in0,$in0,$leperm
2960 vperm $in0,$in0,$in0,$inpperm
2961 vxor $tmp,$tmp,$twk0
2962 Lxts_enc6x_steal:
2963 vxor $in0,$in0,$twk0
2964 vxor $out0,$out0,$out0
2965 vspltisb $out1,-1
2966 vperm $out0,$out0,$out1,$inpperm
2967 vsel $out0,$in0,$tmp,$out0 # $tmp is last block, remember?
2968
2969 subi r30,$out,17
2970 subi $out,$out,16
2971 mtctr $taillen
2972 Loop_xts_enc6x_steal:
2973 lbzu r0,1(r30)
2974 stb r0,16(r30)
2975 bdnz Loop_xts_enc6x_steal
2976
2977 li $taillen,0
2978 mtctr $rounds
2979 b Loop_xts_enc1x # one more time...
2980
2981 .align 4
2982 Lxts_enc6x_done:
2983 ${UCMP}i $ivp,0
2984 beq Lxts_enc6x_ret
2985
2986 vxor $tweak,$twk0,$rndkey0
2987 le?vperm $tweak,$tweak,$tweak,$leperm
2988 stvx_u $tweak,0,$ivp
2989
2990 Lxts_enc6x_ret:
2991 mtlr r11
2992 li r10,`$FRAME+15`
2993 li r11,`$FRAME+31`
2994 stvx $seven,r10,$sp # wipe copies of round keys
2995 addi r10,r10,32
2996 stvx $seven,r11,$sp
2997 addi r11,r11,32
2998 stvx $seven,r10,$sp
2999 addi r10,r10,32
3000 stvx $seven,r11,$sp
3001 addi r11,r11,32
3002 stvx $seven,r10,$sp
3003 addi r10,r10,32
3004 stvx $seven,r11,$sp
3005 addi r11,r11,32
3006 stvx $seven,r10,$sp
3007 addi r10,r10,32
3008 stvx $seven,r11,$sp
3009 addi r11,r11,32
3010
3011 mtspr 256,$vrsave
3012 lvx v20,r10,$sp # ABI says so
3013 addi r10,r10,32
3014 lvx v21,r11,$sp
3015 addi r11,r11,32
3016 lvx v22,r10,$sp
3017 addi r10,r10,32
3018 lvx v23,r11,$sp
3019 addi r11,r11,32
3020 lvx v24,r10,$sp
3021 addi r10,r10,32
3022 lvx v25,r11,$sp
3023 addi r11,r11,32
3024 lvx v26,r10,$sp
3025 addi r10,r10,32
3026 lvx v27,r11,$sp
3027 addi r11,r11,32
3028 lvx v28,r10,$sp
3029 addi r10,r10,32
3030 lvx v29,r11,$sp
3031 addi r11,r11,32
3032 lvx v30,r10,$sp
3033 lvx v31,r11,$sp
3034 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3035 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3036 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3037 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3038 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3039 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3040 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3041 blr
3042 .long 0
3043 .byte 0,12,0x04,1,0x80,6,6,0
3044 .long 0
3045
3046 .align 5
3047 _aesp8_xts_enc5x:
3048 vcipher $out0,$out0,v24
3049 vcipher $out1,$out1,v24
3050 vcipher $out2,$out2,v24
3051 vcipher $out3,$out3,v24
3052 vcipher $out4,$out4,v24
3053 lvx v24,$x20,$key_ # round[3]
3054 addi $key_,$key_,0x20
3055
3056 vcipher $out0,$out0,v25
3057 vcipher $out1,$out1,v25
3058 vcipher $out2,$out2,v25
3059 vcipher $out3,$out3,v25
3060 vcipher $out4,$out4,v25
3061 lvx v25,$x10,$key_ # round[4]
3062 bdnz _aesp8_xts_enc5x
3063
3064 add $inp,$inp,$taillen
3065 cmpwi $taillen,0
3066 vcipher $out0,$out0,v24
3067 vcipher $out1,$out1,v24
3068 vcipher $out2,$out2,v24
3069 vcipher $out3,$out3,v24
3070 vcipher $out4,$out4,v24
3071
3072 subi $inp,$inp,16
3073 vcipher $out0,$out0,v25
3074 vcipher $out1,$out1,v25
3075 vcipher $out2,$out2,v25
3076 vcipher $out3,$out3,v25
3077 vcipher $out4,$out4,v25
3078 vxor $twk0,$twk0,v31
3079
3080 vcipher $out0,$out0,v26
3081 lvsr $inpperm,r0,$taillen # $in5 is no more
3082 vcipher $out1,$out1,v26
3083 vcipher $out2,$out2,v26
3084 vcipher $out3,$out3,v26
3085 vcipher $out4,$out4,v26
3086 vxor $in1,$twk1,v31
3087
3088 vcipher $out0,$out0,v27
3089 lvx_u $in0,0,$inp
3090 vcipher $out1,$out1,v27
3091 vcipher $out2,$out2,v27
3092 vcipher $out3,$out3,v27
3093 vcipher $out4,$out4,v27
3094 vxor $in2,$twk2,v31
3095
3096 addi $key_,$sp,$FRAME+15 # rewind $key_
3097 vcipher $out0,$out0,v28
3098 vcipher $out1,$out1,v28
3099 vcipher $out2,$out2,v28
3100 vcipher $out3,$out3,v28
3101 vcipher $out4,$out4,v28
3102 lvx v24,$x00,$key_ # re-pre-load round[1]
3103 vxor $in3,$twk3,v31
3104
3105 vcipher $out0,$out0,v29
3106 le?vperm $in0,$in0,$in0,$leperm
3107 vcipher $out1,$out1,v29
3108 vcipher $out2,$out2,v29
3109 vcipher $out3,$out3,v29
3110 vcipher $out4,$out4,v29
3111 lvx v25,$x10,$key_ # re-pre-load round[2]
3112 vxor $in4,$twk4,v31
3113
3114 vcipher $out0,$out0,v30
3115 vperm $in0,$in0,$in0,$inpperm
3116 vcipher $out1,$out1,v30
3117 vcipher $out2,$out2,v30
3118 vcipher $out3,$out3,v30
3119 vcipher $out4,$out4,v30
3120
3121 vcipherlast $out0,$out0,$twk0
3122 vcipherlast $out1,$out1,$in1
3123 vcipherlast $out2,$out2,$in2
3124 vcipherlast $out3,$out3,$in3
3125 vcipherlast $out4,$out4,$in4
3126 blr
3127 .long 0
3128 .byte 0,12,0x14,0,0,0,0,0
3129
3130 .align 5
3131 _aesp8_xts_decrypt6x:
3132 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3133 mflr r11
3134 li r7,`$FRAME+8*16+15`
3135 li r3,`$FRAME+8*16+31`
3136 $PUSH r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
3137 stvx v20,r7,$sp # ABI says so
3138 addi r7,r7,32
3139 stvx v21,r3,$sp
3140 addi r3,r3,32
3141 stvx v22,r7,$sp
3142 addi r7,r7,32
3143 stvx v23,r3,$sp
3144 addi r3,r3,32
3145 stvx v24,r7,$sp
3146 addi r7,r7,32
3147 stvx v25,r3,$sp
3148 addi r3,r3,32
3149 stvx v26,r7,$sp
3150 addi r7,r7,32
3151 stvx v27,r3,$sp
3152 addi r3,r3,32
3153 stvx v28,r7,$sp
3154 addi r7,r7,32
3155 stvx v29,r3,$sp
3156 addi r3,r3,32
3157 stvx v30,r7,$sp
3158 stvx v31,r3,$sp
3159 li r0,-1
3160 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3161 li $x10,0x10
3162 $PUSH r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3163 li $x20,0x20
3164 $PUSH r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3165 li $x30,0x30
3166 $PUSH r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3167 li $x40,0x40
3168 $PUSH r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3169 li $x50,0x50
3170 $PUSH r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3171 li $x60,0x60
3172 $PUSH r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3173 li $x70,0x70
3174 mtspr 256,r0
3175
3176 xxlor 2, 32+$eighty7, 32+$eighty7
3177 vsldoi $eighty7,$tmp,$eighty7,1 # 0x010101..87
3178 xxlor 1, 32+$eighty7, 32+$eighty7
3179
3180 # Load XOR Lconsts.
3181 mr $x70, r6
3182 bl Lconsts
3183 lxvw4x 0, $x40, r6 # load XOR contents
3184 mr r6, $x70
3185 li $x70,0x70
3186
3187 subi $rounds,$rounds,3 # -4 in total
3188
3189 lvx $rndkey0,$x00,$key1 # load key schedule
3190 lvx v30,$x10,$key1
3191 addi $key1,$key1,0x20
3192 lvx v31,$x00,$key1
3193 ?vperm $rndkey0,$rndkey0,v30,$keyperm
3194 addi $key_,$sp,$FRAME+15
3195 mtctr $rounds
3196
3197 Load_xts_dec_key:
3198 ?vperm v24,v30,v31,$keyperm
3199 lvx v30,$x10,$key1
3200 addi $key1,$key1,0x20
3201 stvx v24,$x00,$key_ # off-load round[1]
3202 ?vperm v25,v31,v30,$keyperm
3203 lvx v31,$x00,$key1
3204 stvx v25,$x10,$key_ # off-load round[2]
3205 addi $key_,$key_,0x20
3206 bdnz Load_xts_dec_key
3207
3208 lvx v26,$x10,$key1
3209 ?vperm v24,v30,v31,$keyperm
3210 lvx v27,$x20,$key1
3211 stvx v24,$x00,$key_ # off-load round[3]
3212 ?vperm v25,v31,v26,$keyperm
3213 lvx v28,$x30,$key1
3214 stvx v25,$x10,$key_ # off-load round[4]
3215 addi $key_,$sp,$FRAME+15 # rewind $key_
3216 ?vperm v26,v26,v27,$keyperm
3217 lvx v29,$x40,$key1
3218 ?vperm v27,v27,v28,$keyperm
3219 lvx v30,$x50,$key1
3220 ?vperm v28,v28,v29,$keyperm
3221 lvx v31,$x60,$key1
3222 ?vperm v29,v29,v30,$keyperm
3223 lvx $twk5,$x70,$key1 # borrow $twk5
3224 ?vperm v30,v30,v31,$keyperm
3225 lvx v24,$x00,$key_ # pre-load round[1]
3226 ?vperm v31,v31,$twk5,$keyperm
3227 lvx v25,$x10,$key_ # pre-load round[2]
3228
3229 vperm $in0,$inout,$inptail,$inpperm
3230 subi $inp,$inp,31 # undo "caller"
3231 vxor $twk0,$tweak,$rndkey0
3232 vsrab $tmp,$tweak,$seven # next tweak value
3233 vaddubm $tweak,$tweak,$tweak
3234 vand $tmp,$tmp,$eighty7
3235 vxor $out0,$in0,$twk0
3236 xxlor 32+$in1, 0, 0
3237 vpermxor $tweak, $tweak, $tmp, $in1
3238
3239 lvx_u $in1,$x10,$inp
3240 vxor $twk1,$tweak,$rndkey0
3241 vsrab $tmp,$tweak,$seven # next tweak value
3242 vaddubm $tweak,$tweak,$tweak
3243 le?vperm $in1,$in1,$in1,$leperm
3244 vand $tmp,$tmp,$eighty7
3245 vxor $out1,$in1,$twk1
3246 xxlor 32+$in2, 0, 0
3247 vpermxor $tweak, $tweak, $tmp, $in2
3248
3249 lvx_u $in2,$x20,$inp
3250 andi. $taillen,$len,15
3251 vxor $twk2,$tweak,$rndkey0
3252 vsrab $tmp,$tweak,$seven # next tweak value
3253 vaddubm $tweak,$tweak,$tweak
3254 le?vperm $in2,$in2,$in2,$leperm
3255 vand $tmp,$tmp,$eighty7
3256 vxor $out2,$in2,$twk2
3257 xxlor 32+$in3, 0, 0
3258 vpermxor $tweak, $tweak, $tmp, $in3
3259
3260 lvx_u $in3,$x30,$inp
3261 sub $len,$len,$taillen
3262 vxor $twk3,$tweak,$rndkey0
3263 vsrab $tmp,$tweak,$seven # next tweak value
3264 vaddubm $tweak,$tweak,$tweak
3265 le?vperm $in3,$in3,$in3,$leperm
3266 vand $tmp,$tmp,$eighty7
3267 vxor $out3,$in3,$twk3
3268 xxlor 32+$in4, 0, 0
3269 vpermxor $tweak, $tweak, $tmp, $in4
3270
3271 lvx_u $in4,$x40,$inp
3272 subi $len,$len,0x60
3273 vxor $twk4,$tweak,$rndkey0
3274 vsrab $tmp,$tweak,$seven # next tweak value
3275 vaddubm $tweak,$tweak,$tweak
3276 le?vperm $in4,$in4,$in4,$leperm
3277 vand $tmp,$tmp,$eighty7
3278 vxor $out4,$in4,$twk4
3279 xxlor 32+$in5, 0, 0
3280 vpermxor $tweak, $tweak, $tmp, $in5
3281
3282 lvx_u $in5,$x50,$inp
3283 addi $inp,$inp,0x60
3284 vxor $twk5,$tweak,$rndkey0
3285 vsrab $tmp,$tweak,$seven # next tweak value
3286 vaddubm $tweak,$tweak,$tweak
3287 le?vperm $in5,$in5,$in5,$leperm
3288 vand $tmp,$tmp,$eighty7
3289 vxor $out5,$in5,$twk5
3290 xxlor 32+$in0, 0, 0
3291 vpermxor $tweak, $tweak, $tmp, $in0
3292
3293 vxor v31,v31,$rndkey0
3294 mtctr $rounds
3295 b Loop_xts_dec6x
3296
3297 .align 5
3298 Loop_xts_dec6x:
3299 vncipher $out0,$out0,v24
3300 vncipher $out1,$out1,v24
3301 vncipher $out2,$out2,v24
3302 vncipher $out3,$out3,v24
3303 vncipher $out4,$out4,v24
3304 vncipher $out5,$out5,v24
3305 lvx v24,$x20,$key_ # round[3]
3306 addi $key_,$key_,0x20
3307
3308 vncipher $out0,$out0,v25
3309 vncipher $out1,$out1,v25
3310 vncipher $out2,$out2,v25
3311 vncipher $out3,$out3,v25
3312 vncipher $out4,$out4,v25
3313 vncipher $out5,$out5,v25
3314 lvx v25,$x10,$key_ # round[4]
3315 bdnz Loop_xts_dec6x
3316
3317 xxlor 32+$eighty7, 1, 1 # 0x010101..87
3318
3319 subic $len,$len,96 # $len-=96
3320 vxor $in0,$twk0,v31 # xor with last round key
3321 vncipher $out0,$out0,v24
3322 vncipher $out1,$out1,v24
3323 vsrab $tmp,$tweak,$seven # next tweak value
3324 vxor $twk0,$tweak,$rndkey0
3325 vaddubm $tweak,$tweak,$tweak
3326 vncipher $out2,$out2,v24
3327 vncipher $out3,$out3,v24
3328 vncipher $out4,$out4,v24
3329 vncipher $out5,$out5,v24
3330
3331 subfe. r0,r0,r0 # borrow?-1:0
3332 vand $tmp,$tmp,$eighty7
3333 vncipher $out0,$out0,v25
3334 vncipher $out1,$out1,v25
3335 xxlor 32+$in1, 0, 0
3336 vpermxor $tweak, $tweak, $tmp, $in1
3337 vncipher $out2,$out2,v25
3338 vncipher $out3,$out3,v25
3339 vxor $in1,$twk1,v31
3340 vsrab $tmp,$tweak,$seven # next tweak value
3341 vxor $twk1,$tweak,$rndkey0
3342 vncipher $out4,$out4,v25
3343 vncipher $out5,$out5,v25
3344
3345 and r0,r0,$len
3346 vaddubm $tweak,$tweak,$tweak
3347 vncipher $out0,$out0,v26
3348 vncipher $out1,$out1,v26
3349 vand $tmp,$tmp,$eighty7
3350 vncipher $out2,$out2,v26
3351 vncipher $out3,$out3,v26
3352 xxlor 32+$in2, 0, 0
3353 vpermxor $tweak, $tweak, $tmp, $in2
3354 vncipher $out4,$out4,v26
3355 vncipher $out5,$out5,v26
3356
3357 add $inp,$inp,r0 # $inp is adjusted in such
3358 # way that at exit from the
3359 # loop inX-in5 are loaded
3360 # with last "words"
3361 vxor $in2,$twk2,v31
3362 vsrab $tmp,$tweak,$seven # next tweak value
3363 vxor $twk2,$tweak,$rndkey0
3364 vaddubm $tweak,$tweak,$tweak
3365 vncipher $out0,$out0,v27
3366 vncipher $out1,$out1,v27
3367 vncipher $out2,$out2,v27
3368 vncipher $out3,$out3,v27
3369 vand $tmp,$tmp,$eighty7
3370 vncipher $out4,$out4,v27
3371 vncipher $out5,$out5,v27
3372
3373 addi $key_,$sp,$FRAME+15 # rewind $key_
3374 xxlor 32+$in3, 0, 0
3375 vpermxor $tweak, $tweak, $tmp, $in3
3376 vncipher $out0,$out0,v28
3377 vncipher $out1,$out1,v28
3378 vxor $in3,$twk3,v31
3379 vsrab $tmp,$tweak,$seven # next tweak value
3380 vxor $twk3,$tweak,$rndkey0
3381 vncipher $out2,$out2,v28
3382 vncipher $out3,$out3,v28
3383 vaddubm $tweak,$tweak,$tweak
3384 vncipher $out4,$out4,v28
3385 vncipher $out5,$out5,v28
3386 lvx v24,$x00,$key_ # re-pre-load round[1]
3387 vand $tmp,$tmp,$eighty7
3388
3389 vncipher $out0,$out0,v29
3390 vncipher $out1,$out1,v29
3391 xxlor 32+$in4, 0, 0
3392 vpermxor $tweak, $tweak, $tmp, $in4
3393 vncipher $out2,$out2,v29
3394 vncipher $out3,$out3,v29
3395 vxor $in4,$twk4,v31
3396 vsrab $tmp,$tweak,$seven # next tweak value
3397 vxor $twk4,$tweak,$rndkey0
3398 vncipher $out4,$out4,v29
3399 vncipher $out5,$out5,v29
3400 lvx v25,$x10,$key_ # re-pre-load round[2]
3401 vaddubm $tweak,$tweak,$tweak
3402
3403 vncipher $out0,$out0,v30
3404 vncipher $out1,$out1,v30
3405 vand $tmp,$tmp,$eighty7
3406 vncipher $out2,$out2,v30
3407 vncipher $out3,$out3,v30
3408 xxlor 32+$in5, 0, 0
3409 vpermxor $tweak, $tweak, $tmp, $in5
3410 vncipher $out4,$out4,v30
3411 vncipher $out5,$out5,v30
3412 vxor $in5,$twk5,v31
3413 vsrab $tmp,$tweak,$seven # next tweak value
3414 vxor $twk5,$tweak,$rndkey0
3415
3416 vncipherlast $out0,$out0,$in0
3417 lvx_u $in0,$x00,$inp # load next input block
3418 vaddubm $tweak,$tweak,$tweak
3419 vncipherlast $out1,$out1,$in1
3420 lvx_u $in1,$x10,$inp
3421 vncipherlast $out2,$out2,$in2
3422 le?vperm $in0,$in0,$in0,$leperm
3423 lvx_u $in2,$x20,$inp
3424 vand $tmp,$tmp,$eighty7
3425 vncipherlast $out3,$out3,$in3
3426 le?vperm $in1,$in1,$in1,$leperm
3427 lvx_u $in3,$x30,$inp
3428 vncipherlast $out4,$out4,$in4
3429 le?vperm $in2,$in2,$in2,$leperm
3430 lvx_u $in4,$x40,$inp
3431 xxlor 10, 32+$in0, 32+$in0
3432 xxlor 32+$in0, 0, 0
3433 vpermxor $tweak, $tweak, $tmp, $in0
3434 xxlor 32+$in0, 10, 10
3435 vncipherlast $out5,$out5,$in5
3436 le?vperm $in3,$in3,$in3,$leperm
3437 lvx_u $in5,$x50,$inp
3438 addi $inp,$inp,0x60
3439 le?vperm $in4,$in4,$in4,$leperm
3440 le?vperm $in5,$in5,$in5,$leperm
3441
3442 le?vperm $out0,$out0,$out0,$leperm
3443 le?vperm $out1,$out1,$out1,$leperm
3444 stvx_u $out0,$x00,$out # store output
3445 vxor $out0,$in0,$twk0
3446 le?vperm $out2,$out2,$out2,$leperm
3447 stvx_u $out1,$x10,$out
3448 vxor $out1,$in1,$twk1
3449 le?vperm $out3,$out3,$out3,$leperm
3450 stvx_u $out2,$x20,$out
3451 vxor $out2,$in2,$twk2
3452 le?vperm $out4,$out4,$out4,$leperm
3453 stvx_u $out3,$x30,$out
3454 vxor $out3,$in3,$twk3
3455 le?vperm $out5,$out5,$out5,$leperm
3456 stvx_u $out4,$x40,$out
3457 vxor $out4,$in4,$twk4
3458 stvx_u $out5,$x50,$out
3459 vxor $out5,$in5,$twk5
3460 addi $out,$out,0x60
3461
3462 mtctr $rounds
3463 beq Loop_xts_dec6x # did $len-=96 borrow?
3464
3465 xxlor 32+$eighty7, 2, 2 # 0x010101..87
3466
3467 addic. $len,$len,0x60
3468 beq Lxts_dec6x_zero
3469 cmpwi $len,0x20
3470 blt Lxts_dec6x_one
3471 nop
3472 beq Lxts_dec6x_two
3473 cmpwi $len,0x40
3474 blt Lxts_dec6x_three
3475 nop
3476 beq Lxts_dec6x_four
3477
3478 Lxts_dec6x_five:
3479 vxor $out0,$in1,$twk0
3480 vxor $out1,$in2,$twk1
3481 vxor $out2,$in3,$twk2
3482 vxor $out3,$in4,$twk3
3483 vxor $out4,$in5,$twk4
3484
3485 bl _aesp8_xts_dec5x
3486
3487 le?vperm $out0,$out0,$out0,$leperm
3488 vmr $twk0,$twk5 # unused tweak
3489 vxor $twk1,$tweak,$rndkey0
3490 le?vperm $out1,$out1,$out1,$leperm
3491 stvx_u $out0,$x00,$out # store output
3492 vxor $out0,$in0,$twk1
3493 le?vperm $out2,$out2,$out2,$leperm
3494 stvx_u $out1,$x10,$out
3495 le?vperm $out3,$out3,$out3,$leperm
3496 stvx_u $out2,$x20,$out
3497 le?vperm $out4,$out4,$out4,$leperm
3498 stvx_u $out3,$x30,$out
3499 stvx_u $out4,$x40,$out
3500 addi $out,$out,0x50
3501 bne Lxts_dec6x_steal
3502 b Lxts_dec6x_done
3503
3504 .align 4
3505 Lxts_dec6x_four:
3506 vxor $out0,$in2,$twk0
3507 vxor $out1,$in3,$twk1
3508 vxor $out2,$in4,$twk2
3509 vxor $out3,$in5,$twk3
3510 vxor $out4,$out4,$out4
3511
3512 bl _aesp8_xts_dec5x
3513
3514 le?vperm $out0,$out0,$out0,$leperm
3515 vmr $twk0,$twk4 # unused tweak
3516 vmr $twk1,$twk5
3517 le?vperm $out1,$out1,$out1,$leperm
3518 stvx_u $out0,$x00,$out # store output
3519 vxor $out0,$in0,$twk5
3520 le?vperm $out2,$out2,$out2,$leperm
3521 stvx_u $out1,$x10,$out
3522 le?vperm $out3,$out3,$out3,$leperm
3523 stvx_u $out2,$x20,$out
3524 stvx_u $out3,$x30,$out
3525 addi $out,$out,0x40
3526 bne Lxts_dec6x_steal
3527 b Lxts_dec6x_done
3528
3529 .align 4
3530 Lxts_dec6x_three:
3531 vxor $out0,$in3,$twk0
3532 vxor $out1,$in4,$twk1
3533 vxor $out2,$in5,$twk2
3534 vxor $out3,$out3,$out3
3535 vxor $out4,$out4,$out4
3536
3537 bl _aesp8_xts_dec5x
3538
3539 le?vperm $out0,$out0,$out0,$leperm
3540 vmr $twk0,$twk3 # unused tweak
3541 vmr $twk1,$twk4
3542 le?vperm $out1,$out1,$out1,$leperm
3543 stvx_u $out0,$x00,$out # store output
3544 vxor $out0,$in0,$twk4
3545 le?vperm $out2,$out2,$out2,$leperm
3546 stvx_u $out1,$x10,$out
3547 stvx_u $out2,$x20,$out
3548 addi $out,$out,0x30
3549 bne Lxts_dec6x_steal
3550 b Lxts_dec6x_done
3551
3552 .align 4
3553 Lxts_dec6x_two:
3554 vxor $out0,$in4,$twk0
3555 vxor $out1,$in5,$twk1
3556 vxor $out2,$out2,$out2
3557 vxor $out3,$out3,$out3
3558 vxor $out4,$out4,$out4
3559
3560 bl _aesp8_xts_dec5x
3561
3562 le?vperm $out0,$out0,$out0,$leperm
3563 vmr $twk0,$twk2 # unused tweak
3564 vmr $twk1,$twk3
3565 le?vperm $out1,$out1,$out1,$leperm
3566 stvx_u $out0,$x00,$out # store output
3567 vxor $out0,$in0,$twk3
3568 stvx_u $out1,$x10,$out
3569 addi $out,$out,0x20
3570 bne Lxts_dec6x_steal
3571 b Lxts_dec6x_done
3572
3573 .align 4
3574 Lxts_dec6x_one:
3575 vxor $out0,$in5,$twk0
3576 nop
3577 Loop_xts_dec1x:
3578 vncipher $out0,$out0,v24
3579 lvx v24,$x20,$key_ # round[3]
3580 addi $key_,$key_,0x20
3581
3582 vncipher $out0,$out0,v25
3583 lvx v25,$x10,$key_ # round[4]
3584 bdnz Loop_xts_dec1x
3585
3586 subi r0,$taillen,1
3587 vncipher $out0,$out0,v24
3588
3589 andi. r0,r0,16
3590 cmpwi $taillen,0
3591 vncipher $out0,$out0,v25
3592
3593 sub $inp,$inp,r0
3594 vncipher $out0,$out0,v26
3595
3596 lvx_u $in0,0,$inp
3597 vncipher $out0,$out0,v27
3598
3599 addi $key_,$sp,$FRAME+15 # rewind $key_
3600 vncipher $out0,$out0,v28
3601 lvx v24,$x00,$key_ # re-pre-load round[1]
3602
3603 vncipher $out0,$out0,v29
3604 lvx v25,$x10,$key_ # re-pre-load round[2]
3605 vxor $twk0,$twk0,v31
3606
3607 le?vperm $in0,$in0,$in0,$leperm
3608 vncipher $out0,$out0,v30
3609
3610 mtctr $rounds
3611 vncipherlast $out0,$out0,$twk0
3612
3613 vmr $twk0,$twk1 # unused tweak
3614 vmr $twk1,$twk2
3615 le?vperm $out0,$out0,$out0,$leperm
3616 stvx_u $out0,$x00,$out # store output
3617 addi $out,$out,0x10
3618 vxor $out0,$in0,$twk2
3619 bne Lxts_dec6x_steal
3620 b Lxts_dec6x_done
3621
3622 .align 4
3623 Lxts_dec6x_zero:
3624 cmpwi $taillen,0
3625 beq Lxts_dec6x_done
3626
3627 lvx_u $in0,0,$inp
3628 le?vperm $in0,$in0,$in0,$leperm
3629 vxor $out0,$in0,$twk1
3630 Lxts_dec6x_steal:
3631 vncipher $out0,$out0,v24
3632 lvx v24,$x20,$key_ # round[3]
3633 addi $key_,$key_,0x20
3634
3635 vncipher $out0,$out0,v25
3636 lvx v25,$x10,$key_ # round[4]
3637 bdnz Lxts_dec6x_steal
3638
3639 add $inp,$inp,$taillen
3640 vncipher $out0,$out0,v24
3641
3642 cmpwi $taillen,0
3643 vncipher $out0,$out0,v25
3644
3645 lvx_u $in0,0,$inp
3646 vncipher $out0,$out0,v26
3647
3648 lvsr $inpperm,0,$taillen # $in5 is no more
3649 vncipher $out0,$out0,v27
3650
3651 addi $key_,$sp,$FRAME+15 # rewind $key_
3652 vncipher $out0,$out0,v28
3653 lvx v24,$x00,$key_ # re-pre-load round[1]
3654
3655 vncipher $out0,$out0,v29
3656 lvx v25,$x10,$key_ # re-pre-load round[2]
3657 vxor $twk1,$twk1,v31
3658
3659 le?vperm $in0,$in0,$in0,$leperm
3660 vncipher $out0,$out0,v30
3661
3662 vperm $in0,$in0,$in0,$inpperm
3663 vncipherlast $tmp,$out0,$twk1
3664
3665 le?vperm $out0,$tmp,$tmp,$leperm
3666 le?stvx_u $out0,0,$out
3667 be?stvx_u $tmp,0,$out
3668
3669 vxor $out0,$out0,$out0
3670 vspltisb $out1,-1
3671 vperm $out0,$out0,$out1,$inpperm
3672 vsel $out0,$in0,$tmp,$out0
3673 vxor $out0,$out0,$twk0
3674
3675 subi r30,$out,1
3676 mtctr $taillen
3677 Loop_xts_dec6x_steal:
3678 lbzu r0,1(r30)
3679 stb r0,16(r30)
3680 bdnz Loop_xts_dec6x_steal
3681
3682 li $taillen,0
3683 mtctr $rounds
3684 b Loop_xts_dec1x # one more time...
3685
3686 .align 4
3687 Lxts_dec6x_done:
3688 ${UCMP}i $ivp,0
3689 beq Lxts_dec6x_ret
3690
3691 vxor $tweak,$twk0,$rndkey0
3692 le?vperm $tweak,$tweak,$tweak,$leperm
3693 stvx_u $tweak,0,$ivp
3694
3695 Lxts_dec6x_ret:
3696 mtlr r11
3697 li r10,`$FRAME+15`
3698 li r11,`$FRAME+31`
3699 stvx $seven,r10,$sp # wipe copies of round keys
3700 addi r10,r10,32
3701 stvx $seven,r11,$sp
3702 addi r11,r11,32
3703 stvx $seven,r10,$sp
3704 addi r10,r10,32
3705 stvx $seven,r11,$sp
3706 addi r11,r11,32
3707 stvx $seven,r10,$sp
3708 addi r10,r10,32
3709 stvx $seven,r11,$sp
3710 addi r11,r11,32
3711 stvx $seven,r10,$sp
3712 addi r10,r10,32
3713 stvx $seven,r11,$sp
3714 addi r11,r11,32
3715
3716 mtspr 256,$vrsave
3717 lvx v20,r10,$sp # ABI says so
3718 addi r10,r10,32
3719 lvx v21,r11,$sp
3720 addi r11,r11,32
3721 lvx v22,r10,$sp
3722 addi r10,r10,32
3723 lvx v23,r11,$sp
3724 addi r11,r11,32
3725 lvx v24,r10,$sp
3726 addi r10,r10,32
3727 lvx v25,r11,$sp
3728 addi r11,r11,32
3729 lvx v26,r10,$sp
3730 addi r10,r10,32
3731 lvx v27,r11,$sp
3732 addi r11,r11,32
3733 lvx v28,r10,$sp
3734 addi r10,r10,32
3735 lvx v29,r11,$sp
3736 addi r11,r11,32
3737 lvx v30,r10,$sp
3738 lvx v31,r11,$sp
3739 $POP r26,`$FRAME+21*16+0*$SIZE_T`($sp)
3740 $POP r27,`$FRAME+21*16+1*$SIZE_T`($sp)
3741 $POP r28,`$FRAME+21*16+2*$SIZE_T`($sp)
3742 $POP r29,`$FRAME+21*16+3*$SIZE_T`($sp)
3743 $POP r30,`$FRAME+21*16+4*$SIZE_T`($sp)
3744 $POP r31,`$FRAME+21*16+5*$SIZE_T`($sp)
3745 addi $sp,$sp,`$FRAME+21*16+6*$SIZE_T`
3746 blr
3747 .long 0
3748 .byte 0,12,0x04,1,0x80,6,6,0
3749 .long 0
3750
3751 .align 5
3752 _aesp8_xts_dec5x:
3753 vncipher $out0,$out0,v24
3754 vncipher $out1,$out1,v24
3755 vncipher $out2,$out2,v24
3756 vncipher $out3,$out3,v24
3757 vncipher $out4,$out4,v24
3758 lvx v24,$x20,$key_ # round[3]
3759 addi $key_,$key_,0x20
3760
3761 vncipher $out0,$out0,v25
3762 vncipher $out1,$out1,v25
3763 vncipher $out2,$out2,v25
3764 vncipher $out3,$out3,v25
3765 vncipher $out4,$out4,v25
3766 lvx v25,$x10,$key_ # round[4]
3767 bdnz _aesp8_xts_dec5x
3768
3769 subi r0,$taillen,1
3770 vncipher $out0,$out0,v24
3771 vncipher $out1,$out1,v24
3772 vncipher $out2,$out2,v24
3773 vncipher $out3,$out3,v24
3774 vncipher $out4,$out4,v24
3775
3776 andi. r0,r0,16
3777 cmpwi $taillen,0
3778 vncipher $out0,$out0,v25
3779 vncipher $out1,$out1,v25
3780 vncipher $out2,$out2,v25
3781 vncipher $out3,$out3,v25
3782 vncipher $out4,$out4,v25
3783 vxor $twk0,$twk0,v31
3784
3785 sub $inp,$inp,r0
3786 vncipher $out0,$out0,v26
3787 vncipher $out1,$out1,v26
3788 vncipher $out2,$out2,v26
3789 vncipher $out3,$out3,v26
3790 vncipher $out4,$out4,v26
3791 vxor $in1,$twk1,v31
3792
3793 vncipher $out0,$out0,v27
3794 lvx_u $in0,0,$inp
3795 vncipher $out1,$out1,v27
3796 vncipher $out2,$out2,v27
3797 vncipher $out3,$out3,v27
3798 vncipher $out4,$out4,v27
3799 vxor $in2,$twk2,v31
3800
3801 addi $key_,$sp,$FRAME+15 # rewind $key_
3802 vncipher $out0,$out0,v28
3803 vncipher $out1,$out1,v28
3804 vncipher $out2,$out2,v28
3805 vncipher $out3,$out3,v28
3806 vncipher $out4,$out4,v28
3807 lvx v24,$x00,$key_ # re-pre-load round[1]
3808 vxor $in3,$twk3,v31
3809
3810 vncipher $out0,$out0,v29
3811 le?vperm $in0,$in0,$in0,$leperm
3812 vncipher $out1,$out1,v29
3813 vncipher $out2,$out2,v29
3814 vncipher $out3,$out3,v29
3815 vncipher $out4,$out4,v29
3816 lvx v25,$x10,$key_ # re-pre-load round[2]
3817 vxor $in4,$twk4,v31
3818
3819 vncipher $out0,$out0,v30
3820 vncipher $out1,$out1,v30
3821 vncipher $out2,$out2,v30
3822 vncipher $out3,$out3,v30
3823 vncipher $out4,$out4,v30
3824
3825 vncipherlast $out0,$out0,$twk0
3826 vncipherlast $out1,$out1,$in1
3827 vncipherlast $out2,$out2,$in2
3828 vncipherlast $out3,$out3,$in3
3829 vncipherlast $out4,$out4,$in4
3830 mtctr $rounds
3831 blr
3832 .long 0
3833 .byte 0,12,0x14,0,0,0,0,0
3834 ___
3835 }} }}}
3836
3837 my $consts=1;
3838 foreach(split("\n",$code)) {
3839 s/\`([^\`]*)\`/eval($1)/geo;
3840
3841 # constants table endian-specific conversion
3842 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3843 my $conv=$3;
3844 my @bytes=();
3845
3846 # convert to endian-agnostic format
3847 if ($1 eq "long") {
3848 foreach (split(/,\s*/,$2)) {
3849 my $l = /^0/?oct:int;
3850 push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
3851 }
3852 } else {
3853 @bytes = map(/^0/?oct:int,split(/,\s*/,$2));
3854 }
3855
3856 # little-endian conversion
3857 if ($flavour =~ /le$/o) {
3858 SWITCH: for($conv) {
3859 /\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
3860 /\?rev/ && do { @bytes=reverse(@bytes); last; };
3861 }
3862 }
3863
3864 #emit
3865 print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
3866 next;
3867 }
3868 $consts=0 if (m/Lconsts:/o); # end of table
3869
3870 # instructions prefixed with '?' are endian-specific and need
3871 # to be adjusted accordingly...
3872 if ($flavour =~ /le$/o) { # little-endian
3873 s/le\?//o or
3874 s/be\?/#be#/o or
3875 s/\?lvsr/lvsl/o or
3876 s/\?lvsl/lvsr/o or
3877 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3878 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3879 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3880 } else { # big-endian
3881 s/le\?/#le#/o or
3882 s/be\?//o or
3883 s/\?([a-z]+)/$1/o;
3884 }
3885
3886 print $_,"\n";
3887 }
3888
3889 close STDOUT;
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.