1 #!/usr/bin/env perl
2 # SPDX-License-Identifier: GPL-2.0
3
4 # This code is taken from the OpenSSL project but the author (Andy Polyakov)
5 # has relicensed it under the GPLv2. Therefore this program is free software;
6 # you can redistribute it and/or modify it under the terms of the GNU General
7 # Public License version 2 as published by the Free Software Foundation.
8 #
9 # The original headers, including the original license headers, are
10 # included below for completeness.
11
12 # ====================================================================
13 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
14 # project. The module is, however, dual licensed under OpenSSL and
15 # CRYPTOGAMS licenses depending on where you obtain it. For further
16 # details see https://www.openssl.org/~appro/cryptogams/.
17 # ====================================================================
18 #
19 # GHASH for PowerISA v2.07.
20 #
21 # July 2014
22 #
23 # Accurate performance measurements are problematic, because it's
24 # always virtualized setup with possibly throttled processor.
25 # Relative comparison is therefore more informative. This initial
26 # version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
27 # faster than "4-bit" integer-only compiler-generated 64-bit code.
28 # "Initial version" means that there is room for futher improvement.
29
30 $flavour=shift;
31 $output =shift;
32
33 if ($flavour =~ /64/) {
34 $SIZE_T=8;
35 $LRSAVE=2*$SIZE_T;
36 $STU="stdu";
37 $POP="ld";
38 $PUSH="std";
39 } elsif ($flavour =~ /32/) {
40 $SIZE_T=4;
41 $LRSAVE=$SIZE_T;
42 $STU="stwu";
43 $POP="lwz";
44 $PUSH="stw";
45 } else { die "nonsense $flavour"; }
46
47 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
48 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
49 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
50 die "can't locate ppc-xlate.pl";
51
52 open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
53
54 my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block
55
56 my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
57 my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
58 my $vrsave="r12";
59
60 $code=<<___;
61 .machine "any"
62
63 .text
64
65 .globl .gcm_init_p8
66 lis r0,0xfff0
67 li r8,0x10
68 mfspr $vrsave,256
69 li r9,0x20
70 mtspr 256,r0
71 li r10,0x30
72 lvx_u $H,0,r4 # load H
73 le?xor r7,r7,r7
74 le?addi r7,r7,0x8 # need a vperm start with 08
75 le?lvsr 5,0,r7
76 le?vspltisb 6,0x0f
77 le?vxor 5,5,6 # set a b-endian mask
78 le?vperm $H,$H,$H,5
79
80 vspltisb $xC2,-16 # 0xf0
81 vspltisb $t0,1 # one
82 vaddubm $xC2,$xC2,$xC2 # 0xe0
83 vxor $zero,$zero,$zero
84 vor $xC2,$xC2,$t0 # 0xe1
85 vsldoi $xC2,$xC2,$zero,15 # 0xe1...
86 vsldoi $t1,$zero,$t0,1 # ...1
87 vaddubm $xC2,$xC2,$xC2 # 0xc2...
88 vspltisb $t2,7
89 vor $xC2,$xC2,$t1 # 0xc2....01
90 vspltb $t1,$H,0 # most significant byte
91 vsl $H,$H,$t0 # H<<=1
92 vsrab $t1,$t1,$t2 # broadcast carry bit
93 vand $t1,$t1,$xC2
94 vxor $H,$H,$t1 # twisted H
95
96 vsldoi $H,$H,$H,8 # twist even more ...
97 vsldoi $xC2,$zero,$xC2,8 # 0xc2.0
98 vsldoi $Hl,$zero,$H,8 # ... and split
99 vsldoi $Hh,$H,$zero,8
100
101 stvx_u $xC2,0,r3 # save pre-computed table
102 stvx_u $Hl,r8,r3
103 stvx_u $H, r9,r3
104 stvx_u $Hh,r10,r3
105
106 mtspr 256,$vrsave
107 blr
108 .long 0
109 .byte 0,12,0x14,0,0,0,2,0
110 .long 0
111 .size .gcm_init_p8,.-.gcm_init_p8
112
113 .globl .gcm_gmult_p8
114 lis r0,0xfff8
115 li r8,0x10
116 mfspr $vrsave,256
117 li r9,0x20
118 mtspr 256,r0
119 li r10,0x30
120 lvx_u $IN,0,$Xip # load Xi
121
122 lvx_u $Hl,r8,$Htbl # load pre-computed table
123 le?lvsl $lemask,r0,r0
124 lvx_u $H, r9,$Htbl
125 le?vspltisb $t0,0x07
126 lvx_u $Hh,r10,$Htbl
127 le?vxor $lemask,$lemask,$t0
128 lvx_u $xC2,0,$Htbl
129 le?vperm $IN,$IN,$IN,$lemask
130 vxor $zero,$zero,$zero
131
132 vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
133 vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
134 vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
135
136 vpmsumd $t2,$Xl,$xC2 # 1st phase
137
138 vsldoi $t0,$Xm,$zero,8
139 vsldoi $t1,$zero,$Xm,8
140 vxor $Xl,$Xl,$t0
141 vxor $Xh,$Xh,$t1
142
143 vsldoi $Xl,$Xl,$Xl,8
144 vxor $Xl,$Xl,$t2
145
146 vsldoi $t1,$Xl,$Xl,8 # 2nd phase
147 vpmsumd $Xl,$Xl,$xC2
148 vxor $t1,$t1,$Xh
149 vxor $Xl,$Xl,$t1
150
151 le?vperm $Xl,$Xl,$Xl,$lemask
152 stvx_u $Xl,0,$Xip # write out Xi
153
154 mtspr 256,$vrsave
155 blr
156 .long 0
157 .byte 0,12,0x14,0,0,0,2,0
158 .long 0
159 .size .gcm_gmult_p8,.-.gcm_gmult_p8
160
161 .globl .gcm_ghash_p8
162 lis r0,0xfff8
163 li r8,0x10
164 mfspr $vrsave,256
165 li r9,0x20
166 mtspr 256,r0
167 li r10,0x30
168 lvx_u $Xl,0,$Xip # load Xi
169
170 lvx_u $Hl,r8,$Htbl # load pre-computed table
171 le?lvsl $lemask,r0,r0
172 lvx_u $H, r9,$Htbl
173 le?vspltisb $t0,0x07
174 lvx_u $Hh,r10,$Htbl
175 le?vxor $lemask,$lemask,$t0
176 lvx_u $xC2,0,$Htbl
177 le?vperm $Xl,$Xl,$Xl,$lemask
178 vxor $zero,$zero,$zero
179
180 lvx_u $IN,0,$inp
181 addi $inp,$inp,16
182 subi $len,$len,16
183 le?vperm $IN,$IN,$IN,$lemask
184 vxor $IN,$IN,$Xl
185 b Loop
186
187 .align 5
188 Loop:
189 subic $len,$len,16
190 vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
191 subfe. r0,r0,r0 # borrow?-1:0
192 vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
193 and r0,r0,$len
194 vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
195 add $inp,$inp,r0
196
197 vpmsumd $t2,$Xl,$xC2 # 1st phase
198
199 vsldoi $t0,$Xm,$zero,8
200 vsldoi $t1,$zero,$Xm,8
201 vxor $Xl,$Xl,$t0
202 vxor $Xh,$Xh,$t1
203
204 vsldoi $Xl,$Xl,$Xl,8
205 vxor $Xl,$Xl,$t2
206 lvx_u $IN,0,$inp
207 addi $inp,$inp,16
208
209 vsldoi $t1,$Xl,$Xl,8 # 2nd phase
210 vpmsumd $Xl,$Xl,$xC2
211 le?vperm $IN,$IN,$IN,$lemask
212 vxor $t1,$t1,$Xh
213 vxor $IN,$IN,$t1
214 vxor $IN,$IN,$Xl
215 beq Loop # did $len-=16 borrow?
216
217 vxor $Xl,$Xl,$t1
218 le?vperm $Xl,$Xl,$Xl,$lemask
219 stvx_u $Xl,0,$Xip # write out Xi
220
221 mtspr 256,$vrsave
222 blr
223 .long 0
224 .byte 0,12,0x14,0,0,0,4,0
225 .long 0
226 .size .gcm_ghash_p8,.-.gcm_ghash_p8
227
228 .asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
229 .align 2
230 ___
231
232 foreach (split("\n",$code)) {
233 if ($flavour =~ /le$/o) { # little-endian
234 s/le\?//o or
235 s/be\?/#be#/o;
236 } else {
237 s/le\?/#le#/o or
238 s/be\?//o;
239 }
240 print $_,"\n";
241 }
242
243 close STDOUT; # enforce flush
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.