1 /*
2 * Intel SHA Extensions optimized implementati
3 *
4 * This file is provided under a dual BSD/GPLv
5 * redistributing this file, you may do so und
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2015 Intel Corporation.
10 *
11 * This program is free software; you can redi
12 * it under the terms of version 2 of the GNU
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope tha
16 * WITHOUT ANY WARRANTY; without even the impl
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * Sean Gulley <sean.m.gulley@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2015 Intel Corporation.
27 *
28 * Redistribution and use in source and binary
29 * modification, are permitted provided that t
30 * are met:
31 *
32 * * Redistributions of source code must
33 * notice, this list of conditions and
34 * * Redistributions in binary form must
35 * notice, this list of conditions and
36 * the documentation and/or other mater
37 * distribution.
38 * * Neither the name of Intel Corporatio
39 * contributors may be used to endorse
40 * from this software without specific
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTI
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCH
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIR
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGE
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, S
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE PO
53 *
54 */
55
56 #include <linux/linkage.h>
57 #include <linux/cfi_types.h>
58
59 #define DIGEST_PTR %rdi /* 1st arg */
60 #define DATA_PTR %rsi /* 2nd arg */
61 #define NUM_BLKS %rdx /* 3rd arg */
62
63 #define SHA256CONSTANTS %rax
64
65 #define MSG %xmm0 /* sha256rnds2
66 #define STATE0 %xmm1
67 #define STATE1 %xmm2
68 #define MSG0 %xmm3
69 #define MSG1 %xmm4
70 #define MSG2 %xmm5
71 #define MSG3 %xmm6
72 #define TMP %xmm7
73
74 #define SHUF_MASK %xmm8
75
76 #define ABEF_SAVE %xmm9
77 #define CDGH_SAVE %xmm10
78
79 .macro do_4rounds i, m0, m1, m2, m3
80 .if \i < 16
81 movdqu \i*4(DATA_PTR), \m0
82 pshufb SHUF_MASK, \m0
83 .endif
84 movdqa (\i-32)*4(SHA256CONSTA
85 paddd \m0, MSG
86 sha256rnds2 STATE0, STATE1
87 .if \i >= 12 && \i < 60
88 movdqa \m0, TMP
89 palignr $4, \m3, TMP
90 paddd TMP, \m1
91 sha256msg2 \m0, \m1
92 .endif
93 punpckhqdq MSG, MSG
94 sha256rnds2 STATE1, STATE0
95 .if \i >= 4 && \i < 52
96 sha256msg1 \m0, \m3
97 .endif
98 .endm
99
100 /*
101 * Intel SHA Extensions optimized implementati
102 *
103 * The function takes a pointer to the current
104 * input data, and a number of 64 byte blocks
105 * been processed, the digest pointer is upda
106 * The function only processes complete blocks
107 * store partial blocks. All message padding
108 * be done outside the update function.
109 *
110 * void sha256_ni_transform(uint32_t *digest,
111 uint32_t numBlocks);
112 * digest : pointer to digest
113 * data: pointer to input data
114 * numBlocks: Number of blocks to process
115 */
116
117 .text
118 SYM_TYPED_FUNC_START(sha256_ni_transform)
119
120 shl $6, NUM_BLKS
121 jz .Ldone_hash
122 add DATA_PTR, NUM_BLKS
123
124 /*
125 * load initial hash values
126 * Need to reorder these appropriately
127 * DCBA, HGFE -> ABEF, CDGH
128 */
129 movdqu 0*16(DIGEST_PTR), STAT
130 movdqu 1*16(DIGEST_PTR), STAT
131
132 movdqa STATE0, TMP
133 punpcklqdq STATE1, STATE0
134 punpckhqdq TMP, STATE1
135 pshufd $0x1B, STATE0, STATE0
136 pshufd $0xB1, STATE1, STATE1
137
138 movdqa PSHUFFLE_BYTE_FLIP_MAS
139 lea K256+32*4(%rip), SHA25
140
141 .Lloop0:
142 /* Save hash values for addition after
143 movdqa STATE0, ABEF_SAVE
144 movdqa STATE1, CDGH_SAVE
145
146 .irp i, 0, 16, 32, 48
147 do_4rounds (\i + 0), MSG0, MSG1,
148 do_4rounds (\i + 4), MSG1, MSG2,
149 do_4rounds (\i + 8), MSG2, MSG3,
150 do_4rounds (\i + 12), MSG3, MSG0,
151 .endr
152
153 /* Add current hash values with previo
154 paddd ABEF_SAVE, STATE0
155 paddd CDGH_SAVE, STATE1
156
157 /* Increment data pointer and loop if
158 add $64, DATA_PTR
159 cmp NUM_BLKS, DATA_PTR
160 jne .Lloop0
161
162 /* Write hash values back in the corre
163 movdqa STATE0, TMP
164 punpcklqdq STATE1, STATE0
165 punpckhqdq TMP, STATE1
166 pshufd $0xB1, STATE0, STATE0
167 pshufd $0x1B, STATE1, STATE1
168
169 movdqu STATE1, 0*16(DIGEST_PT
170 movdqu STATE0, 1*16(DIGEST_PT
171
172 .Ldone_hash:
173
174 RET
175 SYM_FUNC_END(sha256_ni_transform)
176
177 .section .rodata.cst256.K256, "aM", @pr
178 .align 64
179 K256:
180 .long 0x428a2f98,0x71374491,0xb5c0fb
181 .long 0x3956c25b,0x59f111f1,0x923f82
182 .long 0xd807aa98,0x12835b01,0x243185
183 .long 0x72be5d74,0x80deb1fe,0x9bdc06
184 .long 0xe49b69c1,0xefbe4786,0x0fc19d
185 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9
186 .long 0x983e5152,0xa831c66d,0xb00327
187 .long 0xc6e00bf3,0xd5a79147,0x06ca63
188 .long 0x27b70a85,0x2e1b2138,0x4d2c6d
189 .long 0x650a7354,0x766a0abb,0x81c2c9
190 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b
191 .long 0xd192e819,0xd6990624,0xf40e35
192 .long 0x19a4c116,0x1e376c08,0x274877
193 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca
194 .long 0x748f82ee,0x78a5636f,0x84c878
195 .long 0x90befffa,0xa4506ceb,0xbef9a3
196
197 .section .rodata.cst16.PSHUFFLE_BYTE_FL
198 .align 16
199 PSHUFFLE_BYTE_FLIP_MASK:
200 .octa 0x0c0d0e0f08090a0b04050607000102
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.