Gitiles
Code Review Sign In
review.shift-gmbh.com / SHIFTPHONES / kernel / common / 2e5d2f33d1dbd551f14634483c7ea81e5119d689 / . / arch / arm64 / crypto / aes-modes.S
blob: 039738ae23f662f7ad656815e0a922485bc54428 [file] [log] [blame]
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]1/*
2 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
3 *
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11/* included by aes-ce.S and aes-neon.S */
12
13 .text
14 .align 4
15
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]16aes_encrypt_block4x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]17 encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]18 ret
19ENDPROC(aes_encrypt_block4x)
20
21aes_decrypt_block4x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]22 decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]23 ret
24ENDPROC(aes_decrypt_block4x)
25
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]26 /*
27 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]28 * int blocks)
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]29 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]30 * int blocks)
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]31 */
32
33AES_ENTRY(aes_ecb_encrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]34 stp x29, x30, [sp, #-16]!
35 mov x29, sp
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]36
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]37 enc_prepare w3, x2, x5
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]38
39.LecbencloopNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]40 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]41 bmi .Lecbenc1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]42 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]43 bl aes_encrypt_block4x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]44 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]45 b .LecbencloopNx
46.Lecbenc1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]47 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]48 beq .Lecbencout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]49.Lecbencloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]50 ld1 {v0.16b}, [x1], #16 /* get next pt block */
51 encrypt_block v0, w3, x2, x5, w6
52 st1 {v0.16b}, [x0], #16
53 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]54 bne .Lecbencloop
55.Lecbencout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]56 ldp x29, x30, [sp], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]57 ret
58AES_ENDPROC(aes_ecb_encrypt)
59
60
61AES_ENTRY(aes_ecb_decrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]62 stp x29, x30, [sp, #-16]!
63 mov x29, sp
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]64
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]65 dec_prepare w3, x2, x5
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]66
67.LecbdecloopNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]68 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]69 bmi .Lecbdec1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]70 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]71 bl aes_decrypt_block4x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]72 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]73 b .LecbdecloopNx
74.Lecbdec1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]75 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]76 beq .Lecbdecout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]77.Lecbdecloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]78 ld1 {v0.16b}, [x1], #16 /* get next ct block */
79 decrypt_block v0, w3, x2, x5, w6
80 st1 {v0.16b}, [x0], #16
81 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]82 bne .Lecbdecloop
83.Lecbdecout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]84 ldp x29, x30, [sp], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]85 ret
86AES_ENDPROC(aes_ecb_decrypt)
87
88
89 /*
90 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]91 * int blocks, u8 iv[])
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]92 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]93 * int blocks, u8 iv[])
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]94 */
95
96AES_ENTRY(aes_cbc_encrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]97 ld1 {v4.16b}, [x5] /* get iv */
98 enc_prepare w3, x2, x6
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]99
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]100.Lcbcencloop4x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]101 subs w4, w4, #4
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]102 bmi .Lcbcenc1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]103 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]104 eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]105 encrypt_block v0, w3, x2, x6, w7
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]106 eor v1.16b, v1.16b, v0.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]107 encrypt_block v1, w3, x2, x6, w7
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]108 eor v2.16b, v2.16b, v1.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]109 encrypt_block v2, w3, x2, x6, w7
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]110 eor v3.16b, v3.16b, v2.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]111 encrypt_block v3, w3, x2, x6, w7
112 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]113 mov v4.16b, v3.16b
114 b .Lcbcencloop4x
115.Lcbcenc1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]116 adds w4, w4, #4
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]117 beq .Lcbcencout
118.Lcbcencloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]119 ld1 {v0.16b}, [x1], #16 /* get next pt block */
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]120 eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]121 encrypt_block v4, w3, x2, x6, w7
122 st1 {v4.16b}, [x0], #16
123 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]124 bne .Lcbcencloop
Ard Biesheuvela8f8a692018-03-10 15:21:52 +0000[diff] [blame]125.Lcbcencout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]126 st1 {v4.16b}, [x5] /* return iv */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]127 ret
128AES_ENDPROC(aes_cbc_encrypt)
129
130
131AES_ENTRY(aes_cbc_decrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]132 stp x29, x30, [sp, #-16]!
133 mov x29, sp
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]134
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]135 ld1 {v7.16b}, [x5] /* get iv */
136 dec_prepare w3, x2, x6
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]137
138.LcbcdecloopNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]139 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]140 bmi .Lcbcdec1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]141 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]142 mov v4.16b, v0.16b
143 mov v5.16b, v1.16b
144 mov v6.16b, v2.16b
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]145 bl aes_decrypt_block4x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]146 sub x1, x1, #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]147 eor v0.16b, v0.16b, v7.16b
148 eor v1.16b, v1.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]149 ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]150 eor v2.16b, v2.16b, v5.16b
151 eor v3.16b, v3.16b, v6.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]152 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]153 b .LcbcdecloopNx
154.Lcbcdec1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]155 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]156 beq .Lcbcdecout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]157.Lcbcdecloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]158 ld1 {v1.16b}, [x1], #16 /* get next ct block */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]159 mov v0.16b, v1.16b /* ...and copy to v0 */
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]160 decrypt_block v0, w3, x2, x6, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]161 eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */
162 mov v7.16b, v1.16b /* ct is next iv */
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]163 st1 {v0.16b}, [x0], #16
164 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]165 bne .Lcbcdecloop
166.Lcbcdecout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]167 st1 {v7.16b}, [x5] /* return iv */
168 ldp x29, x30, [sp], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]169 ret
170AES_ENDPROC(aes_cbc_decrypt)
171
172
173 /*
Ard Biesheuveldd597fb2018-09-10 16:41:14 +0200[diff] [blame]174 * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
175 * int rounds, int bytes, u8 const iv[])
176 * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
177 * int rounds, int bytes, u8 const iv[])
178 */
179
180AES_ENTRY(aes_cbc_cts_encrypt)
181 adr_l x8, .Lcts_permute_table
182 sub x4, x4, #16
183 add x9, x8, #32
184 add x8, x8, x4
185 sub x9, x9, x4
186 ld1 {v3.16b}, [x8]
187 ld1 {v4.16b}, [x9]
188
189 ld1 {v0.16b}, [x1], x4 /* overlapping loads */
190 ld1 {v1.16b}, [x1]
191
192 ld1 {v5.16b}, [x5] /* get iv */
193 enc_prepare w3, x2, x6
194
195 eor v0.16b, v0.16b, v5.16b /* xor with iv */
196 tbl v1.16b, {v1.16b}, v4.16b
197 encrypt_block v0, w3, x2, x6, w7
198
199 eor v1.16b, v1.16b, v0.16b
200 tbl v0.16b, {v0.16b}, v3.16b
201 encrypt_block v1, w3, x2, x6, w7
202
203 add x4, x0, x4
204 st1 {v0.16b}, [x4] /* overlapping stores */
205 st1 {v1.16b}, [x0]
206 ret
207AES_ENDPROC(aes_cbc_cts_encrypt)
208
209AES_ENTRY(aes_cbc_cts_decrypt)
210 adr_l x8, .Lcts_permute_table
211 sub x4, x4, #16
212 add x9, x8, #32
213 add x8, x8, x4
214 sub x9, x9, x4
215 ld1 {v3.16b}, [x8]
216 ld1 {v4.16b}, [x9]
217
218 ld1 {v0.16b}, [x1], x4 /* overlapping loads */
219 ld1 {v1.16b}, [x1]
220
221 ld1 {v5.16b}, [x5] /* get iv */
222 dec_prepare w3, x2, x6
223
224 tbl v2.16b, {v1.16b}, v4.16b
225 decrypt_block v0, w3, x2, x6, w7
226 eor v2.16b, v2.16b, v0.16b
227
228 tbx v0.16b, {v1.16b}, v4.16b
229 tbl v2.16b, {v2.16b}, v3.16b
230 decrypt_block v0, w3, x2, x6, w7
231 eor v0.16b, v0.16b, v5.16b /* xor with iv */
232
233 add x4, x0, x4
234 st1 {v2.16b}, [x4] /* overlapping stores */
235 st1 {v0.16b}, [x0]
236 ret
237AES_ENDPROC(aes_cbc_cts_decrypt)
238
239 .section ".rodata", "a"
240 .align 6
241.Lcts_permute_table:
242 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
243 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
244 .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
245 .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
246 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
247 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
248 .previous
249
250
251 /*
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]252 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]253 * int blocks, u8 ctr[])
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]254 */
255
256AES_ENTRY(aes_ctr_encrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]257 stp x29, x30, [sp, #-16]!
258 mov x29, sp
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]259
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]260 enc_prepare w3, x2, x6
261 ld1 {v4.16b}, [x5]
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]262
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]263 umov x6, v4.d[1] /* keep swabbed ctr in reg */
264 rev x6, x6
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]265 cmn w6, w4 /* 32 bit overflow? */
266 bcs .Lctrloop
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]267.LctrloopNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]268 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]269 bmi .Lctr1x
Ard Biesheuveled6ed112018-08-23 17:48:45 +0100[diff] [blame]270 add w7, w6, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]271 mov v0.16b, v4.16b
Ard Biesheuveled6ed112018-08-23 17:48:45 +0100[diff] [blame]272 add w8, w6, #2
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]273 mov v1.16b, v4.16b
Ard Biesheuveled6ed112018-08-23 17:48:45 +0100[diff] [blame]274 add w9, w6, #3
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]275 mov v2.16b, v4.16b
Ard Biesheuveled6ed112018-08-23 17:48:45 +0100[diff] [blame]276 rev w7, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]277 mov v3.16b, v4.16b
Ard Biesheuveled6ed112018-08-23 17:48:45 +0100[diff] [blame]278 rev w8, w8
279 mov v1.s[3], w7
280 rev w9, w9
281 mov v2.s[3], w8
282 mov v3.s[3], w9
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]283 ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]284 bl aes_encrypt_block4x
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]285 eor v0.16b, v5.16b, v0.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]286 ld1 {v5.16b}, [x1], #16 /* get 1 input block */
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]287 eor v1.16b, v6.16b, v1.16b
288 eor v2.16b, v7.16b, v2.16b
289 eor v3.16b, v5.16b, v3.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]290 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]291 add x6, x6, #4
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]292 rev x7, x6
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]293 ins v4.d[1], x7
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]294 cbz w4, .Lctrout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]295 b .LctrloopNx
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]296.Lctr1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]297 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]298 beq .Lctrout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]299.Lctrloop:
300 mov v0.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]301 encrypt_block v0, w3, x2, x8, w7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]302
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]303 adds x6, x6, #1 /* increment BE ctr */
304 rev x7, x6
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]305 ins v4.d[1], x7
306 bcs .Lctrcarry /* overflow? */
307
308.Lctrcarrydone:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]309 subs w4, w4, #1
Ard Biesheuvelccc5d512017-01-28 23:25:34 +0000[diff] [blame]310 bmi .Lctrtailblock /* blocks <0 means tail block */
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]311 ld1 {v3.16b}, [x1], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]312 eor v3.16b, v0.16b, v3.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]313 st1 {v3.16b}, [x0], #16
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]314 bne .Lctrloop
315
316.Lctrout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]317 st1 {v4.16b}, [x5] /* return next CTR value */
318 ldp x29, x30, [sp], #16
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]319 ret
320
Ard Biesheuvelccc5d512017-01-28 23:25:34 +0000[diff] [blame]321.Lctrtailblock:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]322 st1 {v0.16b}, [x0]
323 ldp x29, x30, [sp], #16
324 ret
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]325
326.Lctrcarry:
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]327 umov x7, v4.d[0] /* load upper word of ctr */
328 rev x7, x7 /* ... to handle the carry */
329 add x7, x7, #1
330 rev x7, x7
331 ins v4.d[0], x7
Ard Biesheuvel11e3b722017-01-17 13:46:29 +0000[diff] [blame]332 b .Lctrcarrydone
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]333AES_ENDPROC(aes_ctr_encrypt)
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]334
335
336 /*
337 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
338 * int blocks, u8 const rk2[], u8 iv[], int first)
339 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
340 * int blocks, u8 const rk2[], u8 iv[], int first)
341 */
342
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]343 .macro next_tweak, out, in, tmp
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]344 sshr \tmp\().2d, \in\().2d, #63
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]345 and \tmp\().16b, \tmp\().16b, xtsmask.16b
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]346 add \out\().2d, \in\().2d, \in\().2d
347 ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
348 eor \out\().16b, \out\().16b, \tmp\().16b
349 .endm
350
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]351 .macro xts_load_mask, tmp
352 movi xtsmask.2s, #0x1
353 movi \tmp\().2s, #0x87
354 uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s
355 .endm
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]356
357AES_ENTRY(aes_xts_encrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]358 stp x29, x30, [sp, #-16]!
359 mov x29, sp
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]360
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]361 ld1 {v4.16b}, [x6]
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]362 cbz w7, .Lxtsencnotfirst
363
364 enc_prepare w3, x5, x8
365 encrypt_block v4, w3, x5, x8, w7 /* first tweak */
366 enc_switch_key w3, x2, x8
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]367 xts_load_mask v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]368 b .LxtsencNx
369
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]370.Lxtsencnotfirst:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]371 enc_prepare w3, x2, x8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]372.LxtsencloopNx:
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]373 xts_reload_mask v8
374 next_tweak v4, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]375.LxtsencNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]376 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]377 bmi .Lxtsenc1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]378 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]379 next_tweak v5, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]380 eor v0.16b, v0.16b, v4.16b
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]381 next_tweak v6, v5, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]382 eor v1.16b, v1.16b, v5.16b
383 eor v2.16b, v2.16b, v6.16b
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]384 next_tweak v7, v6, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]385 eor v3.16b, v3.16b, v7.16b
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]386 bl aes_encrypt_block4x
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]387 eor v3.16b, v3.16b, v7.16b
388 eor v0.16b, v0.16b, v4.16b
389 eor v1.16b, v1.16b, v5.16b
390 eor v2.16b, v2.16b, v6.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]391 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]392 mov v4.16b, v7.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]393 cbz w4, .Lxtsencout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]394 b .LxtsencloopNx
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]395.Lxtsenc1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]396 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]397 beq .Lxtsencout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]398.Lxtsencloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]399 ld1 {v1.16b}, [x1], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]400 eor v0.16b, v1.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]401 encrypt_block v0, w3, x2, x8, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]402 eor v0.16b, v0.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]403 st1 {v0.16b}, [x0], #16
404 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]405 beq .Lxtsencout
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]406 next_tweak v4, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]407 b .Lxtsencloop
408.Lxtsencout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]409 st1 {v4.16b}, [x6]
410 ldp x29, x30, [sp], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]411 ret
412AES_ENDPROC(aes_xts_encrypt)
413
414
415AES_ENTRY(aes_xts_decrypt)
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]416 stp x29, x30, [sp, #-16]!
417 mov x29, sp
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]418
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]419 ld1 {v4.16b}, [x6]
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]420 cbz w7, .Lxtsdecnotfirst
421
422 enc_prepare w3, x5, x8
423 encrypt_block v4, w3, x5, x8, w7 /* first tweak */
424 dec_prepare w3, x2, x8
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]425 xts_load_mask v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]426 b .LxtsdecNx
427
Ard Biesheuvel68338172018-03-10 15:21:48 +0000[diff] [blame]428.Lxtsdecnotfirst:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]429 dec_prepare w3, x2, x8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]430.LxtsdecloopNx:
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]431 xts_reload_mask v8
432 next_tweak v4, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]433.LxtsdecNx:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]434 subs w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]435 bmi .Lxtsdec1x
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]436 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]437 next_tweak v5, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]438 eor v0.16b, v0.16b, v4.16b
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]439 next_tweak v6, v5, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]440 eor v1.16b, v1.16b, v5.16b
441 eor v2.16b, v2.16b, v6.16b
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]442 next_tweak v7, v6, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]443 eor v3.16b, v3.16b, v7.16b
Ard Biesheuvel55868b42018-03-10 15:21:51 +0000[diff] [blame]444 bl aes_decrypt_block4x
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]445 eor v3.16b, v3.16b, v7.16b
446 eor v0.16b, v0.16b, v4.16b
447 eor v1.16b, v1.16b, v5.16b
448 eor v2.16b, v2.16b, v6.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]449 st1 {v0.16b-v3.16b}, [x0], #64
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]450 mov v4.16b, v7.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]451 cbz w4, .Lxtsdecout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]452 b .LxtsdecloopNx
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]453.Lxtsdec1x:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]454 adds w4, w4, #4
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]455 beq .Lxtsdecout
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]456.Lxtsdecloop:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]457 ld1 {v1.16b}, [x1], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]458 eor v0.16b, v1.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]459 decrypt_block v0, w3, x2, x8, w7
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]460 eor v0.16b, v0.16b, v4.16b
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]461 st1 {v0.16b}, [x0], #16
462 subs w4, w4, #1
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]463 beq .Lxtsdecout
Ard Biesheuvel2e5d2f32018-09-10 16:41:15 +0200[diff] [blame^]464 next_tweak v4, v4, v8
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]465 b .Lxtsdecloop
466.Lxtsdecout:
Ard Biesheuvel6e7de6a2018-09-10 16:41:13 +0200[diff] [blame]467 st1 {v4.16b}, [x6]
468 ldp x29, x30, [sp], #16
Ard Biesheuvel49788fe2014-03-21 10:19:17 +0100[diff] [blame]469 ret
470AES_ENDPROC(aes_xts_decrypt)
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]471
472 /*
473 * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
474 * int blocks, u8 dg[], int enc_before, int enc_after)
475 */
476AES_ENTRY(aes_mac_update)
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]477 frame_push 6
478
479 mov x19, x0
480 mov x20, x1
481 mov x21, x2
482 mov x22, x3
483 mov x23, x4
484 mov x24, x6
485
486 ld1 {v0.16b}, [x23] /* get dg */
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]487 enc_prepare w2, x1, x7
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]488 cbz w5, .Lmacloop4x
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]489
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]490 encrypt_block v0, w2, x1, x7, w8
491
492.Lmacloop4x:
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]493 subs w22, w22, #4
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]494 bmi .Lmac1x
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]495 ld1 {v1.16b-v4.16b}, [x19], #64 /* get next pt block */
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]496 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]497 encrypt_block v0, w21, x20, x7, w8
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]498 eor v0.16b, v0.16b, v2.16b
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]499 encrypt_block v0, w21, x20, x7, w8
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]500 eor v0.16b, v0.16b, v3.16b
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]501 encrypt_block v0, w21, x20, x7, w8
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]502 eor v0.16b, v0.16b, v4.16b
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]503 cmp w22, wzr
504 csinv x5, x24, xzr, eq
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]505 cbz w5, .Lmacout
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]506 encrypt_block v0, w21, x20, x7, w8
507 st1 {v0.16b}, [x23] /* return dg */
508 cond_yield_neon .Lmacrestart
Ard Biesheuvel870c1632018-03-10 15:21:53 +0000[diff] [blame]509 b .Lmacloop4x
510.Lmac1x:
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]511 add w22, w22, #4
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]512.Lmacloop:
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]513 cbz w22, .Lmacout
514 ld1 {v1.16b}, [x19], #16 /* get next pt block */
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]515 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
516
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]517 subs w22, w22, #1
518 csinv x5, x24, xzr, eq
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]519 cbz w5, .Lmacout
520
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]521.Lmacenc:
522 encrypt_block v0, w21, x20, x7, w8
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]523 b .Lmacloop
524
525.Lmacout:
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]526 st1 {v0.16b}, [x23] /* return dg */
527 frame_pop
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]528 ret
Ard Biesheuvel0c8f8382018-04-30 18:18:24 +0200[diff] [blame]529
530.Lmacrestart:
531 ld1 {v0.16b}, [x23] /* get dg */
532 enc_prepare w21, x20, x0
533 b .Lmacloop4x
Ard Biesheuvel48606202017-02-03 14:49:37 +0000[diff] [blame]534AES_ENDPROC(aes_mac_update)