Blame - arch/x86/crypto/aesni-intel_avx-x86_64.S - SHIFTPHONES/kernel/common

blob: faecb1518bf8164831d4d44099c11e36fc45f411 [file] [log] [blame]

Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1	########################################################################
				2	# Copyright (c) 2013, Intel Corporation
				3	#
				4	# This software is available to you under a choice of one of two
				5	# licenses. You may choose to be licensed under the terms of the GNU
				6	# General Public License (GPL) Version 2, available from the file
				7	# COPYING in the main directory of this source tree, or the
				8	# OpenIB.org BSD license below:
				9	#
				10	# Redistribution and use in source and binary forms, with or without
				11	# modification, are permitted provided that the following conditions are
				12	# met:
				13	#
				14	# * Redistributions of source code must retain the above copyright
				15	# notice, this list of conditions and the following disclaimer.
				16	#
				17	# * Redistributions in binary form must reproduce the above copyright
				18	# notice, this list of conditions and the following disclaimer in the
				19	# documentation and/or other materials provided with the
				20	# distribution.
				21	#
				22	# * Neither the name of the Intel Corporation nor the names of its
				23	# contributors may be used to endorse or promote products derived from
				24	# this software without specific prior written permission.
				25	#
				26	#
				27	# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
				28	# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
				29	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
				30	# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
				31	# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
				32	# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
				33	# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES# LOSS OF USE, DATA, OR
				34	# PROFITS# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
				35	# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
				36	# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
				37	# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
				38	########################################################################
				39	##
				40	## Authors:
				41	## Erdinc Ozturk <erdinc.ozturk@intel.com>
				42	## Vinodh Gopal <vinodh.gopal@intel.com>
				43	## James Guilford <james.guilford@intel.com>
				44	## Tim Chen <tim.c.chen@linux.intel.com>
				45	##
				46	## References:
				47	## This code was derived and highly optimized from the code described in paper:
				48	## Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation
				49	## on Intel Architecture Processors. August, 2010
				50	## The details of the implementation is explained in:
				51	## Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode
				52	## on Intel Architecture Processors. October, 2012.
				53	##
				54	## Assumptions:
				55	##
				56	##
				57	##
				58	## iv:
				59	## 0 1 2 3
				60	## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
				61	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				62	## \| Salt (From the SA) \|
				63	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				64	## \| Initialization Vector \|
				65	## \| (This is the sequence number from IPSec header) \|
				66	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				67	## \| 0x1 \|
				68	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				69	##
				70	##
				71	##
				72	## AAD:
				73	## AAD padded to 128 bits with 0
				74	## for example, assume AAD is a u32 vector
				75	##
				76	## if AAD is 8 bytes:
				77	## AAD[3] = {A0, A1}#
				78	## padded AAD in xmm register = {A1 A0 0 0}
				79	##
				80	## 0 1 2 3
				81	## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
				82	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				83	## \| SPI (A1) \|
				84	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				85	## \| 32-bit Sequence Number (A0) \|
				86	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				87	## \| 0x0 \|
				88	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				89	##
				90	## AAD Format with 32-bit Sequence Number
				91	##
				92	## if AAD is 12 bytes:
				93	## AAD[3] = {A0, A1, A2}#
				94	## padded AAD in xmm register = {A2 A1 A0 0}
				95	##
				96	## 0 1 2 3
				97	## 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
				98	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				99	## \| SPI (A2) \|
				100	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				101	## \| 64-bit Extended Sequence Number {A1,A0} \|
				102	## \| \|
				103	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				104	## \| 0x0 \|
				105	## +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
				106	##
				107	## AAD Format with 64-bit Extended Sequence Number
				108	##
				109	##
				110	## aadLen:
				111	## from the definition of the spec, aadLen can only be 8 or 12 bytes.
				112	## The code additionally supports aadLen of length 16 bytes.
				113	##
				114	## TLen:
				115	## from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
				116	##
				117	## poly = x^128 + x^127 + x^126 + x^121 + 1
				118	## throughout the code, one tab and two tab indentations are used. one tab is
				119	## for GHASH part, two tabs is for AES part.
				120	##
				121
				122	#include <linux/linkage.h>
				123	#include <asm/inst.h>
				124
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	125	# constants in mergeable sections, linker can reorder and merge
				126	.section .rodata.cst16.POLY, "aM", @progbits, 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	127	.align 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	128	POLY: .octa 0xC2000000000000000000000000000001
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	129
				130	.section .rodata.cst16.POLY2, "aM", @progbits, 16
				131	.align 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	132	POLY2: .octa 0xC20000000000000000000001C2000000
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	133
				134	.section .rodata.cst16.TWOONE, "aM", @progbits, 16
				135	.align 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	136	TWOONE: .octa 0x00000001000000000000000000000001
				137
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	138	.section .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
				139	.align 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	140	SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	141
				142	.section .rodata.cst16.ONE, "aM", @progbits, 16
				143	.align 16
				144	ONE: .octa 0x00000000000000000000000000000001
				145
				146	.section .rodata.cst16.ONEf, "aM", @progbits, 16
				147	.align 16
				148	ONEf: .octa 0x01000000000000000000000000000000
				149
				150	# order of these constants should not change.
				151	# more specifically, ALL_F should follow SHIFT_MASK, and zero should follow ALL_F
				152	.section .rodata, "a", @progbits
				153	.align 16
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	154	SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
				155	ALL_F: .octa 0xffffffffffffffffffffffffffffffff
Denys Vlasenko	e183914	2017-01-19 22:33:04 +0100	[diff] [blame]	156	.octa 0x00000000000000000000000000000000
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	157
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	158	.section .rodata
				159	.align 16
				160	.type aad_shift_arr, @object
				161	.size aad_shift_arr, 272
				162	aad_shift_arr:
				163	.octa 0xffffffffffffffffffffffffffffffff
				164	.octa 0xffffffffffffffffffffffffffffff0C
				165	.octa 0xffffffffffffffffffffffffffff0D0C
				166	.octa 0xffffffffffffffffffffffffff0E0D0C
				167	.octa 0xffffffffffffffffffffffff0F0E0D0C
				168	.octa 0xffffffffffffffffffffff0C0B0A0908
				169	.octa 0xffffffffffffffffffff0D0C0B0A0908
				170	.octa 0xffffffffffffffffff0E0D0C0B0A0908
				171	.octa 0xffffffffffffffff0F0E0D0C0B0A0908
				172	.octa 0xffffffffffffff0C0B0A090807060504
				173	.octa 0xffffffffffff0D0C0B0A090807060504
				174	.octa 0xffffffffff0E0D0C0B0A090807060504
				175	.octa 0xffffffff0F0E0D0C0B0A090807060504
				176	.octa 0xffffff0C0B0A09080706050403020100
				177	.octa 0xffff0D0C0B0A09080706050403020100
				178	.octa 0xff0E0D0C0B0A09080706050403020100
				179	.octa 0x0F0E0D0C0B0A09080706050403020100
				180
				181
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	182	.text
				183
				184
				185	##define the fields of the gcm aes context
				186	#{
				187	# u8 expanded_keys[16*11] store expanded keys
				188	# u8 shifted_hkey_1[16] store HashKey <<1 mod poly here
				189	# u8 shifted_hkey_2[16] store HashKey^2 <<1 mod poly here
				190	# u8 shifted_hkey_3[16] store HashKey^3 <<1 mod poly here
				191	# u8 shifted_hkey_4[16] store HashKey^4 <<1 mod poly here
				192	# u8 shifted_hkey_5[16] store HashKey^5 <<1 mod poly here
				193	# u8 shifted_hkey_6[16] store HashKey^6 <<1 mod poly here
				194	# u8 shifted_hkey_7[16] store HashKey^7 <<1 mod poly here
				195	# u8 shifted_hkey_8[16] store HashKey^8 <<1 mod poly here
				196	# u8 shifted_hkey_1_k[16] store XOR HashKey <<1 mod poly here (for Karatsuba purposes)
				197	# u8 shifted_hkey_2_k[16] store XOR HashKey^2 <<1 mod poly here (for Karatsuba purposes)
				198	# u8 shifted_hkey_3_k[16] store XOR HashKey^3 <<1 mod poly here (for Karatsuba purposes)
				199	# u8 shifted_hkey_4_k[16] store XOR HashKey^4 <<1 mod poly here (for Karatsuba purposes)
				200	# u8 shifted_hkey_5_k[16] store XOR HashKey^5 <<1 mod poly here (for Karatsuba purposes)
				201	# u8 shifted_hkey_6_k[16] store XOR HashKey^6 <<1 mod poly here (for Karatsuba purposes)
				202	# u8 shifted_hkey_7_k[16] store XOR HashKey^7 <<1 mod poly here (for Karatsuba purposes)
				203	# u8 shifted_hkey_8_k[16] store XOR HashKey^8 <<1 mod poly here (for Karatsuba purposes)
				204	#} gcm_ctx#
				205
				206	HashKey = 16*11 # store HashKey <<1 mod poly here
				207	HashKey_2 = 16*12 # store HashKey^2 <<1 mod poly here
				208	HashKey_3 = 16*13 # store HashKey^3 <<1 mod poly here
				209	HashKey_4 = 16*14 # store HashKey^4 <<1 mod poly here
				210	HashKey_5 = 16*15 # store HashKey^5 <<1 mod poly here
				211	HashKey_6 = 16*16 # store HashKey^6 <<1 mod poly here
				212	HashKey_7 = 16*17 # store HashKey^7 <<1 mod poly here
				213	HashKey_8 = 16*18 # store HashKey^8 <<1 mod poly here
				214	HashKey_k = 16*19 # store XOR of HashKey <<1 mod poly here (for Karatsuba purposes)
				215	HashKey_2_k = 16*20 # store XOR of HashKey^2 <<1 mod poly here (for Karatsuba purposes)
				216	HashKey_3_k = 16*21 # store XOR of HashKey^3 <<1 mod poly here (for Karatsuba purposes)
				217	HashKey_4_k = 16*22 # store XOR of HashKey^4 <<1 mod poly here (for Karatsuba purposes)
				218	HashKey_5_k = 16*23 # store XOR of HashKey^5 <<1 mod poly here (for Karatsuba purposes)
				219	HashKey_6_k = 16*24 # store XOR of HashKey^6 <<1 mod poly here (for Karatsuba purposes)
				220	HashKey_7_k = 16*25 # store XOR of HashKey^7 <<1 mod poly here (for Karatsuba purposes)
				221	HashKey_8_k = 16*26 # store XOR of HashKey^8 <<1 mod poly here (for Karatsuba purposes)
				222
				223	#define arg1 %rdi
				224	#define arg2 %rsi
				225	#define arg3 %rdx
				226	#define arg4 %rcx
				227	#define arg5 %r8
				228	#define arg6 %r9
				229	#define arg7 STACK_OFFSET+8*1(%r14)
				230	#define arg8 STACK_OFFSET+8*2(%r14)
				231	#define arg9 STACK_OFFSET+8*3(%r14)
				232
				233	i = 0
				234	j = 0
				235
				236	out_order = 0
				237	in_order = 1
				238	DEC = 0
				239	ENC = 1
				240
				241	.macro define_reg r n
				242	reg_\r = %xmm\n
				243	.endm
				244
				245	.macro setreg
				246	.altmacro
				247	define_reg i %i
				248	define_reg j %j
				249	.noaltmacro
				250	.endm
				251
				252	# need to push 4 registers into stack to maintain
				253	STACK_OFFSET = 8*4
				254
				255	TMP1 = 16*0 # Temporary storage for AAD
				256	TMP2 = 16*1 # Temporary storage for AES State 2 (State 1 is stored in an XMM register)
				257	TMP3 = 16*2 # Temporary storage for AES State 3
				258	TMP4 = 16*3 # Temporary storage for AES State 4
				259	TMP5 = 16*4 # Temporary storage for AES State 5
				260	TMP6 = 16*5 # Temporary storage for AES State 6
				261	TMP7 = 16*6 # Temporary storage for AES State 7
				262	TMP8 = 16*7 # Temporary storage for AES State 8
				263
				264	VARIABLE_OFFSET = 16*8
				265
				266	################################
				267	# Utility Macros
				268	################################
				269
				270	# Encryption of a single block
				271	.macro ENCRYPT_SINGLE_BLOCK XMM0
				272	vpxor (arg1), \XMM0, \XMM0
				273	i = 1
				274	setreg
				275	.rep 9
				276	vaesenc 16*i(arg1), \XMM0, \XMM0
				277	i = (i+1)
				278	setreg
				279	.endr
				280	vaesenclast 16*10(arg1), \XMM0, \XMM0
				281	.endm
				282
				283	#ifdef CONFIG_AS_AVX
				284	###############################################################################
				285	# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
				286	# Input: A and B (128-bits each, bit-reflected)
				287	# Output: C = ABx mod poly, (i.e. >>1 )
				288	# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
				289	# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
				290	###############################################################################
				291	.macro GHASH_MUL_AVX GH HK T1 T2 T3 T4 T5
				292
				293	vpshufd $0b01001110, \GH, \T2
				294	vpshufd $0b01001110, \HK, \T3
				295	vpxor \GH , \T2, \T2 # T2 = (a1+a0)
				296	vpxor \HK , \T3, \T3 # T3 = (b1+b0)
				297
				298	vpclmulqdq $0x11, \HK, \GH, \T1 # T1 = a1*b1
				299	vpclmulqdq $0x00, \HK, \GH, \GH # GH = a0*b0
				300	vpclmulqdq $0x00, \T3, \T2, \T2 # T2 = (a1+a0)*(b1+b0)
				301	vpxor \GH, \T2,\T2
				302	vpxor \T1, \T2,\T2 # T2 = a0b1+a1b0
				303
				304	vpslldq $8, \T2,\T3 # shift-L T3 2 DWs
				305	vpsrldq $8, \T2,\T2 # shift-R T2 2 DWs
				306	vpxor \T3, \GH, \GH
				307	vpxor \T2, \T1, \T1 # <T1:GH> = GH x HK
				308
				309	#first phase of the reduction
				310	vpslld $31, \GH, \T2 # packed right shifting << 31
				311	vpslld $30, \GH, \T3 # packed right shifting shift << 30
				312	vpslld $25, \GH, \T4 # packed right shifting shift << 25
				313
				314	vpxor \T3, \T2, \T2 # xor the shifted versions
				315	vpxor \T4, \T2, \T2
				316
				317	vpsrldq $4, \T2, \T5 # shift-R T5 1 DW
				318
				319	vpslldq $12, \T2, \T2 # shift-L T2 3 DWs
				320	vpxor \T2, \GH, \GH # first phase of the reduction complete
				321
				322	#second phase of the reduction
				323
				324	vpsrld $1,\GH, \T2 # packed left shifting >> 1
				325	vpsrld $2,\GH, \T3 # packed left shifting >> 2
				326	vpsrld $7,\GH, \T4 # packed left shifting >> 7
				327	vpxor \T3, \T2, \T2 # xor the shifted versions
				328	vpxor \T4, \T2, \T2
				329
				330	vpxor \T5, \T2, \T2
				331	vpxor \T2, \GH, \GH
				332	vpxor \T1, \GH, \GH # the result is in GH
				333
				334
				335	.endm
				336
				337	.macro PRECOMPUTE_AVX HK T1 T2 T3 T4 T5 T6
				338
				339	# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
				340	vmovdqa \HK, \T5
				341
				342	vpshufd $0b01001110, \T5, \T1
				343	vpxor \T5, \T1, \T1
				344	vmovdqa \T1, HashKey_k(arg1)
				345
				346	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly
				347	vmovdqa \T5, HashKey_2(arg1) # [HashKey_2] = HashKey^2<<1 mod poly
				348	vpshufd $0b01001110, \T5, \T1
				349	vpxor \T5, \T1, \T1
				350	vmovdqa \T1, HashKey_2_k(arg1)
				351
				352	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly
				353	vmovdqa \T5, HashKey_3(arg1)
				354	vpshufd $0b01001110, \T5, \T1
				355	vpxor \T5, \T1, \T1
				356	vmovdqa \T1, HashKey_3_k(arg1)
				357
				358	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly
				359	vmovdqa \T5, HashKey_4(arg1)
				360	vpshufd $0b01001110, \T5, \T1
				361	vpxor \T5, \T1, \T1
				362	vmovdqa \T1, HashKey_4_k(arg1)
				363
				364	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly
				365	vmovdqa \T5, HashKey_5(arg1)
				366	vpshufd $0b01001110, \T5, \T1
				367	vpxor \T5, \T1, \T1
				368	vmovdqa \T1, HashKey_5_k(arg1)
				369
				370	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly
				371	vmovdqa \T5, HashKey_6(arg1)
				372	vpshufd $0b01001110, \T5, \T1
				373	vpxor \T5, \T1, \T1
				374	vmovdqa \T1, HashKey_6_k(arg1)
				375
				376	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly
				377	vmovdqa \T5, HashKey_7(arg1)
				378	vpshufd $0b01001110, \T5, \T1
				379	vpxor \T5, \T1, \T1
				380	vmovdqa \T1, HashKey_7_k(arg1)
				381
				382	GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly
				383	vmovdqa \T5, HashKey_8(arg1)
				384	vpshufd $0b01001110, \T5, \T1
				385	vpxor \T5, \T1, \T1
				386	vmovdqa \T1, HashKey_8_k(arg1)
				387
				388	.endm
				389
				390	## if a = number of total plaintext bytes
				391	## b = floor(a/16)
				392	## num_initial_blocks = b mod 4#
				393	## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
				394	## r10, r11, r12, rax are clobbered
				395	## arg1, arg2, arg3, r14 are used as a pointer only, not modified
				396
				397	.macro INITIAL_BLOCKS_AVX num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC
				398	i = (8-\num_initial_blocks)
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	399	j = 0
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	400	setreg
				401
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	402	mov arg6, %r10 # r10 = AAD
				403	mov arg7, %r12 # r12 = aadLen
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	404
				405
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	406	mov %r12, %r11
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	407
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	408	vpxor reg_j, reg_j, reg_j
				409	vpxor reg_i, reg_i, reg_i
				410	cmp $16, %r11
				411	jl _get_AAD_rest8\@
				412	_get_AAD_blocks\@:
				413	vmovdqu (%r10), reg_i
				414	vpshufb SHUF_MASK(%rip), reg_i, reg_i
				415	vpxor reg_i, reg_j, reg_j
				416	GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6
				417	add $16, %r10
				418	sub $16, %r12
				419	sub $16, %r11
				420	cmp $16, %r11
				421	jge _get_AAD_blocks\@
				422	vmovdqu reg_j, reg_i
				423	cmp $0, %r11
				424	je _get_AAD_done\@
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	425
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	426	vpxor reg_i, reg_i, reg_i
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	427
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	428	/* read the last <16B of AAD. since we have at least 4B of
				429	data right after the AAD (the ICV, and maybe some CT), we can
				430	read 4B/8B blocks safely, and then get rid of the extra stuff */
				431	_get_AAD_rest8\@:
				432	cmp $4, %r11
				433	jle _get_AAD_rest4\@
				434	movq (%r10), \T1
				435	add $8, %r10
				436	sub $8, %r11
				437	vpslldq $8, \T1, \T1
				438	vpsrldq $8, reg_i, reg_i
				439	vpxor \T1, reg_i, reg_i
				440	jmp _get_AAD_rest8\@
				441	_get_AAD_rest4\@:
				442	cmp $0, %r11
				443	jle _get_AAD_rest0\@
				444	mov (%r10), %eax
				445	movq %rax, \T1
				446	add $4, %r10
				447	sub $4, %r11
				448	vpslldq $12, \T1, \T1
				449	vpsrldq $4, reg_i, reg_i
				450	vpxor \T1, reg_i, reg_i
				451	_get_AAD_rest0\@:
				452	/* finalize: shift out the extra bytes we read, and align
				453	left. since pslldq can only shift by an immediate, we use
				454	vpshufb and an array of shuffle masks */
				455	movq %r12, %r11
				456	salq $4, %r11
				457	movdqu aad_shift_arr(%r11), \T1
				458	vpshufb \T1, reg_i, reg_i
				459	_get_AAD_rest_final\@:
				460	vpshufb SHUF_MASK(%rip), reg_i, reg_i
				461	vpxor reg_j, reg_i, reg_i
				462	GHASH_MUL_AVX reg_i, \T2, \T1, \T3, \T4, \T5, \T6
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	463
Sabrina Dubroca	e10f9cb	2017-04-28 18:11:58 +0200	[diff] [blame]	464	_get_AAD_done\@:
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	465	# initialize the data pointer offset as zero
				466	xor %r11, %r11
				467
				468	# start AES for num_initial_blocks blocks
				469	mov arg5, %rax # rax = *Y0
				470	vmovdqu (%rax), \CTR # CTR = Y0
				471	vpshufb SHUF_MASK(%rip), \CTR, \CTR
				472
				473
				474	i = (9-\num_initial_blocks)
				475	setreg
				476	.rep \num_initial_blocks
				477	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				478	vmovdqa \CTR, reg_i
				479	vpshufb SHUF_MASK(%rip), reg_i, reg_i # perform a 16Byte swap
				480	i = (i+1)
				481	setreg
				482	.endr
				483
				484	vmovdqa (arg1), \T_key
				485	i = (9-\num_initial_blocks)
				486	setreg
				487	.rep \num_initial_blocks
				488	vpxor \T_key, reg_i, reg_i
				489	i = (i+1)
				490	setreg
				491	.endr
				492
				493	j = 1
				494	setreg
				495	.rep 9
				496	vmovdqa 16*j(arg1), \T_key
				497	i = (9-\num_initial_blocks)
				498	setreg
				499	.rep \num_initial_blocks
				500	vaesenc \T_key, reg_i, reg_i
				501	i = (i+1)
				502	setreg
				503	.endr
				504
				505	j = (j+1)
				506	setreg
				507	.endr
				508
				509
				510	vmovdqa 16*10(arg1), \T_key
				511	i = (9-\num_initial_blocks)
				512	setreg
				513	.rep \num_initial_blocks
				514	vaesenclast \T_key, reg_i, reg_i
				515	i = (i+1)
				516	setreg
				517	.endr
				518
				519	i = (9-\num_initial_blocks)
				520	setreg
				521	.rep \num_initial_blocks
				522	vmovdqu (arg3, %r11), \T1
				523	vpxor \T1, reg_i, reg_i
				524	vmovdqu reg_i, (arg2 , %r11) # write back ciphertext for num_initial_blocks blocks
				525	add $16, %r11
				526	.if \ENC_DEC == DEC
				527	vmovdqa \T1, reg_i
				528	.endif
				529	vpshufb SHUF_MASK(%rip), reg_i, reg_i # prepare ciphertext for GHASH computations
				530	i = (i+1)
				531	setreg
				532	.endr
				533
				534
				535	i = (8-\num_initial_blocks)
				536	j = (9-\num_initial_blocks)
				537	setreg
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	538
				539	.rep \num_initial_blocks
				540	vpxor reg_i, reg_j, reg_j
				541	GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks
				542	i = (i+1)
				543	j = (j+1)
				544	setreg
				545	.endr
				546	# XMM8 has the combined result here
				547
				548	vmovdqa \XMM8, TMP1(%rsp)
				549	vmovdqa \XMM8, \T3
				550
				551	cmp $128, %r13
				552	jl _initial_blocks_done\@ # no need for precomputed constants
				553
				554	###############################################################################
				555	# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
				556	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				557	vmovdqa \CTR, \XMM1
				558	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				559
				560	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				561	vmovdqa \CTR, \XMM2
				562	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				563
				564	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				565	vmovdqa \CTR, \XMM3
				566	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				567
				568	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				569	vmovdqa \CTR, \XMM4
				570	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				571
				572	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				573	vmovdqa \CTR, \XMM5
				574	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				575
				576	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				577	vmovdqa \CTR, \XMM6
				578	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				579
				580	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				581	vmovdqa \CTR, \XMM7
				582	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				583
				584	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				585	vmovdqa \CTR, \XMM8
				586	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				587
				588	vmovdqa (arg1), \T_key
				589	vpxor \T_key, \XMM1, \XMM1
				590	vpxor \T_key, \XMM2, \XMM2
				591	vpxor \T_key, \XMM3, \XMM3
				592	vpxor \T_key, \XMM4, \XMM4
				593	vpxor \T_key, \XMM5, \XMM5
				594	vpxor \T_key, \XMM6, \XMM6
				595	vpxor \T_key, \XMM7, \XMM7
				596	vpxor \T_key, \XMM8, \XMM8
				597
				598	i = 1
				599	setreg
				600	.rep 9 # do 9 rounds
				601	vmovdqa 16*i(arg1), \T_key
				602	vaesenc \T_key, \XMM1, \XMM1
				603	vaesenc \T_key, \XMM2, \XMM2
				604	vaesenc \T_key, \XMM3, \XMM3
				605	vaesenc \T_key, \XMM4, \XMM4
				606	vaesenc \T_key, \XMM5, \XMM5
				607	vaesenc \T_key, \XMM6, \XMM6
				608	vaesenc \T_key, \XMM7, \XMM7
				609	vaesenc \T_key, \XMM8, \XMM8
				610	i = (i+1)
				611	setreg
				612	.endr
				613
				614
				615	vmovdqa 16*i(arg1), \T_key
				616	vaesenclast \T_key, \XMM1, \XMM1
				617	vaesenclast \T_key, \XMM2, \XMM2
				618	vaesenclast \T_key, \XMM3, \XMM3
				619	vaesenclast \T_key, \XMM4, \XMM4
				620	vaesenclast \T_key, \XMM5, \XMM5
				621	vaesenclast \T_key, \XMM6, \XMM6
				622	vaesenclast \T_key, \XMM7, \XMM7
				623	vaesenclast \T_key, \XMM8, \XMM8
				624
				625	vmovdqu (arg3, %r11), \T1
				626	vpxor \T1, \XMM1, \XMM1
				627	vmovdqu \XMM1, (arg2 , %r11)
				628	.if \ENC_DEC == DEC
				629	vmovdqa \T1, \XMM1
				630	.endif
				631
				632	vmovdqu 16*1(arg3, %r11), \T1
				633	vpxor \T1, \XMM2, \XMM2
				634	vmovdqu \XMM2, 16*1(arg2 , %r11)
				635	.if \ENC_DEC == DEC
				636	vmovdqa \T1, \XMM2
				637	.endif
				638
				639	vmovdqu 16*2(arg3, %r11), \T1
				640	vpxor \T1, \XMM3, \XMM3
				641	vmovdqu \XMM3, 16*2(arg2 , %r11)
				642	.if \ENC_DEC == DEC
				643	vmovdqa \T1, \XMM3
				644	.endif
				645
				646	vmovdqu 16*3(arg3, %r11), \T1
				647	vpxor \T1, \XMM4, \XMM4
				648	vmovdqu \XMM4, 16*3(arg2 , %r11)
				649	.if \ENC_DEC == DEC
				650	vmovdqa \T1, \XMM4
				651	.endif
				652
				653	vmovdqu 16*4(arg3, %r11), \T1
				654	vpxor \T1, \XMM5, \XMM5
				655	vmovdqu \XMM5, 16*4(arg2 , %r11)
				656	.if \ENC_DEC == DEC
				657	vmovdqa \T1, \XMM5
				658	.endif
				659
				660	vmovdqu 16*5(arg3, %r11), \T1
				661	vpxor \T1, \XMM6, \XMM6
				662	vmovdqu \XMM6, 16*5(arg2 , %r11)
				663	.if \ENC_DEC == DEC
				664	vmovdqa \T1, \XMM6
				665	.endif
				666
				667	vmovdqu 16*6(arg3, %r11), \T1
				668	vpxor \T1, \XMM7, \XMM7
				669	vmovdqu \XMM7, 16*6(arg2 , %r11)
				670	.if \ENC_DEC == DEC
				671	vmovdqa \T1, \XMM7
				672	.endif
				673
				674	vmovdqu 16*7(arg3, %r11), \T1
				675	vpxor \T1, \XMM8, \XMM8
				676	vmovdqu \XMM8, 16*7(arg2 , %r11)
				677	.if \ENC_DEC == DEC
				678	vmovdqa \T1, \XMM8
				679	.endif
				680
				681	add $128, %r11
				682
				683	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				684	vpxor TMP1(%rsp), \XMM1, \XMM1 # combine GHASHed value with the corresponding ciphertext
				685	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				686	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				687	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				688	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				689	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				690	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				691	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				692
				693	###############################################################################
				694
				695	_initial_blocks_done\@:
				696
				697	.endm
				698
				699	# encrypt 8 blocks at a time
				700	# ghash the 8 previously encrypted ciphertext blocks
				701	# arg1, arg2, arg3 are used as pointers only, not modified
				702	# r11 is the data offset value
				703	.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
				704
				705	vmovdqa \XMM1, \T2
				706	vmovdqa \XMM2, TMP2(%rsp)
				707	vmovdqa \XMM3, TMP3(%rsp)
				708	vmovdqa \XMM4, TMP4(%rsp)
				709	vmovdqa \XMM5, TMP5(%rsp)
				710	vmovdqa \XMM6, TMP6(%rsp)
				711	vmovdqa \XMM7, TMP7(%rsp)
				712	vmovdqa \XMM8, TMP8(%rsp)
				713
				714	.if \loop_idx == in_order
				715	vpaddd ONE(%rip), \CTR, \XMM1 # INCR CNT
				716	vpaddd ONE(%rip), \XMM1, \XMM2
				717	vpaddd ONE(%rip), \XMM2, \XMM3
				718	vpaddd ONE(%rip), \XMM3, \XMM4
				719	vpaddd ONE(%rip), \XMM4, \XMM5
				720	vpaddd ONE(%rip), \XMM5, \XMM6
				721	vpaddd ONE(%rip), \XMM6, \XMM7
				722	vpaddd ONE(%rip), \XMM7, \XMM8
				723	vmovdqa \XMM8, \CTR
				724
				725	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				726	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				727	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				728	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				729	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				730	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				731	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				732	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				733	.else
				734	vpaddd ONEf(%rip), \CTR, \XMM1 # INCR CNT
				735	vpaddd ONEf(%rip), \XMM1, \XMM2
				736	vpaddd ONEf(%rip), \XMM2, \XMM3
				737	vpaddd ONEf(%rip), \XMM3, \XMM4
				738	vpaddd ONEf(%rip), \XMM4, \XMM5
				739	vpaddd ONEf(%rip), \XMM5, \XMM6
				740	vpaddd ONEf(%rip), \XMM6, \XMM7
				741	vpaddd ONEf(%rip), \XMM7, \XMM8
				742	vmovdqa \XMM8, \CTR
				743	.endif
				744
				745
				746	#######################################################################
				747
				748	vmovdqu (arg1), \T1
				749	vpxor \T1, \XMM1, \XMM1
				750	vpxor \T1, \XMM2, \XMM2
				751	vpxor \T1, \XMM3, \XMM3
				752	vpxor \T1, \XMM4, \XMM4
				753	vpxor \T1, \XMM5, \XMM5
				754	vpxor \T1, \XMM6, \XMM6
				755	vpxor \T1, \XMM7, \XMM7
				756	vpxor \T1, \XMM8, \XMM8
				757
				758	#######################################################################
				759
				760
				761
				762
				763
				764	vmovdqu 16*1(arg1), \T1
				765	vaesenc \T1, \XMM1, \XMM1
				766	vaesenc \T1, \XMM2, \XMM2
				767	vaesenc \T1, \XMM3, \XMM3
				768	vaesenc \T1, \XMM4, \XMM4
				769	vaesenc \T1, \XMM5, \XMM5
				770	vaesenc \T1, \XMM6, \XMM6
				771	vaesenc \T1, \XMM7, \XMM7
				772	vaesenc \T1, \XMM8, \XMM8
				773
				774	vmovdqu 16*2(arg1), \T1
				775	vaesenc \T1, \XMM1, \XMM1
				776	vaesenc \T1, \XMM2, \XMM2
				777	vaesenc \T1, \XMM3, \XMM3
				778	vaesenc \T1, \XMM4, \XMM4
				779	vaesenc \T1, \XMM5, \XMM5
				780	vaesenc \T1, \XMM6, \XMM6
				781	vaesenc \T1, \XMM7, \XMM7
				782	vaesenc \T1, \XMM8, \XMM8
				783
				784
				785	#######################################################################
				786
				787	vmovdqa HashKey_8(arg1), \T5
				788	vpclmulqdq $0x11, \T5, \T2, \T4 # T4 = a1*b1
				789	vpclmulqdq $0x00, \T5, \T2, \T7 # T7 = a0*b0
				790
				791	vpshufd $0b01001110, \T2, \T6
				792	vpxor \T2, \T6, \T6
				793
				794	vmovdqa HashKey_8_k(arg1), \T5
				795	vpclmulqdq $0x00, \T5, \T6, \T6
				796
				797	vmovdqu 16*3(arg1), \T1
				798	vaesenc \T1, \XMM1, \XMM1
				799	vaesenc \T1, \XMM2, \XMM2
				800	vaesenc \T1, \XMM3, \XMM3
				801	vaesenc \T1, \XMM4, \XMM4
				802	vaesenc \T1, \XMM5, \XMM5
				803	vaesenc \T1, \XMM6, \XMM6
				804	vaesenc \T1, \XMM7, \XMM7
				805	vaesenc \T1, \XMM8, \XMM8
				806
				807	vmovdqa TMP2(%rsp), \T1
				808	vmovdqa HashKey_7(arg1), \T5
				809	vpclmulqdq $0x11, \T5, \T1, \T3
				810	vpxor \T3, \T4, \T4
				811	vpclmulqdq $0x00, \T5, \T1, \T3
				812	vpxor \T3, \T7, \T7
				813
				814	vpshufd $0b01001110, \T1, \T3
				815	vpxor \T1, \T3, \T3
				816	vmovdqa HashKey_7_k(arg1), \T5
				817	vpclmulqdq $0x10, \T5, \T3, \T3
				818	vpxor \T3, \T6, \T6
				819
				820	vmovdqu 16*4(arg1), \T1
				821	vaesenc \T1, \XMM1, \XMM1
				822	vaesenc \T1, \XMM2, \XMM2
				823	vaesenc \T1, \XMM3, \XMM3
				824	vaesenc \T1, \XMM4, \XMM4
				825	vaesenc \T1, \XMM5, \XMM5
				826	vaesenc \T1, \XMM6, \XMM6
				827	vaesenc \T1, \XMM7, \XMM7
				828	vaesenc \T1, \XMM8, \XMM8
				829
				830	#######################################################################
				831
				832	vmovdqa TMP3(%rsp), \T1
				833	vmovdqa HashKey_6(arg1), \T5
				834	vpclmulqdq $0x11, \T5, \T1, \T3
				835	vpxor \T3, \T4, \T4
				836	vpclmulqdq $0x00, \T5, \T1, \T3
				837	vpxor \T3, \T7, \T7
				838
				839	vpshufd $0b01001110, \T1, \T3
				840	vpxor \T1, \T3, \T3
				841	vmovdqa HashKey_6_k(arg1), \T5
				842	vpclmulqdq $0x10, \T5, \T3, \T3
				843	vpxor \T3, \T6, \T6
				844
				845	vmovdqu 16*5(arg1), \T1
				846	vaesenc \T1, \XMM1, \XMM1
				847	vaesenc \T1, \XMM2, \XMM2
				848	vaesenc \T1, \XMM3, \XMM3
				849	vaesenc \T1, \XMM4, \XMM4
				850	vaesenc \T1, \XMM5, \XMM5
				851	vaesenc \T1, \XMM6, \XMM6
				852	vaesenc \T1, \XMM7, \XMM7
				853	vaesenc \T1, \XMM8, \XMM8
				854
				855	vmovdqa TMP4(%rsp), \T1
				856	vmovdqa HashKey_5(arg1), \T5
				857	vpclmulqdq $0x11, \T5, \T1, \T3
				858	vpxor \T3, \T4, \T4
				859	vpclmulqdq $0x00, \T5, \T1, \T3
				860	vpxor \T3, \T7, \T7
				861
				862	vpshufd $0b01001110, \T1, \T3
				863	vpxor \T1, \T3, \T3
				864	vmovdqa HashKey_5_k(arg1), \T5
				865	vpclmulqdq $0x10, \T5, \T3, \T3
				866	vpxor \T3, \T6, \T6
				867
				868	vmovdqu 16*6(arg1), \T1
				869	vaesenc \T1, \XMM1, \XMM1
				870	vaesenc \T1, \XMM2, \XMM2
				871	vaesenc \T1, \XMM3, \XMM3
				872	vaesenc \T1, \XMM4, \XMM4
				873	vaesenc \T1, \XMM5, \XMM5
				874	vaesenc \T1, \XMM6, \XMM6
				875	vaesenc \T1, \XMM7, \XMM7
				876	vaesenc \T1, \XMM8, \XMM8
				877
				878
				879	vmovdqa TMP5(%rsp), \T1
				880	vmovdqa HashKey_4(arg1), \T5
				881	vpclmulqdq $0x11, \T5, \T1, \T3
				882	vpxor \T3, \T4, \T4
				883	vpclmulqdq $0x00, \T5, \T1, \T3
				884	vpxor \T3, \T7, \T7
				885
				886	vpshufd $0b01001110, \T1, \T3
				887	vpxor \T1, \T3, \T3
				888	vmovdqa HashKey_4_k(arg1), \T5
				889	vpclmulqdq $0x10, \T5, \T3, \T3
				890	vpxor \T3, \T6, \T6
				891
				892	vmovdqu 16*7(arg1), \T1
				893	vaesenc \T1, \XMM1, \XMM1
				894	vaesenc \T1, \XMM2, \XMM2
				895	vaesenc \T1, \XMM3, \XMM3
				896	vaesenc \T1, \XMM4, \XMM4
				897	vaesenc \T1, \XMM5, \XMM5
				898	vaesenc \T1, \XMM6, \XMM6
				899	vaesenc \T1, \XMM7, \XMM7
				900	vaesenc \T1, \XMM8, \XMM8
				901
				902	vmovdqa TMP6(%rsp), \T1
				903	vmovdqa HashKey_3(arg1), \T5
				904	vpclmulqdq $0x11, \T5, \T1, \T3
				905	vpxor \T3, \T4, \T4
				906	vpclmulqdq $0x00, \T5, \T1, \T3
				907	vpxor \T3, \T7, \T7
				908
				909	vpshufd $0b01001110, \T1, \T3
				910	vpxor \T1, \T3, \T3
				911	vmovdqa HashKey_3_k(arg1), \T5
				912	vpclmulqdq $0x10, \T5, \T3, \T3
				913	vpxor \T3, \T6, \T6
				914
				915
				916	vmovdqu 16*8(arg1), \T1
				917	vaesenc \T1, \XMM1, \XMM1
				918	vaesenc \T1, \XMM2, \XMM2
				919	vaesenc \T1, \XMM3, \XMM3
				920	vaesenc \T1, \XMM4, \XMM4
				921	vaesenc \T1, \XMM5, \XMM5
				922	vaesenc \T1, \XMM6, \XMM6
				923	vaesenc \T1, \XMM7, \XMM7
				924	vaesenc \T1, \XMM8, \XMM8
				925
				926	vmovdqa TMP7(%rsp), \T1
				927	vmovdqa HashKey_2(arg1), \T5
				928	vpclmulqdq $0x11, \T5, \T1, \T3
				929	vpxor \T3, \T4, \T4
				930	vpclmulqdq $0x00, \T5, \T1, \T3
				931	vpxor \T3, \T7, \T7
				932
				933	vpshufd $0b01001110, \T1, \T3
				934	vpxor \T1, \T3, \T3
				935	vmovdqa HashKey_2_k(arg1), \T5
				936	vpclmulqdq $0x10, \T5, \T3, \T3
				937	vpxor \T3, \T6, \T6
				938
				939	#######################################################################
				940
				941	vmovdqu 16*9(arg1), \T5
				942	vaesenc \T5, \XMM1, \XMM1
				943	vaesenc \T5, \XMM2, \XMM2
				944	vaesenc \T5, \XMM3, \XMM3
				945	vaesenc \T5, \XMM4, \XMM4
				946	vaesenc \T5, \XMM5, \XMM5
				947	vaesenc \T5, \XMM6, \XMM6
				948	vaesenc \T5, \XMM7, \XMM7
				949	vaesenc \T5, \XMM8, \XMM8
				950
				951	vmovdqa TMP8(%rsp), \T1
				952	vmovdqa HashKey(arg1), \T5
				953	vpclmulqdq $0x11, \T5, \T1, \T3
				954	vpxor \T3, \T4, \T4
				955	vpclmulqdq $0x00, \T5, \T1, \T3
				956	vpxor \T3, \T7, \T7
				957
				958	vpshufd $0b01001110, \T1, \T3
				959	vpxor \T1, \T3, \T3
				960	vmovdqa HashKey_k(arg1), \T5
				961	vpclmulqdq $0x10, \T5, \T3, \T3
				962	vpxor \T3, \T6, \T6
				963
				964	vpxor \T4, \T6, \T6
				965	vpxor \T7, \T6, \T6
				966
				967	vmovdqu 16*10(arg1), \T5
				968
				969	i = 0
				970	j = 1
				971	setreg
				972	.rep 8
				973	vpxor 16*i(arg3, %r11), \T5, \T2
				974	.if \ENC_DEC == ENC
				975	vaesenclast \T2, reg_j, reg_j
				976	.else
				977	vaesenclast \T2, reg_j, \T3
				978	vmovdqu 16*i(arg3, %r11), reg_j
				979	vmovdqu \T3, 16*i(arg2, %r11)
				980	.endif
				981	i = (i+1)
				982	j = (j+1)
				983	setreg
				984	.endr
				985	#######################################################################
				986
				987
				988	vpslldq $8, \T6, \T3 # shift-L T3 2 DWs
				989	vpsrldq $8, \T6, \T6 # shift-R T2 2 DWs
				990	vpxor \T3, \T7, \T7
				991	vpxor \T4, \T6, \T6 # accumulate the results in T6:T7
				992
				993
				994
				995	#######################################################################
				996	#first phase of the reduction
				997	#######################################################################
				998	vpslld $31, \T7, \T2 # packed right shifting << 31
				999	vpslld $30, \T7, \T3 # packed right shifting shift << 30
				1000	vpslld $25, \T7, \T4 # packed right shifting shift << 25
				1001
				1002	vpxor \T3, \T2, \T2 # xor the shifted versions
				1003	vpxor \T4, \T2, \T2
				1004
				1005	vpsrldq $4, \T2, \T1 # shift-R T1 1 DW
				1006
				1007	vpslldq $12, \T2, \T2 # shift-L T2 3 DWs
				1008	vpxor \T2, \T7, \T7 # first phase of the reduction complete
				1009	#######################################################################
				1010	.if \ENC_DEC == ENC
				1011	vmovdqu \XMM1, 16*0(arg2,%r11) # Write to the Ciphertext buffer
				1012	vmovdqu \XMM2, 16*1(arg2,%r11) # Write to the Ciphertext buffer
				1013	vmovdqu \XMM3, 16*2(arg2,%r11) # Write to the Ciphertext buffer
				1014	vmovdqu \XMM4, 16*3(arg2,%r11) # Write to the Ciphertext buffer
				1015	vmovdqu \XMM5, 16*4(arg2,%r11) # Write to the Ciphertext buffer
				1016	vmovdqu \XMM6, 16*5(arg2,%r11) # Write to the Ciphertext buffer
				1017	vmovdqu \XMM7, 16*6(arg2,%r11) # Write to the Ciphertext buffer
				1018	vmovdqu \XMM8, 16*7(arg2,%r11) # Write to the Ciphertext buffer
				1019	.endif
				1020
				1021	#######################################################################
				1022	#second phase of the reduction
				1023	vpsrld $1, \T7, \T2 # packed left shifting >> 1
				1024	vpsrld $2, \T7, \T3 # packed left shifting >> 2
				1025	vpsrld $7, \T7, \T4 # packed left shifting >> 7
				1026	vpxor \T3, \T2, \T2 # xor the shifted versions
				1027	vpxor \T4, \T2, \T2
				1028
				1029	vpxor \T1, \T2, \T2
				1030	vpxor \T2, \T7, \T7
				1031	vpxor \T7, \T6, \T6 # the result is in T6
				1032	#######################################################################
				1033
				1034	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				1035	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				1036	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				1037	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				1038	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				1039	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				1040	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				1041	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				1042
				1043
				1044	vpxor \T6, \XMM1, \XMM1
				1045
				1046
				1047
				1048	.endm
				1049
				1050
				1051	# GHASH the last 4 ciphertext blocks.
				1052	.macro GHASH_LAST_8_AVX T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
				1053
				1054	## Karatsuba Method
				1055
				1056
				1057	vpshufd $0b01001110, \XMM1, \T2
				1058	vpxor \XMM1, \T2, \T2
				1059	vmovdqa HashKey_8(arg1), \T5
				1060	vpclmulqdq $0x11, \T5, \XMM1, \T6
				1061	vpclmulqdq $0x00, \T5, \XMM1, \T7
				1062
				1063	vmovdqa HashKey_8_k(arg1), \T3
				1064	vpclmulqdq $0x00, \T3, \T2, \XMM1
				1065
				1066	######################
				1067
				1068	vpshufd $0b01001110, \XMM2, \T2
				1069	vpxor \XMM2, \T2, \T2
				1070	vmovdqa HashKey_7(arg1), \T5
				1071	vpclmulqdq $0x11, \T5, \XMM2, \T4
				1072	vpxor \T4, \T6, \T6
				1073
				1074	vpclmulqdq $0x00, \T5, \XMM2, \T4
				1075	vpxor \T4, \T7, \T7
				1076
				1077	vmovdqa HashKey_7_k(arg1), \T3
				1078	vpclmulqdq $0x00, \T3, \T2, \T2
				1079	vpxor \T2, \XMM1, \XMM1
				1080
				1081	######################
				1082
				1083	vpshufd $0b01001110, \XMM3, \T2
				1084	vpxor \XMM3, \T2, \T2
				1085	vmovdqa HashKey_6(arg1), \T5
				1086	vpclmulqdq $0x11, \T5, \XMM3, \T4
				1087	vpxor \T4, \T6, \T6
				1088
				1089	vpclmulqdq $0x00, \T5, \XMM3, \T4
				1090	vpxor \T4, \T7, \T7
				1091
				1092	vmovdqa HashKey_6_k(arg1), \T3
				1093	vpclmulqdq $0x00, \T3, \T2, \T2
				1094	vpxor \T2, \XMM1, \XMM1
				1095
				1096	######################
				1097
				1098	vpshufd $0b01001110, \XMM4, \T2
				1099	vpxor \XMM4, \T2, \T2
				1100	vmovdqa HashKey_5(arg1), \T5
				1101	vpclmulqdq $0x11, \T5, \XMM4, \T4
				1102	vpxor \T4, \T6, \T6
				1103
				1104	vpclmulqdq $0x00, \T5, \XMM4, \T4
				1105	vpxor \T4, \T7, \T7
				1106
				1107	vmovdqa HashKey_5_k(arg1), \T3
				1108	vpclmulqdq $0x00, \T3, \T2, \T2
				1109	vpxor \T2, \XMM1, \XMM1
				1110
				1111	######################
				1112
				1113	vpshufd $0b01001110, \XMM5, \T2
				1114	vpxor \XMM5, \T2, \T2
				1115	vmovdqa HashKey_4(arg1), \T5
				1116	vpclmulqdq $0x11, \T5, \XMM5, \T4
				1117	vpxor \T4, \T6, \T6
				1118
				1119	vpclmulqdq $0x00, \T5, \XMM5, \T4
				1120	vpxor \T4, \T7, \T7
				1121
				1122	vmovdqa HashKey_4_k(arg1), \T3
				1123	vpclmulqdq $0x00, \T3, \T2, \T2
				1124	vpxor \T2, \XMM1, \XMM1
				1125
				1126	######################
				1127
				1128	vpshufd $0b01001110, \XMM6, \T2
				1129	vpxor \XMM6, \T2, \T2
				1130	vmovdqa HashKey_3(arg1), \T5
				1131	vpclmulqdq $0x11, \T5, \XMM6, \T4
				1132	vpxor \T4, \T6, \T6
				1133
				1134	vpclmulqdq $0x00, \T5, \XMM6, \T4
				1135	vpxor \T4, \T7, \T7
				1136
				1137	vmovdqa HashKey_3_k(arg1), \T3
				1138	vpclmulqdq $0x00, \T3, \T2, \T2
				1139	vpxor \T2, \XMM1, \XMM1
				1140
				1141	######################
				1142
				1143	vpshufd $0b01001110, \XMM7, \T2
				1144	vpxor \XMM7, \T2, \T2
				1145	vmovdqa HashKey_2(arg1), \T5
				1146	vpclmulqdq $0x11, \T5, \XMM7, \T4
				1147	vpxor \T4, \T6, \T6
				1148
				1149	vpclmulqdq $0x00, \T5, \XMM7, \T4
				1150	vpxor \T4, \T7, \T7
				1151
				1152	vmovdqa HashKey_2_k(arg1), \T3
				1153	vpclmulqdq $0x00, \T3, \T2, \T2
				1154	vpxor \T2, \XMM1, \XMM1
				1155
				1156	######################
				1157
				1158	vpshufd $0b01001110, \XMM8, \T2
				1159	vpxor \XMM8, \T2, \T2
				1160	vmovdqa HashKey(arg1), \T5
				1161	vpclmulqdq $0x11, \T5, \XMM8, \T4
				1162	vpxor \T4, \T6, \T6
				1163
				1164	vpclmulqdq $0x00, \T5, \XMM8, \T4
				1165	vpxor \T4, \T7, \T7
				1166
				1167	vmovdqa HashKey_k(arg1), \T3
				1168	vpclmulqdq $0x00, \T3, \T2, \T2
				1169
				1170	vpxor \T2, \XMM1, \XMM1
				1171	vpxor \T6, \XMM1, \XMM1
				1172	vpxor \T7, \XMM1, \T2
				1173
				1174
				1175
				1176
				1177	vpslldq $8, \T2, \T4
				1178	vpsrldq $8, \T2, \T2
				1179
				1180	vpxor \T4, \T7, \T7
				1181	vpxor \T2, \T6, \T6 # <T6:T7> holds the result of
				1182	# the accumulated carry-less multiplications
				1183
				1184	#######################################################################
				1185	#first phase of the reduction
				1186	vpslld $31, \T7, \T2 # packed right shifting << 31
				1187	vpslld $30, \T7, \T3 # packed right shifting shift << 30
				1188	vpslld $25, \T7, \T4 # packed right shifting shift << 25
				1189
				1190	vpxor \T3, \T2, \T2 # xor the shifted versions
				1191	vpxor \T4, \T2, \T2
				1192
				1193	vpsrldq $4, \T2, \T1 # shift-R T1 1 DW
				1194
				1195	vpslldq $12, \T2, \T2 # shift-L T2 3 DWs
				1196	vpxor \T2, \T7, \T7 # first phase of the reduction complete
				1197	#######################################################################
				1198
				1199
				1200	#second phase of the reduction
				1201	vpsrld $1, \T7, \T2 # packed left shifting >> 1
				1202	vpsrld $2, \T7, \T3 # packed left shifting >> 2
				1203	vpsrld $7, \T7, \T4 # packed left shifting >> 7
				1204	vpxor \T3, \T2, \T2 # xor the shifted versions
				1205	vpxor \T4, \T2, \T2
				1206
				1207	vpxor \T1, \T2, \T2
				1208	vpxor \T2, \T7, \T7
				1209	vpxor \T7, \T6, \T6 # the result is in T6
				1210
				1211	.endm
				1212
				1213
				1214	# combined for GCM encrypt and decrypt functions
				1215	# clobbering all xmm registers
				1216	# clobbering r10, r11, r12, r13, r14, r15
				1217	.macro GCM_ENC_DEC_AVX ENC_DEC
				1218
				1219	#the number of pushes must equal STACK_OFFSET
				1220	push %r12
				1221	push %r13
				1222	push %r14
				1223	push %r15
				1224
				1225	mov %rsp, %r14
				1226
				1227
				1228
				1229
				1230	sub $VARIABLE_OFFSET, %rsp
				1231	and $~63, %rsp # align rsp to 64 bytes
				1232
				1233
				1234	vmovdqu HashKey(arg1), %xmm13 # xmm13 = HashKey
				1235
				1236	mov arg4, %r13 # save the number of bytes of plaintext/ciphertext
				1237	and $-16, %r13 # r13 = r13 - (r13 mod 16)
				1238
				1239	mov %r13, %r12
				1240	shr $4, %r12
				1241	and $7, %r12
				1242	jz _initial_num_blocks_is_0\@
				1243
				1244	cmp $7, %r12
				1245	je _initial_num_blocks_is_7\@
				1246	cmp $6, %r12
				1247	je _initial_num_blocks_is_6\@
				1248	cmp $5, %r12
				1249	je _initial_num_blocks_is_5\@
				1250	cmp $4, %r12
				1251	je _initial_num_blocks_is_4\@
				1252	cmp $3, %r12
				1253	je _initial_num_blocks_is_3\@
				1254	cmp $2, %r12
				1255	je _initial_num_blocks_is_2\@
				1256
				1257	jmp _initial_num_blocks_is_1\@
				1258
				1259	_initial_num_blocks_is_7\@:
				1260	INITIAL_BLOCKS_AVX 7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1261	sub $16*7, %r13
				1262	jmp _initial_blocks_encrypted\@
				1263
				1264	_initial_num_blocks_is_6\@:
				1265	INITIAL_BLOCKS_AVX 6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1266	sub $16*6, %r13
				1267	jmp _initial_blocks_encrypted\@
				1268
				1269	_initial_num_blocks_is_5\@:
				1270	INITIAL_BLOCKS_AVX 5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1271	sub $16*5, %r13
				1272	jmp _initial_blocks_encrypted\@
				1273
				1274	_initial_num_blocks_is_4\@:
				1275	INITIAL_BLOCKS_AVX 4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1276	sub $16*4, %r13
				1277	jmp _initial_blocks_encrypted\@
				1278
				1279	_initial_num_blocks_is_3\@:
				1280	INITIAL_BLOCKS_AVX 3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1281	sub $16*3, %r13
				1282	jmp _initial_blocks_encrypted\@
				1283
				1284	_initial_num_blocks_is_2\@:
				1285	INITIAL_BLOCKS_AVX 2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1286	sub $16*2, %r13
				1287	jmp _initial_blocks_encrypted\@
				1288
				1289	_initial_num_blocks_is_1\@:
				1290	INITIAL_BLOCKS_AVX 1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1291	sub $16*1, %r13
				1292	jmp _initial_blocks_encrypted\@
				1293
				1294	_initial_num_blocks_is_0\@:
				1295	INITIAL_BLOCKS_AVX 0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				1296
				1297
				1298	_initial_blocks_encrypted\@:
				1299	cmp $0, %r13
				1300	je _zero_cipher_left\@
				1301
				1302	sub $128, %r13
				1303	je _eight_cipher_left\@
				1304
				1305
				1306
				1307
				1308	vmovd %xmm9, %r15d
				1309	and $255, %r15d
				1310	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1311
				1312
				1313	_encrypt_by_8_new\@:
				1314	cmp $(255-8), %r15d
				1315	jg _encrypt_by_8\@
				1316
				1317
				1318
				1319	add $8, %r15b
				1320	GHASH_8_ENCRYPT_8_PARALLEL_AVX %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC
				1321	add $128, %r11
				1322	sub $128, %r13
				1323	jne _encrypt_by_8_new\@
				1324
				1325	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1326	jmp _eight_cipher_left\@
				1327
				1328	_encrypt_by_8\@:
				1329	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1330	add $8, %r15b
				1331	GHASH_8_ENCRYPT_8_PARALLEL_AVX %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC
				1332	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1333	add $128, %r11
				1334	sub $128, %r13
				1335	jne _encrypt_by_8_new\@
				1336
				1337	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1338
				1339
				1340
				1341
				1342	_eight_cipher_left\@:
				1343	GHASH_LAST_8_AVX %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8
				1344
				1345
				1346	_zero_cipher_left\@:
				1347	cmp $16, arg4
				1348	jl _only_less_than_16\@
				1349
				1350	mov arg4, %r13
				1351	and $15, %r13 # r13 = (arg4 mod 16)
				1352
				1353	je _multiple_of_16_bytes\@
				1354
				1355	# handle the last <16 Byte block seperately
				1356
				1357
				1358	vpaddd ONE(%rip), %xmm9, %xmm9 # INCR CNT to get Yn
				1359	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1360	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Yn)
				1361
				1362	sub $16, %r11
				1363	add %r13, %r11
				1364	vmovdqu (arg3, %r11), %xmm1 # receive the last <16 Byte block
				1365
				1366	lea SHIFT_MASK+16(%rip), %r12
				1367	sub %r13, %r12 # adjust the shuffle mask pointer to be
				1368	# able to shift 16-r13 bytes (r13 is the
				1369	# number of bytes in plaintext mod 16)
				1370	vmovdqu (%r12), %xmm2 # get the appropriate shuffle mask
				1371	vpshufb %xmm2, %xmm1, %xmm1 # shift right 16-r13 bytes
				1372	jmp _final_ghash_mul\@
				1373
				1374	_only_less_than_16\@:
				1375	# check for 0 length
				1376	mov arg4, %r13
				1377	and $15, %r13 # r13 = (arg4 mod 16)
				1378
				1379	je _multiple_of_16_bytes\@
				1380
				1381	# handle the last <16 Byte block seperately
				1382
				1383
				1384	vpaddd ONE(%rip), %xmm9, %xmm9 # INCR CNT to get Yn
				1385	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1386	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Yn)
				1387
				1388
				1389	lea SHIFT_MASK+16(%rip), %r12
				1390	sub %r13, %r12 # adjust the shuffle mask pointer to be
				1391	# able to shift 16-r13 bytes (r13 is the
				1392	# number of bytes in plaintext mod 16)
				1393
				1394	_get_last_16_byte_loop\@:
				1395	movb (arg3, %r11), %al
				1396	movb %al, TMP1 (%rsp , %r11)
				1397	add $1, %r11
				1398	cmp %r13, %r11
				1399	jne _get_last_16_byte_loop\@
				1400
				1401	vmovdqu TMP1(%rsp), %xmm1
				1402
				1403	sub $16, %r11
				1404
				1405	_final_ghash_mul\@:
				1406	.if \ENC_DEC == DEC
				1407	vmovdqa %xmm1, %xmm2
				1408	vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn)
				1409	vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to
				1410	# mask out top 16-r13 bytes of xmm9
				1411	vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9
				1412	vpand %xmm1, %xmm2, %xmm2
				1413	vpshufb SHUF_MASK(%rip), %xmm2, %xmm2
				1414	vpxor %xmm2, %xmm14, %xmm14
				1415	#GHASH computation for the last <16 Byte block
				1416	GHASH_MUL_AVX %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
				1417	sub %r13, %r11
				1418	add $16, %r11
				1419	.else
				1420	vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn)
				1421	vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to
				1422	# mask out top 16-r13 bytes of xmm9
				1423	vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9
				1424	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				1425	vpxor %xmm9, %xmm14, %xmm14
				1426	#GHASH computation for the last <16 Byte block
				1427	GHASH_MUL_AVX %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
				1428	sub %r13, %r11
				1429	add $16, %r11
				1430	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 # shuffle xmm9 back to output as ciphertext
				1431	.endif
				1432
				1433
				1434	#############################
				1435	# output r13 Bytes
				1436	vmovq %xmm9, %rax
				1437	cmp $8, %r13
				1438	jle _less_than_8_bytes_left\@
				1439
				1440	mov %rax, (arg2 , %r11)
				1441	add $8, %r11
				1442	vpsrldq $8, %xmm9, %xmm9
				1443	vmovq %xmm9, %rax
				1444	sub $8, %r13
				1445
				1446	_less_than_8_bytes_left\@:
				1447	movb %al, (arg2 , %r11)
				1448	add $1, %r11
				1449	shr $8, %rax
				1450	sub $1, %r13
				1451	jne _less_than_8_bytes_left\@
				1452	#############################
				1453
				1454	_multiple_of_16_bytes\@:
				1455	mov arg7, %r12 # r12 = aadLen (number of bytes)
				1456	shl $3, %r12 # convert into number of bits
				1457	vmovd %r12d, %xmm15 # len(A) in xmm15
				1458
				1459	shl $3, arg4 # len(C) in bits (*128)
				1460	vmovq arg4, %xmm1
				1461	vpslldq $8, %xmm15, %xmm15 # xmm15 = len(A)\|\| 0x0000000000000000
				1462	vpxor %xmm1, %xmm15, %xmm15 # xmm15 = len(A)\|\|len(C)
				1463
				1464	vpxor %xmm15, %xmm14, %xmm14
				1465	GHASH_MUL_AVX %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 # final GHASH computation
				1466	vpshufb SHUF_MASK(%rip), %xmm14, %xmm14 # perform a 16Byte swap
				1467
				1468	mov arg5, %rax # rax = *Y0
				1469	vmovdqu (%rax), %xmm9 # xmm9 = Y0
				1470
				1471	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Y0)
				1472
				1473	vpxor %xmm14, %xmm9, %xmm9
				1474
				1475
				1476
				1477	_return_T\@:
				1478	mov arg8, %r10 # r10 = authTag
				1479	mov arg9, %r11 # r11 = auth_tag_len
				1480
				1481	cmp $16, %r11
				1482	je _T_16\@
				1483
Sabrina Dubroca	0120af7	2017-04-28 18:11:59 +0200	[diff] [blame]	1484	cmp $8, %r11
				1485	jl _T_4\@
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1486
				1487	_T_8\@:
				1488	vmovq %xmm9, %rax
				1489	mov %rax, (%r10)
Sabrina Dubroca	0120af7	2017-04-28 18:11:59 +0200	[diff] [blame]	1490	add $8, %r10
				1491	sub $8, %r11
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1492	vpsrldq $8, %xmm9, %xmm9
Sabrina Dubroca	0120af7	2017-04-28 18:11:59 +0200	[diff] [blame]	1493	cmp $0, %r11
				1494	je _return_T_done\@
				1495	_T_4\@:
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1496	vmovd %xmm9, %eax
Sabrina Dubroca	0120af7	2017-04-28 18:11:59 +0200	[diff] [blame]	1497	mov %eax, (%r10)
				1498	add $4, %r10
				1499	sub $4, %r11
				1500	vpsrldq $4, %xmm9, %xmm9
				1501	cmp $0, %r11
				1502	je _return_T_done\@
				1503	_T_123\@:
				1504	vmovd %xmm9, %eax
				1505	cmp $2, %r11
				1506	jl _T_1\@
				1507	mov %ax, (%r10)
				1508	cmp $2, %r11
				1509	je _return_T_done\@
				1510	add $2, %r10
				1511	sar $16, %eax
				1512	_T_1\@:
				1513	mov %al, (%r10)
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1514	jmp _return_T_done\@
				1515
				1516	_T_16\@:
				1517	vmovdqu %xmm9, (%r10)
				1518
				1519	_return_T_done\@:
				1520	mov %r14, %rsp
				1521
				1522	pop %r15
				1523	pop %r14
				1524	pop %r13
				1525	pop %r12
				1526	.endm
				1527
				1528
				1529	#############################################################
				1530	#void aesni_gcm_precomp_avx_gen2
				1531	# (gcm_data *my_ctx_data,
				1532	# u8 hash_subkey)# / H, the Hash sub key input. Data starts on a 16-byte boundary. */
				1533	#############################################################
				1534	ENTRY(aesni_gcm_precomp_avx_gen2)
				1535	#the number of pushes must equal STACK_OFFSET
				1536	push %r12
				1537	push %r13
				1538	push %r14
				1539	push %r15
				1540
				1541	mov %rsp, %r14
				1542
				1543
				1544
				1545	sub $VARIABLE_OFFSET, %rsp
				1546	and $~63, %rsp # align rsp to 64 bytes
				1547
				1548	vmovdqu (arg2), %xmm6 # xmm6 = HashKey
				1549
				1550	vpshufb SHUF_MASK(%rip), %xmm6, %xmm6
				1551	############### PRECOMPUTATION of HashKey<<1 mod poly from the HashKey
				1552	vmovdqa %xmm6, %xmm2
				1553	vpsllq $1, %xmm6, %xmm6
				1554	vpsrlq $63, %xmm2, %xmm2
				1555	vmovdqa %xmm2, %xmm1
				1556	vpslldq $8, %xmm2, %xmm2
				1557	vpsrldq $8, %xmm1, %xmm1
				1558	vpor %xmm2, %xmm6, %xmm6
				1559	#reduction
				1560	vpshufd $0b00100100, %xmm1, %xmm2
				1561	vpcmpeqd TWOONE(%rip), %xmm2, %xmm2
				1562	vpand POLY(%rip), %xmm2, %xmm2
				1563	vpxor %xmm2, %xmm6, %xmm6 # xmm6 holds the HashKey<<1 mod poly
				1564	#######################################################################
				1565	vmovdqa %xmm6, HashKey(arg1) # store HashKey<<1 mod poly
				1566
				1567
				1568	PRECOMPUTE_AVX %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5
				1569
				1570	mov %r14, %rsp
				1571
				1572	pop %r15
				1573	pop %r14
				1574	pop %r13
				1575	pop %r12
				1576	ret
				1577	ENDPROC(aesni_gcm_precomp_avx_gen2)
				1578
				1579	###############################################################################
				1580	#void aesni_gcm_enc_avx_gen2(
				1581	# gcm_data my_ctx_data, / aligned to 16 Bytes */
				1582	# u8 out, / Ciphertext output. Encrypt in-place is allowed. */
				1583	# const u8 in, / Plaintext input */
				1584	# u64 plaintext_len, /* Length of data in Bytes for encryption. */
				1585	# u8 iv, / Pre-counter block j0: 4 byte salt
				1586	# (from Security Association) concatenated with 8 byte
				1587	# Initialisation Vector (from IPSec ESP Payload)
				1588	# concatenated with 0x00000001. 16-byte aligned pointer. */
				1589	# const u8 aad, / Additional Authentication Data (AAD)*/
				1590	# u64 aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
				1591	# u8 auth_tag, / Authenticated Tag output. */
				1592	# u64 auth_tag_len)# /* Authenticated Tag Length in bytes.
				1593	# Valid values are 16 (most likely), 12 or 8. */
				1594	###############################################################################
				1595	ENTRY(aesni_gcm_enc_avx_gen2)
				1596	GCM_ENC_DEC_AVX ENC
				1597	ret
				1598	ENDPROC(aesni_gcm_enc_avx_gen2)
				1599
				1600	###############################################################################
				1601	#void aesni_gcm_dec_avx_gen2(
				1602	# gcm_data my_ctx_data, / aligned to 16 Bytes */
				1603	# u8 out, / Plaintext output. Decrypt in-place is allowed. */
				1604	# const u8 in, / Ciphertext input */
				1605	# u64 plaintext_len, /* Length of data in Bytes for encryption. */
				1606	# u8 iv, / Pre-counter block j0: 4 byte salt
				1607	# (from Security Association) concatenated with 8 byte
				1608	# Initialisation Vector (from IPSec ESP Payload)
				1609	# concatenated with 0x00000001. 16-byte aligned pointer. */
				1610	# const u8 aad, / Additional Authentication Data (AAD)*/
				1611	# u64 aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
				1612	# u8 auth_tag, / Authenticated Tag output. */
				1613	# u64 auth_tag_len)# /* Authenticated Tag Length in bytes.
				1614	# Valid values are 16 (most likely), 12 or 8. */
				1615	###############################################################################
				1616	ENTRY(aesni_gcm_dec_avx_gen2)
				1617	GCM_ENC_DEC_AVX DEC
				1618	ret
				1619	ENDPROC(aesni_gcm_dec_avx_gen2)
				1620	#endif /* CONFIG_AS_AVX */
				1621
				1622	#ifdef CONFIG_AS_AVX2
				1623	###############################################################################
				1624	# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
				1625	# Input: A and B (128-bits each, bit-reflected)
				1626	# Output: C = ABx mod poly, (i.e. >>1 )
				1627	# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
				1628	# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
				1629	###############################################################################
				1630	.macro GHASH_MUL_AVX2 GH HK T1 T2 T3 T4 T5
				1631
				1632	vpclmulqdq $0x11,\HK,\GH,\T1 # T1 = a1*b1
				1633	vpclmulqdq $0x00,\HK,\GH,\T2 # T2 = a0*b0
				1634	vpclmulqdq $0x01,\HK,\GH,\T3 # T3 = a1*b0
				1635	vpclmulqdq $0x10,\HK,\GH,\GH # GH = a0*b1
				1636	vpxor \T3, \GH, \GH
				1637
				1638
				1639	vpsrldq $8 , \GH, \T3 # shift-R GH 2 DWs
				1640	vpslldq $8 , \GH, \GH # shift-L GH 2 DWs
				1641
				1642	vpxor \T3, \T1, \T1
				1643	vpxor \T2, \GH, \GH
				1644
				1645	#######################################################################
				1646	#first phase of the reduction
				1647	vmovdqa POLY2(%rip), \T3
				1648
				1649	vpclmulqdq $0x01, \GH, \T3, \T2
				1650	vpslldq $8, \T2, \T2 # shift-L T2 2 DWs
				1651
				1652	vpxor \T2, \GH, \GH # first phase of the reduction complete
				1653	#######################################################################
				1654	#second phase of the reduction
				1655	vpclmulqdq $0x00, \GH, \T3, \T2
				1656	vpsrldq $4, \T2, \T2 # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
				1657
				1658	vpclmulqdq $0x10, \GH, \T3, \GH
				1659	vpslldq $4, \GH, \GH # shift-L GH 1 DW (Shift-L 1-DW to obtain result with no shifts)
				1660
				1661	vpxor \T2, \GH, \GH # second phase of the reduction complete
				1662	#######################################################################
				1663	vpxor \T1, \GH, \GH # the result is in GH
				1664
				1665
				1666	.endm
				1667
				1668	.macro PRECOMPUTE_AVX2 HK T1 T2 T3 T4 T5 T6
				1669
				1670	# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
				1671	vmovdqa \HK, \T5
				1672	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly
				1673	vmovdqa \T5, HashKey_2(arg1) # [HashKey_2] = HashKey^2<<1 mod poly
				1674
				1675	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly
				1676	vmovdqa \T5, HashKey_3(arg1)
				1677
				1678	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly
				1679	vmovdqa \T5, HashKey_4(arg1)
				1680
				1681	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly
				1682	vmovdqa \T5, HashKey_5(arg1)
				1683
				1684	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly
				1685	vmovdqa \T5, HashKey_6(arg1)
				1686
				1687	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly
				1688	vmovdqa \T5, HashKey_7(arg1)
				1689
				1690	GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly
				1691	vmovdqa \T5, HashKey_8(arg1)
				1692
				1693	.endm
				1694
				1695
				1696	## if a = number of total plaintext bytes
				1697	## b = floor(a/16)
				1698	## num_initial_blocks = b mod 4#
				1699	## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
				1700	## r10, r11, r12, rax are clobbered
				1701	## arg1, arg2, arg3, r14 are used as a pointer only, not modified
				1702
				1703	.macro INITIAL_BLOCKS_AVX2 num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER
				1704	i = (8-\num_initial_blocks)
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1705	j = 0
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1706	setreg
				1707
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1708	mov arg6, %r10 # r10 = AAD
				1709	mov arg7, %r12 # r12 = aadLen
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1710
				1711
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1712	mov %r12, %r11
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1713
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1714	vpxor reg_j, reg_j, reg_j
				1715	vpxor reg_i, reg_i, reg_i
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1716
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1717	cmp $16, %r11
				1718	jl _get_AAD_rest8\@
				1719	_get_AAD_blocks\@:
				1720	vmovdqu (%r10), reg_i
				1721	vpshufb SHUF_MASK(%rip), reg_i, reg_i
				1722	vpxor reg_i, reg_j, reg_j
				1723	GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6
				1724	add $16, %r10
				1725	sub $16, %r12
				1726	sub $16, %r11
				1727	cmp $16, %r11
				1728	jge _get_AAD_blocks\@
				1729	vmovdqu reg_j, reg_i
				1730	cmp $0, %r11
				1731	je _get_AAD_done\@
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1732
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1733	vpxor reg_i, reg_i, reg_i
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1734
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1735	/* read the last <16B of AAD. since we have at least 4B of
				1736	data right after the AAD (the ICV, and maybe some CT), we can
				1737	read 4B/8B blocks safely, and then get rid of the extra stuff */
				1738	_get_AAD_rest8\@:
				1739	cmp $4, %r11
				1740	jle _get_AAD_rest4\@
				1741	movq (%r10), \T1
				1742	add $8, %r10
				1743	sub $8, %r11
				1744	vpslldq $8, \T1, \T1
				1745	vpsrldq $8, reg_i, reg_i
				1746	vpxor \T1, reg_i, reg_i
				1747	jmp _get_AAD_rest8\@
				1748	_get_AAD_rest4\@:
				1749	cmp $0, %r11
				1750	jle _get_AAD_rest0\@
				1751	mov (%r10), %eax
				1752	movq %rax, \T1
				1753	add $4, %r10
				1754	sub $4, %r11
				1755	vpslldq $12, \T1, \T1
				1756	vpsrldq $4, reg_i, reg_i
				1757	vpxor \T1, reg_i, reg_i
				1758	_get_AAD_rest0\@:
				1759	/* finalize: shift out the extra bytes we read, and align
				1760	left. since pslldq can only shift by an immediate, we use
				1761	vpshufb and an array of shuffle masks */
				1762	movq %r12, %r11
				1763	salq $4, %r11
				1764	movdqu aad_shift_arr(%r11), \T1
				1765	vpshufb \T1, reg_i, reg_i
				1766	_get_AAD_rest_final\@:
				1767	vpshufb SHUF_MASK(%rip), reg_i, reg_i
				1768	vpxor reg_j, reg_i, reg_i
				1769	GHASH_MUL_AVX2 reg_i, \T2, \T1, \T3, \T4, \T5, \T6
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1770
Sabrina Dubroca	27352c4	2017-04-28 18:12:00 +0200	[diff] [blame]	1771	_get_AAD_done\@:
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1772	# initialize the data pointer offset as zero
				1773	xor %r11, %r11
				1774
				1775	# start AES for num_initial_blocks blocks
				1776	mov arg5, %rax # rax = *Y0
				1777	vmovdqu (%rax), \CTR # CTR = Y0
				1778	vpshufb SHUF_MASK(%rip), \CTR, \CTR
				1779
				1780
				1781	i = (9-\num_initial_blocks)
				1782	setreg
				1783	.rep \num_initial_blocks
				1784	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1785	vmovdqa \CTR, reg_i
				1786	vpshufb SHUF_MASK(%rip), reg_i, reg_i # perform a 16Byte swap
				1787	i = (i+1)
				1788	setreg
				1789	.endr
				1790
				1791	vmovdqa (arg1), \T_key
				1792	i = (9-\num_initial_blocks)
				1793	setreg
				1794	.rep \num_initial_blocks
				1795	vpxor \T_key, reg_i, reg_i
				1796	i = (i+1)
				1797	setreg
				1798	.endr
				1799
				1800	j = 1
				1801	setreg
				1802	.rep 9
				1803	vmovdqa 16*j(arg1), \T_key
				1804	i = (9-\num_initial_blocks)
				1805	setreg
				1806	.rep \num_initial_blocks
				1807	vaesenc \T_key, reg_i, reg_i
				1808	i = (i+1)
				1809	setreg
				1810	.endr
				1811
				1812	j = (j+1)
				1813	setreg
				1814	.endr
				1815
				1816
				1817	vmovdqa 16*10(arg1), \T_key
				1818	i = (9-\num_initial_blocks)
				1819	setreg
				1820	.rep \num_initial_blocks
				1821	vaesenclast \T_key, reg_i, reg_i
				1822	i = (i+1)
				1823	setreg
				1824	.endr
				1825
				1826	i = (9-\num_initial_blocks)
				1827	setreg
				1828	.rep \num_initial_blocks
				1829	vmovdqu (arg3, %r11), \T1
				1830	vpxor \T1, reg_i, reg_i
				1831	vmovdqu reg_i, (arg2 , %r11) # write back ciphertext for
				1832	# num_initial_blocks blocks
				1833	add $16, %r11
				1834	.if \ENC_DEC == DEC
				1835	vmovdqa \T1, reg_i
				1836	.endif
				1837	vpshufb SHUF_MASK(%rip), reg_i, reg_i # prepare ciphertext for GHASH computations
				1838	i = (i+1)
				1839	setreg
				1840	.endr
				1841
				1842
				1843	i = (8-\num_initial_blocks)
				1844	j = (9-\num_initial_blocks)
				1845	setreg
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	1846
				1847	.rep \num_initial_blocks
				1848	vpxor reg_i, reg_j, reg_j
				1849	GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks
				1850	i = (i+1)
				1851	j = (j+1)
				1852	setreg
				1853	.endr
				1854	# XMM8 has the combined result here
				1855
				1856	vmovdqa \XMM8, TMP1(%rsp)
				1857	vmovdqa \XMM8, \T3
				1858
				1859	cmp $128, %r13
				1860	jl _initial_blocks_done\@ # no need for precomputed constants
				1861
				1862	###############################################################################
				1863	# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
				1864	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1865	vmovdqa \CTR, \XMM1
				1866	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				1867
				1868	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1869	vmovdqa \CTR, \XMM2
				1870	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				1871
				1872	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1873	vmovdqa \CTR, \XMM3
				1874	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				1875
				1876	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1877	vmovdqa \CTR, \XMM4
				1878	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				1879
				1880	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1881	vmovdqa \CTR, \XMM5
				1882	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				1883
				1884	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1885	vmovdqa \CTR, \XMM6
				1886	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				1887
				1888	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1889	vmovdqa \CTR, \XMM7
				1890	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				1891
				1892	vpaddd ONE(%rip), \CTR, \CTR # INCR Y0
				1893	vmovdqa \CTR, \XMM8
				1894	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				1895
				1896	vmovdqa (arg1), \T_key
				1897	vpxor \T_key, \XMM1, \XMM1
				1898	vpxor \T_key, \XMM2, \XMM2
				1899	vpxor \T_key, \XMM3, \XMM3
				1900	vpxor \T_key, \XMM4, \XMM4
				1901	vpxor \T_key, \XMM5, \XMM5
				1902	vpxor \T_key, \XMM6, \XMM6
				1903	vpxor \T_key, \XMM7, \XMM7
				1904	vpxor \T_key, \XMM8, \XMM8
				1905
				1906	i = 1
				1907	setreg
				1908	.rep 9 # do 9 rounds
				1909	vmovdqa 16*i(arg1), \T_key
				1910	vaesenc \T_key, \XMM1, \XMM1
				1911	vaesenc \T_key, \XMM2, \XMM2
				1912	vaesenc \T_key, \XMM3, \XMM3
				1913	vaesenc \T_key, \XMM4, \XMM4
				1914	vaesenc \T_key, \XMM5, \XMM5
				1915	vaesenc \T_key, \XMM6, \XMM6
				1916	vaesenc \T_key, \XMM7, \XMM7
				1917	vaesenc \T_key, \XMM8, \XMM8
				1918	i = (i+1)
				1919	setreg
				1920	.endr
				1921
				1922
				1923	vmovdqa 16*i(arg1), \T_key
				1924	vaesenclast \T_key, \XMM1, \XMM1
				1925	vaesenclast \T_key, \XMM2, \XMM2
				1926	vaesenclast \T_key, \XMM3, \XMM3
				1927	vaesenclast \T_key, \XMM4, \XMM4
				1928	vaesenclast \T_key, \XMM5, \XMM5
				1929	vaesenclast \T_key, \XMM6, \XMM6
				1930	vaesenclast \T_key, \XMM7, \XMM7
				1931	vaesenclast \T_key, \XMM8, \XMM8
				1932
				1933	vmovdqu (arg3, %r11), \T1
				1934	vpxor \T1, \XMM1, \XMM1
				1935	vmovdqu \XMM1, (arg2 , %r11)
				1936	.if \ENC_DEC == DEC
				1937	vmovdqa \T1, \XMM1
				1938	.endif
				1939
				1940	vmovdqu 16*1(arg3, %r11), \T1
				1941	vpxor \T1, \XMM2, \XMM2
				1942	vmovdqu \XMM2, 16*1(arg2 , %r11)
				1943	.if \ENC_DEC == DEC
				1944	vmovdqa \T1, \XMM2
				1945	.endif
				1946
				1947	vmovdqu 16*2(arg3, %r11), \T1
				1948	vpxor \T1, \XMM3, \XMM3
				1949	vmovdqu \XMM3, 16*2(arg2 , %r11)
				1950	.if \ENC_DEC == DEC
				1951	vmovdqa \T1, \XMM3
				1952	.endif
				1953
				1954	vmovdqu 16*3(arg3, %r11), \T1
				1955	vpxor \T1, \XMM4, \XMM4
				1956	vmovdqu \XMM4, 16*3(arg2 , %r11)
				1957	.if \ENC_DEC == DEC
				1958	vmovdqa \T1, \XMM4
				1959	.endif
				1960
				1961	vmovdqu 16*4(arg3, %r11), \T1
				1962	vpxor \T1, \XMM5, \XMM5
				1963	vmovdqu \XMM5, 16*4(arg2 , %r11)
				1964	.if \ENC_DEC == DEC
				1965	vmovdqa \T1, \XMM5
				1966	.endif
				1967
				1968	vmovdqu 16*5(arg3, %r11), \T1
				1969	vpxor \T1, \XMM6, \XMM6
				1970	vmovdqu \XMM6, 16*5(arg2 , %r11)
				1971	.if \ENC_DEC == DEC
				1972	vmovdqa \T1, \XMM6
				1973	.endif
				1974
				1975	vmovdqu 16*6(arg3, %r11), \T1
				1976	vpxor \T1, \XMM7, \XMM7
				1977	vmovdqu \XMM7, 16*6(arg2 , %r11)
				1978	.if \ENC_DEC == DEC
				1979	vmovdqa \T1, \XMM7
				1980	.endif
				1981
				1982	vmovdqu 16*7(arg3, %r11), \T1
				1983	vpxor \T1, \XMM8, \XMM8
				1984	vmovdqu \XMM8, 16*7(arg2 , %r11)
				1985	.if \ENC_DEC == DEC
				1986	vmovdqa \T1, \XMM8
				1987	.endif
				1988
				1989	add $128, %r11
				1990
				1991	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				1992	vpxor TMP1(%rsp), \XMM1, \XMM1 # combine GHASHed value with
				1993	# the corresponding ciphertext
				1994	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				1995	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				1996	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				1997	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				1998	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				1999	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				2000	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				2001
				2002	###############################################################################
				2003
				2004	_initial_blocks_done\@:
				2005
				2006
				2007	.endm
				2008
				2009
				2010
				2011	# encrypt 8 blocks at a time
				2012	# ghash the 8 previously encrypted ciphertext blocks
				2013	# arg1, arg2, arg3 are used as pointers only, not modified
				2014	# r11 is the data offset value
				2015	.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
				2016
				2017	vmovdqa \XMM1, \T2
				2018	vmovdqa \XMM2, TMP2(%rsp)
				2019	vmovdqa \XMM3, TMP3(%rsp)
				2020	vmovdqa \XMM4, TMP4(%rsp)
				2021	vmovdqa \XMM5, TMP5(%rsp)
				2022	vmovdqa \XMM6, TMP6(%rsp)
				2023	vmovdqa \XMM7, TMP7(%rsp)
				2024	vmovdqa \XMM8, TMP8(%rsp)
				2025
				2026	.if \loop_idx == in_order
				2027	vpaddd ONE(%rip), \CTR, \XMM1 # INCR CNT
				2028	vpaddd ONE(%rip), \XMM1, \XMM2
				2029	vpaddd ONE(%rip), \XMM2, \XMM3
				2030	vpaddd ONE(%rip), \XMM3, \XMM4
				2031	vpaddd ONE(%rip), \XMM4, \XMM5
				2032	vpaddd ONE(%rip), \XMM5, \XMM6
				2033	vpaddd ONE(%rip), \XMM6, \XMM7
				2034	vpaddd ONE(%rip), \XMM7, \XMM8
				2035	vmovdqa \XMM8, \CTR
				2036
				2037	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				2038	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				2039	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				2040	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				2041	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				2042	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				2043	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				2044	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				2045	.else
				2046	vpaddd ONEf(%rip), \CTR, \XMM1 # INCR CNT
				2047	vpaddd ONEf(%rip), \XMM1, \XMM2
				2048	vpaddd ONEf(%rip), \XMM2, \XMM3
				2049	vpaddd ONEf(%rip), \XMM3, \XMM4
				2050	vpaddd ONEf(%rip), \XMM4, \XMM5
				2051	vpaddd ONEf(%rip), \XMM5, \XMM6
				2052	vpaddd ONEf(%rip), \XMM6, \XMM7
				2053	vpaddd ONEf(%rip), \XMM7, \XMM8
				2054	vmovdqa \XMM8, \CTR
				2055	.endif
				2056
				2057
				2058	#######################################################################
				2059
				2060	vmovdqu (arg1), \T1
				2061	vpxor \T1, \XMM1, \XMM1
				2062	vpxor \T1, \XMM2, \XMM2
				2063	vpxor \T1, \XMM3, \XMM3
				2064	vpxor \T1, \XMM4, \XMM4
				2065	vpxor \T1, \XMM5, \XMM5
				2066	vpxor \T1, \XMM6, \XMM6
				2067	vpxor \T1, \XMM7, \XMM7
				2068	vpxor \T1, \XMM8, \XMM8
				2069
				2070	#######################################################################
				2071
				2072
				2073
				2074
				2075
				2076	vmovdqu 16*1(arg1), \T1
				2077	vaesenc \T1, \XMM1, \XMM1
				2078	vaesenc \T1, \XMM2, \XMM2
				2079	vaesenc \T1, \XMM3, \XMM3
				2080	vaesenc \T1, \XMM4, \XMM4
				2081	vaesenc \T1, \XMM5, \XMM5
				2082	vaesenc \T1, \XMM6, \XMM6
				2083	vaesenc \T1, \XMM7, \XMM7
				2084	vaesenc \T1, \XMM8, \XMM8
				2085
				2086	vmovdqu 16*2(arg1), \T1
				2087	vaesenc \T1, \XMM1, \XMM1
				2088	vaesenc \T1, \XMM2, \XMM2
				2089	vaesenc \T1, \XMM3, \XMM3
				2090	vaesenc \T1, \XMM4, \XMM4
				2091	vaesenc \T1, \XMM5, \XMM5
				2092	vaesenc \T1, \XMM6, \XMM6
				2093	vaesenc \T1, \XMM7, \XMM7
				2094	vaesenc \T1, \XMM8, \XMM8
				2095
				2096
				2097	#######################################################################
				2098
				2099	vmovdqa HashKey_8(arg1), \T5
				2100	vpclmulqdq $0x11, \T5, \T2, \T4 # T4 = a1*b1
				2101	vpclmulqdq $0x00, \T5, \T2, \T7 # T7 = a0*b0
				2102	vpclmulqdq $0x01, \T5, \T2, \T6 # T6 = a1*b0
				2103	vpclmulqdq $0x10, \T5, \T2, \T5 # T5 = a0*b1
				2104	vpxor \T5, \T6, \T6
				2105
				2106	vmovdqu 16*3(arg1), \T1
				2107	vaesenc \T1, \XMM1, \XMM1
				2108	vaesenc \T1, \XMM2, \XMM2
				2109	vaesenc \T1, \XMM3, \XMM3
				2110	vaesenc \T1, \XMM4, \XMM4
				2111	vaesenc \T1, \XMM5, \XMM5
				2112	vaesenc \T1, \XMM6, \XMM6
				2113	vaesenc \T1, \XMM7, \XMM7
				2114	vaesenc \T1, \XMM8, \XMM8
				2115
				2116	vmovdqa TMP2(%rsp), \T1
				2117	vmovdqa HashKey_7(arg1), \T5
				2118	vpclmulqdq $0x11, \T5, \T1, \T3
				2119	vpxor \T3, \T4, \T4
				2120
				2121	vpclmulqdq $0x00, \T5, \T1, \T3
				2122	vpxor \T3, \T7, \T7
				2123
				2124	vpclmulqdq $0x01, \T5, \T1, \T3
				2125	vpxor \T3, \T6, \T6
				2126
				2127	vpclmulqdq $0x10, \T5, \T1, \T3
				2128	vpxor \T3, \T6, \T6
				2129
				2130	vmovdqu 16*4(arg1), \T1
				2131	vaesenc \T1, \XMM1, \XMM1
				2132	vaesenc \T1, \XMM2, \XMM2
				2133	vaesenc \T1, \XMM3, \XMM3
				2134	vaesenc \T1, \XMM4, \XMM4
				2135	vaesenc \T1, \XMM5, \XMM5
				2136	vaesenc \T1, \XMM6, \XMM6
				2137	vaesenc \T1, \XMM7, \XMM7
				2138	vaesenc \T1, \XMM8, \XMM8
				2139
				2140	#######################################################################
				2141
				2142	vmovdqa TMP3(%rsp), \T1
				2143	vmovdqa HashKey_6(arg1), \T5
				2144	vpclmulqdq $0x11, \T5, \T1, \T3
				2145	vpxor \T3, \T4, \T4
				2146
				2147	vpclmulqdq $0x00, \T5, \T1, \T3
				2148	vpxor \T3, \T7, \T7
				2149
				2150	vpclmulqdq $0x01, \T5, \T1, \T3
				2151	vpxor \T3, \T6, \T6
				2152
				2153	vpclmulqdq $0x10, \T5, \T1, \T3
				2154	vpxor \T3, \T6, \T6
				2155
				2156	vmovdqu 16*5(arg1), \T1
				2157	vaesenc \T1, \XMM1, \XMM1
				2158	vaesenc \T1, \XMM2, \XMM2
				2159	vaesenc \T1, \XMM3, \XMM3
				2160	vaesenc \T1, \XMM4, \XMM4
				2161	vaesenc \T1, \XMM5, \XMM5
				2162	vaesenc \T1, \XMM6, \XMM6
				2163	vaesenc \T1, \XMM7, \XMM7
				2164	vaesenc \T1, \XMM8, \XMM8
				2165
				2166	vmovdqa TMP4(%rsp), \T1
				2167	vmovdqa HashKey_5(arg1), \T5
				2168	vpclmulqdq $0x11, \T5, \T1, \T3
				2169	vpxor \T3, \T4, \T4
				2170
				2171	vpclmulqdq $0x00, \T5, \T1, \T3
				2172	vpxor \T3, \T7, \T7
				2173
				2174	vpclmulqdq $0x01, \T5, \T1, \T3
				2175	vpxor \T3, \T6, \T6
				2176
				2177	vpclmulqdq $0x10, \T5, \T1, \T3
				2178	vpxor \T3, \T6, \T6
				2179
				2180	vmovdqu 16*6(arg1), \T1
				2181	vaesenc \T1, \XMM1, \XMM1
				2182	vaesenc \T1, \XMM2, \XMM2
				2183	vaesenc \T1, \XMM3, \XMM3
				2184	vaesenc \T1, \XMM4, \XMM4
				2185	vaesenc \T1, \XMM5, \XMM5
				2186	vaesenc \T1, \XMM6, \XMM6
				2187	vaesenc \T1, \XMM7, \XMM7
				2188	vaesenc \T1, \XMM8, \XMM8
				2189
				2190
				2191	vmovdqa TMP5(%rsp), \T1
				2192	vmovdqa HashKey_4(arg1), \T5
				2193	vpclmulqdq $0x11, \T5, \T1, \T3
				2194	vpxor \T3, \T4, \T4
				2195
				2196	vpclmulqdq $0x00, \T5, \T1, \T3
				2197	vpxor \T3, \T7, \T7
				2198
				2199	vpclmulqdq $0x01, \T5, \T1, \T3
				2200	vpxor \T3, \T6, \T6
				2201
				2202	vpclmulqdq $0x10, \T5, \T1, \T3
				2203	vpxor \T3, \T6, \T6
				2204
				2205	vmovdqu 16*7(arg1), \T1
				2206	vaesenc \T1, \XMM1, \XMM1
				2207	vaesenc \T1, \XMM2, \XMM2
				2208	vaesenc \T1, \XMM3, \XMM3
				2209	vaesenc \T1, \XMM4, \XMM4
				2210	vaesenc \T1, \XMM5, \XMM5
				2211	vaesenc \T1, \XMM6, \XMM6
				2212	vaesenc \T1, \XMM7, \XMM7
				2213	vaesenc \T1, \XMM8, \XMM8
				2214
				2215	vmovdqa TMP6(%rsp), \T1
				2216	vmovdqa HashKey_3(arg1), \T5
				2217	vpclmulqdq $0x11, \T5, \T1, \T3
				2218	vpxor \T3, \T4, \T4
				2219
				2220	vpclmulqdq $0x00, \T5, \T1, \T3
				2221	vpxor \T3, \T7, \T7
				2222
				2223	vpclmulqdq $0x01, \T5, \T1, \T3
				2224	vpxor \T3, \T6, \T6
				2225
				2226	vpclmulqdq $0x10, \T5, \T1, \T3
				2227	vpxor \T3, \T6, \T6
				2228
				2229	vmovdqu 16*8(arg1), \T1
				2230	vaesenc \T1, \XMM1, \XMM1
				2231	vaesenc \T1, \XMM2, \XMM2
				2232	vaesenc \T1, \XMM3, \XMM3
				2233	vaesenc \T1, \XMM4, \XMM4
				2234	vaesenc \T1, \XMM5, \XMM5
				2235	vaesenc \T1, \XMM6, \XMM6
				2236	vaesenc \T1, \XMM7, \XMM7
				2237	vaesenc \T1, \XMM8, \XMM8
				2238
				2239	vmovdqa TMP7(%rsp), \T1
				2240	vmovdqa HashKey_2(arg1), \T5
				2241	vpclmulqdq $0x11, \T5, \T1, \T3
				2242	vpxor \T3, \T4, \T4
				2243
				2244	vpclmulqdq $0x00, \T5, \T1, \T3
				2245	vpxor \T3, \T7, \T7
				2246
				2247	vpclmulqdq $0x01, \T5, \T1, \T3
				2248	vpxor \T3, \T6, \T6
				2249
				2250	vpclmulqdq $0x10, \T5, \T1, \T3
				2251	vpxor \T3, \T6, \T6
				2252
				2253
				2254	#######################################################################
				2255
				2256	vmovdqu 16*9(arg1), \T5
				2257	vaesenc \T5, \XMM1, \XMM1
				2258	vaesenc \T5, \XMM2, \XMM2
				2259	vaesenc \T5, \XMM3, \XMM3
				2260	vaesenc \T5, \XMM4, \XMM4
				2261	vaesenc \T5, \XMM5, \XMM5
				2262	vaesenc \T5, \XMM6, \XMM6
				2263	vaesenc \T5, \XMM7, \XMM7
				2264	vaesenc \T5, \XMM8, \XMM8
				2265
				2266	vmovdqa TMP8(%rsp), \T1
				2267	vmovdqa HashKey(arg1), \T5
				2268
				2269	vpclmulqdq $0x00, \T5, \T1, \T3
				2270	vpxor \T3, \T7, \T7
				2271
				2272	vpclmulqdq $0x01, \T5, \T1, \T3
				2273	vpxor \T3, \T6, \T6
				2274
				2275	vpclmulqdq $0x10, \T5, \T1, \T3
				2276	vpxor \T3, \T6, \T6
				2277
				2278	vpclmulqdq $0x11, \T5, \T1, \T3
				2279	vpxor \T3, \T4, \T1
				2280
				2281
				2282	vmovdqu 16*10(arg1), \T5
				2283
				2284	i = 0
				2285	j = 1
				2286	setreg
				2287	.rep 8
				2288	vpxor 16*i(arg3, %r11), \T5, \T2
				2289	.if \ENC_DEC == ENC
				2290	vaesenclast \T2, reg_j, reg_j
				2291	.else
				2292	vaesenclast \T2, reg_j, \T3
				2293	vmovdqu 16*i(arg3, %r11), reg_j
				2294	vmovdqu \T3, 16*i(arg2, %r11)
				2295	.endif
				2296	i = (i+1)
				2297	j = (j+1)
				2298	setreg
				2299	.endr
				2300	#######################################################################
				2301
				2302
				2303	vpslldq $8, \T6, \T3 # shift-L T3 2 DWs
				2304	vpsrldq $8, \T6, \T6 # shift-R T2 2 DWs
				2305	vpxor \T3, \T7, \T7
				2306	vpxor \T6, \T1, \T1 # accumulate the results in T1:T7
				2307
				2308
				2309
				2310	#######################################################################
				2311	#first phase of the reduction
				2312	vmovdqa POLY2(%rip), \T3
				2313
				2314	vpclmulqdq $0x01, \T7, \T3, \T2
				2315	vpslldq $8, \T2, \T2 # shift-L xmm2 2 DWs
				2316
				2317	vpxor \T2, \T7, \T7 # first phase of the reduction complete
				2318	#######################################################################
				2319	.if \ENC_DEC == ENC
				2320	vmovdqu \XMM1, 16*0(arg2,%r11) # Write to the Ciphertext buffer
				2321	vmovdqu \XMM2, 16*1(arg2,%r11) # Write to the Ciphertext buffer
				2322	vmovdqu \XMM3, 16*2(arg2,%r11) # Write to the Ciphertext buffer
				2323	vmovdqu \XMM4, 16*3(arg2,%r11) # Write to the Ciphertext buffer
				2324	vmovdqu \XMM5, 16*4(arg2,%r11) # Write to the Ciphertext buffer
				2325	vmovdqu \XMM6, 16*5(arg2,%r11) # Write to the Ciphertext buffer
				2326	vmovdqu \XMM7, 16*6(arg2,%r11) # Write to the Ciphertext buffer
				2327	vmovdqu \XMM8, 16*7(arg2,%r11) # Write to the Ciphertext buffer
				2328	.endif
				2329
				2330	#######################################################################
				2331	#second phase of the reduction
				2332	vpclmulqdq $0x00, \T7, \T3, \T2
				2333	vpsrldq $4, \T2, \T2 # shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
				2334
				2335	vpclmulqdq $0x10, \T7, \T3, \T4
				2336	vpslldq $4, \T4, \T4 # shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts)
				2337
				2338	vpxor \T2, \T4, \T4 # second phase of the reduction complete
				2339	#######################################################################
				2340	vpxor \T4, \T1, \T1 # the result is in T1
				2341
				2342	vpshufb SHUF_MASK(%rip), \XMM1, \XMM1 # perform a 16Byte swap
				2343	vpshufb SHUF_MASK(%rip), \XMM2, \XMM2 # perform a 16Byte swap
				2344	vpshufb SHUF_MASK(%rip), \XMM3, \XMM3 # perform a 16Byte swap
				2345	vpshufb SHUF_MASK(%rip), \XMM4, \XMM4 # perform a 16Byte swap
				2346	vpshufb SHUF_MASK(%rip), \XMM5, \XMM5 # perform a 16Byte swap
				2347	vpshufb SHUF_MASK(%rip), \XMM6, \XMM6 # perform a 16Byte swap
				2348	vpshufb SHUF_MASK(%rip), \XMM7, \XMM7 # perform a 16Byte swap
				2349	vpshufb SHUF_MASK(%rip), \XMM8, \XMM8 # perform a 16Byte swap
				2350
				2351
				2352	vpxor \T1, \XMM1, \XMM1
				2353
				2354
				2355
				2356	.endm
				2357
				2358
				2359	# GHASH the last 4 ciphertext blocks.
				2360	.macro GHASH_LAST_8_AVX2 T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
				2361
				2362	## Karatsuba Method
				2363
				2364	vmovdqa HashKey_8(arg1), \T5
				2365
				2366	vpshufd $0b01001110, \XMM1, \T2
				2367	vpshufd $0b01001110, \T5, \T3
				2368	vpxor \XMM1, \T2, \T2
				2369	vpxor \T5, \T3, \T3
				2370
				2371	vpclmulqdq $0x11, \T5, \XMM1, \T6
				2372	vpclmulqdq $0x00, \T5, \XMM1, \T7
				2373
				2374	vpclmulqdq $0x00, \T3, \T2, \XMM1
				2375
				2376	######################
				2377
				2378	vmovdqa HashKey_7(arg1), \T5
				2379	vpshufd $0b01001110, \XMM2, \T2
				2380	vpshufd $0b01001110, \T5, \T3
				2381	vpxor \XMM2, \T2, \T2
				2382	vpxor \T5, \T3, \T3
				2383
				2384	vpclmulqdq $0x11, \T5, \XMM2, \T4
				2385	vpxor \T4, \T6, \T6
				2386
				2387	vpclmulqdq $0x00, \T5, \XMM2, \T4
				2388	vpxor \T4, \T7, \T7
				2389
				2390	vpclmulqdq $0x00, \T3, \T2, \T2
				2391
				2392	vpxor \T2, \XMM1, \XMM1
				2393
				2394	######################
				2395
				2396	vmovdqa HashKey_6(arg1), \T5
				2397	vpshufd $0b01001110, \XMM3, \T2
				2398	vpshufd $0b01001110, \T5, \T3
				2399	vpxor \XMM3, \T2, \T2
				2400	vpxor \T5, \T3, \T3
				2401
				2402	vpclmulqdq $0x11, \T5, \XMM3, \T4
				2403	vpxor \T4, \T6, \T6
				2404
				2405	vpclmulqdq $0x00, \T5, \XMM3, \T4
				2406	vpxor \T4, \T7, \T7
				2407
				2408	vpclmulqdq $0x00, \T3, \T2, \T2
				2409
				2410	vpxor \T2, \XMM1, \XMM1
				2411
				2412	######################
				2413
				2414	vmovdqa HashKey_5(arg1), \T5
				2415	vpshufd $0b01001110, \XMM4, \T2
				2416	vpshufd $0b01001110, \T5, \T3
				2417	vpxor \XMM4, \T2, \T2
				2418	vpxor \T5, \T3, \T3
				2419
				2420	vpclmulqdq $0x11, \T5, \XMM4, \T4
				2421	vpxor \T4, \T6, \T6
				2422
				2423	vpclmulqdq $0x00, \T5, \XMM4, \T4
				2424	vpxor \T4, \T7, \T7
				2425
				2426	vpclmulqdq $0x00, \T3, \T2, \T2
				2427
				2428	vpxor \T2, \XMM1, \XMM1
				2429
				2430	######################
				2431
				2432	vmovdqa HashKey_4(arg1), \T5
				2433	vpshufd $0b01001110, \XMM5, \T2
				2434	vpshufd $0b01001110, \T5, \T3
				2435	vpxor \XMM5, \T2, \T2
				2436	vpxor \T5, \T3, \T3
				2437
				2438	vpclmulqdq $0x11, \T5, \XMM5, \T4
				2439	vpxor \T4, \T6, \T6
				2440
				2441	vpclmulqdq $0x00, \T5, \XMM5, \T4
				2442	vpxor \T4, \T7, \T7
				2443
				2444	vpclmulqdq $0x00, \T3, \T2, \T2
				2445
				2446	vpxor \T2, \XMM1, \XMM1
				2447
				2448	######################
				2449
				2450	vmovdqa HashKey_3(arg1), \T5
				2451	vpshufd $0b01001110, \XMM6, \T2
				2452	vpshufd $0b01001110, \T5, \T3
				2453	vpxor \XMM6, \T2, \T2
				2454	vpxor \T5, \T3, \T3
				2455
				2456	vpclmulqdq $0x11, \T5, \XMM6, \T4
				2457	vpxor \T4, \T6, \T6
				2458
				2459	vpclmulqdq $0x00, \T5, \XMM6, \T4
				2460	vpxor \T4, \T7, \T7
				2461
				2462	vpclmulqdq $0x00, \T3, \T2, \T2
				2463
				2464	vpxor \T2, \XMM1, \XMM1
				2465
				2466	######################
				2467
				2468	vmovdqa HashKey_2(arg1), \T5
				2469	vpshufd $0b01001110, \XMM7, \T2
				2470	vpshufd $0b01001110, \T5, \T3
				2471	vpxor \XMM7, \T2, \T2
				2472	vpxor \T5, \T3, \T3
				2473
				2474	vpclmulqdq $0x11, \T5, \XMM7, \T4
				2475	vpxor \T4, \T6, \T6
				2476
				2477	vpclmulqdq $0x00, \T5, \XMM7, \T4
				2478	vpxor \T4, \T7, \T7
				2479
				2480	vpclmulqdq $0x00, \T3, \T2, \T2
				2481
				2482	vpxor \T2, \XMM1, \XMM1
				2483
				2484	######################
				2485
				2486	vmovdqa HashKey(arg1), \T5
				2487	vpshufd $0b01001110, \XMM8, \T2
				2488	vpshufd $0b01001110, \T5, \T3
				2489	vpxor \XMM8, \T2, \T2
				2490	vpxor \T5, \T3, \T3
				2491
				2492	vpclmulqdq $0x11, \T5, \XMM8, \T4
				2493	vpxor \T4, \T6, \T6
				2494
				2495	vpclmulqdq $0x00, \T5, \XMM8, \T4
				2496	vpxor \T4, \T7, \T7
				2497
				2498	vpclmulqdq $0x00, \T3, \T2, \T2
				2499
				2500	vpxor \T2, \XMM1, \XMM1
				2501	vpxor \T6, \XMM1, \XMM1
				2502	vpxor \T7, \XMM1, \T2
				2503
				2504
				2505
				2506
				2507	vpslldq $8, \T2, \T4
				2508	vpsrldq $8, \T2, \T2
				2509
				2510	vpxor \T4, \T7, \T7
				2511	vpxor \T2, \T6, \T6 # <T6:T7> holds the result of the
				2512	# accumulated carry-less multiplications
				2513
				2514	#######################################################################
				2515	#first phase of the reduction
				2516	vmovdqa POLY2(%rip), \T3
				2517
				2518	vpclmulqdq $0x01, \T7, \T3, \T2
				2519	vpslldq $8, \T2, \T2 # shift-L xmm2 2 DWs
				2520
				2521	vpxor \T2, \T7, \T7 # first phase of the reduction complete
				2522	#######################################################################
				2523
				2524
				2525	#second phase of the reduction
				2526	vpclmulqdq $0x00, \T7, \T3, \T2
				2527	vpsrldq $4, \T2, \T2 # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
				2528
				2529	vpclmulqdq $0x10, \T7, \T3, \T4
				2530	vpslldq $4, \T4, \T4 # shift-L T4 1 DW (Shift-L 1-DW to obtain result with no shifts)
				2531
				2532	vpxor \T2, \T4, \T4 # second phase of the reduction complete
				2533	#######################################################################
				2534	vpxor \T4, \T6, \T6 # the result is in T6
				2535	.endm
				2536
				2537
				2538
				2539	# combined for GCM encrypt and decrypt functions
				2540	# clobbering all xmm registers
				2541	# clobbering r10, r11, r12, r13, r14, r15
				2542	.macro GCM_ENC_DEC_AVX2 ENC_DEC
				2543
				2544	#the number of pushes must equal STACK_OFFSET
				2545	push %r12
				2546	push %r13
				2547	push %r14
				2548	push %r15
				2549
				2550	mov %rsp, %r14
				2551
				2552
				2553
				2554
				2555	sub $VARIABLE_OFFSET, %rsp
				2556	and $~63, %rsp # align rsp to 64 bytes
				2557
				2558
				2559	vmovdqu HashKey(arg1), %xmm13 # xmm13 = HashKey
				2560
				2561	mov arg4, %r13 # save the number of bytes of plaintext/ciphertext
				2562	and $-16, %r13 # r13 = r13 - (r13 mod 16)
				2563
				2564	mov %r13, %r12
				2565	shr $4, %r12
				2566	and $7, %r12
				2567	jz _initial_num_blocks_is_0\@
				2568
				2569	cmp $7, %r12
				2570	je _initial_num_blocks_is_7\@
				2571	cmp $6, %r12
				2572	je _initial_num_blocks_is_6\@
				2573	cmp $5, %r12
				2574	je _initial_num_blocks_is_5\@
				2575	cmp $4, %r12
				2576	je _initial_num_blocks_is_4\@
				2577	cmp $3, %r12
				2578	je _initial_num_blocks_is_3\@
				2579	cmp $2, %r12
				2580	je _initial_num_blocks_is_2\@
				2581
				2582	jmp _initial_num_blocks_is_1\@
				2583
				2584	_initial_num_blocks_is_7\@:
				2585	INITIAL_BLOCKS_AVX2 7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2586	sub $16*7, %r13
				2587	jmp _initial_blocks_encrypted\@
				2588
				2589	_initial_num_blocks_is_6\@:
				2590	INITIAL_BLOCKS_AVX2 6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2591	sub $16*6, %r13
				2592	jmp _initial_blocks_encrypted\@
				2593
				2594	_initial_num_blocks_is_5\@:
				2595	INITIAL_BLOCKS_AVX2 5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2596	sub $16*5, %r13
				2597	jmp _initial_blocks_encrypted\@
				2598
				2599	_initial_num_blocks_is_4\@:
				2600	INITIAL_BLOCKS_AVX2 4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2601	sub $16*4, %r13
				2602	jmp _initial_blocks_encrypted\@
				2603
				2604	_initial_num_blocks_is_3\@:
				2605	INITIAL_BLOCKS_AVX2 3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2606	sub $16*3, %r13
				2607	jmp _initial_blocks_encrypted\@
				2608
				2609	_initial_num_blocks_is_2\@:
				2610	INITIAL_BLOCKS_AVX2 2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2611	sub $16*2, %r13
				2612	jmp _initial_blocks_encrypted\@
				2613
				2614	_initial_num_blocks_is_1\@:
				2615	INITIAL_BLOCKS_AVX2 1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2616	sub $16*1, %r13
				2617	jmp _initial_blocks_encrypted\@
				2618
				2619	_initial_num_blocks_is_0\@:
				2620	INITIAL_BLOCKS_AVX2 0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
				2621
				2622
				2623	_initial_blocks_encrypted\@:
				2624	cmp $0, %r13
				2625	je _zero_cipher_left\@
				2626
				2627	sub $128, %r13
				2628	je _eight_cipher_left\@
				2629
				2630
				2631
				2632
				2633	vmovd %xmm9, %r15d
				2634	and $255, %r15d
				2635	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2636
				2637
				2638	_encrypt_by_8_new\@:
				2639	cmp $(255-8), %r15d
				2640	jg _encrypt_by_8\@
				2641
				2642
				2643
				2644	add $8, %r15b
				2645	GHASH_8_ENCRYPT_8_PARALLEL_AVX2 %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC
				2646	add $128, %r11
				2647	sub $128, %r13
				2648	jne _encrypt_by_8_new\@
				2649
				2650	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2651	jmp _eight_cipher_left\@
				2652
				2653	_encrypt_by_8\@:
				2654	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2655	add $8, %r15b
				2656	GHASH_8_ENCRYPT_8_PARALLEL_AVX2 %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC
				2657	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2658	add $128, %r11
				2659	sub $128, %r13
				2660	jne _encrypt_by_8_new\@
				2661
				2662	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2663
				2664
				2665
				2666
				2667	_eight_cipher_left\@:
				2668	GHASH_LAST_8_AVX2 %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8
				2669
				2670
				2671	_zero_cipher_left\@:
				2672	cmp $16, arg4
				2673	jl _only_less_than_16\@
				2674
				2675	mov arg4, %r13
				2676	and $15, %r13 # r13 = (arg4 mod 16)
				2677
				2678	je _multiple_of_16_bytes\@
				2679
				2680	# handle the last <16 Byte block seperately
				2681
				2682
				2683	vpaddd ONE(%rip), %xmm9, %xmm9 # INCR CNT to get Yn
				2684	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2685	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Yn)
				2686
				2687	sub $16, %r11
				2688	add %r13, %r11
				2689	vmovdqu (arg3, %r11), %xmm1 # receive the last <16 Byte block
				2690
				2691	lea SHIFT_MASK+16(%rip), %r12
				2692	sub %r13, %r12 # adjust the shuffle mask pointer
				2693	# to be able to shift 16-r13 bytes
				2694	# (r13 is the number of bytes in plaintext mod 16)
				2695	vmovdqu (%r12), %xmm2 # get the appropriate shuffle mask
				2696	vpshufb %xmm2, %xmm1, %xmm1 # shift right 16-r13 bytes
				2697	jmp _final_ghash_mul\@
				2698
				2699	_only_less_than_16\@:
				2700	# check for 0 length
				2701	mov arg4, %r13
				2702	and $15, %r13 # r13 = (arg4 mod 16)
				2703
				2704	je _multiple_of_16_bytes\@
				2705
				2706	# handle the last <16 Byte block seperately
				2707
				2708
				2709	vpaddd ONE(%rip), %xmm9, %xmm9 # INCR CNT to get Yn
				2710	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2711	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Yn)
				2712
				2713
				2714	lea SHIFT_MASK+16(%rip), %r12
				2715	sub %r13, %r12 # adjust the shuffle mask pointer to be
				2716	# able to shift 16-r13 bytes (r13 is the
				2717	# number of bytes in plaintext mod 16)
				2718
				2719	_get_last_16_byte_loop\@:
				2720	movb (arg3, %r11), %al
				2721	movb %al, TMP1 (%rsp , %r11)
				2722	add $1, %r11
				2723	cmp %r13, %r11
				2724	jne _get_last_16_byte_loop\@
				2725
				2726	vmovdqu TMP1(%rsp), %xmm1
				2727
				2728	sub $16, %r11
				2729
				2730	_final_ghash_mul\@:
				2731	.if \ENC_DEC == DEC
				2732	vmovdqa %xmm1, %xmm2
				2733	vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn)
				2734	vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to mask out top 16-r13 bytes of xmm9
				2735	vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9
				2736	vpand %xmm1, %xmm2, %xmm2
				2737	vpshufb SHUF_MASK(%rip), %xmm2, %xmm2
				2738	vpxor %xmm2, %xmm14, %xmm14
				2739	#GHASH computation for the last <16 Byte block
				2740	GHASH_MUL_AVX2 %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
				2741	sub %r13, %r11
				2742	add $16, %r11
				2743	.else
				2744	vpxor %xmm1, %xmm9, %xmm9 # Plaintext XOR E(K, Yn)
				2745	vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1 # get the appropriate mask to mask out top 16-r13 bytes of xmm9
				2746	vpand %xmm1, %xmm9, %xmm9 # mask out top 16-r13 bytes of xmm9
				2747	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
				2748	vpxor %xmm9, %xmm14, %xmm14
				2749	#GHASH computation for the last <16 Byte block
				2750	GHASH_MUL_AVX2 %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
				2751	sub %r13, %r11
				2752	add $16, %r11
				2753	vpshufb SHUF_MASK(%rip), %xmm9, %xmm9 # shuffle xmm9 back to output as ciphertext
				2754	.endif
				2755
				2756
				2757	#############################
				2758	# output r13 Bytes
				2759	vmovq %xmm9, %rax
				2760	cmp $8, %r13
				2761	jle _less_than_8_bytes_left\@
				2762
				2763	mov %rax, (arg2 , %r11)
				2764	add $8, %r11
				2765	vpsrldq $8, %xmm9, %xmm9
				2766	vmovq %xmm9, %rax
				2767	sub $8, %r13
				2768
				2769	_less_than_8_bytes_left\@:
				2770	movb %al, (arg2 , %r11)
				2771	add $1, %r11
				2772	shr $8, %rax
				2773	sub $1, %r13
				2774	jne _less_than_8_bytes_left\@
				2775	#############################
				2776
				2777	_multiple_of_16_bytes\@:
				2778	mov arg7, %r12 # r12 = aadLen (number of bytes)
				2779	shl $3, %r12 # convert into number of bits
				2780	vmovd %r12d, %xmm15 # len(A) in xmm15
				2781
				2782	shl $3, arg4 # len(C) in bits (*128)
				2783	vmovq arg4, %xmm1
				2784	vpslldq $8, %xmm15, %xmm15 # xmm15 = len(A)\|\| 0x0000000000000000
				2785	vpxor %xmm1, %xmm15, %xmm15 # xmm15 = len(A)\|\|len(C)
				2786
				2787	vpxor %xmm15, %xmm14, %xmm14
				2788	GHASH_MUL_AVX2 %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6 # final GHASH computation
				2789	vpshufb SHUF_MASK(%rip), %xmm14, %xmm14 # perform a 16Byte swap
				2790
				2791	mov arg5, %rax # rax = *Y0
				2792	vmovdqu (%rax), %xmm9 # xmm9 = Y0
				2793
				2794	ENCRYPT_SINGLE_BLOCK %xmm9 # E(K, Y0)
				2795
				2796	vpxor %xmm14, %xmm9, %xmm9
				2797
				2798
				2799
				2800	_return_T\@:
				2801	mov arg8, %r10 # r10 = authTag
				2802	mov arg9, %r11 # r11 = auth_tag_len
				2803
				2804	cmp $16, %r11
				2805	je _T_16\@
				2806
Sabrina Dubroca	2df7e81	2017-04-28 18:12:01 +0200	[diff] [blame]	2807	cmp $8, %r11
				2808	jl _T_4\@
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	2809
				2810	_T_8\@:
				2811	vmovq %xmm9, %rax
				2812	mov %rax, (%r10)
Sabrina Dubroca	2df7e81	2017-04-28 18:12:01 +0200	[diff] [blame]	2813	add $8, %r10
				2814	sub $8, %r11
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	2815	vpsrldq $8, %xmm9, %xmm9
Sabrina Dubroca	2df7e81	2017-04-28 18:12:01 +0200	[diff] [blame]	2816	cmp $0, %r11
				2817	je _return_T_done\@
				2818	_T_4\@:
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	2819	vmovd %xmm9, %eax
Sabrina Dubroca	2df7e81	2017-04-28 18:12:01 +0200	[diff] [blame]	2820	mov %eax, (%r10)
				2821	add $4, %r10
				2822	sub $4, %r11
				2823	vpsrldq $4, %xmm9, %xmm9
				2824	cmp $0, %r11
				2825	je _return_T_done\@
				2826	_T_123\@:
				2827	vmovd %xmm9, %eax
				2828	cmp $2, %r11
				2829	jl _T_1\@
				2830	mov %ax, (%r10)
				2831	cmp $2, %r11
				2832	je _return_T_done\@
				2833	add $2, %r10
				2834	sar $16, %eax
				2835	_T_1\@:
				2836	mov %al, (%r10)
Tim Chen	d764593	2013-12-11 14:28:41 -0800	[diff] [blame]	2837	jmp _return_T_done\@
				2838
				2839	_T_16\@:
				2840	vmovdqu %xmm9, (%r10)
				2841
				2842	_return_T_done\@:
				2843	mov %r14, %rsp
				2844
				2845	pop %r15
				2846	pop %r14
				2847	pop %r13
				2848	pop %r12
				2849	.endm
				2850
				2851
				2852	#############################################################
				2853	#void aesni_gcm_precomp_avx_gen4
				2854	# (gcm_data *my_ctx_data,
				2855	# u8 hash_subkey)# / H, the Hash sub key input.
				2856	# Data starts on a 16-byte boundary. */
				2857	#############################################################
				2858	ENTRY(aesni_gcm_precomp_avx_gen4)
				2859	#the number of pushes must equal STACK_OFFSET
				2860	push %r12
				2861	push %r13
				2862	push %r14
				2863	push %r15
				2864
				2865	mov %rsp, %r14
				2866
				2867
				2868
				2869	sub $VARIABLE_OFFSET, %rsp
				2870	and $~63, %rsp # align rsp to 64 bytes
				2871
				2872	vmovdqu (arg2), %xmm6 # xmm6 = HashKey
				2873
				2874	vpshufb SHUF_MASK(%rip), %xmm6, %xmm6
				2875	############### PRECOMPUTATION of HashKey<<1 mod poly from the HashKey
				2876	vmovdqa %xmm6, %xmm2
				2877	vpsllq $1, %xmm6, %xmm6
				2878	vpsrlq $63, %xmm2, %xmm2
				2879	vmovdqa %xmm2, %xmm1
				2880	vpslldq $8, %xmm2, %xmm2
				2881	vpsrldq $8, %xmm1, %xmm1
				2882	vpor %xmm2, %xmm6, %xmm6
				2883	#reduction
				2884	vpshufd $0b00100100, %xmm1, %xmm2
				2885	vpcmpeqd TWOONE(%rip), %xmm2, %xmm2
				2886	vpand POLY(%rip), %xmm2, %xmm2
				2887	vpxor %xmm2, %xmm6, %xmm6 # xmm6 holds the HashKey<<1 mod poly
				2888	#######################################################################
				2889	vmovdqa %xmm6, HashKey(arg1) # store HashKey<<1 mod poly
				2890
				2891
				2892	PRECOMPUTE_AVX2 %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5
				2893
				2894	mov %r14, %rsp
				2895
				2896	pop %r15
				2897	pop %r14
				2898	pop %r13
				2899	pop %r12
				2900	ret
				2901	ENDPROC(aesni_gcm_precomp_avx_gen4)
				2902
				2903
				2904	###############################################################################
				2905	#void aesni_gcm_enc_avx_gen4(
				2906	# gcm_data my_ctx_data, / aligned to 16 Bytes */
				2907	# u8 out, / Ciphertext output. Encrypt in-place is allowed. */
				2908	# const u8 in, / Plaintext input */
				2909	# u64 plaintext_len, /* Length of data in Bytes for encryption. */
				2910	# u8 iv, / Pre-counter block j0: 4 byte salt
				2911	# (from Security Association) concatenated with 8 byte
				2912	# Initialisation Vector (from IPSec ESP Payload)
				2913	# concatenated with 0x00000001. 16-byte aligned pointer. */
				2914	# const u8 aad, / Additional Authentication Data (AAD)*/
				2915	# u64 aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
				2916	# u8 auth_tag, / Authenticated Tag output. */
				2917	# u64 auth_tag_len)# /* Authenticated Tag Length in bytes.
				2918	# Valid values are 16 (most likely), 12 or 8. */
				2919	###############################################################################
				2920	ENTRY(aesni_gcm_enc_avx_gen4)
				2921	GCM_ENC_DEC_AVX2 ENC
				2922	ret
				2923	ENDPROC(aesni_gcm_enc_avx_gen4)
				2924
				2925	###############################################################################
				2926	#void aesni_gcm_dec_avx_gen4(
				2927	# gcm_data my_ctx_data, / aligned to 16 Bytes */
				2928	# u8 out, / Plaintext output. Decrypt in-place is allowed. */
				2929	# const u8 in, / Ciphertext input */
				2930	# u64 plaintext_len, /* Length of data in Bytes for encryption. */
				2931	# u8 iv, / Pre-counter block j0: 4 byte salt
				2932	# (from Security Association) concatenated with 8 byte
				2933	# Initialisation Vector (from IPSec ESP Payload)
				2934	# concatenated with 0x00000001. 16-byte aligned pointer. */
				2935	# const u8 aad, / Additional Authentication Data (AAD)*/
				2936	# u64 aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
				2937	# u8 auth_tag, / Authenticated Tag output. */
				2938	# u64 auth_tag_len)# /* Authenticated Tag Length in bytes.
				2939	# Valid values are 16 (most likely), 12 or 8. */
				2940	###############################################################################
				2941	ENTRY(aesni_gcm_dec_avx_gen4)
				2942	GCM_ENC_DEC_AVX2 DEC
				2943	ret
				2944	ENDPROC(aesni_gcm_dec_avx_gen4)
				2945
				2946	#endif /* CONFIG_AS_AVX2 */