From 8419e84e357405cd9c1843a75c80691f330c5db4 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Sat, 11 Jul 2026 05:28:50 +0100 Subject: [PATCH 1/4] Re-architect ChaCha20/Poly1305 SIMD kernels for streaming throughput ChaCha (x86_64 + i386): - Vertical kernels (AVX2 8-way, SSE2/SSSE3 4-way) now stream AGroups groups per call on a 64B-aligned frame: one-time prologue, light per-group state reload, resident c-pair rounds (2 c-stores + 2 c-loads per round), and a fused add-back/transpose/xor tail written straight to the output (no bounce buffer). - New i386 AVX2 8-way kernel (stack-resident state rows, resident rot16/rot8 masks) roughly doubles 32-bit throughput. - Engine bulk ladder: one WideGroupsSafe wrap guard + TryProcessWide per tier; each wide kernel is entered once per span instead of per group. Poly1305: - x86_64 AVX2 bulk loop is modulo-scheduled: r-power rows load into the freed temps while pending limbs absorb, h2 (folded early in the prior reduce) opens all five accumulators, and the next block's load + limb split thread through the multiply and lazy reduce. - i386 AVX2 and both SSE2 kernels: 64B-aligned frames, k-major multiply with the accumulator resident across its terms, mask26/padbit as memory operands (keeps H0..H4 resident on i386), and a one-time aligned in-frame table copy so SSE2 multiplies take memory operands. ChaCha20Poly1305: - Bulk data now runs one whole-span cipher call and one whole-span MAC call per ProcessBytes; the 512-byte stride loops are gone. --- .../src/Crypto/Engines/ClpChaChaEngine.pas | 87 +- .../src/Crypto/Engines/ClpChaChaSimd.pas | 20 +- .../Crypto/Engines/ClpChaChaX86Backend.pas | 86 +- .../src/Crypto/Modes/ClpChaCha20Poly1305.pas | 25 +- .../ChaCha/ChaCha7539Blocks4Sse2_i386.inc | 447 +++++----- .../ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc | 746 ++++++++-------- .../ChaCha/ChaCha7539Blocks4Ssse3_i386.inc | 499 ++++++----- .../ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc | 678 +++++++------- .../ChaCha/ChaCha7539Blocks8Avx2_i386.inc | 500 +++++++++++ .../ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc | 784 ++++++++-------- .../Poly1305BlocksBulkAvx2Core_i386.inc | 496 +++++----- .../Poly1305BlocksBulkAvx2Core_x86_64.inc | 259 +++--- .../Poly1305BlocksBulkSse2Core_i386.inc | 844 ++++++++---------- .../Poly1305BlocksBulkSse2Core_x86_64.inc | 714 ++++++++------- 14 files changed, 3237 insertions(+), 2948 deletions(-) create mode 100644 CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas index d77e3a1e..e96a57f8 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas @@ -57,11 +57,14 @@ interface TChaChaBaseEngine = class(TSalsa20Engine) strict private - // Pointer-form, unvalidated inner helpers (1 / 2 / 4 / 8 blocks). + // Pointer-form, unvalidated inner helpers (1 / 2 blocks). procedure ProcessBlockFast(AIn, AOut: PByte); procedure ProcessBlocks2Fast(AIn, AOut: PByte); - procedure ProcessBlocks4Fast(AIn, AOut: PByte); - procedure ProcessBlocks8Fast(AIn, AOut: PByte); + // One streaming SIMD tier (AGroupBlocks = 8 or 4): clamps to the wrap-safe + // span, runs the kernel once over every whole group, advances the cursors. + // False = no SIMD kernel for that width or no safe groups (degrade a tier). + function TryProcessWide(AGroupBlocks: Int32; var AIn, AOut: PByte; + var ABlockCount: Int32): Boolean; strict protected // 8 -> 4 -> 2 -> 1 bulk ladder (widest kernel first). @@ -69,9 +72,10 @@ TChaChaBaseEngine = class(TSalsa20Engine) // True for the DJB 64-bit counter (words 12-13); False (IETF). function CounterIs64Bit: Boolean; virtual; - // Wide kernels broadcast word 13; for the 64-bit counter that holds only while - // word 12 does not wrap over the N-block span (else the caller degrades a tier). - function WideBlocksSafe(ABlocks: Int32): Boolean; inline; + // Wide kernels broadcast word 13, valid only while word 12 does not wrap + // inside the processed span: clamps a span of AGroups x AGroupBlocks + // blocks to the groups fully before the wrap (identity for IETF). + function WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32; procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override; @@ -133,12 +137,20 @@ function TChaChaBaseEngine.CounterIs64Bit: Boolean; Result := False; end; -function TChaChaBaseEngine.WideBlocksSafe(ABlocks: Int32): Boolean; +function TChaChaBaseEngine.WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32; +var + LSafeBlocks: UInt64; begin - Result := (not CounterIs64Bit) or - (FEngineState[12] <= (UInt32($FFFFFFFF) - UInt32(ABlocks - 1))); + Result := AGroups; + if CounterIs64Bit then + begin + LSafeBlocks := UInt64($100000000) - FEngineState[12]; + if (UInt64(Result) * UInt32(AGroupBlocks)) > LSafeBlocks then + Result := Int32(LSafeBlocks div UInt32(AGroupBlocks)); + end; end; + procedure TChaChaBaseEngine.GenerateKeyStream(const AOutput: TCryptoLibByteArray); begin ChaChaCore(FRounds, FEngineState, AOutput); @@ -299,26 +311,29 @@ procedure TChaChaBaseEngine.ProcessBlocks2Fast(AIn, AOut: PByte); ProcessBlockFast(AIn + 64, AOut + 64); end; -procedure TChaChaBaseEngine.ProcessBlocks4Fast(AIn, AOut: PByte); -begin - if WideBlocksSafe(4) then - if TChaChaSimd.TryProcessBlocks4(FRounds, PByte(@FEngineState[0]), - AIn, AOut, CounterIs64Bit) then - Exit; - - ProcessBlocks2Fast(AIn, AOut); - ProcessBlocks2Fast(AIn + 128, AOut + 128); -end; - -procedure TChaChaBaseEngine.ProcessBlocks8Fast(AIn, AOut: PByte); +function TChaChaBaseEngine.TryProcessWide(AGroupBlocks: Int32; + var AIn, AOut: PByte; var ABlockCount: Int32): Boolean; +var + LGroups: Int32; begin - if WideBlocksSafe(8) then - if TChaChaSimd.TryProcessBlocks8(FRounds, PByte(@FEngineState[0]), - AIn, AOut, CounterIs64Bit) then - Exit; - - ProcessBlocks4Fast(AIn, AOut); - ProcessBlocks4Fast(AIn + 256, AOut + 256); + Result := False; + LGroups := WideGroupsSafe(ABlockCount div AGroupBlocks, AGroupBlocks); + if LGroups = 0 then + Exit; + case AGroupBlocks of + 8: + Result := TChaChaSimd.TryProcessBlocks8(FRounds, + PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit); + else + Result := TChaChaSimd.TryProcessBlocks4(FRounds, + PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit); + end; + if Result then + begin + AIn := AIn + LGroups * (AGroupBlocks * 64); + AOut := AOut + LGroups * (AGroupBlocks * 64); + System.Dec(ABlockCount, LGroups * AGroupBlocks); + end; end; function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte; @@ -327,14 +342,24 @@ function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte; Result := ABlockCount * 64; while ABlockCount >= 8 do begin - ProcessBlocks8Fast(AIn, AOut); + // Widest streaming tier that lands, over all safe groups at once. + if TryProcessWide(8, AIn, AOut, ABlockCount) then + continue; + if TryProcessWide(4, AIn, AOut, ABlockCount) then + continue; + // No wide SIMD (or at the DJB wrap boundary): one 8-block group via 2/1. + ProcessBlocks2Fast(AIn, AOut); + ProcessBlocks2Fast(AIn + 128, AOut + 128); + ProcessBlocks2Fast(AIn + 256, AOut + 256); + ProcessBlocks2Fast(AIn + 384, AOut + 384); AIn := AIn + 512; AOut := AOut + 512; System.Dec(ABlockCount, 8); end; - if ABlockCount >= 4 then + if (ABlockCount >= 4) and (not TryProcessWide(4, AIn, AOut, ABlockCount)) then begin - ProcessBlocks4Fast(AIn, AOut); + ProcessBlocks2Fast(AIn, AOut); + ProcessBlocks2Fast(AIn + 128, AOut + 128); AIn := AIn + 256; AOut := AOut + 256; System.Dec(ABlockCount, 4); diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas index bf10519e..ac6ca3ee 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas @@ -42,10 +42,14 @@ TChaChaSimd = class sealed /// Two-block ChaCha keystream (128 bytes). ACtr64 = DJB 64-bit /// counter (no SIMD 2-block body; returns False). class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; - /// Four-block ChaCha keystream (256 bytes). ACtr64 = DJB variant. - class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; - /// Eight-block ChaCha keystream (512 bytes). ACtr64 = DJB variant. - class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// Four-block ChaCha keystream, streaming AGroups consecutive + /// 256-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the + /// caller guarantees no counter low-word wrap inside the span. + class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static; + /// Eight-block ChaCha keystream, streaming AGroups consecutive + /// 512-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the + /// caller guarantees no counter low-word wrap inside the span. + class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static; end; implementation @@ -70,19 +74,19 @@ class function TChaChaSimd.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: {$IFEND} end; -class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, ACtr64); + Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, AGroups, ACtr64); {$ELSE} Result := False; {$IFEND} end; -class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, ACtr64); + Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, AGroups, ACtr64); {$ELSE} Result := False; {$IFEND} diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas index 04f13d17..249b0c06 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas @@ -45,12 +45,16 @@ TChaChaX86Backend = class sealed /// 64-bit-counter (DJB) variant; no SIMD 2-block kernel exists for it, so it /// returns False and the caller runs its scalar path. class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; - /// Four-block ChaCha keystream (256 bytes). ACtr64 selects the DJB - /// 64-bit-counter kernel. - class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; - /// AVX2 eight-block ChaCha keystream (512 bytes). ACtr64 selects the - /// DJB 64-bit-counter kernel. - class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// Four-block ChaCha keystream, streaming AGroups consecutive + /// 256-byte groups (AGroups >= 1) in one call. ACtr64 selects the DJB + /// 64-bit-counter kernel; the caller must guarantee the counter low word + /// does not wrap inside the 4*AGroups-block span. + class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static; + /// AVX2 eight-block ChaCha keystream, streaming AGroups consecutive + /// 512-byte groups (AGroups >= 1) in one call. ACtr64 selects the DJB + /// 64-bit-counter kernel; the caller must guarantee the counter low word + /// does not wrap inside the 8*AGroups-block span. + class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static; end; implementation @@ -83,79 +87,89 @@ procedure ChaCha7539ProcessBlocks2Sse2(ARounds: Int32; AState, AIn, AOut: PByte) {$ENDIF} end; -procedure ChaCha7539ProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte); +procedure ChaCha7539ProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc} {$ENDIF} end; -procedure ChaCha7539ProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte); +procedure ChaCha7539ProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc} {$ENDIF} end; -// AVX2 8-way (512B): x86_64 only (needs 16 ymm; i386 has 8, so it falls back to -// two 256B ProcessBlocks4 in the engine). +// AVX2 8-way: streams AGroups x 512B per call. x86_64 keeps the 16 state rows +// in ymm registers; i386 (8 ymm) works them from a 64B-aligned stack frame. +procedure ChaCha7539ProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -procedure ChaCha7539ProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte); -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc} -end; {$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_i386.inc} +{$ENDIF} +end; // DJB ChaCha (64-bit counter, words 12-13): the vertical kernels are identical to // the 7539 ones except the counter advance carries into word 13, which the shared // includes emit under CHACHA_CTR64. Callers guard against a low-word wrap across the -// lanes before invoking these (see the engine's WideBlocksSafe). -procedure ChaChaProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte); +// lanes before invoking these (see the engine's WideGroupsSafe). +procedure ChaChaProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$DEFINE CHACHA_CTR64} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc} {$UNDEF CHACHA_CTR64} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$DEFINE CHACHA_CTR64} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc} {$UNDEF CHACHA_CTR64} {$ENDIF} end; -procedure ChaChaProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte); +procedure ChaChaProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$DEFINE CHACHA_CTR64} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc} {$UNDEF CHACHA_CTR64} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$DEFINE CHACHA_CTR64} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc} {$UNDEF CHACHA_CTR64} {$ENDIF} end; +procedure ChaChaProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} -procedure ChaChaProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte); -{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$DEFINE CHACHA_CTR64} {$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc} {$UNDEF CHACHA_CTR64} -end; {$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_i386.inc} +{$UNDEF CHACHA_CTR64} +{$ENDIF} +end; {$ENDIF CRYPTOLIB_X86_SIMD} { TChaChaX86Backend } @@ -193,7 +207,7 @@ class function TChaChaX86Backend.TryProcessBlocks2(ARounds: Int32; AState, AIn, Result := False; end; -class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} // 4-block (256B) tier: AVX2 gives no extra lanes at 4-way, and AVX2 CPUs run the @@ -202,17 +216,17 @@ class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, TX86SimdLevel.SSSE3: begin if ACtr64 then - ChaChaProcessBlocks4Ssse3(ARounds, AState, AIn, AOut) + ChaChaProcessBlocks4Ssse3(ARounds, AState, AIn, AOut, AGroups) else - ChaCha7539ProcessBlocks4Ssse3(ARounds, AState, AIn, AOut); + ChaCha7539ProcessBlocks4Ssse3(ARounds, AState, AIn, AOut, AGroups); Exit(True); end; TX86SimdLevel.SSE2: begin if ACtr64 then - ChaChaProcessBlocks4Sse2(ARounds, AState, AIn, AOut) + ChaChaProcessBlocks4Sse2(ARounds, AState, AIn, AOut, AGroups) else - ChaCha7539ProcessBlocks4Sse2(ARounds, AState, AIn, AOut); + ChaCha7539ProcessBlocks4Sse2(ARounds, AState, AIn, AOut, AGroups); Exit(True); end; end; @@ -220,16 +234,16 @@ class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, Result := False; end; -class function TChaChaX86Backend.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +class function TChaChaX86Backend.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean; begin -{$IFDEF CRYPTOLIB_X86_64_ASM} +{$IFDEF CRYPTOLIB_X86_SIMD} case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2]) of TX86SimdLevel.AVX2: begin if ACtr64 then - ChaChaProcessBlocks8Avx2(ARounds, AState, AIn, AOut) + ChaChaProcessBlocks8Avx2(ARounds, AState, AIn, AOut, AGroups) else - ChaCha7539ProcessBlocks8Avx2(ARounds, AState, AIn, AOut); + ChaCha7539ProcessBlocks8Avx2(ARounds, AState, AIn, AOut, AGroups); Exit(True); end; end; diff --git a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas index 4193c979..1ebb2aa0 100644 --- a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas +++ b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas @@ -414,7 +414,7 @@ function TChaCha20Poly1305.ProcessByte(AInput: Byte; function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; AInOff, ALen: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LResultLen, LAvailable, LInLimit1, LInLimit2, LRemBlocks: Int32; + LResultLen, LAvailable, LInLimit1, LRemBlocks: Int32; begin if (AInput = nil) then raise EArgumentNilCryptoLibException.CreateRes(@SInBytesNil); @@ -461,7 +461,6 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; end; LInLimit1 := AInOff + ALen - System.Length(FBuf); - LInLimit2 := LInLimit1 - BufSize; LAvailable := BufSize - FBufPos; System.Move(AInput[AInOff], FBuf[FBufPos], LAvailable); @@ -472,15 +471,8 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; if (FBulkChaCha <> nil) then begin - // Bulk: MAC + decrypt in 512B (8-way) strides, then the remaining whole - // blocks (< 8) in a single engine-laddered call. - while (AInOff <= LInLimit2 - (BufSize * 6)) do - begin - FPoly1305.BlockUpdate(AInput, AInOff, BufSize * 8); - ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 8; - LResultLen := LResultLen + BufSize * 8; - end; + // Bulk: MAC then decrypt every whole block, one streaming call each + // (MAC first reads all ciphertext, so in-place decrypt stays safe). if (AInOff <= LInLimit1) then begin LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1; @@ -517,7 +509,6 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; end; LInLimit1 := AInOff + ALen - BufSize; - LInLimit2 := LInLimit1 - BufSize; if (FBufPos > 0) then begin @@ -529,14 +520,8 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; if (FBulkChaCha <> nil) then begin - // Bulk: encrypt in 512B (8-way) strides, then the remaining whole blocks - // (< 8) in a single engine-laddered call; MAC over the ciphertext below. - while (AInOff <= LInLimit2 - (BufSize * 6)) do - begin - ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 8; - LResultLen := LResultLen + BufSize * 8; - end; + // Bulk: encrypt every whole block in one streaming engine call (the + // ladder runs the widest kernels internally); MAC the ciphertext below. if (AInOff <= LInLimit1) then begin LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1; diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc index a5031baf..2ae80d0e 100644 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc @@ -1,95 +1,144 @@ -// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (i386). +// ChaCha7539 (IETF) 4-way SSE2 keystream kernel, vertical layout (i386). // ------------------------------------------------------------------- // -// Four 64-byte ChaCha20 blocks, state words held vertically (one block per -// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame; -// each quarter-round loads its 4 words into xmm0-3 (temp xmm4). +// Four 64-byte ChaCha20 blocks per group, state words held vertically (one +// block per xmm lane). i386 has only 8 xmm, so the 16 state words live in a +// 64B-aligned stack frame (original esp parked inside it); each quarter-round +// loads its 4 words into xmm0-3 (temp xmm4). // rot16/rot8 use shift+or; rot12/rot7 use shift+or. // -// On entry (after ClpSimdProc4Begin_i386.inc): -// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut. +// Streams AGroups x 256B in one call: prologue once, per-group ORIG->STATE +// reload (counter lanes +4), fused accumulate/transpose/xor tail to AOut. +// +// On entry (after ClpSimdProc5Begin_i386.inc): +// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut, +// ecx = AGroups (4-block groups, >= 1). // // On exit: -// AState word-12 counter (byte 48) advanced by 4; the keystream is xored -// with AIn into AOut (256 bytes). +// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is +// xored with AIn into AOut (256*AGroups bytes). // // punpck* / pshufb are db-encoded for broad assembler compatibility. - sub esp, 528 + mov edx, esp + sub esp, 560 + and esp, -64 + mov dword ptr [esp + 544], edx mov dword ptr [esp + 512], 0 mov dword ptr [esp + 516], 1 mov dword ptr [esp + 520], 2 mov dword ptr [esp + 524], 3 + mov dword ptr [esp + 528], 4 + mov dword ptr [esp + 532], 4 + mov dword ptr [esp + 536], 4 + mov dword ptr [esp + 540], 4 movd xmm0, [esi] pshufd xmm0, xmm0, $00 - movdqu [esp], xmm0 - movdqu [esp + 256], xmm0 + movdqa [esp], xmm0 + movdqa [esp + 256], xmm0 movd xmm0, [esi + 4] pshufd xmm0, xmm0, $00 - movdqu [esp + 16], xmm0 - movdqu [esp + 272], xmm0 + movdqa [esp + 16], xmm0 + movdqa [esp + 272], xmm0 movd xmm0, [esi + 8] pshufd xmm0, xmm0, $00 - movdqu [esp + 32], xmm0 - movdqu [esp + 288], xmm0 + movdqa [esp + 32], xmm0 + movdqa [esp + 288], xmm0 movd xmm0, [esi + 12] pshufd xmm0, xmm0, $00 - movdqu [esp + 48], xmm0 - movdqu [esp + 304], xmm0 + movdqa [esp + 48], xmm0 + movdqa [esp + 304], xmm0 movd xmm0, [esi + 16] pshufd xmm0, xmm0, $00 - movdqu [esp + 64], xmm0 - movdqu [esp + 320], xmm0 + movdqa [esp + 64], xmm0 + movdqa [esp + 320], xmm0 movd xmm0, [esi + 20] pshufd xmm0, xmm0, $00 - movdqu [esp + 80], xmm0 - movdqu [esp + 336], xmm0 + movdqa [esp + 80], xmm0 + movdqa [esp + 336], xmm0 movd xmm0, [esi + 24] pshufd xmm0, xmm0, $00 - movdqu [esp + 96], xmm0 - movdqu [esp + 352], xmm0 + movdqa [esp + 96], xmm0 + movdqa [esp + 352], xmm0 movd xmm0, [esi + 28] pshufd xmm0, xmm0, $00 - movdqu [esp + 112], xmm0 - movdqu [esp + 368], xmm0 + movdqa [esp + 112], xmm0 + movdqa [esp + 368], xmm0 movd xmm0, [esi + 32] pshufd xmm0, xmm0, $00 - movdqu [esp + 128], xmm0 - movdqu [esp + 384], xmm0 + movdqa [esp + 128], xmm0 + movdqa [esp + 384], xmm0 movd xmm0, [esi + 36] pshufd xmm0, xmm0, $00 - movdqu [esp + 144], xmm0 - movdqu [esp + 400], xmm0 + movdqa [esp + 144], xmm0 + movdqa [esp + 400], xmm0 movd xmm0, [esi + 40] pshufd xmm0, xmm0, $00 - movdqu [esp + 160], xmm0 - movdqu [esp + 416], xmm0 + movdqa [esp + 160], xmm0 + movdqa [esp + 416], xmm0 movd xmm0, [esi + 44] pshufd xmm0, xmm0, $00 - movdqu [esp + 176], xmm0 - movdqu [esp + 432], xmm0 + movdqa [esp + 176], xmm0 + movdqa [esp + 432], xmm0 movd xmm0, [esi + 48] pshufd xmm0, xmm0, $00 - movdqu xmm1, [esp + 512] - paddd xmm0, xmm1 - movdqu [esp + 192], xmm0 - movdqu [esp + 448], xmm0 + db $66, $0F, $FE, $84, $24, $00, $02, $00, $00 // paddd xmm0, [esp + 512] + movdqa [esp + 192], xmm0 + movdqa [esp + 448], xmm0 movd xmm0, [esi + 52] pshufd xmm0, xmm0, $00 - movdqu [esp + 208], xmm0 - movdqu [esp + 464], xmm0 + movdqa [esp + 208], xmm0 + movdqa [esp + 464], xmm0 movd xmm0, [esi + 56] pshufd xmm0, xmm0, $00 - movdqu [esp + 224], xmm0 - movdqu [esp + 480], xmm0 + movdqa [esp + 224], xmm0 + movdqa [esp + 480], xmm0 movd xmm0, [esi + 60] pshufd xmm0, xmm0, $00 - movdqu [esp + 240], xmm0 - movdqu [esp + 496], xmm0 + movdqa [esp + 240], xmm0 + movdqa [esp + 496], xmm0 + jmp @@c7539_sse2_enter +@@c7539_sse2_outer: + movdqa xmm0, [esp + 256] + movdqa [esp], xmm0 + movdqa xmm0, [esp + 272] + movdqa [esp + 16], xmm0 + movdqa xmm0, [esp + 288] + movdqa [esp + 32], xmm0 + movdqa xmm0, [esp + 304] + movdqa [esp + 48], xmm0 + movdqa xmm0, [esp + 320] + movdqa [esp + 64], xmm0 + movdqa xmm0, [esp + 336] + movdqa [esp + 80], xmm0 + movdqa xmm0, [esp + 352] + movdqa [esp + 96], xmm0 + movdqa xmm0, [esp + 368] + movdqa [esp + 112], xmm0 + movdqa xmm0, [esp + 384] + movdqa [esp + 128], xmm0 + movdqa xmm0, [esp + 400] + movdqa [esp + 144], xmm0 + movdqa xmm0, [esp + 416] + movdqa [esp + 160], xmm0 + movdqa xmm0, [esp + 432] + movdqa [esp + 176], xmm0 + movdqa xmm0, [esp + 448] + db $66, $0F, $FE, $84, $24, $10, $02, $00, $00 // paddd xmm0, [esp + 528] + movdqa [esp + 448], xmm0 + movdqa [esp + 192], xmm0 + movdqa xmm0, [esp + 464] + movdqa [esp + 208], xmm0 + movdqa xmm0, [esp + 480] + movdqa [esp + 224], xmm0 + movdqa xmm0, [esp + 496] + movdqa [esp + 240], xmm0 +@@c7539_sse2_enter: + mov edx, ebx @@c7539_sse2_round: - movdqu xmm0, [esp] - movdqu xmm1, [esp + 64] - movdqu xmm2, [esp + 128] - movdqu xmm3, [esp + 192] + movdqa xmm0, [esp] + movdqa xmm1, [esp + 64] + movdqa xmm2, [esp + 128] + movdqa xmm3, [esp + 192] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -114,14 +163,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp], xmm0 - movdqu [esp + 64], xmm1 - movdqu [esp + 128], xmm2 - movdqu [esp + 192], xmm3 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 144] - movdqu xmm3, [esp + 208] + movdqa [esp], xmm0 + movdqa [esp + 64], xmm1 + movdqa [esp + 128], xmm2 + movdqa [esp + 192], xmm3 + movdqa xmm0, [esp + 16] + movdqa xmm1, [esp + 80] + movdqa xmm2, [esp + 144] + movdqa xmm3, [esp + 208] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -146,14 +195,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 16], xmm0 - movdqu [esp + 80], xmm1 - movdqu [esp + 144], xmm2 - movdqu [esp + 208], xmm3 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 96] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 224] + movdqa [esp + 16], xmm0 + movdqa [esp + 80], xmm1 + movdqa [esp + 144], xmm2 + movdqa [esp + 208], xmm3 + movdqa xmm0, [esp + 32] + movdqa xmm1, [esp + 96] + movdqa xmm2, [esp + 160] + movdqa xmm3, [esp + 224] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -178,14 +227,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 32], xmm0 - movdqu [esp + 96], xmm1 - movdqu [esp + 160], xmm2 - movdqu [esp + 224], xmm3 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 112] - movdqu xmm2, [esp + 176] - movdqu xmm3, [esp + 240] + movdqa [esp + 32], xmm0 + movdqa [esp + 96], xmm1 + movdqa [esp + 160], xmm2 + movdqa [esp + 224], xmm3 + movdqa xmm0, [esp + 48] + movdqa xmm1, [esp + 112] + movdqa xmm2, [esp + 176] + movdqa xmm3, [esp + 240] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -210,14 +259,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 48], xmm0 - movdqu [esp + 112], xmm1 - movdqu [esp + 176], xmm2 - movdqu [esp + 240], xmm3 - movdqu xmm0, [esp] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 240] + movdqa [esp + 48], xmm0 + movdqa [esp + 112], xmm1 + movdqa [esp + 176], xmm2 + movdqa [esp + 240], xmm3 + movdqa xmm0, [esp] + movdqa xmm1, [esp + 80] + movdqa xmm2, [esp + 160] + movdqa xmm3, [esp + 240] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -242,14 +291,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp], xmm0 - movdqu [esp + 80], xmm1 - movdqu [esp + 160], xmm2 - movdqu [esp + 240], xmm3 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 96] - movdqu xmm2, [esp + 176] - movdqu xmm3, [esp + 192] + movdqa [esp], xmm0 + movdqa [esp + 80], xmm1 + movdqa [esp + 160], xmm2 + movdqa [esp + 240], xmm3 + movdqa xmm0, [esp + 16] + movdqa xmm1, [esp + 96] + movdqa xmm2, [esp + 176] + movdqa xmm3, [esp + 192] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -274,14 +323,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 16], xmm0 - movdqu [esp + 96], xmm1 - movdqu [esp + 176], xmm2 - movdqu [esp + 192], xmm3 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 112] - movdqu xmm2, [esp + 128] - movdqu xmm3, [esp + 208] + movdqa [esp + 16], xmm0 + movdqa [esp + 96], xmm1 + movdqa [esp + 176], xmm2 + movdqa [esp + 192], xmm3 + movdqa xmm0, [esp + 32] + movdqa xmm1, [esp + 112] + movdqa xmm2, [esp + 128] + movdqa xmm3, [esp + 208] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -306,14 +355,14 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 32], xmm0 - movdqu [esp + 112], xmm1 - movdqu [esp + 128], xmm2 - movdqu [esp + 208], xmm3 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 64] - movdqu xmm2, [esp + 144] - movdqu xmm3, [esp + 224] + movdqa [esp + 32], xmm0 + movdqa [esp + 112], xmm1 + movdqa [esp + 128], xmm2 + movdqa [esp + 208], xmm3 + movdqa xmm0, [esp + 48] + movdqa xmm1, [esp + 64] + movdqa xmm2, [esp + 144] + movdqa xmm3, [esp + 224] paddd xmm0, xmm1 pxor xmm3, xmm0 movdqa xmm4, xmm3 @@ -338,96 +387,36 @@ pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 48], xmm0 - movdqu [esp + 64], xmm1 - movdqu [esp + 144], xmm2 - movdqu [esp + 224], xmm3 - sub ebx, 2 + movdqa [esp + 48], xmm0 + movdqa [esp + 64], xmm1 + movdqa [esp + 144], xmm2 + movdqa [esp + 224], xmm3 + sub edx, 2 jg @@c7539_sse2_round - movdqu xmm0, [esp] - movdqu xmm1, [esp + 256] - paddd xmm0, xmm1 - movdqu [esp], xmm0 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 272] - paddd xmm0, xmm1 - movdqu [esp + 16], xmm0 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 288] - paddd xmm0, xmm1 - movdqu [esp + 32], xmm0 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 304] - paddd xmm0, xmm1 - movdqu [esp + 48], xmm0 - movdqu xmm0, [esp + 64] - movdqu xmm1, [esp + 320] - paddd xmm0, xmm1 - movdqu [esp + 64], xmm0 - movdqu xmm0, [esp + 80] - movdqu xmm1, [esp + 336] - paddd xmm0, xmm1 - movdqu [esp + 80], xmm0 - movdqu xmm0, [esp + 96] - movdqu xmm1, [esp + 352] - paddd xmm0, xmm1 - movdqu [esp + 96], xmm0 - movdqu xmm0, [esp + 112] - movdqu xmm1, [esp + 368] - paddd xmm0, xmm1 - movdqu [esp + 112], xmm0 - movdqu xmm0, [esp + 128] - movdqu xmm1, [esp + 384] - paddd xmm0, xmm1 - movdqu [esp + 128], xmm0 - movdqu xmm0, [esp + 144] - movdqu xmm1, [esp + 400] - paddd xmm0, xmm1 - movdqu [esp + 144], xmm0 - movdqu xmm0, [esp + 160] - movdqu xmm1, [esp + 416] - paddd xmm0, xmm1 - movdqu [esp + 160], xmm0 - movdqu xmm0, [esp + 176] - movdqu xmm1, [esp + 432] - paddd xmm0, xmm1 - movdqu [esp + 176], xmm0 - movdqu xmm0, [esp + 192] - movdqu xmm1, [esp + 448] - paddd xmm0, xmm1 - movdqu [esp + 192], xmm0 - movdqu xmm0, [esp + 208] - movdqu xmm1, [esp + 464] - paddd xmm0, xmm1 - movdqu [esp + 208], xmm0 - movdqu xmm0, [esp + 224] - movdqu xmm1, [esp + 480] - paddd xmm0, xmm1 - movdqu [esp + 224], xmm0 - movdqu xmm0, [esp + 240] - movdqu xmm1, [esp + 496] - paddd xmm0, xmm1 - movdqu [esp + 240], xmm0 add dword ptr [esi + 48], 4 {$IFDEF CHACHA_CTR64} adc dword ptr [esi + 52], 0 {$ENDIF} - movdqu xmm0, [esp] - movdqu xmm1, [esp + 16] - movdqu xmm2, [esp + 32] - movdqu xmm3, [esp + 48] + movdqa xmm0, [esp] + db $66, $0F, $FE, $84, $24, $00, $01, $00, $00 // paddd xmm0, [esp + 256] + movdqa xmm1, [esp + 16] + db $66, $0F, $FE, $8C, $24, $10, $01, $00, $00 // paddd xmm1, [esp + 272] + movdqa xmm2, [esp + 32] + db $66, $0F, $FE, $94, $24, $20, $01, $00, $00 // paddd xmm2, [esp + 288] + movdqa xmm3, [esp + 48] + db $66, $0F, $FE, $9C, $24, $30, $01, $00, $00 // paddd xmm3, [esp + 304] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi] pxor xmm0, xmm5 movdqu [eax], xmm0 @@ -440,22 +429,26 @@ movdqu xmm5, [edi + 192] pxor xmm3, xmm5 movdqu [eax + 192], xmm3 - movdqu xmm0, [esp + 64] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 96] - movdqu xmm3, [esp + 112] + movdqa xmm0, [esp + 64] + db $66, $0F, $FE, $84, $24, $40, $01, $00, $00 // paddd xmm0, [esp + 320] + movdqa xmm1, [esp + 80] + db $66, $0F, $FE, $8C, $24, $50, $01, $00, $00 // paddd xmm1, [esp + 336] + movdqa xmm2, [esp + 96] + db $66, $0F, $FE, $94, $24, $60, $01, $00, $00 // paddd xmm2, [esp + 352] + movdqa xmm3, [esp + 112] + db $66, $0F, $FE, $9C, $24, $70, $01, $00, $00 // paddd xmm3, [esp + 368] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 16] pxor xmm0, xmm5 movdqu [eax + 16], xmm0 @@ -468,22 +461,26 @@ movdqu xmm5, [edi + 208] pxor xmm3, xmm5 movdqu [eax + 208], xmm3 - movdqu xmm0, [esp + 128] - movdqu xmm1, [esp + 144] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 176] + movdqa xmm0, [esp + 128] + db $66, $0F, $FE, $84, $24, $80, $01, $00, $00 // paddd xmm0, [esp + 384] + movdqa xmm1, [esp + 144] + db $66, $0F, $FE, $8C, $24, $90, $01, $00, $00 // paddd xmm1, [esp + 400] + movdqa xmm2, [esp + 160] + db $66, $0F, $FE, $94, $24, $A0, $01, $00, $00 // paddd xmm2, [esp + 416] + movdqa xmm3, [esp + 176] + db $66, $0F, $FE, $9C, $24, $B0, $01, $00, $00 // paddd xmm3, [esp + 432] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 32] pxor xmm0, xmm5 movdqu [eax + 32], xmm0 @@ -496,22 +493,26 @@ movdqu xmm5, [edi + 224] pxor xmm3, xmm5 movdqu [eax + 224], xmm3 - movdqu xmm0, [esp + 192] - movdqu xmm1, [esp + 208] - movdqu xmm2, [esp + 224] - movdqu xmm3, [esp + 240] + movdqa xmm0, [esp + 192] + db $66, $0F, $FE, $84, $24, $C0, $01, $00, $00 // paddd xmm0, [esp + 448] + movdqa xmm1, [esp + 208] + db $66, $0F, $FE, $8C, $24, $D0, $01, $00, $00 // paddd xmm1, [esp + 464] + movdqa xmm2, [esp + 224] + db $66, $0F, $FE, $94, $24, $E0, $01, $00, $00 // paddd xmm2, [esp + 480] + movdqa xmm3, [esp + 240] + db $66, $0F, $FE, $9C, $24, $F0, $01, $00, $00 // paddd xmm3, [esp + 496] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 48] pxor xmm0, xmm5 movdqu [eax + 48], xmm0 @@ -524,7 +525,11 @@ movdqu xmm5, [edi + 240] pxor xmm3, xmm5 movdqu [eax + 240], xmm3 - add esp, 528 + add edi, 256 + add eax, 256 + dec ecx + jnz @@c7539_sse2_outer + mov esp, dword ptr [esp + 544] pop edi pop esi pop ebx diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc index 080a9a32..0028fc49 100644 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc @@ -1,364 +1,354 @@ -// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (x86-64). +// ChaCha7539 (IETF) 4-way SSE2 keystream kernel, vertical layout (x86-64). // ------------------------------------------------------------------- // -// Four 64-byte ChaCha20 blocks, state words held vertically (one block per -// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words -// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch. -// rot16/rot8 use shift+or; rot12/rot7 use shift+or. +// Four 64-byte ChaCha20 blocks per group, state words held vertically (one +// block per xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), +// xmm12-15 = words d (12-15). Words c (8-11): two are resident in xmm8/xmm9, +// the other two wait in stack slots; quarter-rounds run as two resident pairs +// with one pair swap per round (2 stores + 2 loads). xmm10/xmm11 are the +// rotate temps. rot16/rot8 use shift+or; +// rot12/rot7 use shift+or. The frame is 64B-aligned (r11 = frame register). // -// On entry (after ClpSimdProc4Begin_x86_64.inc): -// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// Streams AGroups x 256B in one call: prologue once, light per-group reload +// (counter lanes +4), fused accumulate/transpose/xor tail straight to AOut. +// +// On entry (after ClpSimdProc5Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut, +// r10 = AGroups (4-block groups, >= 1). // // On exit: -// AState word-12 counter (byte 48) advanced by 4; the keystream is xored -// with AIn into AOut (256 bytes). +// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is +// xored with AIn into AOut (256*AGroups bytes). // // Ops touching the extended xmm8-15 registers are db-encoded for broad // assembler compatibility. - mov r10, rcx + mov r10d, r10d {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - sub rsp, 336 + mov r11, rsp + sub rsp, 352 + and rsp, -64 mov dword ptr [rsp + 320], 0 mov dword ptr [rsp + 324], 1 mov dword ptr [rsp + 328], 2 mov dword ptr [rsp + 332], 3 + mov dword ptr [rsp + 336], 4 + mov dword ptr [rsp + 340], 4 + mov dword ptr [rsp + 344], 4 + mov dword ptr [rsp + 348], 4 movd xmm0, [rdx] pshufd xmm0, xmm0, $00 - movdqu [rsp + 64], xmm0 + movdqa [rsp + 64], xmm0 movd xmm1, [rdx + 4] pshufd xmm1, xmm1, $00 - movdqu [rsp + 80], xmm1 + movdqa [rsp + 80], xmm1 movd xmm2, [rdx + 8] pshufd xmm2, xmm2, $00 - movdqu [rsp + 96], xmm2 + movdqa [rsp + 96], xmm2 movd xmm3, [rdx + 12] pshufd xmm3, xmm3, $00 - movdqu [rsp + 112], xmm3 + movdqa [rsp + 112], xmm3 movd xmm4, [rdx + 16] pshufd xmm4, xmm4, $00 - movdqu [rsp + 128], xmm4 + movdqa [rsp + 128], xmm4 movd xmm5, [rdx + 20] pshufd xmm5, xmm5, $00 - movdqu [rsp + 144], xmm5 + movdqa [rsp + 144], xmm5 movd xmm6, [rdx + 24] pshufd xmm6, xmm6, $00 - movdqu [rsp + 160], xmm6 + movdqa [rsp + 160], xmm6 movd xmm7, [rdx + 28] pshufd xmm7, xmm7, $00 - movdqu [rsp + 176], xmm7 + movdqa [rsp + 176], xmm7 movd xmm8, [rdx + 32] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp], xmm8 - movdqu [rsp + 192], xmm8 - movd xmm8, [rdx + 36] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 16], xmm8 - movdqu [rsp + 208], xmm8 - movd xmm8, [rdx + 40] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 32], xmm8 - movdqu [rsp + 224], xmm8 - movd xmm8, [rdx + 44] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 48], xmm8 - movdqu [rsp + 240], xmm8 + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqa [rsp + 192], xmm8 + movd xmm9, [rdx + 36] + db $66, $45, $0F, $70, $C9, $00 // pshufd xmm9, xmm9, $00 + movdqa [rsp + 208], xmm9 + movd xmm10, [rdx + 40] + db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00 + movdqa [rsp + 224], xmm10 + movdqa [rsp + 32], xmm10 + movd xmm10, [rdx + 44] + db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00 + movdqa [rsp + 240], xmm10 + movdqa [rsp + 48], xmm10 movd xmm12, [rdx + 48] - db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 - movdqu xmm8, [rsp + 320] - db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 - movdqu [rsp + 256], xmm12 + db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 + db $66, $44, $0F, $FE, $A4, $24, $40, $01, $00, $00 // paddd xmm12, [rsp + 320] + movdqa [rsp + 256], xmm12 movd xmm13, [rdx + 52] - db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 - movdqu [rsp + 272], xmm13 + db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 + movdqa [rsp + 272], xmm13 movd xmm14, [rdx + 56] - db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 - movdqu [rsp + 288], xmm14 + db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 + movdqa [rsp + 288], xmm14 movd xmm15, [rdx + 60] - db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 - movdqu [rsp + 304], xmm15 + db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 + movdqa [rsp + 304], xmm15 + jmp @@c7539_sse2_enter +@@c7539_sse2_outer: + movdqa xmm0, [rsp + 64] + movdqa xmm1, [rsp + 80] + movdqa xmm2, [rsp + 96] + movdqa xmm3, [rsp + 112] + movdqa xmm4, [rsp + 128] + movdqa xmm5, [rsp + 144] + movdqa xmm6, [rsp + 160] + movdqa xmm7, [rsp + 176] + movdqa xmm8, [rsp + 192] + movdqa xmm9, [rsp + 208] + movdqa xmm10, [rsp + 224] + movdqa [rsp + 32], xmm10 + movdqa xmm10, [rsp + 240] + movdqa [rsp + 48], xmm10 + movdqa xmm12, [rsp + 256] + db $66, $44, $0F, $FE, $A4, $24, $50, $01, $00, $00 // paddd xmm12, [rsp + 336] + movdqa [rsp + 256], xmm12 + movdqa xmm13, [rsp + 272] + movdqa xmm14, [rsp + 288] + movdqa xmm15, [rsp + 304] +@@c7539_sse2_enter: + mov eax, ecx @@c7539_sse2_round: - movdqu xmm8, [rsp] - paddd xmm0, xmm4 - db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 - db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 - db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 - pslld xmm4, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 paddd xmm0, xmm4 - db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 - db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 - db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 - pslld xmm4, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 - movdqu [rsp], xmm8 - movdqu xmm8, [rsp + 16] + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $6F, $D4 // movdqa xmm10, xmm12 + db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 + db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16 + db $66, $45, $0F, $EB, $E2 // por xmm12, xmm10 paddd xmm1, xmm5 - db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 - db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 - db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $6F, $DD // movdqa xmm11, xmm13 + db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 + db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16 + db $66, $45, $0F, $EB, $EB // por xmm13, xmm11 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10 + db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13 + db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9 + db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5 pslld xmm5, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $EB // por xmm5, xmm11 + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $6F, $D4 // movdqa xmm10, xmm12 + db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 + db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24 + db $66, $45, $0F, $EB, $E2 // por xmm12, xmm10 paddd xmm1, xmm5 - db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 - db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 - db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $6F, $DD // movdqa xmm11, xmm13 + db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 + db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24 + db $66, $45, $0F, $EB, $EB // por xmm13, xmm11 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11 + db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13 + db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9 + db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5 pslld xmm5, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - movdqu [rsp + 16], xmm8 - movdqu xmm8, [rsp + 32] - paddd xmm2, xmm6 - db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 - db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 - db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 - pslld xmm6, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $EA // por xmm5, xmm10 + movdqa [rsp], xmm8 + movdqa [rsp + 16], xmm9 + movdqa xmm8, [rsp + 32] + movdqa xmm9, [rsp + 48] paddd xmm2, xmm6 - db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 - db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 - db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 - pslld xmm6, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 - movdqu [rsp + 32], xmm8 - movdqu xmm8, [rsp + 48] + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $6F, $D6 // movdqa xmm10, xmm14 + db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 + db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16 + db $66, $45, $0F, $EB, $F2 // por xmm14, xmm10 paddd xmm3, xmm7 - db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 - db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 - db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $6F, $DF // movdqa xmm11, xmm15 + db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 + db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16 + db $66, $45, $0F, $EB, $FB // por xmm15, xmm11 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10 + db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15 + db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9 + db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7 pslld xmm7, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $FB // por xmm7, xmm11 + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $6F, $D6 // movdqa xmm10, xmm14 + db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 + db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24 + db $66, $45, $0F, $EB, $F2 // por xmm14, xmm10 paddd xmm3, xmm7 - db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 - db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 - db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $6F, $DF // movdqa xmm11, xmm15 + db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 + db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24 + db $66, $45, $0F, $EB, $FB // por xmm15, xmm11 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11 + db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15 + db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9 + db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7 pslld xmm7, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 - movdqu [rsp + 48], xmm8 - movdqu xmm8, [rsp + 32] + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $FA // por xmm7, xmm10 paddd xmm0, xmm5 - db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 - db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 - db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 - pslld xmm5, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - paddd xmm0, xmm5 - db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 - db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 - db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 - pslld xmm5, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - movdqu [rsp + 32], xmm8 - movdqu xmm8, [rsp + 48] + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $6F, $D7 // movdqa xmm10, xmm15 + db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 + db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16 + db $66, $45, $0F, $EB, $FA // por xmm15, xmm10 paddd xmm1, xmm6 - db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 - db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 - db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $6F, $DC // movdqa xmm11, xmm12 + db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 + db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16 + db $66, $45, $0F, $EB, $E3 // por xmm12, xmm11 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $EA // por xmm5, xmm10 + db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12 + db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 pslld xmm6, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11 + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $6F, $D7 // movdqa xmm10, xmm15 + db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 + db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24 + db $66, $45, $0F, $EB, $FA // por xmm15, xmm10 paddd xmm1, xmm6 - db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 - db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 - db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $6F, $DC // movdqa xmm11, xmm12 + db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 + db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24 + db $66, $45, $0F, $EB, $E3 // por xmm12, xmm11 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $EB // por xmm5, xmm11 + db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12 + db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9 + db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6 pslld xmm6, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 - movdqu [rsp + 48], xmm8 - movdqu xmm8, [rsp] - paddd xmm2, xmm7 - db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 - db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 - db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 - pslld xmm7, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10 + movdqa [rsp + 32], xmm8 + movdqa [rsp + 48], xmm9 + movdqa xmm8, [rsp] + movdqa xmm9, [rsp + 16] paddd xmm2, xmm7 - db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 - db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 - db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 - pslld xmm7, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 - movdqu [rsp], xmm8 - movdqu xmm8, [rsp + 16] + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $6F, $D5 // movdqa xmm10, xmm13 + db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 + db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16 + db $66, $45, $0F, $EB, $EA // por xmm13, xmm10 paddd xmm3, xmm4 - db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 - db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 - db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 - db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 - db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $6F, $DE // movdqa xmm11, xmm14 + db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 + db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16 + db $66, $45, $0F, $EB, $F3 // por xmm14, xmm11 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $FA // por xmm7, xmm10 + db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14 + db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9 + db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4 pslld xmm4, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11 + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $6F, $D5 // movdqa xmm10, xmm13 + db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 + db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24 + db $66, $45, $0F, $EB, $EA // por xmm13, xmm10 paddd xmm3, xmm4 - db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 - db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 - db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 - db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 - db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $6F, $DE // movdqa xmm11, xmm14 + db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 + db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24 + db $66, $45, $0F, $EB, $F3 // por xmm14, xmm11 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $FB // por xmm7, xmm11 + db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14 + db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9 + db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4 pslld xmm4, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 - movdqu [rsp + 16], xmm8 - sub r10, 2 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10 + sub eax, 2 jg @@c7539_sse2_round - movdqu xmm8, [rsp + 64] - db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8 - movdqu [rsp + 64], xmm0 - movdqu xmm8, [rsp + 80] - db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8 - movdqu [rsp + 80], xmm1 - movdqu xmm8, [rsp + 96] - db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8 - movdqu [rsp + 96], xmm2 - movdqu xmm8, [rsp + 112] - db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8 - movdqu [rsp + 112], xmm3 - movdqu xmm8, [rsp + 128] - db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8 - movdqu [rsp + 128], xmm4 - movdqu xmm8, [rsp + 144] - db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8 - movdqu [rsp + 144], xmm5 - movdqu xmm8, [rsp + 160] - db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8 - movdqu [rsp + 160], xmm6 - movdqu xmm8, [rsp + 176] - db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8 - movdqu [rsp + 176], xmm7 - movdqu xmm8, [rsp + 256] - db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 - movdqu [rsp + 256], xmm12 - movdqu xmm8, [rsp + 272] - db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8 - movdqu [rsp + 272], xmm13 - movdqu xmm8, [rsp + 288] - db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8 - movdqu [rsp + 288], xmm14 - movdqu xmm8, [rsp + 304] - db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8 - movdqu [rsp + 304], xmm15 - movdqu xmm8, [rsp] - movdqu xmm9, [rsp + 192] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 192], xmm8 - movdqu xmm8, [rsp + 16] - movdqu xmm9, [rsp + 208] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 208], xmm8 - movdqu xmm8, [rsp + 32] - movdqu xmm9, [rsp + 224] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 224], xmm8 - movdqu xmm8, [rsp + 48] - movdqu xmm9, [rsp + 240] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 240], xmm8 add dword ptr [rdx + 48], 4 {$IFDEF CHACHA_CTR64} adc dword ptr [rdx + 52], 0 {$ENDIF} - movdqu xmm0, [rsp + 64] - movdqu xmm1, [rsp + 80] - movdqu xmm2, [rsp + 96] - movdqu xmm3, [rsp + 112] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $FE, $44, $24, $40 // paddd xmm0, [rsp + 64] + db $66, $0F, $FE, $4C, $24, $50 // paddd xmm1, [rsp + 80] + db $66, $0F, $FE, $54, $24, $60 // paddd xmm2, [rsp + 96] + db $66, $0F, $FE, $5C, $24, $70 // paddd xmm3, [rsp + 112] + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $44, $0F, $6A, $D1 // punpckhdq xmm10, xmm1 + db $66, $44, $0F, $6F, $DA // movdqa xmm11, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $44, $0F, $6A, $DB // punpckhdq xmm11, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $6C, $D3 // punpcklqdq xmm10, xmm11 + db $66, $41, $0F, $6D, $DB // punpckhqdq xmm3, xmm11 + db $66, $0F, $FE, $A4, $24, $80, $00, $00, $00 // paddd xmm4, [rsp + 128] + db $66, $0F, $FE, $AC, $24, $90, $00, $00, $00 // paddd xmm5, [rsp + 144] + db $66, $0F, $FE, $B4, $24, $A0, $00, $00, $00 // paddd xmm6, [rsp + 160] + db $66, $0F, $FE, $BC, $24, $B0, $00, $00, $00 // paddd xmm7, [rsp + 176] + movdqa xmm2, xmm4 + db $66, $0F, $62, $E5 // punpckldq xmm4, xmm5 + db $66, $0F, $6A, $D5 // punpckhdq xmm2, xmm5 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 + db $66, $0F, $62, $F7 // punpckldq xmm6, xmm7 + db $66, $44, $0F, $6A, $DF // punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + db $66, $0F, $6C, $E6 // punpcklqdq xmm4, xmm6 + db $66, $0F, $6D, $EE // punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm2 + db $66, $41, $0F, $6C, $D3 // punpcklqdq xmm2, xmm11 + db $66, $41, $0F, $6D, $FB // punpckhqdq xmm7, xmm11 movdqu xmm6, [r8] pxor xmm0, xmm6 movdqu [r9], xmm0 @@ -366,95 +356,85 @@ pxor xmm1, xmm6 movdqu [r9 + 64], xmm1 movdqu xmm6, [r8 + 128] - pxor xmm4, xmm6 - movdqu [r9 + 128], xmm4 + db $66, $44, $0F, $EF, $D6 // pxor xmm10, xmm6 + movdqu [r9 + 128], xmm10 movdqu xmm6, [r8 + 192] pxor xmm3, xmm6 movdqu [r9 + 192], xmm3 - movdqu xmm0, [rsp + 128] - movdqu xmm1, [rsp + 144] - movdqu xmm2, [rsp + 160] - movdqu xmm3, [rsp + 176] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm6, [r8 + 16] - pxor xmm0, xmm6 - movdqu [r9 + 16], xmm0 + pxor xmm4, xmm6 + movdqu [r9 + 16], xmm4 movdqu xmm6, [r8 + 80] - pxor xmm1, xmm6 - movdqu [r9 + 80], xmm1 + pxor xmm5, xmm6 + movdqu [r9 + 80], xmm5 movdqu xmm6, [r8 + 144] - pxor xmm4, xmm6 - movdqu [r9 + 144], xmm4 + pxor xmm2, xmm6 + movdqu [r9 + 144], xmm2 movdqu xmm6, [r8 + 208] - pxor xmm3, xmm6 - movdqu [r9 + 208], xmm3 - movdqu xmm0, [rsp + 192] - movdqu xmm1, [rsp + 208] - movdqu xmm2, [rsp + 224] - movdqu xmm3, [rsp + 240] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 - movdqu xmm6, [r8 + 32] - pxor xmm0, xmm6 - movdqu [r9 + 32], xmm0 - movdqu xmm6, [r8 + 96] - pxor xmm1, xmm6 - movdqu [r9 + 96], xmm1 - movdqu xmm6, [r8 + 160] - pxor xmm4, xmm6 - movdqu [r9 + 160], xmm4 - movdqu xmm6, [r8 + 224] - pxor xmm3, xmm6 - movdqu [r9 + 224], xmm3 - movdqu xmm0, [rsp + 256] - movdqu xmm1, [rsp + 272] - movdqu xmm2, [rsp + 288] - movdqu xmm3, [rsp + 304] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 - movdqu xmm6, [r8 + 48] - pxor xmm0, xmm6 - movdqu [r9 + 48], xmm0 - movdqu xmm6, [r8 + 112] - pxor xmm1, xmm6 - movdqu [r9 + 112], xmm1 - movdqu xmm6, [r8 + 176] - pxor xmm4, xmm6 - movdqu [r9 + 176], xmm4 - movdqu xmm6, [r8 + 240] - pxor xmm3, xmm6 - movdqu [r9 + 240], xmm3 - add rsp, 336 + pxor xmm7, xmm6 + movdqu [r9 + 208], xmm7 + movdqa xmm0, [rsp + 32] + movdqa xmm1, [rsp + 48] + db $66, $44, $0F, $FE, $84, $24, $C0, $00, $00, $00 // paddd xmm8, [rsp + 192] + db $66, $44, $0F, $FE, $8C, $24, $D0, $00, $00, $00 // paddd xmm9, [rsp + 208] + db $66, $0F, $FE, $84, $24, $E0, $00, $00, $00 // paddd xmm0, [rsp + 224] + db $66, $0F, $FE, $8C, $24, $F0, $00, $00, $00 // paddd xmm1, [rsp + 240] + db $66, $41, $0F, $6F, $D0 // movdqa xmm2, xmm8 + db $66, $45, $0F, $62, $C1 // punpckldq xmm8, xmm9 + db $66, $41, $0F, $6A, $D1 // punpckhdq xmm2, xmm9 + movdqa xmm3, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $D9 // punpckhdq xmm3, xmm1 + db $66, $45, $0F, $6F, $C8 // movdqa xmm9, xmm8 + db $66, $44, $0F, $6C, $C0 // punpcklqdq xmm8, xmm0 + db $66, $44, $0F, $6D, $C8 // punpckhqdq xmm9, xmm0 + movdqa xmm1, xmm2 + db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 + db $66, $0F, $6D, $CB // punpckhqdq xmm1, xmm3 + db $66, $44, $0F, $FE, $A4, $24, $00, $01, $00, $00 // paddd xmm12, [rsp + 256] + db $66, $44, $0F, $FE, $AC, $24, $10, $01, $00, $00 // paddd xmm13, [rsp + 272] + db $66, $44, $0F, $FE, $B4, $24, $20, $01, $00, $00 // paddd xmm14, [rsp + 288] + db $66, $44, $0F, $FE, $BC, $24, $30, $01, $00, $00 // paddd xmm15, [rsp + 304] + db $66, $41, $0F, $6F, $C4 // movdqa xmm0, xmm12 + db $66, $45, $0F, $62, $E5 // punpckldq xmm12, xmm13 + db $66, $41, $0F, $6A, $C5 // punpckhdq xmm0, xmm13 + db $66, $41, $0F, $6F, $DE // movdqa xmm3, xmm14 + db $66, $45, $0F, $62, $F7 // punpckldq xmm14, xmm15 + db $66, $41, $0F, $6A, $DF // punpckhdq xmm3, xmm15 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $45, $0F, $6C, $E6 // punpcklqdq xmm12, xmm14 + db $66, $45, $0F, $6D, $EE // punpckhqdq xmm13, xmm14 + db $66, $44, $0F, $6F, $F8 // movdqa xmm15, xmm0 + db $66, $0F, $6C, $C3 // punpcklqdq xmm0, xmm3 + db $66, $44, $0F, $6D, $FB // punpckhqdq xmm15, xmm3 + movdqu xmm3, [r8 + 32] + db $66, $44, $0F, $EF, $C3 // pxor xmm8, xmm3 + movdqu [r9 + 32], xmm8 + movdqu xmm3, [r8 + 96] + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + movdqu [r9 + 96], xmm9 + movdqu xmm3, [r8 + 160] + pxor xmm2, xmm3 + movdqu [r9 + 160], xmm2 + movdqu xmm3, [r8 + 224] + pxor xmm1, xmm3 + movdqu [r9 + 224], xmm1 + movdqu xmm3, [r8 + 48] + db $66, $44, $0F, $EF, $E3 // pxor xmm12, xmm3 + movdqu [r9 + 48], xmm12 + movdqu xmm3, [r8 + 112] + db $66, $44, $0F, $EF, $EB // pxor xmm13, xmm3 + movdqu [r9 + 112], xmm13 + movdqu xmm3, [r8 + 176] + pxor xmm0, xmm3 + movdqu [r9 + 176], xmm0 + movdqu xmm3, [r8 + 240] + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + movdqu [r9 + 240], xmm15 + add r8, 256 + add r9, 256 + dec r10 + jnz @@c7539_sse2_outer + mov rsp, r11 {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc index 7af122fb..c02a316a 100644 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc @@ -1,108 +1,157 @@ -// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (i386). +// ChaCha7539 (IETF) 4-way SSSE3 keystream kernel, vertical layout (i386). // ------------------------------------------------------------------- // -// Four 64-byte ChaCha20 blocks, state words held vertically (one block per -// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame; -// each quarter-round loads its 4 words into xmm0-3 (temp xmm4) and the rot16/rot8 masks in xmm6-7. +// Four 64-byte ChaCha20 blocks per group, state words held vertically (one +// block per xmm lane). i386 has only 8 xmm, so the 16 state words live in a +// 64B-aligned stack frame (original esp parked inside it); each quarter-round +// loads its 4 words into xmm0-3 (temp xmm4) and the rot16/rot8 masks in xmm6-7. // rot16/rot8 use pshufb; rot12/rot7 use shift+or. // -// On entry (after ClpSimdProc4Begin_i386.inc): -// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut. +// Streams AGroups x 256B in one call: prologue once, per-group ORIG->STATE +// reload (counter lanes +4), fused accumulate/transpose/xor tail to AOut. +// +// On entry (after ClpSimdProc5Begin_i386.inc): +// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut, +// ecx = AGroups (4-block groups, >= 1). // // On exit: -// AState word-12 counter (byte 48) advanced by 4; the keystream is xored -// with AIn into AOut (256 bytes). +// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is +// xored with AIn into AOut (256*AGroups bytes). // // punpck* / pshufb are db-encoded for broad assembler compatibility. - sub esp, 560 - mov dword ptr [esp + 528], $01000302 - mov dword ptr [esp + 532], $05040706 - mov dword ptr [esp + 536], $09080B0A - mov dword ptr [esp + 540], $0D0C0F0E - mov dword ptr [esp + 544], $02010003 - mov dword ptr [esp + 548], $06050407 - mov dword ptr [esp + 552], $0A09080B - mov dword ptr [esp + 556], $0E0D0C0F - movdqu xmm6, [esp + 528] - movdqu xmm7, [esp + 544] + mov edx, esp + sub esp, 592 + and esp, -64 + mov dword ptr [esp + 576], edx + mov dword ptr [esp + 544], $01000302 + mov dword ptr [esp + 548], $05040706 + mov dword ptr [esp + 552], $09080B0A + mov dword ptr [esp + 556], $0D0C0F0E + mov dword ptr [esp + 560], $02010003 + mov dword ptr [esp + 564], $06050407 + mov dword ptr [esp + 568], $0A09080B + mov dword ptr [esp + 572], $0E0D0C0F + movdqa xmm6, [esp + 544] + movdqa xmm7, [esp + 560] mov dword ptr [esp + 512], 0 mov dword ptr [esp + 516], 1 mov dword ptr [esp + 520], 2 mov dword ptr [esp + 524], 3 + mov dword ptr [esp + 528], 4 + mov dword ptr [esp + 532], 4 + mov dword ptr [esp + 536], 4 + mov dword ptr [esp + 540], 4 movd xmm0, [esi] pshufd xmm0, xmm0, $00 - movdqu [esp], xmm0 - movdqu [esp + 256], xmm0 + movdqa [esp], xmm0 + movdqa [esp + 256], xmm0 movd xmm0, [esi + 4] pshufd xmm0, xmm0, $00 - movdqu [esp + 16], xmm0 - movdqu [esp + 272], xmm0 + movdqa [esp + 16], xmm0 + movdqa [esp + 272], xmm0 movd xmm0, [esi + 8] pshufd xmm0, xmm0, $00 - movdqu [esp + 32], xmm0 - movdqu [esp + 288], xmm0 + movdqa [esp + 32], xmm0 + movdqa [esp + 288], xmm0 movd xmm0, [esi + 12] pshufd xmm0, xmm0, $00 - movdqu [esp + 48], xmm0 - movdqu [esp + 304], xmm0 + movdqa [esp + 48], xmm0 + movdqa [esp + 304], xmm0 movd xmm0, [esi + 16] pshufd xmm0, xmm0, $00 - movdqu [esp + 64], xmm0 - movdqu [esp + 320], xmm0 + movdqa [esp + 64], xmm0 + movdqa [esp + 320], xmm0 movd xmm0, [esi + 20] pshufd xmm0, xmm0, $00 - movdqu [esp + 80], xmm0 - movdqu [esp + 336], xmm0 + movdqa [esp + 80], xmm0 + movdqa [esp + 336], xmm0 movd xmm0, [esi + 24] pshufd xmm0, xmm0, $00 - movdqu [esp + 96], xmm0 - movdqu [esp + 352], xmm0 + movdqa [esp + 96], xmm0 + movdqa [esp + 352], xmm0 movd xmm0, [esi + 28] pshufd xmm0, xmm0, $00 - movdqu [esp + 112], xmm0 - movdqu [esp + 368], xmm0 + movdqa [esp + 112], xmm0 + movdqa [esp + 368], xmm0 movd xmm0, [esi + 32] pshufd xmm0, xmm0, $00 - movdqu [esp + 128], xmm0 - movdqu [esp + 384], xmm0 + movdqa [esp + 128], xmm0 + movdqa [esp + 384], xmm0 movd xmm0, [esi + 36] pshufd xmm0, xmm0, $00 - movdqu [esp + 144], xmm0 - movdqu [esp + 400], xmm0 + movdqa [esp + 144], xmm0 + movdqa [esp + 400], xmm0 movd xmm0, [esi + 40] pshufd xmm0, xmm0, $00 - movdqu [esp + 160], xmm0 - movdqu [esp + 416], xmm0 + movdqa [esp + 160], xmm0 + movdqa [esp + 416], xmm0 movd xmm0, [esi + 44] pshufd xmm0, xmm0, $00 - movdqu [esp + 176], xmm0 - movdqu [esp + 432], xmm0 + movdqa [esp + 176], xmm0 + movdqa [esp + 432], xmm0 movd xmm0, [esi + 48] pshufd xmm0, xmm0, $00 - movdqu xmm1, [esp + 512] - paddd xmm0, xmm1 - movdqu [esp + 192], xmm0 - movdqu [esp + 448], xmm0 + db $66, $0F, $FE, $84, $24, $00, $02, $00, $00 // paddd xmm0, [esp + 512] + movdqa [esp + 192], xmm0 + movdqa [esp + 448], xmm0 movd xmm0, [esi + 52] pshufd xmm0, xmm0, $00 - movdqu [esp + 208], xmm0 - movdqu [esp + 464], xmm0 + movdqa [esp + 208], xmm0 + movdqa [esp + 464], xmm0 movd xmm0, [esi + 56] pshufd xmm0, xmm0, $00 - movdqu [esp + 224], xmm0 - movdqu [esp + 480], xmm0 + movdqa [esp + 224], xmm0 + movdqa [esp + 480], xmm0 movd xmm0, [esi + 60] pshufd xmm0, xmm0, $00 - movdqu [esp + 240], xmm0 - movdqu [esp + 496], xmm0 + movdqa [esp + 240], xmm0 + movdqa [esp + 496], xmm0 + jmp @@c7539_ssse3_enter +@@c7539_ssse3_outer: + movdqa xmm0, [esp + 256] + movdqa [esp], xmm0 + movdqa xmm0, [esp + 272] + movdqa [esp + 16], xmm0 + movdqa xmm0, [esp + 288] + movdqa [esp + 32], xmm0 + movdqa xmm0, [esp + 304] + movdqa [esp + 48], xmm0 + movdqa xmm0, [esp + 320] + movdqa [esp + 64], xmm0 + movdqa xmm0, [esp + 336] + movdqa [esp + 80], xmm0 + movdqa xmm0, [esp + 352] + movdqa [esp + 96], xmm0 + movdqa xmm0, [esp + 368] + movdqa [esp + 112], xmm0 + movdqa xmm0, [esp + 384] + movdqa [esp + 128], xmm0 + movdqa xmm0, [esp + 400] + movdqa [esp + 144], xmm0 + movdqa xmm0, [esp + 416] + movdqa [esp + 160], xmm0 + movdqa xmm0, [esp + 432] + movdqa [esp + 176], xmm0 + movdqa xmm0, [esp + 448] + db $66, $0F, $FE, $84, $24, $10, $02, $00, $00 // paddd xmm0, [esp + 528] + movdqa [esp + 448], xmm0 + movdqa [esp + 192], xmm0 + movdqa xmm0, [esp + 464] + movdqa [esp + 208], xmm0 + movdqa xmm0, [esp + 480] + movdqa [esp + 224], xmm0 + movdqa xmm0, [esp + 496] + movdqa [esp + 240], xmm0 +@@c7539_ssse3_enter: + mov edx, ebx @@c7539_ssse3_round: - movdqu xmm0, [esp] - movdqu xmm1, [esp + 64] - movdqu xmm2, [esp + 128] - movdqu xmm3, [esp + 192] + movdqa xmm0, [esp] + movdqa xmm1, [esp + 64] + movdqa xmm2, [esp + 128] + movdqa xmm3, [esp + 192] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -111,24 +160,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp], xmm0 - movdqu [esp + 64], xmm1 - movdqu [esp + 128], xmm2 - movdqu [esp + 192], xmm3 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 144] - movdqu xmm3, [esp + 208] + movdqa [esp], xmm0 + movdqa [esp + 64], xmm1 + movdqa [esp + 128], xmm2 + movdqa [esp + 192], xmm3 + movdqa xmm0, [esp + 16] + movdqa xmm1, [esp + 80] + movdqa xmm2, [esp + 144] + movdqa xmm3, [esp + 208] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -137,24 +186,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 16], xmm0 - movdqu [esp + 80], xmm1 - movdqu [esp + 144], xmm2 - movdqu [esp + 208], xmm3 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 96] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 224] + movdqa [esp + 16], xmm0 + movdqa [esp + 80], xmm1 + movdqa [esp + 144], xmm2 + movdqa [esp + 208], xmm3 + movdqa xmm0, [esp + 32] + movdqa xmm1, [esp + 96] + movdqa xmm2, [esp + 160] + movdqa xmm3, [esp + 224] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -163,24 +212,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 32], xmm0 - movdqu [esp + 96], xmm1 - movdqu [esp + 160], xmm2 - movdqu [esp + 224], xmm3 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 112] - movdqu xmm2, [esp + 176] - movdqu xmm3, [esp + 240] + movdqa [esp + 32], xmm0 + movdqa [esp + 96], xmm1 + movdqa [esp + 160], xmm2 + movdqa [esp + 224], xmm3 + movdqa xmm0, [esp + 48] + movdqa xmm1, [esp + 112] + movdqa xmm2, [esp + 176] + movdqa xmm3, [esp + 240] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -189,24 +238,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 48], xmm0 - movdqu [esp + 112], xmm1 - movdqu [esp + 176], xmm2 - movdqu [esp + 240], xmm3 - movdqu xmm0, [esp] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 240] + movdqa [esp + 48], xmm0 + movdqa [esp + 112], xmm1 + movdqa [esp + 176], xmm2 + movdqa [esp + 240], xmm3 + movdqa xmm0, [esp] + movdqa xmm1, [esp + 80] + movdqa xmm2, [esp + 160] + movdqa xmm3, [esp + 240] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -215,24 +264,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp], xmm0 - movdqu [esp + 80], xmm1 - movdqu [esp + 160], xmm2 - movdqu [esp + 240], xmm3 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 96] - movdqu xmm2, [esp + 176] - movdqu xmm3, [esp + 192] + movdqa [esp], xmm0 + movdqa [esp + 80], xmm1 + movdqa [esp + 160], xmm2 + movdqa [esp + 240], xmm3 + movdqa xmm0, [esp + 16] + movdqa xmm1, [esp + 96] + movdqa xmm2, [esp + 176] + movdqa xmm3, [esp + 192] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -241,24 +290,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 16], xmm0 - movdqu [esp + 96], xmm1 - movdqu [esp + 176], xmm2 - movdqu [esp + 192], xmm3 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 112] - movdqu xmm2, [esp + 128] - movdqu xmm3, [esp + 208] + movdqa [esp + 16], xmm0 + movdqa [esp + 96], xmm1 + movdqa [esp + 176], xmm2 + movdqa [esp + 192], xmm3 + movdqa xmm0, [esp + 32] + movdqa xmm1, [esp + 112] + movdqa xmm2, [esp + 128] + movdqa xmm3, [esp + 208] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -267,24 +316,24 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 32], xmm0 - movdqu [esp + 112], xmm1 - movdqu [esp + 128], xmm2 - movdqu [esp + 208], xmm3 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 64] - movdqu xmm2, [esp + 144] - movdqu xmm3, [esp + 224] + movdqa [esp + 32], xmm0 + movdqa [esp + 112], xmm1 + movdqa [esp + 128], xmm2 + movdqa [esp + 208], xmm3 + movdqa xmm0, [esp + 48] + movdqa xmm1, [esp + 64] + movdqa xmm2, [esp + 144] + movdqa xmm3, [esp + 224] paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 @@ -293,103 +342,43 @@ por xmm1, xmm4 paddd xmm0, xmm1 pxor xmm3, xmm0 - db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 paddd xmm2, xmm3 pxor xmm1, xmm2 movdqa xmm4, xmm1 pslld xmm1, 7 psrld xmm4, 25 por xmm1, xmm4 - movdqu [esp + 48], xmm0 - movdqu [esp + 64], xmm1 - movdqu [esp + 144], xmm2 - movdqu [esp + 224], xmm3 - sub ebx, 2 + movdqa [esp + 48], xmm0 + movdqa [esp + 64], xmm1 + movdqa [esp + 144], xmm2 + movdqa [esp + 224], xmm3 + sub edx, 2 jg @@c7539_ssse3_round - movdqu xmm0, [esp] - movdqu xmm1, [esp + 256] - paddd xmm0, xmm1 - movdqu [esp], xmm0 - movdqu xmm0, [esp + 16] - movdqu xmm1, [esp + 272] - paddd xmm0, xmm1 - movdqu [esp + 16], xmm0 - movdqu xmm0, [esp + 32] - movdqu xmm1, [esp + 288] - paddd xmm0, xmm1 - movdqu [esp + 32], xmm0 - movdqu xmm0, [esp + 48] - movdqu xmm1, [esp + 304] - paddd xmm0, xmm1 - movdqu [esp + 48], xmm0 - movdqu xmm0, [esp + 64] - movdqu xmm1, [esp + 320] - paddd xmm0, xmm1 - movdqu [esp + 64], xmm0 - movdqu xmm0, [esp + 80] - movdqu xmm1, [esp + 336] - paddd xmm0, xmm1 - movdqu [esp + 80], xmm0 - movdqu xmm0, [esp + 96] - movdqu xmm1, [esp + 352] - paddd xmm0, xmm1 - movdqu [esp + 96], xmm0 - movdqu xmm0, [esp + 112] - movdqu xmm1, [esp + 368] - paddd xmm0, xmm1 - movdqu [esp + 112], xmm0 - movdqu xmm0, [esp + 128] - movdqu xmm1, [esp + 384] - paddd xmm0, xmm1 - movdqu [esp + 128], xmm0 - movdqu xmm0, [esp + 144] - movdqu xmm1, [esp + 400] - paddd xmm0, xmm1 - movdqu [esp + 144], xmm0 - movdqu xmm0, [esp + 160] - movdqu xmm1, [esp + 416] - paddd xmm0, xmm1 - movdqu [esp + 160], xmm0 - movdqu xmm0, [esp + 176] - movdqu xmm1, [esp + 432] - paddd xmm0, xmm1 - movdqu [esp + 176], xmm0 - movdqu xmm0, [esp + 192] - movdqu xmm1, [esp + 448] - paddd xmm0, xmm1 - movdqu [esp + 192], xmm0 - movdqu xmm0, [esp + 208] - movdqu xmm1, [esp + 464] - paddd xmm0, xmm1 - movdqu [esp + 208], xmm0 - movdqu xmm0, [esp + 224] - movdqu xmm1, [esp + 480] - paddd xmm0, xmm1 - movdqu [esp + 224], xmm0 - movdqu xmm0, [esp + 240] - movdqu xmm1, [esp + 496] - paddd xmm0, xmm1 - movdqu [esp + 240], xmm0 add dword ptr [esi + 48], 4 {$IFDEF CHACHA_CTR64} adc dword ptr [esi + 52], 0 {$ENDIF} - movdqu xmm0, [esp] - movdqu xmm1, [esp + 16] - movdqu xmm2, [esp + 32] - movdqu xmm3, [esp + 48] + movdqa xmm0, [esp] + db $66, $0F, $FE, $84, $24, $00, $01, $00, $00 // paddd xmm0, [esp + 256] + movdqa xmm1, [esp + 16] + db $66, $0F, $FE, $8C, $24, $10, $01, $00, $00 // paddd xmm1, [esp + 272] + movdqa xmm2, [esp + 32] + db $66, $0F, $FE, $94, $24, $20, $01, $00, $00 // paddd xmm2, [esp + 288] + movdqa xmm3, [esp + 48] + db $66, $0F, $FE, $9C, $24, $30, $01, $00, $00 // paddd xmm3, [esp + 304] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi] pxor xmm0, xmm5 movdqu [eax], xmm0 @@ -402,22 +391,26 @@ movdqu xmm5, [edi + 192] pxor xmm3, xmm5 movdqu [eax + 192], xmm3 - movdqu xmm0, [esp + 64] - movdqu xmm1, [esp + 80] - movdqu xmm2, [esp + 96] - movdqu xmm3, [esp + 112] + movdqa xmm0, [esp + 64] + db $66, $0F, $FE, $84, $24, $40, $01, $00, $00 // paddd xmm0, [esp + 320] + movdqa xmm1, [esp + 80] + db $66, $0F, $FE, $8C, $24, $50, $01, $00, $00 // paddd xmm1, [esp + 336] + movdqa xmm2, [esp + 96] + db $66, $0F, $FE, $94, $24, $60, $01, $00, $00 // paddd xmm2, [esp + 352] + movdqa xmm3, [esp + 112] + db $66, $0F, $FE, $9C, $24, $70, $01, $00, $00 // paddd xmm3, [esp + 368] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 16] pxor xmm0, xmm5 movdqu [eax + 16], xmm0 @@ -430,22 +423,26 @@ movdqu xmm5, [edi + 208] pxor xmm3, xmm5 movdqu [eax + 208], xmm3 - movdqu xmm0, [esp + 128] - movdqu xmm1, [esp + 144] - movdqu xmm2, [esp + 160] - movdqu xmm3, [esp + 176] + movdqa xmm0, [esp + 128] + db $66, $0F, $FE, $84, $24, $80, $01, $00, $00 // paddd xmm0, [esp + 384] + movdqa xmm1, [esp + 144] + db $66, $0F, $FE, $8C, $24, $90, $01, $00, $00 // paddd xmm1, [esp + 400] + movdqa xmm2, [esp + 160] + db $66, $0F, $FE, $94, $24, $A0, $01, $00, $00 // paddd xmm2, [esp + 416] + movdqa xmm3, [esp + 176] + db $66, $0F, $FE, $9C, $24, $B0, $01, $00, $00 // paddd xmm3, [esp + 432] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 32] pxor xmm0, xmm5 movdqu [eax + 32], xmm0 @@ -458,22 +455,26 @@ movdqu xmm5, [edi + 224] pxor xmm3, xmm5 movdqu [eax + 224], xmm3 - movdqu xmm0, [esp + 192] - movdqu xmm1, [esp + 208] - movdqu xmm2, [esp + 224] - movdqu xmm3, [esp + 240] + movdqa xmm0, [esp + 192] + db $66, $0F, $FE, $84, $24, $C0, $01, $00, $00 // paddd xmm0, [esp + 448] + movdqa xmm1, [esp + 208] + db $66, $0F, $FE, $8C, $24, $D0, $01, $00, $00 // paddd xmm1, [esp + 464] + movdqa xmm2, [esp + 224] + db $66, $0F, $FE, $94, $24, $E0, $01, $00, $00 // paddd xmm2, [esp + 480] + movdqa xmm3, [esp + 240] + db $66, $0F, $FE, $9C, $24, $F0, $01, $00, $00 // paddd xmm3, [esp + 496] movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm5, [edi + 48] pxor xmm0, xmm5 movdqu [eax + 48], xmm0 @@ -486,7 +487,11 @@ movdqu xmm5, [edi + 240] pxor xmm3, xmm5 movdqu [eax + 240], xmm3 - add esp, 560 + add edi, 256 + add eax, 256 + dec ecx + jnz @@c7539_ssse3_outer + mov esp, dword ptr [esp + 576] pop edi pop esi pop ebx diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc index ac08b118..0a64aaf2 100644 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc @@ -1,326 +1,324 @@ -// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (x86-64). +// ChaCha7539 (IETF) 4-way SSSE3 keystream kernel, vertical layout (x86-64). // ------------------------------------------------------------------- // -// Four 64-byte ChaCha20 blocks, state words held vertically (one block per -// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words -// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch, xmm10-11 the rot16/rot8 pshufb masks. -// rot16/rot8 use pshufb; rot12/rot7 use shift+or. +// Four 64-byte ChaCha20 blocks per group, state words held vertically (one +// block per xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), +// xmm12-15 = words d (12-15). Words c (8-11): two are resident in xmm8/xmm9, +// the other two wait in stack slots; quarter-rounds run as two resident pairs +// with one pair swap per round (2 stores + 2 loads). xmm10/xmm11 are the +// rotate temps. rot16/rot8 use pshufb (mask reloaded into the just-freed temp); +// rot12/rot7 use shift+or. The frame is 64B-aligned (r11 = frame register). // -// On entry (after ClpSimdProc4Begin_x86_64.inc): -// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// Streams AGroups x 256B in one call: prologue once, light per-group reload +// (counter lanes +4), fused accumulate/transpose/xor tail straight to AOut. +// +// On entry (after ClpSimdProc5Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut, +// r10 = AGroups (4-block groups, >= 1). // // On exit: -// AState word-12 counter (byte 48) advanced by 4; the keystream is xored -// with AIn into AOut (256 bytes). +// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is +// xored with AIn into AOut (256*AGroups bytes). // // Ops touching the extended xmm8-15 registers are db-encoded for broad // assembler compatibility. - mov r10, rcx + mov r10d, r10d {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - sub rsp, 368 - mov dword ptr [rsp + 336], $01000302 - mov dword ptr [rsp + 340], $05040706 - mov dword ptr [rsp + 344], $09080B0A - mov dword ptr [rsp + 348], $0D0C0F0E - mov dword ptr [rsp + 352], $02010003 - mov dword ptr [rsp + 356], $06050407 - mov dword ptr [rsp + 360], $0A09080B - mov dword ptr [rsp + 364], $0E0D0C0F - movdqu xmm10, [rsp + 336] - movdqu xmm11, [rsp + 352] + mov r11, rsp + sub rsp, 384 + and rsp, -64 mov dword ptr [rsp + 320], 0 mov dword ptr [rsp + 324], 1 mov dword ptr [rsp + 328], 2 mov dword ptr [rsp + 332], 3 + mov dword ptr [rsp + 336], 4 + mov dword ptr [rsp + 340], 4 + mov dword ptr [rsp + 344], 4 + mov dword ptr [rsp + 348], 4 + mov dword ptr [rsp + 352], $01000302 + mov dword ptr [rsp + 356], $05040706 + mov dword ptr [rsp + 360], $09080B0A + mov dword ptr [rsp + 364], $0D0C0F0E + mov dword ptr [rsp + 368], $02010003 + mov dword ptr [rsp + 372], $06050407 + mov dword ptr [rsp + 376], $0A09080B + mov dword ptr [rsp + 380], $0E0D0C0F movd xmm0, [rdx] pshufd xmm0, xmm0, $00 - movdqu [rsp + 64], xmm0 + movdqa [rsp + 64], xmm0 movd xmm1, [rdx + 4] pshufd xmm1, xmm1, $00 - movdqu [rsp + 80], xmm1 + movdqa [rsp + 80], xmm1 movd xmm2, [rdx + 8] pshufd xmm2, xmm2, $00 - movdqu [rsp + 96], xmm2 + movdqa [rsp + 96], xmm2 movd xmm3, [rdx + 12] pshufd xmm3, xmm3, $00 - movdqu [rsp + 112], xmm3 + movdqa [rsp + 112], xmm3 movd xmm4, [rdx + 16] pshufd xmm4, xmm4, $00 - movdqu [rsp + 128], xmm4 + movdqa [rsp + 128], xmm4 movd xmm5, [rdx + 20] pshufd xmm5, xmm5, $00 - movdqu [rsp + 144], xmm5 + movdqa [rsp + 144], xmm5 movd xmm6, [rdx + 24] pshufd xmm6, xmm6, $00 - movdqu [rsp + 160], xmm6 + movdqa [rsp + 160], xmm6 movd xmm7, [rdx + 28] pshufd xmm7, xmm7, $00 - movdqu [rsp + 176], xmm7 + movdqa [rsp + 176], xmm7 movd xmm8, [rdx + 32] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp], xmm8 - movdqu [rsp + 192], xmm8 - movd xmm8, [rdx + 36] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 16], xmm8 - movdqu [rsp + 208], xmm8 - movd xmm8, [rdx + 40] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 32], xmm8 - movdqu [rsp + 224], xmm8 - movd xmm8, [rdx + 44] - db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 - movdqu [rsp + 48], xmm8 - movdqu [rsp + 240], xmm8 + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqa [rsp + 192], xmm8 + movd xmm9, [rdx + 36] + db $66, $45, $0F, $70, $C9, $00 // pshufd xmm9, xmm9, $00 + movdqa [rsp + 208], xmm9 + movd xmm10, [rdx + 40] + db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00 + movdqa [rsp + 224], xmm10 + movdqa [rsp + 32], xmm10 + movd xmm10, [rdx + 44] + db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00 + movdqa [rsp + 240], xmm10 + movdqa [rsp + 48], xmm10 movd xmm12, [rdx + 48] - db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 - movdqu xmm8, [rsp + 320] - db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 - movdqu [rsp + 256], xmm12 + db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 + db $66, $44, $0F, $FE, $A4, $24, $40, $01, $00, $00 // paddd xmm12, [rsp + 320] + movdqa [rsp + 256], xmm12 movd xmm13, [rdx + 52] - db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 - movdqu [rsp + 272], xmm13 + db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 + movdqa [rsp + 272], xmm13 movd xmm14, [rdx + 56] - db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 - movdqu [rsp + 288], xmm14 + db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 + movdqa [rsp + 288], xmm14 movd xmm15, [rdx + 60] - db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 - movdqu [rsp + 304], xmm15 + db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 + movdqa [rsp + 304], xmm15 + movdqa xmm11, [rsp + 352] + jmp @@c7539_ssse3_enter +@@c7539_ssse3_outer: + movdqa xmm0, [rsp + 64] + movdqa xmm1, [rsp + 80] + movdqa xmm2, [rsp + 96] + movdqa xmm3, [rsp + 112] + movdqa xmm4, [rsp + 128] + movdqa xmm5, [rsp + 144] + movdqa xmm6, [rsp + 160] + movdqa xmm7, [rsp + 176] + movdqa xmm8, [rsp + 192] + movdqa xmm9, [rsp + 208] + movdqa xmm10, [rsp + 224] + movdqa [rsp + 32], xmm10 + movdqa xmm10, [rsp + 240] + movdqa [rsp + 48], xmm10 + movdqa xmm12, [rsp + 256] + db $66, $44, $0F, $FE, $A4, $24, $50, $01, $00, $00 // paddd xmm12, [rsp + 336] + movdqa [rsp + 256], xmm12 + movdqa xmm13, [rsp + 272] + movdqa xmm14, [rsp + 288] + movdqa xmm15, [rsp + 304] + movdqa xmm11, [rsp + 352] +@@c7539_ssse3_enter: + mov eax, ecx @@c7539_ssse3_round: - movdqu xmm8, [rsp] - paddd xmm0, xmm4 - db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 - db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 - pslld xmm4, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 paddd xmm0, xmm4 - db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 - db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 - pslld xmm4, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 - movdqu [rsp], xmm8 - movdqu xmm8, [rsp + 16] + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 paddd xmm1, xmm5 - db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 - db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10 + movdqa xmm10, [rsp + 368] + db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13 + db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9 + db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5 pslld xmm5, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $EB // por xmm5, xmm11 + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 paddd xmm1, xmm5 - db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 - db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11 + movdqa xmm11, [rsp + 352] + db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13 + db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9 + db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5 pslld xmm5, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - movdqu [rsp + 16], xmm8 - movdqu xmm8, [rsp + 32] - paddd xmm2, xmm6 - db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 - db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 - pslld xmm6, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $EA // por xmm5, xmm10 + movdqa [rsp], xmm8 + movdqa [rsp + 16], xmm9 + movdqa xmm8, [rsp + 32] + movdqa xmm9, [rsp + 48] paddd xmm2, xmm6 - db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 - db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 - pslld xmm6, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 - movdqu [rsp + 32], xmm8 - movdqu xmm8, [rsp + 48] + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 paddd xmm3, xmm7 - db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 - db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10 + movdqa xmm10, [rsp + 368] + db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15 + db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9 + db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7 pslld xmm7, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $FB // por xmm7, xmm11 + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 paddd xmm3, xmm7 - db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 - db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11 + movdqa xmm11, [rsp + 352] + db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15 + db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9 + db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7 pslld xmm7, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 - movdqu [rsp + 48], xmm8 - movdqu xmm8, [rsp + 32] + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $FA // por xmm7, xmm10 paddd xmm0, xmm5 - db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 - db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 - pslld xmm5, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - paddd xmm0, xmm5 - db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 - db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 - db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 - pslld xmm5, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 - movdqu [rsp + 32], xmm8 - movdqu xmm8, [rsp + 48] + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 paddd xmm1, xmm6 - db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 - db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $EA // por xmm5, xmm10 + movdqa xmm10, [rsp + 368] + db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12 + db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 pslld xmm6, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11 + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 paddd xmm1, xmm6 - db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 - db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 - db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $EB // por xmm5, xmm11 + movdqa xmm11, [rsp + 352] + db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12 + db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9 + db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6 pslld xmm6, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 - movdqu [rsp + 48], xmm8 - movdqu xmm8, [rsp] - paddd xmm2, xmm7 - db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 - db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 - pslld xmm7, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10 + movdqa [rsp + 32], xmm8 + movdqa [rsp + 48], xmm9 + movdqa xmm8, [rsp] + movdqa xmm9, [rsp + 16] paddd xmm2, xmm7 - db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 - db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 - db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 - pslld xmm7, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 - movdqu [rsp], xmm8 - movdqu xmm8, [rsp + 16] + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 paddd xmm3, xmm4 - db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 - db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20 + db $66, $41, $0F, $EB, $FA // por xmm7, xmm10 + movdqa xmm10, [rsp + 368] + db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14 + db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9 + db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4 pslld xmm4, 12 - db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20 + db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11 + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 paddd xmm3, xmm4 - db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 - db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 - db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25 + db $66, $41, $0F, $EB, $FB // por xmm7, xmm11 + movdqa xmm11, [rsp + 352] + db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14 + db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9 + db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4 pslld xmm4, 7 - db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 - db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 - movdqu [rsp + 16], xmm8 - sub r10, 2 + db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25 + db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10 + sub eax, 2 jg @@c7539_ssse3_round - movdqu xmm8, [rsp + 64] - db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8 - movdqu [rsp + 64], xmm0 - movdqu xmm8, [rsp + 80] - db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8 - movdqu [rsp + 80], xmm1 - movdqu xmm8, [rsp + 96] - db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8 - movdqu [rsp + 96], xmm2 - movdqu xmm8, [rsp + 112] - db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8 - movdqu [rsp + 112], xmm3 - movdqu xmm8, [rsp + 128] - db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8 - movdqu [rsp + 128], xmm4 - movdqu xmm8, [rsp + 144] - db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8 - movdqu [rsp + 144], xmm5 - movdqu xmm8, [rsp + 160] - db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8 - movdqu [rsp + 160], xmm6 - movdqu xmm8, [rsp + 176] - db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8 - movdqu [rsp + 176], xmm7 - movdqu xmm8, [rsp + 256] - db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 - movdqu [rsp + 256], xmm12 - movdqu xmm8, [rsp + 272] - db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8 - movdqu [rsp + 272], xmm13 - movdqu xmm8, [rsp + 288] - db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8 - movdqu [rsp + 288], xmm14 - movdqu xmm8, [rsp + 304] - db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8 - movdqu [rsp + 304], xmm15 - movdqu xmm8, [rsp] - movdqu xmm9, [rsp + 192] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 192], xmm8 - movdqu xmm8, [rsp + 16] - movdqu xmm9, [rsp + 208] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 208], xmm8 - movdqu xmm8, [rsp + 32] - movdqu xmm9, [rsp + 224] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 224], xmm8 - movdqu xmm8, [rsp + 48] - movdqu xmm9, [rsp + 240] - db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 - movdqu [rsp + 240], xmm8 add dword ptr [rdx + 48], 4 {$IFDEF CHACHA_CTR64} adc dword ptr [rdx + 52], 0 {$ENDIF} - movdqu xmm0, [rsp + 64] - movdqu xmm1, [rsp + 80] - movdqu xmm2, [rsp + 96] - movdqu xmm3, [rsp + 112] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + db $66, $0F, $FE, $44, $24, $40 // paddd xmm0, [rsp + 64] + db $66, $0F, $FE, $4C, $24, $50 // paddd xmm1, [rsp + 80] + db $66, $0F, $FE, $54, $24, $60 // paddd xmm2, [rsp + 96] + db $66, $0F, $FE, $5C, $24, $70 // paddd xmm3, [rsp + 112] + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $44, $0F, $6A, $D1 // punpckhdq xmm10, xmm1 + db $66, $44, $0F, $6F, $DA // movdqa xmm11, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $44, $0F, $6A, $DB // punpckhdq xmm11, xmm3 movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $6C, $D3 // punpcklqdq xmm10, xmm11 + db $66, $41, $0F, $6D, $DB // punpckhqdq xmm3, xmm11 + db $66, $0F, $FE, $A4, $24, $80, $00, $00, $00 // paddd xmm4, [rsp + 128] + db $66, $0F, $FE, $AC, $24, $90, $00, $00, $00 // paddd xmm5, [rsp + 144] + db $66, $0F, $FE, $B4, $24, $A0, $00, $00, $00 // paddd xmm6, [rsp + 160] + db $66, $0F, $FE, $BC, $24, $B0, $00, $00, $00 // paddd xmm7, [rsp + 176] + movdqa xmm2, xmm4 + db $66, $0F, $62, $E5 // punpckldq xmm4, xmm5 + db $66, $0F, $6A, $D5 // punpckhdq xmm2, xmm5 + db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6 + db $66, $0F, $62, $F7 // punpckldq xmm6, xmm7 + db $66, $44, $0F, $6A, $DF // punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + db $66, $0F, $6C, $E6 // punpcklqdq xmm4, xmm6 + db $66, $0F, $6D, $EE // punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm2 + db $66, $41, $0F, $6C, $D3 // punpcklqdq xmm2, xmm11 + db $66, $41, $0F, $6D, $FB // punpckhqdq xmm7, xmm11 movdqu xmm6, [r8] pxor xmm0, xmm6 movdqu [r9], xmm0 @@ -328,95 +326,85 @@ pxor xmm1, xmm6 movdqu [r9 + 64], xmm1 movdqu xmm6, [r8 + 128] - pxor xmm4, xmm6 - movdqu [r9 + 128], xmm4 + db $66, $44, $0F, $EF, $D6 // pxor xmm10, xmm6 + movdqu [r9 + 128], xmm10 movdqu xmm6, [r8 + 192] pxor xmm3, xmm6 movdqu [r9 + 192], xmm3 - movdqu xmm0, [rsp + 128] - movdqu xmm1, [rsp + 144] - movdqu xmm2, [rsp + 160] - movdqu xmm3, [rsp + 176] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 movdqu xmm6, [r8 + 16] - pxor xmm0, xmm6 - movdqu [r9 + 16], xmm0 + pxor xmm4, xmm6 + movdqu [r9 + 16], xmm4 movdqu xmm6, [r8 + 80] - pxor xmm1, xmm6 - movdqu [r9 + 80], xmm1 + pxor xmm5, xmm6 + movdqu [r9 + 80], xmm5 movdqu xmm6, [r8 + 144] - pxor xmm4, xmm6 - movdqu [r9 + 144], xmm4 + pxor xmm2, xmm6 + movdqu [r9 + 144], xmm2 movdqu xmm6, [r8 + 208] - pxor xmm3, xmm6 - movdqu [r9 + 208], xmm3 - movdqu xmm0, [rsp + 192] - movdqu xmm1, [rsp + 208] - movdqu xmm2, [rsp + 224] - movdqu xmm3, [rsp + 240] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 - movdqu xmm6, [r8 + 32] - pxor xmm0, xmm6 - movdqu [r9 + 32], xmm0 - movdqu xmm6, [r8 + 96] - pxor xmm1, xmm6 - movdqu [r9 + 96], xmm1 - movdqu xmm6, [r8 + 160] - pxor xmm4, xmm6 - movdqu [r9 + 160], xmm4 - movdqu xmm6, [r8 + 224] - pxor xmm3, xmm6 - movdqu [r9 + 224], xmm3 - movdqu xmm0, [rsp + 256] - movdqu xmm1, [rsp + 272] - movdqu xmm2, [rsp + 288] - movdqu xmm3, [rsp + 304] - movdqa xmm4, xmm0 - db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 - db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 - movdqa xmm5, xmm2 - db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 - db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 - movdqa xmm1, xmm0 - db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 - db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 - movdqa xmm3, xmm4 - db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 - db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 - movdqu xmm6, [r8 + 48] - pxor xmm0, xmm6 - movdqu [r9 + 48], xmm0 - movdqu xmm6, [r8 + 112] - pxor xmm1, xmm6 - movdqu [r9 + 112], xmm1 - movdqu xmm6, [r8 + 176] - pxor xmm4, xmm6 - movdqu [r9 + 176], xmm4 - movdqu xmm6, [r8 + 240] - pxor xmm3, xmm6 - movdqu [r9 + 240], xmm3 - add rsp, 368 + pxor xmm7, xmm6 + movdqu [r9 + 208], xmm7 + movdqa xmm0, [rsp + 32] + movdqa xmm1, [rsp + 48] + db $66, $44, $0F, $FE, $84, $24, $C0, $00, $00, $00 // paddd xmm8, [rsp + 192] + db $66, $44, $0F, $FE, $8C, $24, $D0, $00, $00, $00 // paddd xmm9, [rsp + 208] + db $66, $0F, $FE, $84, $24, $E0, $00, $00, $00 // paddd xmm0, [rsp + 224] + db $66, $0F, $FE, $8C, $24, $F0, $00, $00, $00 // paddd xmm1, [rsp + 240] + db $66, $41, $0F, $6F, $D0 // movdqa xmm2, xmm8 + db $66, $45, $0F, $62, $C1 // punpckldq xmm8, xmm9 + db $66, $41, $0F, $6A, $D1 // punpckhdq xmm2, xmm9 + movdqa xmm3, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $D9 // punpckhdq xmm3, xmm1 + db $66, $45, $0F, $6F, $C8 // movdqa xmm9, xmm8 + db $66, $44, $0F, $6C, $C0 // punpcklqdq xmm8, xmm0 + db $66, $44, $0F, $6D, $C8 // punpckhqdq xmm9, xmm0 + movdqa xmm1, xmm2 + db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 + db $66, $0F, $6D, $CB // punpckhqdq xmm1, xmm3 + db $66, $44, $0F, $FE, $A4, $24, $00, $01, $00, $00 // paddd xmm12, [rsp + 256] + db $66, $44, $0F, $FE, $AC, $24, $10, $01, $00, $00 // paddd xmm13, [rsp + 272] + db $66, $44, $0F, $FE, $B4, $24, $20, $01, $00, $00 // paddd xmm14, [rsp + 288] + db $66, $44, $0F, $FE, $BC, $24, $30, $01, $00, $00 // paddd xmm15, [rsp + 304] + db $66, $41, $0F, $6F, $C4 // movdqa xmm0, xmm12 + db $66, $45, $0F, $62, $E5 // punpckldq xmm12, xmm13 + db $66, $41, $0F, $6A, $C5 // punpckhdq xmm0, xmm13 + db $66, $41, $0F, $6F, $DE // movdqa xmm3, xmm14 + db $66, $45, $0F, $62, $F7 // punpckldq xmm14, xmm15 + db $66, $41, $0F, $6A, $DF // punpckhdq xmm3, xmm15 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $45, $0F, $6C, $E6 // punpcklqdq xmm12, xmm14 + db $66, $45, $0F, $6D, $EE // punpckhqdq xmm13, xmm14 + db $66, $44, $0F, $6F, $F8 // movdqa xmm15, xmm0 + db $66, $0F, $6C, $C3 // punpcklqdq xmm0, xmm3 + db $66, $44, $0F, $6D, $FB // punpckhqdq xmm15, xmm3 + movdqu xmm3, [r8 + 32] + db $66, $44, $0F, $EF, $C3 // pxor xmm8, xmm3 + movdqu [r9 + 32], xmm8 + movdqu xmm3, [r8 + 96] + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + movdqu [r9 + 96], xmm9 + movdqu xmm3, [r8 + 160] + pxor xmm2, xmm3 + movdqu [r9 + 160], xmm2 + movdqu xmm3, [r8 + 224] + pxor xmm1, xmm3 + movdqu [r9 + 224], xmm1 + movdqu xmm3, [r8 + 48] + db $66, $44, $0F, $EF, $E3 // pxor xmm12, xmm3 + movdqu [r9 + 48], xmm12 + movdqu xmm3, [r8 + 112] + db $66, $44, $0F, $EF, $EB // pxor xmm13, xmm3 + movdqu [r9 + 112], xmm13 + movdqu xmm3, [r8 + 176] + pxor xmm0, xmm3 + movdqu [r9 + 176], xmm0 + movdqu xmm3, [r8 + 240] + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + movdqu [r9 + 240], xmm15 + add r8, 256 + add r9, 256 + dec r10 + jnz @@c7539_ssse3_outer + mov rsp, r11 {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc new file mode 100644 index 00000000..e20d288a --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc @@ -0,0 +1,500 @@ +// ChaCha7539 (IETF) 8-way AVX2 keystream kernel, vertical layout (i386). +// ------------------------------------------------------------------- +// +// Eight 64-byte ChaCha20 blocks per group, state words held vertically (one +// block per ymm lane). i386 has only 8 ymm, so the 16 state rows live in a +// 64B-aligned stack frame (original esp parked inside it); each quarter-round +// loads its 4 rows into ymm0-3 (temp ymm4), with the rot16/rot8 vpshufb masks +// resident in ymm6/ymm7. rot12/rot7 use vpslld/vpsrld/vpor. +// +// Streams AGroups x 512B in one call: prologue once, per-group ORIG->STATE +// reload (counter lanes +8), fused accumulate/transpose/xor tail to AOut +// (vextracti128 lifts each row's high block piece). +// +// On entry (after ClpSimdProc5Begin_i386.inc): +// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut, +// ecx = AGroups (8-block groups, >= 1). +// +// On exit: +// AState word-12 counter (byte 48) advanced by 8*AGroups; the keystream is +// xored with AIn into AOut (512*AGroups bytes). +// +// VEX-encoded instructions are db-encoded for broad assembler compatibility. + mov edx, esp + sub esp, 1168 + and esp, -64 + mov dword ptr [esp + 1152], edx + mov dword ptr [esp + 1024], 0 + mov dword ptr [esp + 1028], 1 + mov dword ptr [esp + 1032], 2 + mov dword ptr [esp + 1036], 3 + mov dword ptr [esp + 1040], 4 + mov dword ptr [esp + 1044], 5 + mov dword ptr [esp + 1048], 6 + mov dword ptr [esp + 1052], 7 + mov dword ptr [esp + 1056], 8 + mov dword ptr [esp + 1060], 8 + mov dword ptr [esp + 1064], 8 + mov dword ptr [esp + 1068], 8 + mov dword ptr [esp + 1072], 8 + mov dword ptr [esp + 1076], 8 + mov dword ptr [esp + 1080], 8 + mov dword ptr [esp + 1084], 8 + mov dword ptr [esp + 1088], $01000302 + mov dword ptr [esp + 1092], $05040706 + mov dword ptr [esp + 1096], $09080B0A + mov dword ptr [esp + 1100], $0D0C0F0E + mov dword ptr [esp + 1104], $01000302 + mov dword ptr [esp + 1108], $05040706 + mov dword ptr [esp + 1112], $09080B0A + mov dword ptr [esp + 1116], $0D0C0F0E + mov dword ptr [esp + 1120], $02010003 + mov dword ptr [esp + 1124], $06050407 + mov dword ptr [esp + 1128], $0A09080B + mov dword ptr [esp + 1132], $0E0D0C0F + mov dword ptr [esp + 1136], $02010003 + mov dword ptr [esp + 1140], $06050407 + mov dword ptr [esp + 1144], $0A09080B + mov dword ptr [esp + 1148], $0E0D0C0F + db $C5, $FD, $6F, $B4, $24, $40, $04, $00, $00 // vmovdqa ymm6, [esp + 1088] + db $C5, $FD, $6F, $BC, $24, $60, $04, $00, $00 // vmovdqa ymm7, [esp + 1120] + db $C4, $E2, $7D, $58, $06 // vpbroadcastd ymm0, [esi + 0] + db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0 + db $C5, $FD, $7F, $84, $24, $00, $02, $00, $00 // vmovdqa [esp + 512], ymm0 + db $C4, $E2, $7D, $58, $46, $04 // vpbroadcastd ymm0, [esi + 4] + db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0 + db $C5, $FD, $7F, $84, $24, $20, $02, $00, $00 // vmovdqa [esp + 544], ymm0 + db $C4, $E2, $7D, $58, $46, $08 // vpbroadcastd ymm0, [esi + 8] + db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0 + db $C5, $FD, $7F, $84, $24, $40, $02, $00, $00 // vmovdqa [esp + 576], ymm0 + db $C4, $E2, $7D, $58, $46, $0C // vpbroadcastd ymm0, [esi + 12] + db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0 + db $C5, $FD, $7F, $84, $24, $60, $02, $00, $00 // vmovdqa [esp + 608], ymm0 + db $C4, $E2, $7D, $58, $46, $10 // vpbroadcastd ymm0, [esi + 16] + db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm0 + db $C5, $FD, $7F, $84, $24, $80, $02, $00, $00 // vmovdqa [esp + 640], ymm0 + db $C4, $E2, $7D, $58, $46, $14 // vpbroadcastd ymm0, [esi + 20] + db $C5, $FD, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm0 + db $C5, $FD, $7F, $84, $24, $A0, $02, $00, $00 // vmovdqa [esp + 672], ymm0 + db $C4, $E2, $7D, $58, $46, $18 // vpbroadcastd ymm0, [esi + 24] + db $C5, $FD, $7F, $84, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm0 + db $C5, $FD, $7F, $84, $24, $C0, $02, $00, $00 // vmovdqa [esp + 704], ymm0 + db $C4, $E2, $7D, $58, $46, $1C // vpbroadcastd ymm0, [esi + 28] + db $C5, $FD, $7F, $84, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm0 + db $C5, $FD, $7F, $84, $24, $E0, $02, $00, $00 // vmovdqa [esp + 736], ymm0 + db $C4, $E2, $7D, $58, $46, $20 // vpbroadcastd ymm0, [esi + 32] + db $C5, $FD, $7F, $84, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm0 + db $C5, $FD, $7F, $84, $24, $00, $03, $00, $00 // vmovdqa [esp + 768], ymm0 + db $C4, $E2, $7D, $58, $46, $24 // vpbroadcastd ymm0, [esi + 36] + db $C5, $FD, $7F, $84, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm0 + db $C5, $FD, $7F, $84, $24, $20, $03, $00, $00 // vmovdqa [esp + 800], ymm0 + db $C4, $E2, $7D, $58, $46, $28 // vpbroadcastd ymm0, [esi + 40] + db $C5, $FD, $7F, $84, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm0 + db $C5, $FD, $7F, $84, $24, $40, $03, $00, $00 // vmovdqa [esp + 832], ymm0 + db $C4, $E2, $7D, $58, $46, $2C // vpbroadcastd ymm0, [esi + 44] + db $C5, $FD, $7F, $84, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm0 + db $C5, $FD, $7F, $84, $24, $60, $03, $00, $00 // vmovdqa [esp + 864], ymm0 + db $C4, $E2, $7D, $58, $46, $30 // vpbroadcastd ymm0, [esi + 48] + db $C5, $FD, $FE, $84, $24, $00, $04, $00, $00 // vpaddd ymm0, ymm0, [esp + 1024] + db $C5, $FD, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm0 + db $C5, $FD, $7F, $84, $24, $80, $03, $00, $00 // vmovdqa [esp + 896], ymm0 + db $C4, $E2, $7D, $58, $46, $34 // vpbroadcastd ymm0, [esi + 52] + db $C5, $FD, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm0 + db $C5, $FD, $7F, $84, $24, $A0, $03, $00, $00 // vmovdqa [esp + 928], ymm0 + db $C4, $E2, $7D, $58, $46, $38 // vpbroadcastd ymm0, [esi + 56] + db $C5, $FD, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm0 + db $C5, $FD, $7F, $84, $24, $C0, $03, $00, $00 // vmovdqa [esp + 960], ymm0 + db $C4, $E2, $7D, $58, $46, $3C // vpbroadcastd ymm0, [esi + 60] + db $C5, $FD, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm0 + db $C5, $FD, $7F, $84, $24, $E0, $03, $00, $00 // vmovdqa [esp + 992], ymm0 + jmp @@c7539_avx2_8x_i386_enter +@@c7539_avx2_8x_i386_outer: + db $C5, $FD, $6F, $84, $24, $00, $02, $00, $00 // vmovdqa ymm0, [esp + 512] + db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0 + db $C5, $FD, $6F, $84, $24, $20, $02, $00, $00 // vmovdqa ymm0, [esp + 544] + db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0 + db $C5, $FD, $6F, $84, $24, $40, $02, $00, $00 // vmovdqa ymm0, [esp + 576] + db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0 + db $C5, $FD, $6F, $84, $24, $60, $02, $00, $00 // vmovdqa ymm0, [esp + 608] + db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0 + db $C5, $FD, $6F, $84, $24, $80, $02, $00, $00 // vmovdqa ymm0, [esp + 640] + db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm0 + db $C5, $FD, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqa ymm0, [esp + 672] + db $C5, $FD, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm0 + db $C5, $FD, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqa ymm0, [esp + 704] + db $C5, $FD, $7F, $84, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm0 + db $C5, $FD, $6F, $84, $24, $E0, $02, $00, $00 // vmovdqa ymm0, [esp + 736] + db $C5, $FD, $7F, $84, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm0 + db $C5, $FD, $6F, $84, $24, $00, $03, $00, $00 // vmovdqa ymm0, [esp + 768] + db $C5, $FD, $7F, $84, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm0 + db $C5, $FD, $6F, $84, $24, $20, $03, $00, $00 // vmovdqa ymm0, [esp + 800] + db $C5, $FD, $7F, $84, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm0 + db $C5, $FD, $6F, $84, $24, $40, $03, $00, $00 // vmovdqa ymm0, [esp + 832] + db $C5, $FD, $7F, $84, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm0 + db $C5, $FD, $6F, $84, $24, $60, $03, $00, $00 // vmovdqa ymm0, [esp + 864] + db $C5, $FD, $7F, $84, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm0 + db $C5, $FD, $6F, $84, $24, $80, $03, $00, $00 // vmovdqa ymm0, [esp + 896] + db $C5, $FD, $FE, $84, $24, $20, $04, $00, $00 // vpaddd ymm0, ymm0, [esp + 1056] + db $C5, $FD, $7F, $84, $24, $80, $03, $00, $00 // vmovdqa [esp + 896], ymm0 + db $C5, $FD, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm0 + db $C5, $FD, $6F, $84, $24, $A0, $03, $00, $00 // vmovdqa ymm0, [esp + 928] + db $C5, $FD, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm0 + db $C5, $FD, $6F, $84, $24, $C0, $03, $00, $00 // vmovdqa ymm0, [esp + 960] + db $C5, $FD, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm0 + db $C5, $FD, $6F, $84, $24, $E0, $03, $00, $00 // vmovdqa ymm0, [esp + 992] + db $C5, $FD, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm0 +@@c7539_avx2_8x_i386_enter: + mov edx, ebx +@@c7539_avx2_8x_i386_round: + db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0] + db $C5, $FD, $6F, $8C, $24, $80, $00, $00, $00 // vmovdqa ymm1, [esp + 128] + db $C5, $FD, $6F, $94, $24, $00, $01, $00, $00 // vmovdqa ymm2, [esp + 256] + db $C5, $FD, $6F, $9C, $24, $80, $01, $00, $00 // vmovdqa ymm3, [esp + 384] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0 + db $C5, $FD, $7F, $8C, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm1 + db $C5, $FD, $7F, $94, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm2 + db $C5, $FD, $7F, $9C, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm3 + db $C5, $FD, $6F, $44, $24, $20 // vmovdqa ymm0, [esp + 32] + db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160] + db $C5, $FD, $6F, $94, $24, $20, $01, $00, $00 // vmovdqa ymm2, [esp + 288] + db $C5, $FD, $6F, $9C, $24, $A0, $01, $00, $00 // vmovdqa ymm3, [esp + 416] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0 + db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm1 + db $C5, $FD, $7F, $94, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm2 + db $C5, $FD, $7F, $9C, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm3 + db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [esp + 64] + db $C5, $FD, $6F, $8C, $24, $C0, $00, $00, $00 // vmovdqa ymm1, [esp + 192] + db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320] + db $C5, $FD, $6F, $9C, $24, $C0, $01, $00, $00 // vmovdqa ymm3, [esp + 448] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0 + db $C5, $FD, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm1 + db $C5, $FD, $7F, $94, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm2 + db $C5, $FD, $7F, $9C, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm3 + db $C5, $FD, $6F, $44, $24, $60 // vmovdqa ymm0, [esp + 96] + db $C5, $FD, $6F, $8C, $24, $E0, $00, $00, $00 // vmovdqa ymm1, [esp + 224] + db $C5, $FD, $6F, $94, $24, $60, $01, $00, $00 // vmovdqa ymm2, [esp + 352] + db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0 + db $C5, $FD, $7F, $8C, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm1 + db $C5, $FD, $7F, $94, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm2 + db $C5, $FD, $7F, $9C, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm3 + db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0] + db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160] + db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320] + db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0 + db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm1 + db $C5, $FD, $7F, $94, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm2 + db $C5, $FD, $7F, $9C, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm3 + db $C5, $FD, $6F, $44, $24, $20 // vmovdqa ymm0, [esp + 32] + db $C5, $FD, $6F, $8C, $24, $C0, $00, $00, $00 // vmovdqa ymm1, [esp + 192] + db $C5, $FD, $6F, $94, $24, $60, $01, $00, $00 // vmovdqa ymm2, [esp + 352] + db $C5, $FD, $6F, $9C, $24, $80, $01, $00, $00 // vmovdqa ymm3, [esp + 384] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0 + db $C5, $FD, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm1 + db $C5, $FD, $7F, $94, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm2 + db $C5, $FD, $7F, $9C, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm3 + db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [esp + 64] + db $C5, $FD, $6F, $8C, $24, $E0, $00, $00, $00 // vmovdqa ymm1, [esp + 224] + db $C5, $FD, $6F, $94, $24, $00, $01, $00, $00 // vmovdqa ymm2, [esp + 256] + db $C5, $FD, $6F, $9C, $24, $A0, $01, $00, $00 // vmovdqa ymm3, [esp + 416] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0 + db $C5, $FD, $7F, $8C, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm1 + db $C5, $FD, $7F, $94, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm2 + db $C5, $FD, $7F, $9C, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm3 + db $C5, $FD, $6F, $44, $24, $60 // vmovdqa ymm0, [esp + 96] + db $C5, $FD, $6F, $8C, $24, $80, $00, $00, $00 // vmovdqa ymm1, [esp + 128] + db $C5, $FD, $6F, $94, $24, $20, $01, $00, $00 // vmovdqa ymm2, [esp + 288] + db $C5, $FD, $6F, $9C, $24, $C0, $01, $00, $00 // vmovdqa ymm3, [esp + 448] + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12 + db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7 + db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25 + db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1 + db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0 + db $C5, $FD, $7F, $8C, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm1 + db $C5, $FD, $7F, $94, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm2 + db $C5, $FD, $7F, $9C, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm3 + sub edx, 2 + jg @@c7539_avx2_8x_i386_round + add dword ptr [esi + 48], 8 +{$IFDEF CHACHA_CTR64} + adc dword ptr [esi + 52], 0 +{$ENDIF} + db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0] + db $C5, $FD, $FE, $84, $24, $00, $02, $00, $00 // vpaddd ymm0, ymm0, [esp + 512] + db $C5, $FD, $6F, $4C, $24, $20 // vmovdqa ymm1, [esp + 32] + db $C5, $F5, $FE, $8C, $24, $20, $02, $00, $00 // vpaddd ymm1, ymm1, [esp + 544] + db $C5, $FD, $6F, $54, $24, $40 // vmovdqa ymm2, [esp + 64] + db $C5, $ED, $FE, $94, $24, $40, $02, $00, $00 // vpaddd ymm2, ymm2, [esp + 576] + db $C5, $FD, $6F, $5C, $24, $60 // vmovdqa ymm3, [esp + 96] + db $C5, $E5, $FE, $9C, $24, $60, $02, $00, $00 // vpaddd ymm3, ymm3, [esp + 608] + db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1 + db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3 + db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5 + db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5 + db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2 + db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2 + db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1 + db $C5, $F1, $EF, $0F // vpxor xmm1, xmm1, [edi] + db $C5, $FA, $7F, $08 // vmovdqu [eax], xmm1 + db $C5, $D1, $EF, $AF, $00, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 256] + db $C5, $FA, $7F, $A8, $00, $01, $00, $00 // vmovdqu [eax + 256], xmm5 + db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1 + db $C5, $D9, $EF, $67, $40 // vpxor xmm4, xmm4, [edi + 64] + db $C5, $FA, $7F, $60, $40 // vmovdqu [eax + 64], xmm4 + db $C5, $D1, $EF, $AF, $40, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 320] + db $C5, $FA, $7F, $A8, $40, $01, $00, $00 // vmovdqu [eax + 320], xmm5 + db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1 + db $C5, $E1, $EF, $9F, $80, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 128] + db $C5, $FA, $7F, $98, $80, $00, $00, $00 // vmovdqu [eax + 128], xmm3 + db $C5, $D1, $EF, $AF, $80, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 384] + db $C5, $FA, $7F, $A8, $80, $01, $00, $00 // vmovdqu [eax + 384], xmm5 + db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1 + db $C5, $F9, $EF, $87, $C0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 192] + db $C5, $FA, $7F, $80, $C0, $00, $00, $00 // vmovdqu [eax + 192], xmm0 + db $C5, $D1, $EF, $AF, $C0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 448] + db $C5, $FA, $7F, $A8, $C0, $01, $00, $00 // vmovdqu [eax + 448], xmm5 + db $C5, $FD, $6F, $84, $24, $80, $00, $00, $00 // vmovdqa ymm0, [esp + 128] + db $C5, $FD, $FE, $84, $24, $80, $02, $00, $00 // vpaddd ymm0, ymm0, [esp + 640] + db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160] + db $C5, $F5, $FE, $8C, $24, $A0, $02, $00, $00 // vpaddd ymm1, ymm1, [esp + 672] + db $C5, $FD, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqa ymm2, [esp + 192] + db $C5, $ED, $FE, $94, $24, $C0, $02, $00, $00 // vpaddd ymm2, ymm2, [esp + 704] + db $C5, $FD, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqa ymm3, [esp + 224] + db $C5, $E5, $FE, $9C, $24, $E0, $02, $00, $00 // vpaddd ymm3, ymm3, [esp + 736] + db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1 + db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3 + db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5 + db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5 + db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2 + db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2 + db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1 + db $C5, $F1, $EF, $4F, $10 // vpxor xmm1, xmm1, [edi + 16] + db $C5, $FA, $7F, $48, $10 // vmovdqu [eax + 16], xmm1 + db $C5, $D1, $EF, $AF, $10, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 272] + db $C5, $FA, $7F, $A8, $10, $01, $00, $00 // vmovdqu [eax + 272], xmm5 + db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1 + db $C5, $D9, $EF, $67, $50 // vpxor xmm4, xmm4, [edi + 80] + db $C5, $FA, $7F, $60, $50 // vmovdqu [eax + 80], xmm4 + db $C5, $D1, $EF, $AF, $50, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 336] + db $C5, $FA, $7F, $A8, $50, $01, $00, $00 // vmovdqu [eax + 336], xmm5 + db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1 + db $C5, $E1, $EF, $9F, $90, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 144] + db $C5, $FA, $7F, $98, $90, $00, $00, $00 // vmovdqu [eax + 144], xmm3 + db $C5, $D1, $EF, $AF, $90, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 400] + db $C5, $FA, $7F, $A8, $90, $01, $00, $00 // vmovdqu [eax + 400], xmm5 + db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1 + db $C5, $F9, $EF, $87, $D0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 208] + db $C5, $FA, $7F, $80, $D0, $00, $00, $00 // vmovdqu [eax + 208], xmm0 + db $C5, $D1, $EF, $AF, $D0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 464] + db $C5, $FA, $7F, $A8, $D0, $01, $00, $00 // vmovdqu [eax + 464], xmm5 + db $C5, $FD, $6F, $84, $24, $00, $01, $00, $00 // vmovdqa ymm0, [esp + 256] + db $C5, $FD, $FE, $84, $24, $00, $03, $00, $00 // vpaddd ymm0, ymm0, [esp + 768] + db $C5, $FD, $6F, $8C, $24, $20, $01, $00, $00 // vmovdqa ymm1, [esp + 288] + db $C5, $F5, $FE, $8C, $24, $20, $03, $00, $00 // vpaddd ymm1, ymm1, [esp + 800] + db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320] + db $C5, $ED, $FE, $94, $24, $40, $03, $00, $00 // vpaddd ymm2, ymm2, [esp + 832] + db $C5, $FD, $6F, $9C, $24, $60, $01, $00, $00 // vmovdqa ymm3, [esp + 352] + db $C5, $E5, $FE, $9C, $24, $60, $03, $00, $00 // vpaddd ymm3, ymm3, [esp + 864] + db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1 + db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3 + db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5 + db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5 + db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2 + db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2 + db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1 + db $C5, $F1, $EF, $4F, $20 // vpxor xmm1, xmm1, [edi + 32] + db $C5, $FA, $7F, $48, $20 // vmovdqu [eax + 32], xmm1 + db $C5, $D1, $EF, $AF, $20, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 288] + db $C5, $FA, $7F, $A8, $20, $01, $00, $00 // vmovdqu [eax + 288], xmm5 + db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1 + db $C5, $D9, $EF, $67, $60 // vpxor xmm4, xmm4, [edi + 96] + db $C5, $FA, $7F, $60, $60 // vmovdqu [eax + 96], xmm4 + db $C5, $D1, $EF, $AF, $60, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 352] + db $C5, $FA, $7F, $A8, $60, $01, $00, $00 // vmovdqu [eax + 352], xmm5 + db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1 + db $C5, $E1, $EF, $9F, $A0, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 160] + db $C5, $FA, $7F, $98, $A0, $00, $00, $00 // vmovdqu [eax + 160], xmm3 + db $C5, $D1, $EF, $AF, $A0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 416] + db $C5, $FA, $7F, $A8, $A0, $01, $00, $00 // vmovdqu [eax + 416], xmm5 + db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1 + db $C5, $F9, $EF, $87, $E0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 224] + db $C5, $FA, $7F, $80, $E0, $00, $00, $00 // vmovdqu [eax + 224], xmm0 + db $C5, $D1, $EF, $AF, $E0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 480] + db $C5, $FA, $7F, $A8, $E0, $01, $00, $00 // vmovdqu [eax + 480], xmm5 + db $C5, $FD, $6F, $84, $24, $80, $01, $00, $00 // vmovdqa ymm0, [esp + 384] + db $C5, $FD, $FE, $84, $24, $80, $03, $00, $00 // vpaddd ymm0, ymm0, [esp + 896] + db $C5, $FD, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqa ymm1, [esp + 416] + db $C5, $F5, $FE, $8C, $24, $A0, $03, $00, $00 // vpaddd ymm1, ymm1, [esp + 928] + db $C5, $FD, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqa ymm2, [esp + 448] + db $C5, $ED, $FE, $94, $24, $C0, $03, $00, $00 // vpaddd ymm2, ymm2, [esp + 960] + db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480] + db $C5, $E5, $FE, $9C, $24, $E0, $03, $00, $00 // vpaddd ymm3, ymm3, [esp + 992] + db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1 + db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3 + db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5 + db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5 + db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2 + db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2 + db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1 + db $C5, $F1, $EF, $4F, $30 // vpxor xmm1, xmm1, [edi + 48] + db $C5, $FA, $7F, $48, $30 // vmovdqu [eax + 48], xmm1 + db $C5, $D1, $EF, $AF, $30, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 304] + db $C5, $FA, $7F, $A8, $30, $01, $00, $00 // vmovdqu [eax + 304], xmm5 + db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1 + db $C5, $D9, $EF, $67, $70 // vpxor xmm4, xmm4, [edi + 112] + db $C5, $FA, $7F, $60, $70 // vmovdqu [eax + 112], xmm4 + db $C5, $D1, $EF, $AF, $70, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 368] + db $C5, $FA, $7F, $A8, $70, $01, $00, $00 // vmovdqu [eax + 368], xmm5 + db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1 + db $C5, $E1, $EF, $9F, $B0, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 176] + db $C5, $FA, $7F, $98, $B0, $00, $00, $00 // vmovdqu [eax + 176], xmm3 + db $C5, $D1, $EF, $AF, $B0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 432] + db $C5, $FA, $7F, $A8, $B0, $01, $00, $00 // vmovdqu [eax + 432], xmm5 + db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1 + db $C5, $F9, $EF, $87, $F0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 240] + db $C5, $FA, $7F, $80, $F0, $00, $00, $00 // vmovdqu [eax + 240], xmm0 + db $C5, $D1, $EF, $AF, $F0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 496] + db $C5, $FA, $7F, $A8, $F0, $01, $00, $00 // vmovdqu [eax + 496], xmm5 + add edi, 512 + add eax, 512 + dec ecx + jnz @@c7539_avx2_8x_i386_outer + db $C5, $F8, $77 // vzeroupper + mov esp, dword ptr [esp + 1152] + pop edi + pop esi + pop ebx + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc index 0fd47d0c..5c51ad83 100644 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc @@ -3,21 +3,33 @@ // // Eight 64-byte ChaCha20 blocks, state words held vertically (one block per // ymm lane). ymm0-3 = words a (0-3), ymm4-7 = words b (4-7), ymm12-15 = words -// d (12-15); words c (8-11) live in a stack slot; ymm8-11 are scratch. The four -// quarter-rounds of each round are step-interleaved; rot16/rot8 use vpshufb with -// the mask read from memory; rot12/rot7 use vpslld/vpsrld/vpor. +// d (12-15). Words c (8-11): two are resident in ymm8/ymm9 while the other two +// wait in stack slots; quarter-rounds run as two resident pairs and the pair a +// round ends with is the pair the next round starts with, so a round costs one +// pair swap (2 stores + 2 loads). ymm10/ymm11 are the rotate temps; rot16/rot8 +// use vpshufb (mask reloaded into the just-freed temp), rot12/rot7 use +// vpslld/vpsrld/vpor. The frame is 64B-aligned (r11 = frame register), so all +// stack traffic is aligned vmovdqa. // -// On entry (after ClpSimdProc4Begin_x86_64.inc): -// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// Streams AGroups x 512B in one call: frame, broadcasts and constants are set +// up once; each group re-enters via a light state reload (counter lanes +8) +// and ends with a fused accumulate/transpose/xor written straight to AOut +// (vperm2i128 pairs whole 32-byte block halves; no bounce buffer). +// +// On entry (after ClpSimdProc5Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut, +// r10 = AGroups (8-block groups, >= 1). // // On exit: -// AState word-12 counter (byte 48) advanced by 8; the keystream is xored -// with AIn into AOut (512 bytes). +// AState word-12 counter (byte 48) advanced by 8*AGroups; the keystream is +// xored with AIn into AOut (512*AGroups bytes). // // VEX-encoded instructions are db-encoded for broad assembler compatibility. - mov r10, rcx + mov r10d, r10d {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - sub rsp, 1248 + mov r11, rsp + sub rsp, 832 + and rsp, -64 mov dword ptr [rsp + 640], 0 mov dword ptr [rsp + 644], 1 mov dword ptr [rsp + 648], 2 @@ -26,427 +38,349 @@ mov dword ptr [rsp + 660], 5 mov dword ptr [rsp + 664], 6 mov dword ptr [rsp + 668], 7 - mov dword ptr [rsp + 672], $01000302 - mov dword ptr [rsp + 676], $05040706 - mov dword ptr [rsp + 680], $09080B0A - mov dword ptr [rsp + 684], $0D0C0F0E - mov dword ptr [rsp + 688], $01000302 - mov dword ptr [rsp + 692], $05040706 - mov dword ptr [rsp + 696], $09080B0A - mov dword ptr [rsp + 700], $0D0C0F0E - mov dword ptr [rsp + 704], $02010003 - mov dword ptr [rsp + 708], $06050407 - mov dword ptr [rsp + 712], $0A09080B - mov dword ptr [rsp + 716], $0E0D0C0F - mov dword ptr [rsp + 720], $02010003 - mov dword ptr [rsp + 724], $06050407 - mov dword ptr [rsp + 728], $0A09080B - mov dword ptr [rsp + 732], $0E0D0C0F - db $C4, $E2, $7D, $58, $02 // vpbroadcastd ymm0, [rdx] - db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0 - db $C4, $E2, $7D, $58, $4A, $04 // vpbroadcastd ymm1, [rdx + 4] - db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1 - db $C4, $E2, $7D, $58, $52, $08 // vpbroadcastd ymm2, [rdx + 8] - db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2 - db $C4, $E2, $7D, $58, $5A, $0C // vpbroadcastd ymm3, [rdx + 12] - db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3 - db $C4, $E2, $7D, $58, $62, $10 // vpbroadcastd ymm4, [rdx + 16] - db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4 - db $C4, $E2, $7D, $58, $6A, $14 // vpbroadcastd ymm5, [rdx + 20] - db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5 - db $C4, $E2, $7D, $58, $72, $18 // vpbroadcastd ymm6, [rdx + 24] - db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6 - db $C4, $E2, $7D, $58, $7A, $1C // vpbroadcastd ymm7, [rdx + 28] - db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7 - db $C4, $62, $7D, $58, $42, $20 // vpbroadcastd ymm8, [rdx + 32] - db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8 - db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 - db $C4, $62, $7D, $58, $42, $24 // vpbroadcastd ymm8, [rdx + 36] - db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8 - db $C5, $7E, $7F, $44, $24, $20 // vmovdqu [rsp + 32], ymm8 - db $C4, $62, $7D, $58, $42, $28 // vpbroadcastd ymm8, [rdx + 40] - db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8 - db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 - db $C4, $62, $7D, $58, $42, $2C // vpbroadcastd ymm8, [rdx + 44] - db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8 - db $C5, $7E, $7F, $44, $24, $60 // vmovdqu [rsp + 96], ymm8 - db $C4, $62, $7D, $58, $62, $30 // vpbroadcastd ymm12, [rdx + 48] - db $C5, $1D, $FE, $A4, $24, $80, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 640] - db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12 - db $C4, $62, $7D, $58, $6A, $34 // vpbroadcastd ymm13, [rdx + 52] - db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13 - db $C4, $62, $7D, $58, $72, $38 // vpbroadcastd ymm14, [rdx + 56] - db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14 - db $C4, $62, $7D, $58, $7A, $3C // vpbroadcastd ymm15, [rdx + 60] - db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15 + mov dword ptr [rsp + 672], 8 + mov dword ptr [rsp + 676], 8 + mov dword ptr [rsp + 680], 8 + mov dword ptr [rsp + 684], 8 + mov dword ptr [rsp + 688], 8 + mov dword ptr [rsp + 692], 8 + mov dword ptr [rsp + 696], 8 + mov dword ptr [rsp + 700], 8 + mov dword ptr [rsp + 704], $01000302 + mov dword ptr [rsp + 708], $05040706 + mov dword ptr [rsp + 712], $09080B0A + mov dword ptr [rsp + 716], $0D0C0F0E + mov dword ptr [rsp + 720], $01000302 + mov dword ptr [rsp + 724], $05040706 + mov dword ptr [rsp + 728], $09080B0A + mov dword ptr [rsp + 732], $0D0C0F0E + mov dword ptr [rsp + 736], $02010003 + mov dword ptr [rsp + 740], $06050407 + mov dword ptr [rsp + 744], $0A09080B + mov dword ptr [rsp + 748], $0E0D0C0F + mov dword ptr [rsp + 752], $02010003 + mov dword ptr [rsp + 756], $06050407 + mov dword ptr [rsp + 760], $0A09080B + mov dword ptr [rsp + 764], $0E0D0C0F + db $C4, $E2, $7D, $58, $02 // vpbroadcastd ymm0, [rdx] + db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [rsp + 128], ymm0 + db $C4, $E2, $7D, $58, $4A, $04 // vpbroadcastd ymm1, [rdx + 4] + db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [rsp + 160], ymm1 + db $C4, $E2, $7D, $58, $52, $08 // vpbroadcastd ymm2, [rdx + 8] + db $C5, $FD, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqa [rsp + 192], ymm2 + db $C4, $E2, $7D, $58, $5A, $0C // vpbroadcastd ymm3, [rdx + 12] + db $C5, $FD, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqa [rsp + 224], ymm3 + db $C4, $E2, $7D, $58, $62, $10 // vpbroadcastd ymm4, [rdx + 16] + db $C5, $FD, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqa [rsp + 256], ymm4 + db $C4, $E2, $7D, $58, $6A, $14 // vpbroadcastd ymm5, [rdx + 20] + db $C5, $FD, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqa [rsp + 288], ymm5 + db $C4, $E2, $7D, $58, $72, $18 // vpbroadcastd ymm6, [rdx + 24] + db $C5, $FD, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqa [rsp + 320], ymm6 + db $C4, $E2, $7D, $58, $7A, $1C // vpbroadcastd ymm7, [rdx + 28] + db $C5, $FD, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqa [rsp + 352], ymm7 + db $C4, $62, $7D, $58, $42, $20 // vpbroadcastd ymm8, [rdx + 32] + db $C5, $7D, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [rsp + 384], ymm8 + db $C4, $62, $7D, $58, $4A, $24 // vpbroadcastd ymm9, [rdx + 36] + db $C5, $7D, $7F, $8C, $24, $A0, $01, $00, $00 // vmovdqa [rsp + 416], ymm9 + db $C4, $62, $7D, $58, $52, $28 // vpbroadcastd ymm10, [rdx + 40] + db $C5, $7D, $7F, $94, $24, $C0, $01, $00, $00 // vmovdqa [rsp + 448], ymm10 + db $C5, $7D, $7F, $54, $24, $40 // vmovdqa [rsp + 64], ymm10 + db $C4, $62, $7D, $58, $52, $2C // vpbroadcastd ymm10, [rdx + 44] + db $C5, $7D, $7F, $94, $24, $E0, $01, $00, $00 // vmovdqa [rsp + 480], ymm10 + db $C5, $7D, $7F, $54, $24, $60 // vmovdqa [rsp + 96], ymm10 + db $C4, $62, $7D, $58, $62, $30 // vpbroadcastd ymm12, [rdx + 48] + db $C5, $1D, $FE, $A4, $24, $80, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 640] + db $C5, $7D, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqa [rsp + 512], ymm12 + db $C4, $62, $7D, $58, $6A, $34 // vpbroadcastd ymm13, [rdx + 52] + db $C5, $7D, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqa [rsp + 544], ymm13 + db $C4, $62, $7D, $58, $72, $38 // vpbroadcastd ymm14, [rdx + 56] + db $C5, $7D, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqa [rsp + 576], ymm14 + db $C4, $62, $7D, $58, $7A, $3C // vpbroadcastd ymm15, [rdx + 60] + db $C5, $7D, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqa [rsp + 608], ymm15 + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] + jmp @@c7539_avx2_8x_enter +@@c7539_avx2_8x_outer: + db $C5, $FD, $6F, $84, $24, $80, $00, $00, $00 // vmovdqa ymm0, [rsp + 128] + db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [rsp + 160] + db $C5, $FD, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqa ymm2, [rsp + 192] + db $C5, $FD, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqa ymm3, [rsp + 224] + db $C5, $FD, $6F, $A4, $24, $00, $01, $00, $00 // vmovdqa ymm4, [rsp + 256] + db $C5, $FD, $6F, $AC, $24, $20, $01, $00, $00 // vmovdqa ymm5, [rsp + 288] + db $C5, $FD, $6F, $B4, $24, $40, $01, $00, $00 // vmovdqa ymm6, [rsp + 320] + db $C5, $FD, $6F, $BC, $24, $60, $01, $00, $00 // vmovdqa ymm7, [rsp + 352] + db $C5, $7D, $6F, $84, $24, $80, $01, $00, $00 // vmovdqa ymm8, [rsp + 384] + db $C5, $7D, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqa ymm9, [rsp + 416] + db $C5, $7D, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqa ymm10, [rsp + 448] + db $C5, $7D, $7F, $54, $24, $40 // vmovdqa [rsp + 64], ymm10 + db $C5, $7D, $6F, $94, $24, $E0, $01, $00, $00 // vmovdqa ymm10, [rsp + 480] + db $C5, $7D, $7F, $54, $24, $60 // vmovdqa [rsp + 96], ymm10 + db $C5, $7D, $6F, $A4, $24, $00, $02, $00, $00 // vmovdqa ymm12, [rsp + 512] + db $C5, $1D, $FE, $A4, $24, $A0, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 672] + db $C5, $7D, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqa [rsp + 512], ymm12 + db $C5, $7D, $6F, $AC, $24, $20, $02, $00, $00 // vmovdqa ymm13, [rsp + 544] + db $C5, $7D, $6F, $B4, $24, $40, $02, $00, $00 // vmovdqa ymm14, [rsp + 576] + db $C5, $7D, $6F, $BC, $24, $60, $02, $00, $00 // vmovdqa ymm15, [rsp + 608] + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] +@@c7539_avx2_8x_enter: + mov eax, ecx @@c7539_avx2_8x_round: - db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 - db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 - db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 - db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 - db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 - db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 - db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 - db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672] - db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 - db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 - db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 - db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 - db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] - db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32] - db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64] - db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96] - db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 - db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 - db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14 - db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15 - db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 - db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 - db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 - db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 - db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 - db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9 - db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10 - db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11 - db $C5, $BD, $72, $F4, $0C // vpslld ymm8, ymm4, 12 - db $C5, $B5, $72, $F5, $0C // vpslld ymm9, ymm5, 12 - db $C5, $AD, $72, $F6, $0C // vpslld ymm10, ymm6, 12 - db $C5, $A5, $72, $F7, $0C // vpslld ymm11, ymm7, 12 - db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 - db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 - db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 - db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 - db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4 - db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5 - db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 - db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 - db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 - db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 - db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 - db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 - db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 - db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 - db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 - db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704] - db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 - db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 - db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 - db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 - db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] - db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32] - db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64] - db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96] - db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 - db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 - db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14 - db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15 - db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 - db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 - db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 - db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 - db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 - db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9 - db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10 - db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11 - db $C5, $BD, $72, $F4, $07 // vpslld ymm8, ymm4, 7 - db $C5, $B5, $72, $F5, $07 // vpslld ymm9, ymm5, 7 - db $C5, $AD, $72, $F6, $07 // vpslld ymm10, ymm6, 7 - db $C5, $A5, $72, $F7, $07 // vpslld ymm11, ymm7, 7 - db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 - db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 - db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 - db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 - db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4 - db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5 - db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 - db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 - db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 - db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 - db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 - db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 - db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 - db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 - db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 - db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672] - db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 - db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 - db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 - db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 - db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] - db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96] - db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp] - db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32] - db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 - db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 - db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13 - db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14 - db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 - db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 - db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10 - db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11 - db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 - db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9 - db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10 - db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11 - db $C5, $BD, $72, $F5, $0C // vpslld ymm8, ymm5, 12 - db $C5, $B5, $72, $F6, $0C // vpslld ymm9, ymm6, 12 - db $C5, $AD, $72, $F7, $0C // vpslld ymm10, ymm7, 12 - db $C5, $A5, $72, $F4, $0C // vpslld ymm11, ymm4, 12 - db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 - db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 - db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 - db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 - db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5 - db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6 - db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 - db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 - db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 - db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 - db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 - db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 - db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 - db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 - db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 - db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704] - db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 - db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 - db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 - db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 - db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] - db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96] - db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp] - db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32] - db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 - db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 - db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13 - db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14 - db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 - db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 - db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10 - db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11 - db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 - db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9 - db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10 - db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11 - db $C5, $BD, $72, $F5, $07 // vpslld ymm8, ymm5, 7 - db $C5, $B5, $72, $F6, $07 // vpslld ymm9, ymm6, 7 - db $C5, $AD, $72, $F7, $07 // vpslld ymm10, ymm7, 7 - db $C5, $A5, $72, $F4, $07 // vpslld ymm11, ymm4, 7 - db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 - db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 - db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 - db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 - db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5 - db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6 - db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 - db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 - sub r10, 2 + db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 + db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 + db $C4, $42, $1D, $00, $E3 // vpshufb ymm12, ymm12, ymm11 + db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 + db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 + db $C4, $42, $15, $00, $EB // vpshufb ymm13, ymm13, ymm11 + db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 + db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 + db $C5, $AD, $72, $F4, $0C // vpslld ymm10, ymm4, 12 + db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 + db $C5, $AD, $EB, $E4 // vpor ymm4, ymm10, ymm4 + db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736] + db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 + db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 + db $C5, $A5, $72, $F5, $0C // vpslld ymm11, ymm5, 12 + db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 + db $C5, $A5, $EB, $ED // vpor ymm5, ymm11, ymm5 + db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 + db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 + db $C4, $42, $1D, $00, $E2 // vpshufb ymm12, ymm12, ymm10 + db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 + db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 + db $C4, $42, $15, $00, $EA // vpshufb ymm13, ymm13, ymm10 + db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 + db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 + db $C5, $A5, $72, $F4, $07 // vpslld ymm11, ymm4, 7 + db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 + db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] + db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 + db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 + db $C5, $AD, $72, $F5, $07 // vpslld ymm10, ymm5, 7 + db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 + db $C5, $AD, $EB, $ED // vpor ymm5, ymm10, ymm5 + db $C5, $7D, $7F, $04, $24 // vmovdqa [rsp], ymm8 + db $C5, $7D, $7F, $4C, $24, $20 // vmovdqa [rsp + 32], ymm9 + db $C5, $7D, $6F, $44, $24, $40 // vmovdqa ymm8, [rsp + 64] + db $C5, $7D, $6F, $4C, $24, $60 // vmovdqa ymm9, [rsp + 96] + db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 + db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 + db $C4, $42, $0D, $00, $F3 // vpshufb ymm14, ymm14, ymm11 + db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 + db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 + db $C4, $42, $05, $00, $FB // vpshufb ymm15, ymm15, ymm11 + db $C4, $41, $3D, $FE, $C6 // vpaddd ymm8, ymm8, ymm14 + db $C4, $C1, $4D, $EF, $F0 // vpxor ymm6, ymm6, ymm8 + db $C5, $AD, $72, $F6, $0C // vpslld ymm10, ymm6, 12 + db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 + db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 + db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736] + db $C4, $41, $35, $FE, $CF // vpaddd ymm9, ymm9, ymm15 + db $C4, $C1, $45, $EF, $F9 // vpxor ymm7, ymm7, ymm9 + db $C5, $A5, $72, $F7, $0C // vpslld ymm11, ymm7, 12 + db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 + db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 + db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 + db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 + db $C4, $42, $0D, $00, $F2 // vpshufb ymm14, ymm14, ymm10 + db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 + db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 + db $C4, $42, $05, $00, $FA // vpshufb ymm15, ymm15, ymm10 + db $C4, $41, $3D, $FE, $C6 // vpaddd ymm8, ymm8, ymm14 + db $C4, $C1, $4D, $EF, $F0 // vpxor ymm6, ymm6, ymm8 + db $C5, $A5, $72, $F6, $07 // vpslld ymm11, ymm6, 7 + db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 + db $C5, $A5, $EB, $F6 // vpor ymm6, ymm11, ymm6 + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] + db $C4, $41, $35, $FE, $CF // vpaddd ymm9, ymm9, ymm15 + db $C4, $C1, $45, $EF, $F9 // vpxor ymm7, ymm7, ymm9 + db $C5, $AD, $72, $F7, $07 // vpslld ymm10, ymm7, 7 + db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 + db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 + db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 + db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 + db $C4, $42, $05, $00, $FB // vpshufb ymm15, ymm15, ymm11 + db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 + db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 + db $C4, $42, $1D, $00, $E3 // vpshufb ymm12, ymm12, ymm11 + db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 + db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 + db $C5, $AD, $72, $F5, $0C // vpslld ymm10, ymm5, 12 + db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 + db $C5, $AD, $EB, $ED // vpor ymm5, ymm10, ymm5 + db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736] + db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 + db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 + db $C5, $A5, $72, $F6, $0C // vpslld ymm11, ymm6, 12 + db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 + db $C5, $A5, $EB, $F6 // vpor ymm6, ymm11, ymm6 + db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 + db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 + db $C4, $42, $05, $00, $FA // vpshufb ymm15, ymm15, ymm10 + db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 + db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 + db $C4, $42, $1D, $00, $E2 // vpshufb ymm12, ymm12, ymm10 + db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 + db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 + db $C5, $A5, $72, $F5, $07 // vpslld ymm11, ymm5, 7 + db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 + db $C5, $A5, $EB, $ED // vpor ymm5, ymm11, ymm5 + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] + db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 + db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 + db $C5, $AD, $72, $F6, $07 // vpslld ymm10, ymm6, 7 + db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 + db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 + db $C5, $7D, $7F, $44, $24, $40 // vmovdqa [rsp + 64], ymm8 + db $C5, $7D, $7F, $4C, $24, $60 // vmovdqa [rsp + 96], ymm9 + db $C5, $7D, $6F, $04, $24 // vmovdqa ymm8, [rsp] + db $C5, $7D, $6F, $4C, $24, $20 // vmovdqa ymm9, [rsp + 32] + db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 + db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 + db $C4, $42, $15, $00, $EB // vpshufb ymm13, ymm13, ymm11 + db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 + db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 + db $C4, $42, $0D, $00, $F3 // vpshufb ymm14, ymm14, ymm11 + db $C4, $41, $3D, $FE, $C5 // vpaddd ymm8, ymm8, ymm13 + db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 + db $C5, $AD, $72, $F7, $0C // vpslld ymm10, ymm7, 12 + db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 + db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 + db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736] + db $C4, $41, $35, $FE, $CE // vpaddd ymm9, ymm9, ymm14 + db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 + db $C5, $A5, $72, $F4, $0C // vpslld ymm11, ymm4, 12 + db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 + db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 + db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 + db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 + db $C4, $42, $15, $00, $EA // vpshufb ymm13, ymm13, ymm10 + db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 + db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 + db $C4, $42, $0D, $00, $F2 // vpshufb ymm14, ymm14, ymm10 + db $C4, $41, $3D, $FE, $C5 // vpaddd ymm8, ymm8, ymm13 + db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 + db $C5, $A5, $72, $F7, $07 // vpslld ymm11, ymm7, 7 + db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 + db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 + db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704] + db $C4, $41, $35, $FE, $CE // vpaddd ymm9, ymm9, ymm14 + db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 + db $C5, $AD, $72, $F4, $07 // vpslld ymm10, ymm4, 7 + db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 + db $C5, $AD, $EB, $E4 // vpor ymm4, ymm10, ymm4 + sub eax, 2 jg @@c7539_avx2_8x_round - db $C5, $7E, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm8, [rsp + 128] - db $C4, $C1, $7D, $FE, $C0 // vpaddd ymm0, ymm0, ymm8 - db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0 - db $C5, $7E, $6F, $84, $24, $A0, $00, $00, $00 // vmovdqu ymm8, [rsp + 160] - db $C4, $C1, $75, $FE, $C8 // vpaddd ymm1, ymm1, ymm8 - db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1 - db $C5, $7E, $6F, $84, $24, $C0, $00, $00, $00 // vmovdqu ymm8, [rsp + 192] - db $C4, $C1, $6D, $FE, $D0 // vpaddd ymm2, ymm2, ymm8 - db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2 - db $C5, $7E, $6F, $84, $24, $E0, $00, $00, $00 // vmovdqu ymm8, [rsp + 224] - db $C4, $C1, $65, $FE, $D8 // vpaddd ymm3, ymm3, ymm8 - db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3 - db $C5, $7E, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm8, [rsp + 256] - db $C4, $C1, $5D, $FE, $E0 // vpaddd ymm4, ymm4, ymm8 - db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4 - db $C5, $7E, $6F, $84, $24, $20, $01, $00, $00 // vmovdqu ymm8, [rsp + 288] - db $C4, $C1, $55, $FE, $E8 // vpaddd ymm5, ymm5, ymm8 - db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5 - db $C5, $7E, $6F, $84, $24, $40, $01, $00, $00 // vmovdqu ymm8, [rsp + 320] - db $C4, $C1, $4D, $FE, $F0 // vpaddd ymm6, ymm6, ymm8 - db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6 - db $C5, $7E, $6F, $84, $24, $60, $01, $00, $00 // vmovdqu ymm8, [rsp + 352] - db $C4, $C1, $45, $FE, $F8 // vpaddd ymm7, ymm7, ymm8 - db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7 - db $C5, $7E, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm8, [rsp + 512] - db $C4, $41, $1D, $FE, $E0 // vpaddd ymm12, ymm12, ymm8 - db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12 - db $C5, $7E, $6F, $84, $24, $20, $02, $00, $00 // vmovdqu ymm8, [rsp + 544] - db $C4, $41, $15, $FE, $E8 // vpaddd ymm13, ymm13, ymm8 - db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13 - db $C5, $7E, $6F, $84, $24, $40, $02, $00, $00 // vmovdqu ymm8, [rsp + 576] - db $C4, $41, $0D, $FE, $F0 // vpaddd ymm14, ymm14, ymm8 - db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14 - db $C5, $7E, $6F, $84, $24, $60, $02, $00, $00 // vmovdqu ymm8, [rsp + 608] - db $C4, $41, $05, $FE, $F8 // vpaddd ymm15, ymm15, ymm8 - db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15 - db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] - db $C5, $7E, $6F, $8C, $24, $80, $01, $00, $00 // vmovdqu ymm9, [rsp + 384] - db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 - db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8 - db $C5, $7E, $6F, $44, $24, $20 // vmovdqu ymm8, [rsp + 32] - db $C5, $7E, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm9, [rsp + 416] - db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 - db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8 - db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] - db $C5, $7E, $6F, $8C, $24, $C0, $01, $00, $00 // vmovdqu ymm9, [rsp + 448] - db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 - db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8 - db $C5, $7E, $6F, $44, $24, $60 // vmovdqu ymm8, [rsp + 96] - db $C5, $7E, $6F, $8C, $24, $E0, $01, $00, $00 // vmovdqu ymm9, [rsp + 480] - db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 - db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8 add dword ptr [rdx + 48], 8 {$IFDEF CHACHA_CTR64} adc dword ptr [rdx + 52], 0 {$ENDIF} - db $C5, $FE, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm0, [rsp + 128] - db $C5, $FE, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqu ymm1, [rsp + 160] - db $C5, $FE, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqu ymm2, [rsp + 192] - db $C5, $FE, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqu ymm3, [rsp + 224] - db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 - db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 - db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 - db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 - db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 - db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 - db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 - db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 - db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 - db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 - db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 - db $C5, $FA, $7F, $84, $24, $E0, $02, $00, $00 // vmovdqu [rsp + 736], xmm0 - db $C4, $E3, $7D, $39, $84, $24, $E0, $03, $00, $00, $01 // vextracti128 [rsp + 992], ymm0, 1 - db $C5, $FA, $7F, $8C, $24, $20, $03, $00, $00 // vmovdqu [rsp + 800], xmm1 - db $C4, $E3, $7D, $39, $8C, $24, $20, $04, $00, $00, $01 // vextracti128 [rsp + 1056], ymm1, 1 - db $C5, $FA, $7F, $A4, $24, $60, $03, $00, $00 // vmovdqu [rsp + 864], xmm4 - db $C4, $E3, $7D, $39, $A4, $24, $60, $04, $00, $00, $01 // vextracti128 [rsp + 1120], ymm4, 1 - db $C5, $FA, $7F, $9C, $24, $A0, $03, $00, $00 // vmovdqu [rsp + 928], xmm3 - db $C4, $E3, $7D, $39, $9C, $24, $A0, $04, $00, $00, $01 // vextracti128 [rsp + 1184], ymm3, 1 - db $C5, $FE, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm0, [rsp + 256] - db $C5, $FE, $6F, $8C, $24, $20, $01, $00, $00 // vmovdqu ymm1, [rsp + 288] - db $C5, $FE, $6F, $94, $24, $40, $01, $00, $00 // vmovdqu ymm2, [rsp + 320] - db $C5, $FE, $6F, $9C, $24, $60, $01, $00, $00 // vmovdqu ymm3, [rsp + 352] - db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 - db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 - db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 - db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 - db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 - db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 - db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 - db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 - db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 - db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 - db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 - db $C5, $FA, $7F, $84, $24, $F0, $02, $00, $00 // vmovdqu [rsp + 752], xmm0 - db $C4, $E3, $7D, $39, $84, $24, $F0, $03, $00, $00, $01 // vextracti128 [rsp + 1008], ymm0, 1 - db $C5, $FA, $7F, $8C, $24, $30, $03, $00, $00 // vmovdqu [rsp + 816], xmm1 - db $C4, $E3, $7D, $39, $8C, $24, $30, $04, $00, $00, $01 // vextracti128 [rsp + 1072], ymm1, 1 - db $C5, $FA, $7F, $A4, $24, $70, $03, $00, $00 // vmovdqu [rsp + 880], xmm4 - db $C4, $E3, $7D, $39, $A4, $24, $70, $04, $00, $00, $01 // vextracti128 [rsp + 1136], ymm4, 1 - db $C5, $FA, $7F, $9C, $24, $B0, $03, $00, $00 // vmovdqu [rsp + 944], xmm3 - db $C4, $E3, $7D, $39, $9C, $24, $B0, $04, $00, $00, $01 // vextracti128 [rsp + 1200], ymm3, 1 - db $C5, $FE, $6F, $84, $24, $80, $01, $00, $00 // vmovdqu ymm0, [rsp + 384] - db $C5, $FE, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm1, [rsp + 416] - db $C5, $FE, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqu ymm2, [rsp + 448] - db $C5, $FE, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqu ymm3, [rsp + 480] - db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 - db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 - db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 - db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 - db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 - db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 - db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 - db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 - db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 - db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 - db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 - db $C5, $FA, $7F, $84, $24, $00, $03, $00, $00 // vmovdqu [rsp + 768], xmm0 - db $C4, $E3, $7D, $39, $84, $24, $00, $04, $00, $00, $01 // vextracti128 [rsp + 1024], ymm0, 1 - db $C5, $FA, $7F, $8C, $24, $40, $03, $00, $00 // vmovdqu [rsp + 832], xmm1 - db $C4, $E3, $7D, $39, $8C, $24, $40, $04, $00, $00, $01 // vextracti128 [rsp + 1088], ymm1, 1 - db $C5, $FA, $7F, $A4, $24, $80, $03, $00, $00 // vmovdqu [rsp + 896], xmm4 - db $C4, $E3, $7D, $39, $A4, $24, $80, $04, $00, $00, $01 // vextracti128 [rsp + 1152], ymm4, 1 - db $C5, $FA, $7F, $9C, $24, $C0, $03, $00, $00 // vmovdqu [rsp + 960], xmm3 - db $C4, $E3, $7D, $39, $9C, $24, $C0, $04, $00, $00, $01 // vextracti128 [rsp + 1216], ymm3, 1 - db $C5, $FE, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm0, [rsp + 512] - db $C5, $FE, $6F, $8C, $24, $20, $02, $00, $00 // vmovdqu ymm1, [rsp + 544] - db $C5, $FE, $6F, $94, $24, $40, $02, $00, $00 // vmovdqu ymm2, [rsp + 576] - db $C5, $FE, $6F, $9C, $24, $60, $02, $00, $00 // vmovdqu ymm3, [rsp + 608] - db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 - db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 - db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 - db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 - db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 - db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 - db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 - db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 - db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 - db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 - db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 - db $C5, $FA, $7F, $84, $24, $10, $03, $00, $00 // vmovdqu [rsp + 784], xmm0 - db $C4, $E3, $7D, $39, $84, $24, $10, $04, $00, $00, $01 // vextracti128 [rsp + 1040], ymm0, 1 - db $C5, $FA, $7F, $8C, $24, $50, $03, $00, $00 // vmovdqu [rsp + 848], xmm1 - db $C4, $E3, $7D, $39, $8C, $24, $50, $04, $00, $00, $01 // vextracti128 [rsp + 1104], ymm1, 1 - db $C5, $FA, $7F, $A4, $24, $90, $03, $00, $00 // vmovdqu [rsp + 912], xmm4 - db $C4, $E3, $7D, $39, $A4, $24, $90, $04, $00, $00, $01 // vextracti128 [rsp + 1168], ymm4, 1 - db $C5, $FA, $7F, $9C, $24, $D0, $03, $00, $00 // vmovdqu [rsp + 976], xmm3 - db $C4, $E3, $7D, $39, $9C, $24, $D0, $04, $00, $00, $01 // vextracti128 [rsp + 1232], ymm3, 1 - db $C5, $FE, $6F, $84, $24, $E0, $02, $00, $00 // vmovdqu ymm0, [rsp + 736] - db $C4, $C1, $7D, $EF, $00 // vpxor ymm0, ymm0, [r8 + 0] - db $C4, $C1, $7E, $7F, $01 // vmovdqu [r9], ymm0 - db $C5, $FE, $6F, $84, $24, $00, $03, $00, $00 // vmovdqu ymm0, [rsp + 768] - db $C4, $C1, $7D, $EF, $40, $20 // vpxor ymm0, ymm0, [r8 + 32] - db $C4, $C1, $7E, $7F, $41, $20 // vmovdqu [r9 + 32], ymm0 - db $C5, $FE, $6F, $84, $24, $20, $03, $00, $00 // vmovdqu ymm0, [rsp + 800] - db $C4, $C1, $7D, $EF, $40, $40 // vpxor ymm0, ymm0, [r8 + 64] - db $C4, $C1, $7E, $7F, $41, $40 // vmovdqu [r9 + 64], ymm0 - db $C5, $FE, $6F, $84, $24, $40, $03, $00, $00 // vmovdqu ymm0, [rsp + 832] - db $C4, $C1, $7D, $EF, $40, $60 // vpxor ymm0, ymm0, [r8 + 96] - db $C4, $C1, $7E, $7F, $41, $60 // vmovdqu [r9 + 96], ymm0 - db $C5, $FE, $6F, $84, $24, $60, $03, $00, $00 // vmovdqu ymm0, [rsp + 864] - db $C4, $C1, $7D, $EF, $80, $80, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 128] - db $C4, $C1, $7E, $7F, $81, $80, $00, $00, $00 // vmovdqu [r9 + 128], ymm0 - db $C5, $FE, $6F, $84, $24, $80, $03, $00, $00 // vmovdqu ymm0, [rsp + 896] - db $C4, $C1, $7D, $EF, $80, $A0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 160] - db $C4, $C1, $7E, $7F, $81, $A0, $00, $00, $00 // vmovdqu [r9 + 160], ymm0 - db $C5, $FE, $6F, $84, $24, $A0, $03, $00, $00 // vmovdqu ymm0, [rsp + 928] - db $C4, $C1, $7D, $EF, $80, $C0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 192] - db $C4, $C1, $7E, $7F, $81, $C0, $00, $00, $00 // vmovdqu [r9 + 192], ymm0 - db $C5, $FE, $6F, $84, $24, $C0, $03, $00, $00 // vmovdqu ymm0, [rsp + 960] - db $C4, $C1, $7D, $EF, $80, $E0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 224] - db $C4, $C1, $7E, $7F, $81, $E0, $00, $00, $00 // vmovdqu [r9 + 224], ymm0 - db $C5, $FE, $6F, $84, $24, $E0, $03, $00, $00 // vmovdqu ymm0, [rsp + 992] - db $C4, $C1, $7D, $EF, $80, $00, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 256] - db $C4, $C1, $7E, $7F, $81, $00, $01, $00, $00 // vmovdqu [r9 + 256], ymm0 - db $C5, $FE, $6F, $84, $24, $00, $04, $00, $00 // vmovdqu ymm0, [rsp + 1024] - db $C4, $C1, $7D, $EF, $80, $20, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 288] - db $C4, $C1, $7E, $7F, $81, $20, $01, $00, $00 // vmovdqu [r9 + 288], ymm0 - db $C5, $FE, $6F, $84, $24, $20, $04, $00, $00 // vmovdqu ymm0, [rsp + 1056] - db $C4, $C1, $7D, $EF, $80, $40, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 320] - db $C4, $C1, $7E, $7F, $81, $40, $01, $00, $00 // vmovdqu [r9 + 320], ymm0 - db $C5, $FE, $6F, $84, $24, $40, $04, $00, $00 // vmovdqu ymm0, [rsp + 1088] - db $C4, $C1, $7D, $EF, $80, $60, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 352] - db $C4, $C1, $7E, $7F, $81, $60, $01, $00, $00 // vmovdqu [r9 + 352], ymm0 - db $C5, $FE, $6F, $84, $24, $60, $04, $00, $00 // vmovdqu ymm0, [rsp + 1120] - db $C4, $C1, $7D, $EF, $80, $80, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 384] - db $C4, $C1, $7E, $7F, $81, $80, $01, $00, $00 // vmovdqu [r9 + 384], ymm0 - db $C5, $FE, $6F, $84, $24, $80, $04, $00, $00 // vmovdqu ymm0, [rsp + 1152] - db $C4, $C1, $7D, $EF, $80, $A0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 416] - db $C4, $C1, $7E, $7F, $81, $A0, $01, $00, $00 // vmovdqu [r9 + 416], ymm0 - db $C5, $FE, $6F, $84, $24, $A0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1184] - db $C4, $C1, $7D, $EF, $80, $C0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 448] - db $C4, $C1, $7E, $7F, $81, $C0, $01, $00, $00 // vmovdqu [r9 + 448], ymm0 - db $C5, $FE, $6F, $84, $24, $C0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1216] - db $C4, $C1, $7D, $EF, $80, $E0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 480] - db $C4, $C1, $7E, $7F, $81, $E0, $01, $00, $00 // vmovdqu [r9 + 480], ymm0 - db $C5, $F8, $77 // vzeroupper - add rsp, 1248 + db $C5, $FD, $FE, $84, $24, $80, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp + 128] + db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp + 160] + db $C5, $ED, $FE, $94, $24, $C0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp + 192] + db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp + 224] + db $C5, $7D, $62, $D1 // vpunpckldq ymm10, ymm0, ymm1 + db $C5, $6D, $62, $DB // vpunpckldq ymm11, ymm2, ymm3 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3 + db $C4, $C1, $2D, $6C, $CB // vpunpcklqdq ymm1, ymm10, ymm11 + db $C4, $41, $2D, $6D, $D3 // vpunpckhqdq ymm10, ymm10, ymm11 + db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2 + db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2 + db $C5, $DD, $FE, $A4, $24, $00, $01, $00, $00 // vpaddd ymm4, ymm4, [rsp + 256] + db $C5, $D5, $FE, $AC, $24, $20, $01, $00, $00 // vpaddd ymm5, ymm5, [rsp + 288] + db $C5, $CD, $FE, $B4, $24, $40, $01, $00, $00 // vpaddd ymm6, ymm6, [rsp + 320] + db $C5, $C5, $FE, $BC, $24, $60, $01, $00, $00 // vpaddd ymm7, ymm7, [rsp + 352] + db $C5, $DD, $62, $D5 // vpunpckldq ymm2, ymm4, ymm5 + db $C5, $4D, $62, $DF // vpunpckldq ymm11, ymm6, ymm7 + db $C5, $DD, $6A, $E5 // vpunpckhdq ymm4, ymm4, ymm5 + db $C5, $CD, $6A, $F7 // vpunpckhdq ymm6, ymm6, ymm7 + db $C4, $C1, $6D, $6C, $EB // vpunpcklqdq ymm5, ymm2, ymm11 + db $C4, $C1, $6D, $6D, $D3 // vpunpckhqdq ymm2, ymm2, ymm11 + db $C5, $DD, $6C, $FE // vpunpcklqdq ymm7, ymm4, ymm6 + db $C5, $DD, $6D, $E6 // vpunpckhqdq ymm4, ymm4, ymm6 + db $C4, $E3, $75, $46, $F5, $20 // vperm2i128 ymm6, ymm1, ymm5, $20 + db $C4, $E3, $75, $46, $ED, $31 // vperm2i128 ymm5, ymm1, ymm5, $31 + db $C4, $E3, $2D, $46, $CA, $20 // vperm2i128 ymm1, ymm10, ymm2, $20 + db $C4, $E3, $2D, $46, $D2, $31 // vperm2i128 ymm2, ymm10, ymm2, $31 + db $C4, $63, $65, $46, $D7, $20 // vperm2i128 ymm10, ymm3, ymm7, $20 + db $C4, $E3, $65, $46, $FF, $31 // vperm2i128 ymm7, ymm3, ymm7, $31 + db $C4, $E3, $7D, $46, $DC, $20 // vperm2i128 ymm3, ymm0, ymm4, $20 + db $C4, $E3, $7D, $46, $E4, $31 // vperm2i128 ymm4, ymm0, ymm4, $31 + db $C5, $FD, $7F, $B4, $24, $00, $03, $00, $00 // vmovdqa [rsp + 768], ymm6 + db $C5, $FD, $7F, $8C, $24, $20, $03, $00, $00 // vmovdqa [rsp + 800], ymm1 + db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [rsp + 64] + db $C5, $FD, $6F, $4C, $24, $60 // vmovdqa ymm1, [rsp + 96] + db $C5, $3D, $FE, $84, $24, $80, $01, $00, $00 // vpaddd ymm8, ymm8, [rsp + 384] + db $C5, $35, $FE, $8C, $24, $A0, $01, $00, $00 // vpaddd ymm9, ymm9, [rsp + 416] + db $C5, $FD, $FE, $84, $24, $C0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp + 448] + db $C5, $F5, $FE, $8C, $24, $E0, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp + 480] + db $C4, $C1, $3D, $62, $F1 // vpunpckldq ymm6, ymm8, ymm9 + db $C5, $7D, $62, $D9 // vpunpckldq ymm11, ymm0, ymm1 + db $C4, $41, $3D, $6A, $C1 // vpunpckhdq ymm8, ymm8, ymm9 + db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1 + db $C4, $41, $4D, $6C, $CB // vpunpcklqdq ymm9, ymm6, ymm11 + db $C4, $C1, $4D, $6D, $F3 // vpunpckhqdq ymm6, ymm6, ymm11 + db $C5, $BD, $6C, $C8 // vpunpcklqdq ymm1, ymm8, ymm0 + db $C5, $3D, $6D, $C0 // vpunpckhqdq ymm8, ymm8, ymm0 + db $C5, $1D, $FE, $A4, $24, $00, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 512] + db $C5, $15, $FE, $AC, $24, $20, $02, $00, $00 // vpaddd ymm13, ymm13, [rsp + 544] + db $C5, $0D, $FE, $B4, $24, $40, $02, $00, $00 // vpaddd ymm14, ymm14, [rsp + 576] + db $C5, $05, $FE, $BC, $24, $60, $02, $00, $00 // vpaddd ymm15, ymm15, [rsp + 608] + db $C4, $C1, $1D, $62, $C5 // vpunpckldq ymm0, ymm12, ymm13 + db $C4, $41, $0D, $62, $DF // vpunpckldq ymm11, ymm14, ymm15 + db $C4, $41, $1D, $6A, $E5 // vpunpckhdq ymm12, ymm12, ymm13 + db $C4, $41, $0D, $6A, $F7 // vpunpckhdq ymm14, ymm14, ymm15 + db $C4, $41, $7D, $6C, $EB // vpunpcklqdq ymm13, ymm0, ymm11 + db $C4, $C1, $7D, $6D, $C3 // vpunpckhqdq ymm0, ymm0, ymm11 + db $C4, $41, $1D, $6C, $FE // vpunpcklqdq ymm15, ymm12, ymm14 + db $C4, $41, $1D, $6D, $E6 // vpunpckhqdq ymm12, ymm12, ymm14 + db $C4, $43, $35, $46, $DD, $20 // vperm2i128 ymm11, ymm9, ymm13, $20 + db $C4, $43, $35, $46, $ED, $31 // vperm2i128 ymm13, ymm9, ymm13, $31 + db $C4, $63, $4D, $46, $C8, $20 // vperm2i128 ymm9, ymm6, ymm0, $20 + db $C4, $E3, $4D, $46, $C0, $31 // vperm2i128 ymm0, ymm6, ymm0, $31 + db $C4, $C3, $75, $46, $F7, $20 // vperm2i128 ymm6, ymm1, ymm15, $20 + db $C4, $43, $75, $46, $FF, $31 // vperm2i128 ymm15, ymm1, ymm15, $31 + db $C4, $C3, $3D, $46, $CC, $20 // vperm2i128 ymm1, ymm8, ymm12, $20 + db $C4, $43, $3D, $46, $E4, $31 // vperm2i128 ymm12, ymm8, ymm12, $31 + db $C5, $7D, $6F, $84, $24, $00, $03, $00, $00 // vmovdqa ymm8, [rsp + 768] + db $C4, $41, $3D, $EF, $00 // vpxor ymm8, ymm8, [r8 + 0] + db $C4, $41, $7E, $7F, $01 // vmovdqu [r9], ymm8 + db $C4, $41, $25, $EF, $58, $20 // vpxor ymm11, ymm11, [r8 + 32] + db $C4, $41, $7E, $7F, $59, $20 // vmovdqu [r9 + 32], ymm11 + db $C5, $7D, $6F, $B4, $24, $20, $03, $00, $00 // vmovdqa ymm14, [rsp + 800] + db $C4, $41, $0D, $EF, $70, $40 // vpxor ymm14, ymm14, [r8 + 64] + db $C4, $41, $7E, $7F, $71, $40 // vmovdqu [r9 + 64], ymm14 + db $C4, $41, $35, $EF, $48, $60 // vpxor ymm9, ymm9, [r8 + 96] + db $C4, $41, $7E, $7F, $49, $60 // vmovdqu [r9 + 96], ymm9 + db $C4, $41, $2D, $EF, $90, $80, $00, $00, $00 // vpxor ymm10, ymm10, [r8 + 128] + db $C4, $41, $7E, $7F, $91, $80, $00, $00, $00 // vmovdqu [r9 + 128], ymm10 + db $C4, $C1, $4D, $EF, $B0, $A0, $00, $00, $00 // vpxor ymm6, ymm6, [r8 + 160] + db $C4, $C1, $7E, $7F, $B1, $A0, $00, $00, $00 // vmovdqu [r9 + 160], ymm6 + db $C4, $C1, $65, $EF, $98, $C0, $00, $00, $00 // vpxor ymm3, ymm3, [r8 + 192] + db $C4, $C1, $7E, $7F, $99, $C0, $00, $00, $00 // vmovdqu [r9 + 192], ymm3 + db $C4, $C1, $75, $EF, $88, $E0, $00, $00, $00 // vpxor ymm1, ymm1, [r8 + 224] + db $C4, $C1, $7E, $7F, $89, $E0, $00, $00, $00 // vmovdqu [r9 + 224], ymm1 + db $C4, $C1, $55, $EF, $A8, $00, $01, $00, $00 // vpxor ymm5, ymm5, [r8 + 256] + db $C4, $C1, $7E, $7F, $A9, $00, $01, $00, $00 // vmovdqu [r9 + 256], ymm5 + db $C4, $41, $15, $EF, $A8, $20, $01, $00, $00 // vpxor ymm13, ymm13, [r8 + 288] + db $C4, $41, $7E, $7F, $A9, $20, $01, $00, $00 // vmovdqu [r9 + 288], ymm13 + db $C4, $C1, $6D, $EF, $90, $40, $01, $00, $00 // vpxor ymm2, ymm2, [r8 + 320] + db $C4, $C1, $7E, $7F, $91, $40, $01, $00, $00 // vmovdqu [r9 + 320], ymm2 + db $C4, $C1, $7D, $EF, $80, $60, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 352] + db $C4, $C1, $7E, $7F, $81, $60, $01, $00, $00 // vmovdqu [r9 + 352], ymm0 + db $C4, $C1, $45, $EF, $B8, $80, $01, $00, $00 // vpxor ymm7, ymm7, [r8 + 384] + db $C4, $C1, $7E, $7F, $B9, $80, $01, $00, $00 // vmovdqu [r9 + 384], ymm7 + db $C4, $41, $05, $EF, $B8, $A0, $01, $00, $00 // vpxor ymm15, ymm15, [r8 + 416] + db $C4, $41, $7E, $7F, $B9, $A0, $01, $00, $00 // vmovdqu [r9 + 416], ymm15 + db $C4, $C1, $5D, $EF, $A0, $C0, $01, $00, $00 // vpxor ymm4, ymm4, [r8 + 448] + db $C4, $C1, $7E, $7F, $A1, $C0, $01, $00, $00 // vmovdqu [r9 + 448], ymm4 + db $C4, $41, $1D, $EF, $A0, $E0, $01, $00, $00 // vpxor ymm12, ymm12, [r8 + 480] + db $C4, $41, $7E, $7F, $A1, $E0, $01, $00, $00 // vmovdqu [r9 + 480], ymm12 + add r8, 512 + add r9, 512 + dec r10 + jnz @@c7539_avx2_8x_outer + db $C5, $F8, $77 // vzeroupper + mov rsp, r11 {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc index 89324967..b0b12226 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc @@ -18,283 +18,224 @@ // (5*r), row 9 pad. Bulk reads [+32*j] (broadcast r^4); tail reads // [+32*j+4] ([r^4,r^3,r^2,r^1]). - sub esp, 320 - // ---- Prologue: mask26 (ymm5), padbit (ymm6) ---- + // ---- 64B-aligned frame; mask26/padbit stored as mem operands ---- + mov edx, esp + sub esp, 256 + and esp, -64 + mov dword ptr [esp + 224], edx db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 - db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 - db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 - db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 + db $C5, $FD, $7F, $AC, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm5 + db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 + db $C5, $D5, $73, $D5, $3F // vpsrlq ymm5, ymm5, 63 + db $C5, $D5, $73, $F5, $18 // vpsllq ymm5, ymm5, 24 + db $C5, $FD, $7F, $AC, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm5 // ---- Load H0..H4 into lane 0 ---- db $C5, $F9, $6E, $43, $24 // vmovd xmm0, dword ptr [ebx + 36] db $C5, $F9, $6E, $4B, $28 // vmovd xmm1, dword ptr [ebx + 40] db $C5, $F9, $6E, $53, $2C // vmovd xmm2, dword ptr [ebx + 44] db $C5, $F9, $6E, $5B, $30 // vmovd xmm3, dword ptr [ebx + 48] db $C5, $F9, $6E, $63, $34 // vmovd xmm4, dword ptr [ebx + 52] - // ---- Pre-loop: spill H, absorb first 4 blocks ---- - db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0 - db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1 - db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2 - db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3 - db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4 - db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi] - db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16] - db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1 - db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1 + // ---- Pre-loop: absorb first 4 blocks ---- + db $C5, $FA, $6F, $2F // vmovdqu xmm5, [edi] + db $C5, $FA, $6F, $77, $10 // vmovdqu xmm6, [edi + 16] + db $C4, $E3, $55, $38, $6F, $20, $01 // vinserti128 ymm5, ymm5, [edi + 32], 1 + db $C4, $E3, $4D, $38, $77, $30, $01 // vinserti128 ymm6, ymm6, [edi + 48], 1 add edi, 64 - db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 - db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 - db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 - db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 - db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 - db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 - db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 - db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 - db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 - db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160] - db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192] - db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224] - db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256] - db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288] + db $C5, $D5, $6D, $FE // vpunpckhqdq ymm7, ymm5, ymm6 + db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 + db $C5, $C5, $EB, $BC, $24, $C0, $00, $00, $00 // vpor ymm7, ymm7, [esp + 192] + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6 + db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6 + db $C5, $CD, $73, $DE, $06 // vpsrldq ymm6, ymm6, 6 + db $C5, $C5, $6C, $FE // vpunpcklqdq ymm7, ymm7, ymm6 + db $C5, $CD, $73, $D7, $1E // vpsrlq ymm6, ymm7, 30 + db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160] + db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6 + db $C5, $C5, $73, $D7, $04 // vpsrlq ymm7, ymm7, 4 + db $C5, $C5, $DB, $BC, $24, $A0, $00, $00, $00 // vpand ymm7, ymm7, [esp + 160] + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 + db $C5, $D5, $DB, $AC, $24, $A0, $00, $00, $00 // vpand ymm5, ymm5, [esp + 160] + db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5 sub eax, 64 jz @@poly1305_avx2_i386_tail @@poly1305_avx2_i386_loop: - // ---- Bulk: H *= r^4 (broadcast), D0..D4 spilled ---- - db $C5, $FD, $F4, $BE, $80, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $F5, $F4, $7E, $60 // vpmuludq ymm7, ymm1, [esi + 96] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $ED, $F4, $7E, $40 // vpmuludq ymm7, ymm2, [esi + 64] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $E5, $F4, $7E, $20 // vpmuludq ymm7, ymm3, [esi + 32] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $DD, $F4, $3E // vpmuludq ymm7, ymm4, [esi] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $FD, $F4, $7E, $60 // vpmuludq ymm7, ymm0, [esi + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $F5, $F4, $7E, $40 // vpmuludq ymm7, ymm1, [esi + 64] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $ED, $F4, $7E, $20 // vpmuludq ymm7, ymm2, [esi + 32] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $E5, $F4, $3E // vpmuludq ymm7, ymm3, [esi] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $DD, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 256] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $FD, $F4, $7E, $40 // vpmuludq ymm7, ymm0, [esi + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $F5, $F4, $7E, $20 // vpmuludq ymm7, ymm1, [esi + 32] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $ED, $F4, $3E // vpmuludq ymm7, ymm2, [esi] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $E5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 256] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $DD, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 224] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $FD, $F4, $7E, $20 // vpmuludq ymm7, ymm0, [esi + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $F5, $F4, $3E // vpmuludq ymm7, ymm1, [esi] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $ED, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 256] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $E5, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 224] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $DD, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 192] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $FD, $F4, $3E // vpmuludq ymm7, ymm0, [esi] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $F5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 256] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $ED, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 224] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $E5, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 192] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $DD, $F4, $BE, $A0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 160] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - // ---- Lazy reduce ---- - db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp] - db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32] - db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64] - db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96] - db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128] - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + // ---- Bulk: H *= r^4 (broadcast), k-major resident accumulator ---- + db $C5, $FD, $F4, $AE, $80, $00, $00, $00 // vpmuludq ymm5, ymm0, [esi + 128] + db $C5, $F5, $F4, $76, $60 // vpmuludq ymm6, ymm1, [esi + 96] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $76, $40 // vpmuludq ymm6, ymm2, [esi + 64] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $76, $20 // vpmuludq ymm6, ymm3, [esi + 32] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $36 // vpmuludq ymm6, ymm4, [esi] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $AC, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm5 + db $C5, $FD, $F4, $6E, $60 // vpmuludq ymm5, ymm0, [esi + 96] + db $C5, $F5, $F4, $76, $40 // vpmuludq ymm6, ymm1, [esi + 64] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $76, $20 // vpmuludq ymm6, ymm2, [esi + 32] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $36 // vpmuludq ymm6, ymm3, [esi] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm4, [esi + 256] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $60 // vmovdqa [esp + 96], ymm5 + db $C5, $FD, $F4, $6E, $40 // vpmuludq ymm5, ymm0, [esi + 64] + db $C5, $F5, $F4, $76, $20 // vpmuludq ymm6, ymm1, [esi + 32] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $36 // vpmuludq ymm6, ymm2, [esi] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm3, [esi + 256] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 224] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $40 // vmovdqa [esp + 64], ymm5 + db $C5, $FD, $F4, $6E, $20 // vpmuludq ymm5, ymm0, [esi + 32] + db $C5, $F5, $F4, $36 // vpmuludq ymm6, ymm1, [esi] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm2, [esi + 256] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 224] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $C0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 192] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $20 // vmovdqa [esp + 32], ymm5 + db $C5, $FD, $F4, $2E // vpmuludq ymm5, ymm0, [esi] + db $C5, $F5, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm1, [esi + 256] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm2, [esi + 224] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $C0, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 192] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $A0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 160] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + // ---- Lazy reduce (D0 from ymm5) ---- + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C5, $D5, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm5, [esp + 160] + db $C5, $CD, $D4, $4C, $24, $20 // vpaddq ymm1, ymm6, [esp + 32] + db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26 + db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160] + db $C5, $CD, $D4, $54, $24, $40 // vpaddq ymm2, ymm6, [esp + 64] + db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26 + db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160] + db $C5, $CD, $D4, $5C, $24, $60 // vpaddq ymm3, ymm6, [esp + 96] + db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26 + db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160] + db $C5, $CD, $D4, $A4, $24, $80, $00, $00, $00 // vpaddq ymm4, ymm6, [esp + 128] + db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26 + db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160] + db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2 + db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7 + db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6 + db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26 + db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 // ---- Absorb next 4 blocks ---- - db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0 - db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1 - db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2 - db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3 - db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4 - db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi] - db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16] - db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1 - db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1 + db $C5, $FA, $6F, $2F // vmovdqu xmm5, [edi] + db $C5, $FA, $6F, $77, $10 // vmovdqu xmm6, [edi + 16] + db $C4, $E3, $55, $38, $6F, $20, $01 // vinserti128 ymm5, ymm5, [edi + 32], 1 + db $C4, $E3, $4D, $38, $77, $30, $01 // vinserti128 ymm6, ymm6, [edi + 48], 1 add edi, 64 - db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 - db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 - db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 - db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 - db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 - db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 - db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 - db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 - db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 - db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160] - db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192] - db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224] - db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256] - db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288] + db $C5, $D5, $6D, $FE // vpunpckhqdq ymm7, ymm5, ymm6 + db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 + db $C5, $C5, $EB, $BC, $24, $C0, $00, $00, $00 // vpor ymm7, ymm7, [esp + 192] + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6 + db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6 + db $C5, $CD, $73, $DE, $06 // vpsrldq ymm6, ymm6, 6 + db $C5, $C5, $6C, $FE // vpunpcklqdq ymm7, ymm7, ymm6 + db $C5, $CD, $73, $D7, $1E // vpsrlq ymm6, ymm7, 30 + db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160] + db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6 + db $C5, $C5, $73, $D7, $04 // vpsrlq ymm7, ymm7, 4 + db $C5, $C5, $DB, $BC, $24, $A0, $00, $00, $00 // vpand ymm7, ymm7, [esp + 160] + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 + db $C5, $D5, $DB, $AC, $24, $A0, $00, $00, $00 // vpand ymm5, ymm5, [esp + 160] + db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5 sub eax, 64 jnz @@poly1305_avx2_i386_loop @@poly1305_avx2_i386_tail: // ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ---- - db $C5, $FD, $F4, $BE, $84, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 132] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $F5, $F4, $7E, $64 // vpmuludq ymm7, ymm1, [esi + 100] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $ED, $F4, $7E, $44 // vpmuludq ymm7, ymm2, [esi + 68] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $E5, $F4, $7E, $24 // vpmuludq ymm7, ymm3, [esi + 36] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $DD, $F4, $7E, $04 // vpmuludq ymm7, ymm4, [esi + 4] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 - db $C5, $FD, $F4, $7E, $64 // vpmuludq ymm7, ymm0, [esi + 100] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $F5, $F4, $7E, $44 // vpmuludq ymm7, ymm1, [esi + 68] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $ED, $F4, $7E, $24 // vpmuludq ymm7, ymm2, [esi + 36] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $E5, $F4, $7E, $04 // vpmuludq ymm7, ymm3, [esi + 4] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $DD, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 260] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 - db $C5, $FD, $F4, $7E, $44 // vpmuludq ymm7, ymm0, [esi + 68] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $F5, $F4, $7E, $24 // vpmuludq ymm7, ymm1, [esi + 36] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $ED, $F4, $7E, $04 // vpmuludq ymm7, ymm2, [esi + 4] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $E5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 260] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $DD, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 228] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 - db $C5, $FD, $F4, $7E, $24 // vpmuludq ymm7, ymm0, [esi + 36] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $F5, $F4, $7E, $04 // vpmuludq ymm7, ymm1, [esi + 4] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $ED, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 260] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $E5, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 228] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $DD, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 196] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 - db $C5, $FD, $F4, $7E, $04 // vpmuludq ymm7, ymm0, [esi + 4] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $F5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 260] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $ED, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 228] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $E5, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 196] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $DD, $F4, $BE, $A4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 164] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $FD, $F4, $AE, $84, $00, $00, $00 // vpmuludq ymm5, ymm0, [esi + 132] + db $C5, $F5, $F4, $76, $64 // vpmuludq ymm6, ymm1, [esi + 100] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $76, $44 // vpmuludq ymm6, ymm2, [esi + 68] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $76, $24 // vpmuludq ymm6, ymm3, [esi + 36] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $76, $04 // vpmuludq ymm6, ymm4, [esi + 4] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $AC, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm5 + db $C5, $FD, $F4, $6E, $64 // vpmuludq ymm5, ymm0, [esi + 100] + db $C5, $F5, $F4, $76, $44 // vpmuludq ymm6, ymm1, [esi + 68] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $76, $24 // vpmuludq ymm6, ymm2, [esi + 36] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $76, $04 // vpmuludq ymm6, ymm3, [esi + 4] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm4, [esi + 260] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $60 // vmovdqa [esp + 96], ymm5 + db $C5, $FD, $F4, $6E, $44 // vpmuludq ymm5, ymm0, [esi + 68] + db $C5, $F5, $F4, $76, $24 // vpmuludq ymm6, ymm1, [esi + 36] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $76, $04 // vpmuludq ymm6, ymm2, [esi + 4] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm3, [esi + 260] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 228] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $40 // vmovdqa [esp + 64], ymm5 + db $C5, $FD, $F4, $6E, $24 // vpmuludq ymm5, ymm0, [esi + 36] + db $C5, $F5, $F4, $76, $04 // vpmuludq ymm6, ymm1, [esi + 4] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm2, [esi + 260] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 228] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $C4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 196] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $FD, $7F, $6C, $24, $20 // vmovdqa [esp + 32], ymm5 + db $C5, $FD, $F4, $6E, $04 // vpmuludq ymm5, ymm0, [esi + 4] + db $C5, $F5, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm1, [esi + 260] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $ED, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm2, [esi + 228] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $E5, $F4, $B6, $C4, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 196] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 + db $C5, $DD, $F4, $B6, $A4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 164] + db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6 // ---- Tail lazy reduce ---- - db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp] - db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32] - db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64] - db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96] - db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128] - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C5, $D5, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm5, [esp + 160] + db $C5, $CD, $D4, $4C, $24, $20 // vpaddq ymm1, ymm6, [esp + 32] + db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26 + db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160] + db $C5, $CD, $D4, $54, $24, $40 // vpaddq ymm2, ymm6, [esp + 64] + db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26 + db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160] + db $C5, $CD, $D4, $5C, $24, $60 // vpaddq ymm3, ymm6, [esp + 96] + db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26 + db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160] + db $C5, $CD, $D4, $A4, $24, $80, $00, $00, $00 // vpaddq ymm4, ymm6, [esp + 128] + db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26 + db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160] + db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2 + db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7 + db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6 + db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26 + db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 // ---- Horizontal sum 4 lanes -> lane 0 ---- db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8 db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 @@ -317,27 +258,26 @@ db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 // ---- Final lazy reduce (lane 0 scalars) ---- - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26 + db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 + db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26 + db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160] + db $C5, $ED, $D4, $D6 // vpaddq ymm2, ymm2, ymm6 + db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26 + db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160] + db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6 + db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26 + db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160] + db $C5, $DD, $D4, $E6 // vpaddq ymm4, ymm4, ymm6 + db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26 + db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160] + db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2 + db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7 + db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6 + db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26 + db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 // ---- Store H0..H4 back to ctx ---- db $C5, $F9, $7E, $43, $24 // vmovd dword ptr [ebx + 36], xmm0 db $C5, $F9, $7E, $4B, $28 // vmovd dword ptr [ebx + 40], xmm1 @@ -345,7 +285,7 @@ db $C5, $F9, $7E, $5B, $30 // vmovd dword ptr [ebx + 48], xmm3 db $C5, $F9, $7E, $63, $34 // vmovd dword ptr [ebx + 52], xmm4 db $C5, $F8, $77 // vzeroupper - add esp, 320 + mov esp, dword ptr [esp + 224] pop edi pop esi pop ebx diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc index 50df2c65..0d41bd35 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc @@ -17,146 +17,158 @@ // [+32*j+4] ([r^4,r^3,r^2,r^1]). {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - // ---- Prologue: mask26 (ymm5), padbit (ymm6) ---- - db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 - db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 - db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 - db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 - db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 + // ---- Prologue: mask26 -> ymm15 + [rsp], padbit -> [rsp+32] ---- + mov r11, rsp + sub rsp, 64 + and rsp, -64 + db $C4, $41, $05, $76, $FF // vpcmpeqd ymm15, ymm15, ymm15 + db $C4, $C1, $05, $73, $D7, $26 // vpsrlq ymm15, ymm15, 38 + db $C5, $7D, $7F, $3C, $24 // vmovdqa [rsp + 0], ymm15 + db $C5, $C5, $76, $FF // vpcmpeqd ymm7, ymm7, ymm7 + db $C5, $C5, $73, $D7, $3F // vpsrlq ymm7, ymm7, 63 + db $C5, $C5, $73, $F7, $18 // vpsllq ymm7, ymm7, 24 + db $C5, $FD, $7F, $7C, $24, $20 // vmovdqa [rsp + 32], ymm7 // ---- Load H0..H4 into lane 0 ---- db $C5, $F9, $6E, $41, $24 // vmovd xmm0, dword ptr [rcx + 36] db $C5, $F9, $6E, $49, $28 // vmovd xmm1, dword ptr [rcx + 40] db $C5, $F9, $6E, $51, $2C // vmovd xmm2, dword ptr [rcx + 44] db $C5, $F9, $6E, $59, $30 // vmovd xmm3, dword ptr [rcx + 48] db $C5, $F9, $6E, $61, $34 // vmovd xmm4, dword ptr [rcx + 52] - // ---- Pre-loop: absorb first 4 blocks ---- - db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8] - db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16] - db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1 - db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1 + // ---- Pre-loop: split first 4 blocks into T; h2 absorbs early ---- + db $C4, $C1, $7A, $6F, $28 // vmovdqu xmm5, [r8] + db $C4, $C3, $55, $38, $68, $20, $01 // vinserti128 ymm5, ymm5, [r8 + 32], 1 + db $C4, $C1, $7A, $6F, $70, $10 // vmovdqu xmm6, [r8 + 16] + db $C4, $C3, $4D, $38, $70, $30, $01 // vinserti128 ymm6, ymm6, [r8 + 48], 1 add r8, 64 - db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 - db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 - db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 - db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 - db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 - db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 - db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 - db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 - db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 - db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 - db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 - db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 - db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 - db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 - db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6 + db $C5, $BD, $73, $DE, $06 // vpsrldq ymm8, ymm6, 6 + db $C5, $55, $6D, $CE // vpunpckhqdq ymm9, ymm5, ymm6 + db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6 + db $C4, $41, $45, $6C, $C0 // vpunpcklqdq ymm8, ymm7, ymm8 + db $C4, $C1, $45, $73, $D0, $04 // vpsrlq ymm7, ymm8, 4 + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C4, $C1, $3D, $73, $D0, $1E // vpsrlq ymm8, ymm8, 30 + db $C4, $C1, $35, $73, $D1, $28 // vpsrlq ymm9, ymm9, 40 + db $C4, $C1, $55, $DB, $EF // vpand ymm5, ymm5, ymm15 + db $C4, $C1, $4D, $DB, $F7 // vpand ymm6, ymm6, ymm15 + db $C4, $C1, $45, $DB, $FF // vpand ymm7, ymm7, ymm15 + db $C4, $41, $3D, $DB, $C7 // vpand ymm8, ymm8, ymm15 + db $C5, $35, $EB, $4C, $24, $20 // vpor ymm9, ymm9, [rsp + 32] + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 sub r9, 64 jz @@poly1305_avx2_tail @@poly1305_avx2_loop: - // ---- Bulk: H *= r^4 (broadcast) ---- - db $C5, $7D, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 128] - db $C5, $75, $F4, $6A, $60 // vpmuludq ymm13, ymm1, [rdx + 96] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $6D, $F4, $6A, $40 // vpmuludq ymm13, ymm2, [rdx + 64] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $65, $F4, $6A, $20 // vpmuludq ymm13, ymm3, [rdx + 32] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $5D, $F4, $2A // vpmuludq ymm13, ymm4, [rdx] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $7D, $F4, $5A, $60 // vpmuludq ymm11, ymm0, [rdx + 96] - db $C5, $75, $F4, $6A, $40 // vpmuludq ymm13, ymm1, [rdx + 64] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $6D, $F4, $6A, $20 // vpmuludq ymm13, ymm2, [rdx + 32] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $65, $F4, $2A // vpmuludq ymm13, ymm3, [rdx] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $5D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 256] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $7D, $F4, $52, $40 // vpmuludq ymm10, ymm0, [rdx + 64] - db $C5, $75, $F4, $6A, $20 // vpmuludq ymm13, ymm1, [rdx + 32] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $6D, $F4, $2A // vpmuludq ymm13, ymm2, [rdx] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $65, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 256] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $5D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 224] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $7D, $F4, $4A, $20 // vpmuludq ymm9, ymm0, [rdx + 32] - db $C5, $75, $F4, $2A // vpmuludq ymm13, ymm1, [rdx] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $6D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 256] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $65, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 224] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $5D, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 192] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $7D, $F4, $02 // vpmuludq ymm8, ymm0, [rdx] - db $C5, $75, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm1, [rdx + 256] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $6D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 224] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $65, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 192] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $5D, $F4, $AA, $A0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 160] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - // ---- Lazy reduce (two interleaved carry chains) ---- - db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26 - db $C4, $C1, $0D, $73, $D3, $1A // vpsrlq ymm14, ymm11, 26 - db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5 - db $C5, $25, $DB, $DD // vpand ymm11, ymm11, ymm5 - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C4, $41, $1D, $D4, $E6 // vpaddq ymm12, ymm12, ymm14 - db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26 - db $C4, $C1, $0D, $73, $D4, $1A // vpsrlq ymm14, ymm12, 26 - db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5 - db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5 - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C4, $C1, $05, $73, $F6, $02 // vpsllq ymm15, ymm14, 2 - db $C4, $41, $0D, $D4, $F7 // vpaddq ymm14, ymm14, ymm15 + db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5 + db $C5, $FE, $6F, $2A // vmovdqu ymm5, [rdx] + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 + db $C5, $FE, $6F, $72, $20 // vmovdqu ymm6, [rdx + 32] + db $C4, $C1, $65, $D4, $D8 // vpaddq ymm3, ymm3, ymm8 + db $C5, $FE, $6F, $7A, $40 // vmovdqu ymm7, [rdx + 64] + db $C4, $C1, $5D, $D4, $E1 // vpaddq ymm4, ymm4, ymm9 + db $C5, $7E, $6F, $82, $E0, $00, $00, $00 // vmovdqu ymm8, [rdx + 224] + db $C5, $7E, $6F, $BA, $00, $01, $00, $00 // vmovdqu ymm15, [rdx + 256] + db $C5, $6D, $F4, $E5 // vpmuludq ymm12, ymm2, ymm5 + db $C5, $6D, $F4, $EE // vpmuludq ymm13, ymm2, ymm6 + db $C5, $6D, $F4, $F7 // vpmuludq ymm14, ymm2, ymm7 + db $C4, $41, $6D, $F4, $D0 // vpmuludq ymm10, ymm2, ymm8 + db $C4, $41, $6D, $F4, $DF // vpmuludq ymm11, ymm2, ymm15 + db $C5, $7D, $F4, $CE // vpmuludq ymm9, ymm0, ymm6 + db $C5, $F5, $F4, $D6 // vpmuludq ymm2, ymm1, ymm6 + db $C4, $41, $25, $D4, $D9 // vpaddq ymm11, ymm11, ymm9 + db $C5, $1D, $D4, $E2 // vpaddq ymm12, ymm12, ymm2 + db $C5, $65, $F4, $CE // vpmuludq ymm9, ymm3, ymm6 + db $C5, $DD, $F4, $92, $A0, $00, $00, $00 // vpmuludq ymm2, ymm4, [rdx + 160] + db $C4, $41, $0D, $D4, $F1 // vpaddq ymm14, ymm14, ymm9 + db $C5, $2D, $D4, $D2 // vpaddq ymm10, ymm10, ymm2 + db $C5, $FE, $6F, $B2, $C0, $00, $00, $00 // vmovdqu ymm6, [rdx + 192] + db $C5, $7D, $F4, $CD // vpmuludq ymm9, ymm0, ymm5 + db $C5, $F5, $F4, $D5 // vpmuludq ymm2, ymm1, ymm5 + db $C4, $41, $2D, $D4, $D1 // vpaddq ymm10, ymm10, ymm9 + db $C5, $25, $D4, $DA // vpaddq ymm11, ymm11, ymm2 + db $C5, $65, $F4, $CD // vpmuludq ymm9, ymm3, ymm5 + db $C5, $DD, $F4, $D5 // vpmuludq ymm2, ymm4, ymm5 + db $C4, $C1, $7A, $6F, $28 // vmovdqu xmm5, [r8] + db $C4, $41, $15, $D4, $E9 // vpaddq ymm13, ymm13, ymm9 + db $C5, $0D, $D4, $F2 // vpaddq ymm14, ymm14, ymm2 + db $C4, $C3, $55, $38, $68, $20, $01 // vinserti128 ymm5, ymm5, [r8 + 32], 1 + db $C5, $65, $F4, $CE // vpmuludq ymm9, ymm3, ymm6 + db $C5, $DD, $F4, $D6 // vpmuludq ymm2, ymm4, ymm6 + db $C4, $C1, $7A, $6F, $70, $10 // vmovdqu xmm6, [r8 + 16] + db $C4, $41, $2D, $D4, $D1 // vpaddq ymm10, ymm10, ymm9 + db $C5, $25, $D4, $DA // vpaddq ymm11, ymm11, ymm2 + db $C5, $FE, $6F, $52, $60 // vmovdqu ymm2, [rdx + 96] + db $C5, $75, $F4, $CF // vpmuludq ymm9, ymm1, ymm7 + db $C5, $FD, $F4, $FF // vpmuludq ymm7, ymm0, ymm7 + db $C4, $41, $15, $D4, $E9 // vpaddq ymm13, ymm13, ymm9 + db $C5, $1D, $D4, $E7 // vpaddq ymm12, ymm12, ymm7 + db $C4, $C3, $4D, $38, $70, $30, $01 // vinserti128 ymm6, ymm6, [r8 + 48], 1 + add r8, 64 + db $C5, $75, $F4, $CA // vpmuludq ymm9, ymm1, ymm2 + db $C5, $FD, $F4, $D2 // vpmuludq ymm2, ymm0, ymm2 + db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6 + db $C4, $41, $0D, $D4, $F1 // vpaddq ymm14, ymm14, ymm9 + db $C5, $15, $D4, $EA // vpaddq ymm13, ymm13, ymm2 + db $C4, $41, $65, $F4, $C8 // vpmuludq ymm9, ymm3, ymm8 + db $C4, $C1, $5D, $F4, $D0 // vpmuludq ymm2, ymm4, ymm8 + db $C5, $BD, $73, $DE, $06 // vpsrldq ymm8, ymm6, 6 + db $C4, $41, $25, $D4, $D9 // vpaddq ymm11, ymm11, ymm9 + db $C5, $1D, $D4, $E2 // vpaddq ymm12, ymm12, ymm2 + db $C5, $55, $6D, $CE // vpunpckhqdq ymm9, ymm5, ymm6 + db $C4, $C1, $65, $F4, $DF // vpmuludq ymm3, ymm3, ymm15 + db $C4, $C1, $5D, $F4, $E7 // vpmuludq ymm4, ymm4, ymm15 + db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6 + db $C5, $9D, $D4, $D3 // vpaddq ymm2, ymm12, ymm3 + db $C5, $95, $D4, $DC // vpaddq ymm3, ymm13, ymm4 + db $C4, $41, $45, $6C, $C0 // vpunpcklqdq ymm8, ymm7, ymm8 + db $C5, $FD, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm4, ymm0, [rdx + 128] + db $C4, $C1, $75, $F4, $C7 // vpmuludq ymm0, ymm1, ymm15 + db $C5, $7D, $6F, $3C, $24 // vmovdqa ymm15, [rsp + 0] + db $C4, $C1, $5D, $D4, $E6 // vpaddq ymm4, ymm4, ymm14 + db $C4, $C1, $7D, $D4, $C2 // vpaddq ymm0, ymm0, ymm10 + db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26 + db $C4, $C1, $65, $DB, $DF // vpand ymm3, ymm3, ymm15 + db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13 + db $C5, $AD, $73, $D0, $1A // vpsrlq ymm10, ymm0, 26 + db $C4, $C1, $7D, $DB, $C7 // vpand ymm0, ymm0, ymm15 + db $C4, $C1, $25, $D4, $CA // vpaddq ymm1, ymm11, ymm10 + db $C5, $8D, $73, $D4, $1A // vpsrlq ymm14, ymm4, 26 + db $C4, $C1, $5D, $DB, $E7 // vpand ymm4, ymm4, ymm15 + db $C4, $C1, $45, $73, $D0, $04 // vpsrlq ymm7, ymm8, 4 + db $C5, $A5, $73, $D1, $1A // vpsrlq ymm11, ymm1, 26 + db $C4, $C1, $75, $DB, $CF // vpand ymm1, ymm1, ymm15 + db $C4, $C1, $6D, $D4, $D3 // vpaddq ymm2, ymm2, ymm11 db $C4, $C1, $7D, $D4, $C6 // vpaddq ymm0, ymm0, ymm14 - db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26 - db $C5, $8D, $73, $D0, $1A // vpsrlq ymm14, ymm0, 26 - db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C4, $C1, $75, $D4, $CE // vpaddq ymm1, ymm1, ymm14 - db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26 - db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5 + db $C4, $C1, $0D, $73, $F6, $02 // vpsllq ymm14, ymm14, 2 + db $C4, $C1, $7D, $D4, $C6 // vpaddq ymm0, ymm0, ymm14 + db $C4, $C1, $45, $DB, $FF // vpand ymm7, ymm7, ymm15 + db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26 + db $C5, $9D, $73, $D2, $1A // vpsrlq ymm12, ymm2, 26 + db $C4, $C1, $6D, $DB, $D7 // vpand ymm2, ymm2, ymm15 + db $C4, $C1, $65, $D4, $DC // vpaddq ymm3, ymm3, ymm12 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C4, $C1, $3D, $73, $D0, $1E // vpsrlq ymm8, ymm8, 30 + db $C5, $AD, $73, $D0, $1A // vpsrlq ymm10, ymm0, 26 + db $C4, $C1, $7D, $DB, $C7 // vpand ymm0, ymm0, ymm15 + db $C4, $C1, $75, $D4, $CA // vpaddq ymm1, ymm1, ymm10 + db $C4, $C1, $35, $73, $D1, $28 // vpsrlq ymm9, ymm9, 40 + db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26 + db $C4, $C1, $65, $DB, $DF // vpand ymm3, ymm3, ymm15 db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13 - // ---- Absorb next 4 blocks ---- - db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8] - db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16] - db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1 - db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1 - add r8, 64 - db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 - db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 - db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 - db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 - db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 - db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 - db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 - db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 - db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 - db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 - db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 - db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 - db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 - db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 - db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C4, $C1, $55, $DB, $EF // vpand ymm5, ymm5, ymm15 + db $C4, $C1, $4D, $DB, $F7 // vpand ymm6, ymm6, ymm15 + db $C4, $41, $3D, $DB, $C7 // vpand ymm8, ymm8, ymm15 + db $C5, $35, $EB, $4C, $24, $20 // vpor ymm9, ymm9, [rsp + 32] sub r9, 64 jnz @@poly1305_avx2_loop @@poly1305_avx2_tail: + // ---- Absorb the final pending limbs (T2 already folded) ---- + db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6 + db $C4, $C1, $65, $D4, $D8 // vpaddq ymm3, ymm3, ymm8 + db $C4, $C1, $5D, $D4, $E1 // vpaddq ymm4, ymm4, ymm9 + db $C5, $FD, $6F, $2C, $24 // vmovdqa ymm5, [rsp + 0] // ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ---- db $C5, $7D, $F4, $A2, $84, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 132] db $C5, $75, $F4, $6A, $64 // vpmuludq ymm13, ymm1, [rdx + 100] @@ -273,4 +285,5 @@ db $C5, $F9, $7E, $59, $30 // vmovd dword ptr [rcx + 48], xmm3 db $C5, $F9, $7E, $61, $34 // vmovd dword ptr [rcx + 52], xmm4 db $C5, $F8, $77 // vzeroupper + mov rsp, r11 {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc index 7c288564..adf29b51 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc @@ -5,11 +5,11 @@ // 2 blocks per iteration against an r/r^2 power table, then a single // pshufd-based horizontal fold of the 2 lanes. // -// i386 has only xmm0..7 (no callee-saved xmm), so the accumulator -// H0..H4 and the transient products D0..D4 live in a 160-byte esp -// stack frame; xmm7 = mask26 and xmm6 = padbit stay resident. Every -// table/state access is an unaligned movdqu load/store (SSE memory- -// operand arithmetic would #GP on the non-16-aligned power table). +// i386 has only xmm0..7 (no callee-saved xmm): H0..H4 stay RESIDENT +// in xmm0..4, xmm5..7 are the accumulator/product temps, and the +// 64B-aligned esp frame holds a one-time copy of the bulk table rows +// plus mask26/padbit, making SSE memory-operand arithmetic against +// [esp+off] legal and every hot access aligned. // // On entry (after ClpSimdProc5Begin_i386.inc): // ebx = ACtx (72-byte R/S/H/K state); esi = APowTable (160-byte r/r^2) @@ -19,490 +19,388 @@ // rows 5..8 s-limbs (5*r), row 9 pad. Bulk reads [+16*j] (broadcast r^2); // tail reads [+16*j+4] ([r^2,r^1]). - sub esp, 160 - // ---- Prologue: mask26 (xmm7), padbit (xmm6) ---- - db $66, $0F, $76, $FF // pcmpeqd xmm7, xmm7 - db $66, $0F, $73, $D7, $26 // psrlq xmm7, 38 - db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6 - db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63 - db $66, $0F, $73, $F6, $18 // psllq xmm6, 24 - // ---- Load H0..H4 (lane 0) into the stack frame ---- + // ---- 64B-aligned frame: table copy + D slots + mask/pad ---- + mov edx, esp + sub esp, 256 + and esp, -64 + mov dword ptr [esp + 240], edx + db $F3, $0F, $6F, $2E // movdqu xmm5, [esi] + db $66, $0F, $7F, $2C, $24 // movdqa [esp + 0], xmm5 + db $F3, $0F, $6F, $6E, $10 // movdqu xmm5, [esi + 16] + db $66, $0F, $7F, $6C, $24, $10 // movdqa [esp + 16], xmm5 + db $F3, $0F, $6F, $6E, $20 // movdqu xmm5, [esi + 32] + db $66, $0F, $7F, $6C, $24, $20 // movdqa [esp + 32], xmm5 + db $F3, $0F, $6F, $6E, $30 // movdqu xmm5, [esi + 48] + db $66, $0F, $7F, $6C, $24, $30 // movdqa [esp + 48], xmm5 + db $F3, $0F, $6F, $6E, $40 // movdqu xmm5, [esi + 64] + db $66, $0F, $7F, $6C, $24, $40 // movdqa [esp + 64], xmm5 + db $F3, $0F, $6F, $6E, $50 // movdqu xmm5, [esi + 80] + db $66, $0F, $7F, $6C, $24, $50 // movdqa [esp + 80], xmm5 + db $F3, $0F, $6F, $6E, $60 // movdqu xmm5, [esi + 96] + db $66, $0F, $7F, $6C, $24, $60 // movdqa [esp + 96], xmm5 + db $F3, $0F, $6F, $6E, $70 // movdqu xmm5, [esi + 112] + db $66, $0F, $7F, $6C, $24, $70 // movdqa [esp + 112], xmm5 + db $F3, $0F, $6F, $AE, $80, $00, $00, $00 // movdqu xmm5, [esi + 128] + db $66, $0F, $7F, $AC, $24, $80, $00, $00, $00 // movdqa [esp + 128], xmm5 + db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5 + db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38 + db $66, $0F, $7F, $AC, $24, $D0, $00, $00, $00 // movdqa [esp + 208], xmm5 + db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5 + db $66, $0F, $73, $D5, $3F // psrlq xmm5, 63 + db $66, $0F, $73, $F5, $18 // psllq xmm5, 24 + db $66, $0F, $7F, $AC, $24, $E0, $00, $00, $00 // movdqa [esp + 224], xmm5 + // ---- Load H0..H4 into lane 0 (resident) ---- db $66, $0F, $6E, $43, $24 // movd xmm0, dword ptr [ebx + 36] - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $66, $0F, $6E, $43, $28 // movd xmm0, dword ptr [ebx + 40] - db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 - db $66, $0F, $6E, $43, $2C // movd xmm0, dword ptr [ebx + 44] - db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 - db $66, $0F, $6E, $43, $30 // movd xmm0, dword ptr [ebx + 48] - db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 - db $66, $0F, $6E, $43, $34 // movd xmm0, dword ptr [ebx + 52] - db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + db $66, $0F, $6E, $4B, $28 // movd xmm1, dword ptr [ebx + 40] + db $66, $0F, $6E, $53, $2C // movd xmm2, dword ptr [ebx + 44] + db $66, $0F, $6E, $5B, $30 // movd xmm3, dword ptr [ebx + 48] + db $66, $0F, $6E, $63, $34 // movd xmm4, dword ptr [ebx + 52] // ---- Pre-loop: absorb first 2 blocks ---- - db $F3, $0F, $6F, $07 // movdqu xmm0, [edi] - db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16] + db $F3, $0F, $6F, $2F // movdqu xmm5, [edi] + db $F3, $0F, $6F, $77, $10 // movdqu xmm6, [edi + 16] add edi, 32 - db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0 - db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6 - db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1 - db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6 - db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0 - db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1 - db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1 - db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 - db $66, $0F, $6F, $DA // movdqa xmm3, xmm2 - db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30 - db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4 - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $66, $0F, $DB, $CF // pand xmm1, xmm7 - db $66, $0F, $DB, $D7 // pand xmm2, xmm7 - db $66, $0F, $DB, $DF // pand xmm3, xmm7 - db $66, $0F, $EB, $E6 // por xmm4, xmm6 - db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp] - db $66, $0F, $D4, $E8 // paddq xmm5, xmm0 - db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5 - db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16] - db $66, $0F, $D4, $E9 // paddq xmm5, xmm1 - db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5 - db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32] - db $66, $0F, $D4, $EA // paddq xmm5, xmm2 - db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5 - db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48] - db $66, $0F, $D4, $EB // paddq xmm5, xmm3 - db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5 - db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64] - db $66, $0F, $D4, $EC // paddq xmm5, xmm4 - db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5 + db $66, $0F, $6F, $FD // movdqa xmm7, xmm5 + db $66, $0F, $6D, $FE // punpckhqdq xmm7, xmm6 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $0F, $EB, $BC, $24, $E0, $00, $00, $00 // por xmm7, [esp + 224] + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $FD // movdqa xmm7, xmm5 + db $66, $0F, $73, $DF, $06 // psrldq xmm7, 6 + db $66, $0F, $6C, $EE // punpcklqdq xmm5, xmm6 + db $66, $0F, $73, $DE, $06 // psrldq xmm6, 6 + db $66, $0F, $6C, $FE // punpcklqdq xmm7, xmm6 + db $66, $0F, $6F, $F7 // movdqa xmm6, xmm7 + db $66, $0F, $73, $D6, $1E // psrlq xmm6, 30 + db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208] + db $66, $0F, $D4, $DE // paddq xmm3, xmm6 + db $66, $0F, $73, $D7, $04 // psrlq xmm7, 4 + db $66, $0F, $DB, $BC, $24, $D0, $00, $00, $00 // pand xmm7, [esp + 208] + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 + db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208] + db $66, $0F, $D4, $C5 // paddq xmm0, xmm5 sub eax, 32 jz @@poly1305_sse2_tail @@poly1305_sse2_loop: - // ---- Bulk: H *= r^2 (broadcast) ---- - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $40 // movdqu xmm1, [esi + 64] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $30 // movdqu xmm2, [esi + 48] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $30 // movdqu xmm1, [esi + 48] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $20 // movdqu xmm1, [esi + 32] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $10 // movdqu xmm1, [esi + 16] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $0E // movdqu xmm1, [esi] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $50 // movdqu xmm2, [esi + 80] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0 - // ---- Lazy reduce D -> H ---- - db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2 - db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 - db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2 - db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 - db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2 - db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 - db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2 - db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 - db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 - db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 - db $66, $0F, $D4, $CA // paddq xmm1, xmm2 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + // ---- Bulk: H *= r^2 (broadcast), k-major resident accumulator ---- + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $66, $0F, $F4, $6C, $24, $40 // pmuludq xmm5, [esp + 64] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $F4, $74, $24, $30 // pmuludq xmm6, [esp + 48] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $F4, $74, $24, $20 // pmuludq xmm6, [esp + 32] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $C0, $00, $00, $00 // movdqa [esp + 192], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $66, $0F, $F4, $6C, $24, $30 // pmuludq xmm5, [esp + 48] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $F4, $74, $24, $20 // pmuludq xmm6, [esp + 32] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $B0, $00, $00, $00 // movdqa [esp + 176], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $66, $0F, $F4, $6C, $24, $20 // pmuludq xmm5, [esp + 32] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $A0, $00, $00, $00 // movdqa [esp + 160], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $66, $0F, $F4, $6C, $24, $10 // pmuludq xmm5, [esp + 16] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $F4, $74, $24, $60 // pmuludq xmm6, [esp + 96] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $90, $00, $00, $00 // movdqa [esp + 144], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $66, $0F, $F4, $2C, $24 // pmuludq xmm5, [esp + 0] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $F4, $74, $24, $60 // pmuludq xmm6, [esp + 96] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $F4, $74, $24, $50 // pmuludq xmm6, [esp + 80] + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + // ---- Lazy reduce (D0 from xmm5) ---- + db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208] + db $66, $0F, $6F, $C5 // movdqa xmm0, xmm5 + db $66, $0F, $6F, $CE // movdqa xmm1, xmm6 + db $66, $0F, $D4, $8C, $24, $90, $00, $00, $00 // paddq xmm1, [esp + 144] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208] + db $66, $0F, $6F, $D6 // movdqa xmm2, xmm6 + db $66, $0F, $D4, $94, $24, $A0, $00, $00, $00 // paddq xmm2, [esp + 160] + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208] + db $66, $0F, $6F, $DE // movdqa xmm3, xmm6 + db $66, $0F, $D4, $9C, $24, $B0, $00, $00, $00 // paddq xmm3, [esp + 176] + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208] + db $66, $0F, $6F, $E6 // movdqa xmm4, xmm6 + db $66, $0F, $D4, $A4, $24, $C0, $00, $00, $00 // paddq xmm4, [esp + 192] + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208] + db $66, $0F, $6F, $FE // movdqa xmm7, xmm6 + db $66, $0F, $73, $F7, $02 // psllq xmm7, 2 + db $66, $0F, $D4, $F7 // paddq xmm6, xmm7 + db $66, $0F, $D4, $C6 // paddq xmm0, xmm6 + db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 // ---- Absorb next 2 blocks ---- - db $F3, $0F, $6F, $07 // movdqu xmm0, [edi] - db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16] + db $F3, $0F, $6F, $2F // movdqu xmm5, [edi] + db $F3, $0F, $6F, $77, $10 // movdqu xmm6, [edi + 16] add edi, 32 - db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0 - db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6 - db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1 - db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6 - db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0 - db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1 - db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1 - db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 - db $66, $0F, $6F, $DA // movdqa xmm3, xmm2 - db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30 - db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4 - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $66, $0F, $DB, $CF // pand xmm1, xmm7 - db $66, $0F, $DB, $D7 // pand xmm2, xmm7 - db $66, $0F, $DB, $DF // pand xmm3, xmm7 - db $66, $0F, $EB, $E6 // por xmm4, xmm6 - db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp] - db $66, $0F, $D4, $E8 // paddq xmm5, xmm0 - db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5 - db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16] - db $66, $0F, $D4, $E9 // paddq xmm5, xmm1 - db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5 - db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32] - db $66, $0F, $D4, $EA // paddq xmm5, xmm2 - db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5 - db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48] - db $66, $0F, $D4, $EB // paddq xmm5, xmm3 - db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5 - db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64] - db $66, $0F, $D4, $EC // paddq xmm5, xmm4 - db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5 + db $66, $0F, $6F, $FD // movdqa xmm7, xmm5 + db $66, $0F, $6D, $FE // punpckhqdq xmm7, xmm6 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $0F, $EB, $BC, $24, $E0, $00, $00, $00 // por xmm7, [esp + 224] + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $FD // movdqa xmm7, xmm5 + db $66, $0F, $73, $DF, $06 // psrldq xmm7, 6 + db $66, $0F, $6C, $EE // punpcklqdq xmm5, xmm6 + db $66, $0F, $73, $DE, $06 // psrldq xmm6, 6 + db $66, $0F, $6C, $FE // punpcklqdq xmm7, xmm6 + db $66, $0F, $6F, $F7 // movdqa xmm6, xmm7 + db $66, $0F, $73, $D6, $1E // psrlq xmm6, 30 + db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208] + db $66, $0F, $D4, $DE // paddq xmm3, xmm6 + db $66, $0F, $73, $D7, $04 // psrlq xmm7, 4 + db $66, $0F, $DB, $BC, $24, $D0, $00, $00, $00 // pand xmm7, [esp + 208] + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 + db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208] + db $66, $0F, $D4, $C5 // paddq xmm0, xmm5 sub eax, 32 jnz @@poly1305_sse2_loop @@poly1305_sse2_tail: // ---- Tail: H *= [r^2, r^1] (shifted table read) ---- - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $44 // movdqu xmm1, [esi + 68] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $34 // movdqu xmm2, [esi + 52] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $34 // movdqu xmm1, [esi + 52] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $24 // movdqu xmm1, [esi + 36] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $14 // movdqu xmm1, [esi + 20] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $F3, $0F, $6F, $4E, $04 // movdqu xmm1, [esi + 4] - db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] - db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] - db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] - db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] - db $F3, $0F, $6F, $56, $54 // movdqu xmm2, [esi + 84] - db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0 - db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2 - db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 - db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2 - db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 - db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2 - db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 - db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2 - db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 - db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 - db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 - db $66, $0F, $D4, $CA // paddq xmm1, xmm2 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $F3, $0F, $6F, $7E, $44 // movdqu xmm7, [esi + 68] + db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $F3, $0F, $6F, $7E, $34 // movdqu xmm7, [esi + 52] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $C0, $00, $00, $00 // movdqa [esp + 192], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $F3, $0F, $6F, $7E, $34 // movdqu xmm7, [esi + 52] + db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $B0, $00, $00, $00 // movdqa [esp + 176], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36] + db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $A0, $00, $00, $00 // movdqa [esp + 160], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20] + db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $F3, $0F, $6F, $7E, $64 // movdqu xmm7, [esi + 100] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $7F, $AC, $24, $90, $00, $00, $00 // movdqa [esp + 144], xmm5 + db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0 + db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4] + db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $F3, $0F, $6F, $7E, $64 // movdqu xmm7, [esi + 100] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $F3, $0F, $6F, $7E, $54 // movdqu xmm7, [esi + 84] + db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7 + db $66, $0F, $D4, $EE // paddq xmm5, xmm6 + db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208] + db $66, $0F, $6F, $C5 // movdqa xmm0, xmm5 + db $66, $0F, $6F, $CE // movdqa xmm1, xmm6 + db $66, $0F, $D4, $8C, $24, $90, $00, $00, $00 // paddq xmm1, [esp + 144] + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208] + db $66, $0F, $6F, $D6 // movdqa xmm2, xmm6 + db $66, $0F, $D4, $94, $24, $A0, $00, $00, $00 // paddq xmm2, [esp + 160] + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208] + db $66, $0F, $6F, $DE // movdqa xmm3, xmm6 + db $66, $0F, $D4, $9C, $24, $B0, $00, $00, $00 // paddq xmm3, [esp + 176] + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208] + db $66, $0F, $6F, $E6 // movdqa xmm4, xmm6 + db $66, $0F, $D4, $A4, $24, $C0, $00, $00, $00 // paddq xmm4, [esp + 192] + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208] + db $66, $0F, $6F, $FE // movdqa xmm7, xmm6 + db $66, $0F, $73, $F7, $02 // psllq xmm7, 2 + db $66, $0F, $D4, $F7 // paddq xmm6, xmm7 + db $66, $0F, $D4, $C6 // paddq xmm0, xmm6 + db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 // ---- Horizontal fold 2 lanes -> 1, final reduce, store ---- - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] - db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 - db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] - db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 - db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] - db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 - db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] - db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 - db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 - db $F3, $0F, $6F, $54, $24, $20 // movdqu xmm2, [esp + 32] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $20 // movdqu [esp + 32], xmm2 - db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 - db $F3, $0F, $6F, $54, $24, $30 // movdqu xmm2, [esp + 48] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $30 // movdqu [esp + 48], xmm2 - db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 - db $F3, $0F, $6F, $54, $24, $40 // movdqu xmm2, [esp + 64] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $40 // movdqu [esp + 64], xmm2 - db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 - db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 - db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 - db $66, $0F, $D4, $CA // paddq xmm1, xmm2 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] - db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 - db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 - db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 - db $66, $0F, $DB, $C7 // pand xmm0, xmm7 - db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 - db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] - db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 - db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 - db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $70, $F0, $4E // pshufd xmm6, xmm0, 0x4E + db $66, $0F, $D4, $C6 // paddq xmm0, xmm6 + db $66, $0F, $70, $F1, $4E // pshufd xmm6, xmm1, 0x4E + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 + db $66, $0F, $70, $F2, $4E // pshufd xmm6, xmm2, 0x4E + db $66, $0F, $D4, $D6 // paddq xmm2, xmm6 + db $66, $0F, $70, $F3, $4E // pshufd xmm6, xmm3, 0x4E + db $66, $0F, $D4, $DE // paddq xmm3, xmm6 + db $66, $0F, $70, $F4, $4E // pshufd xmm6, xmm4, 0x4E + db $66, $0F, $D4, $E6 // paddq xmm4, xmm6 + db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 + db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208] + db $66, $0F, $D4, $D6 // paddq xmm2, xmm6 + db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208] + db $66, $0F, $D4, $DE // paddq xmm3, xmm6 + db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208] + db $66, $0F, $D4, $E6 // paddq xmm4, xmm6 + db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208] + db $66, $0F, $6F, $FE // movdqa xmm7, xmm6 + db $66, $0F, $73, $F7, $02 // psllq xmm7, 2 + db $66, $0F, $D4, $F7 // paddq xmm6, xmm7 + db $66, $0F, $D4, $C6 // paddq xmm0, xmm6 + db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0 + db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26 + db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208] + db $66, $0F, $D4, $CE // paddq xmm1, xmm6 db $66, $0F, $7E, $43, $24 // movd dword ptr [ebx + 36], xmm0 - db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] - db $66, $0F, $7E, $43, $28 // movd dword ptr [ebx + 40], xmm0 - db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] - db $66, $0F, $7E, $43, $2C // movd dword ptr [ebx + 44], xmm0 - db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] - db $66, $0F, $7E, $43, $30 // movd dword ptr [ebx + 48], xmm0 - db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] - db $66, $0F, $7E, $43, $34 // movd dword ptr [ebx + 52], xmm0 - add esp, 160 + db $66, $0F, $7E, $4B, $28 // movd dword ptr [ebx + 40], xmm1 + db $66, $0F, $7E, $53, $2C // movd dword ptr [ebx + 44], xmm2 + db $66, $0F, $7E, $5B, $30 // movd dword ptr [ebx + 48], xmm3 + db $66, $0F, $7E, $63, $34 // movd dword ptr [ebx + 52], xmm4 + mov esp, dword ptr [esp + 240] pop edi pop esi pop ebx diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc index e8172d0b..ebb4aae0 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc @@ -15,380 +15,378 @@ // tail reads [+16*j+4] ([r^2,r^1]). {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + // ---- 64B-aligned frame: bulk table rows copied in once ---- + mov r11, rsp + sub rsp, 192 + and rsp, -64 + db $F3, $44, $0F, $6F, $22 // movdqu xmm12, [rdx] + db $66, $44, $0F, $7F, $24, $24 // movdqa [rsp + 0], xmm12 + db $F3, $44, $0F, $6F, $62, $10 // movdqu xmm12, [rdx + 16] + db $66, $44, $0F, $7F, $64, $24, $10 // movdqa [rsp + 16], xmm12 + db $F3, $44, $0F, $6F, $62, $20 // movdqu xmm12, [rdx + 32] + db $66, $44, $0F, $7F, $64, $24, $20 // movdqa [rsp + 32], xmm12 + db $F3, $44, $0F, $6F, $62, $30 // movdqu xmm12, [rdx + 48] + db $66, $44, $0F, $7F, $64, $24, $30 // movdqa [rsp + 48], xmm12 + db $F3, $44, $0F, $6F, $62, $40 // movdqu xmm12, [rdx + 64] + db $66, $44, $0F, $7F, $64, $24, $40 // movdqa [rsp + 64], xmm12 + db $F3, $44, $0F, $6F, $62, $50 // movdqu xmm12, [rdx + 80] + db $66, $44, $0F, $7F, $64, $24, $50 // movdqa [rsp + 80], xmm12 + db $F3, $44, $0F, $6F, $62, $60 // movdqu xmm12, [rdx + 96] + db $66, $44, $0F, $7F, $64, $24, $60 // movdqa [rsp + 96], xmm12 + db $F3, $44, $0F, $6F, $62, $70 // movdqu xmm12, [rdx + 112] + db $66, $44, $0F, $7F, $64, $24, $70 // movdqa [rsp + 112], xmm12 + db $F3, $44, $0F, $6F, $A2, $80, $00, $00, $00 // movdqu xmm12, [rdx + 128] + db $66, $44, $0F, $7F, $A4, $24, $80, $00, $00, $00 // movdqa [rsp + 128], xmm12 // ---- Prologue: mask26 (xmm5), padbit (xmm6) ---- - db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5 - db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38 - db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6 - db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63 - db $66, $0F, $73, $F6, $18 // psllq xmm6, 24 + db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5 + db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38 + db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6 + db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63 + db $66, $0F, $73, $F6, $18 // psllq xmm6, 24 // ---- Load H0..H4 into lane 0 ---- - db $66, $0F, $6E, $41, $24 // movd xmm0, dword ptr [rcx + 36] - db $66, $0F, $6E, $49, $28 // movd xmm1, dword ptr [rcx + 40] - db $66, $0F, $6E, $51, $2C // movd xmm2, dword ptr [rcx + 44] - db $66, $0F, $6E, $59, $30 // movd xmm3, dword ptr [rcx + 48] - db $66, $0F, $6E, $61, $34 // movd xmm4, dword ptr [rcx + 52] + db $66, $0F, $6E, $41, $24 // movd xmm0, dword ptr [rcx + 36] + db $66, $0F, $6E, $49, $28 // movd xmm1, dword ptr [rcx + 40] + db $66, $0F, $6E, $51, $2C // movd xmm2, dword ptr [rcx + 44] + db $66, $0F, $6E, $59, $30 // movd xmm3, dword ptr [rcx + 48] + db $66, $0F, $6E, $61, $34 // movd xmm4, dword ptr [rcx + 52] // ---- Pre-loop: absorb first 2 blocks ---- - db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] - db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] + db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] + db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] add r8, 32 - db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 - db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 - db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 - db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 - db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 - db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 - db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 - db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 - db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 - db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 - db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 - db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 - db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 - db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 - db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 - db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 - db $66, $0F, $EB, $FE // por xmm7, xmm6 - db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 - db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 - db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 - db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 - db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 + db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 + db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 + db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 + db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 + db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 + db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 + db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 + db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 + db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 + db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 + db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 + db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 + db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 + db $66, $0F, $EB, $FE // por xmm7, xmm6 + db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 + db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 sub r9, 32 jz @@poly1305_sse2_tail @@poly1305_sse2_loop: // ---- Bulk: H *= r^2 (broadcast) ---- - db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 - db $F3, $44, $0F, $6F, $6A, $40 // movdqu xmm13, [rdx + 64] - db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 - db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48] - db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 - db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] - db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 - db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] - db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 - db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] - db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $50 // movdqu xmm13, [rdx + 80] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 + db $66, $44, $0F, $F4, $5C, $24, $40 // pmuludq xmm11, [rsp + 64] + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $66, $44, $0F, $F4, $64, $24, $30 // pmuludq xmm12, [rsp + 48] + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $66, $44, $0F, $F4, $64, $24, $20 // pmuludq xmm12, [rsp + 32] + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16] + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0] + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $66, $44, $0F, $F4, $54, $24, $30 // pmuludq xmm10, [rsp + 48] + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $66, $44, $0F, $F4, $64, $24, $20 // pmuludq xmm12, [rsp + 32] + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16] + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0] + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128] + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 + db $66, $44, $0F, $F4, $4C, $24, $20 // pmuludq xmm9, [rsp + 32] + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16] + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0] + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128] + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112] + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 + db $66, $44, $0F, $F4, $44, $24, $10 // pmuludq xmm8, [rsp + 16] + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0] + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128] + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112] + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $66, $44, $0F, $F4, $64, $24, $60 // pmuludq xmm12, [rsp + 96] + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $66, $0F, $F4, $3C, $24 // pmuludq xmm7, [rsp + 0] + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128] + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112] + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $66, $44, $0F, $F4, $64, $24, $60 // pmuludq xmm12, [rsp + 96] + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $66, $44, $0F, $F4, $64, $24, $50 // pmuludq xmm12, [rsp + 80] + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 // ---- Lazy reduce D -> H ---- - db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $DB, $FD // pand xmm7, xmm5 - db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 - db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 - db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 - db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 - db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 - db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 - db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 - db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 - db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 - db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 - db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 - db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 - db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 - db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 - db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 - db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 - db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 - db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $DB, $C5 // pand xmm0, xmm5 - db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $FD // pand xmm7, xmm5 + db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 + db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 + db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 + db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 + db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 + db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 + db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 + db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 + db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 + db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 + db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 + db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 + db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 + db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 // ---- Absorb next 2 blocks ---- - db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] - db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] + db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] + db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] add r8, 32 - db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 - db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 - db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 - db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 - db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 - db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 - db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 - db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 - db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 - db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 - db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 - db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 - db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 - db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 - db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 - db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 - db $66, $0F, $EB, $FE // por xmm7, xmm6 - db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 - db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 - db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 - db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 - db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 + db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 + db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 + db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 + db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 + db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 + db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 + db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 + db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 + db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 + db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 + db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 + db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 + db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 + db $66, $0F, $EB, $FE // por xmm7, xmm6 + db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 + db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 sub r9, 32 jnz @@poly1305_sse2_loop @@poly1305_sse2_tail: // ---- Tail: H *= [r^2, r^1] (shifted table read) ---- - db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 - db $F3, $44, $0F, $6F, $6A, $44 // movdqu xmm13, [rdx + 68] - db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 - db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 - db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] - db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 - db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 - db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] - db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 - db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 - db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] - db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 - db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 - db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] - db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13 - db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 - db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 - db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 - db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 - db $F3, $44, $0F, $6F, $6A, $54 // movdqu xmm13, [rdx + 84] - db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $DB, $FD // pand xmm7, xmm5 - db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 - db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 - db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 - db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 - db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 - db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 - db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 - db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 - db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 - db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 - db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 - db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 - db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 - db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 - db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 - db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 - db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 - db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 - db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 - db $66, $0F, $DB, $C5 // pand xmm0, xmm5 - db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 + db $F3, $44, $0F, $6F, $6A, $44 // movdqu xmm13, [rdx + 68] + db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] + db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $54 // movdqu xmm13, [rdx + 84] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $FD // pand xmm7, xmm5 + db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 + db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 + db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 + db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 + db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 + db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 + db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 + db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 + db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 + db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 + db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 + db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 + db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 + db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 // ---- Horizontal fold 2 lanes -> 1, final reduce, store ---- - db $66, $0F, $70, $F8, $4E // pshufd xmm7, xmm0, 0x4E - db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 - db $66, $0F, $70, $F9, $4E // pshufd xmm7, xmm1, 0x4E - db $66, $0F, $D4, $CF // paddq xmm1, xmm7 - db $66, $0F, $70, $FA, $4E // pshufd xmm7, xmm2, 0x4E - db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 - db $66, $0F, $70, $FB, $4E // pshufd xmm7, xmm3, 0x4E - db $66, $0F, $D4, $DF // paddq xmm3, xmm7 - db $66, $0F, $70, $FC, $4E // pshufd xmm7, xmm4, 0x4E - db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 - db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $C5 // pand xmm0, xmm5 - db $66, $0F, $D4, $CF // paddq xmm1, xmm7 - db $66, $0F, $6F, $F9 // movdqa xmm7, xmm1 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $CD // pand xmm1, xmm5 - db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 - db $66, $0F, $6F, $FA // movdqa xmm7, xmm2 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $D5 // pand xmm2, xmm5 - db $66, $0F, $D4, $DF // paddq xmm3, xmm7 - db $66, $0F, $6F, $FB // movdqa xmm7, xmm3 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $DD // pand xmm3, xmm5 - db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 - db $66, $0F, $6F, $FC // movdqa xmm7, xmm4 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $E5 // pand xmm4, xmm5 - db $66, $44, $0F, $6F, $E7 // movdqa xmm12, xmm7 - db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 - db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 - db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 - db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 - db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 - db $66, $0F, $DB, $C5 // pand xmm0, xmm5 - db $66, $0F, $D4, $CF // paddq xmm1, xmm7 - db $66, $0F, $7E, $41, $24 // movd dword ptr [rcx + 36], xmm0 - db $66, $0F, $7E, $49, $28 // movd dword ptr [rcx + 40], xmm1 - db $66, $0F, $7E, $51, $2C // movd dword ptr [rcx + 44], xmm2 - db $66, $0F, $7E, $59, $30 // movd dword ptr [rcx + 48], xmm3 - db $66, $0F, $7E, $61, $34 // movd dword ptr [rcx + 52], xmm4 + db $66, $0F, $70, $F8, $4E // pshufd xmm7, xmm0, 0x4E + db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 + db $66, $0F, $70, $F9, $4E // pshufd xmm7, xmm1, 0x4E + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $70, $FA, $4E // pshufd xmm7, xmm2, 0x4E + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $70, $FB, $4E // pshufd xmm7, xmm3, 0x4E + db $66, $0F, $D4, $DF // paddq xmm3, xmm7 + db $66, $0F, $70, $FC, $4E // pshufd xmm7, xmm4, 0x4E + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $6F, $F9 // movdqa xmm7, xmm1 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $CD // pand xmm1, xmm5 + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $6F, $FA // movdqa xmm7, xmm2 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $D5 // pand xmm2, xmm5 + db $66, $0F, $D4, $DF // paddq xmm3, xmm7 + db $66, $0F, $6F, $FB // movdqa xmm7, xmm3 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $DD // pand xmm3, xmm5 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $FC // movdqa xmm7, xmm4 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $E5 // pand xmm4, xmm5 + db $66, $44, $0F, $6F, $E7 // movdqa xmm12, xmm7 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $7E, $41, $24 // movd dword ptr [rcx + 36], xmm0 + db $66, $0F, $7E, $49, $28 // movd dword ptr [rcx + 40], xmm1 + db $66, $0F, $7E, $51, $2C // movd dword ptr [rcx + 44], xmm2 + db $66, $0F, $7E, $59, $30 // movd dword ptr [rcx + 48], xmm3 + db $66, $0F, $7E, $61, $34 // movd dword ptr [rcx + 52], xmm4 + mov rsp, r11 {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} From e2d6d7d1465f010e3e7e10b12cd6bc13068ea9a1 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Sat, 11 Jul 2026 12:26:45 +0100 Subject: [PATCH 2/4] Reach reference parity across AES CTR/OCB/CCM and rework the fused GCM subsystem Mode-level: every AEAD mode now has a same-key re-Init fast path, so a fresh nonce per message no longer pays for key-dependent setup: - OCB: skip both key schedules, the L-table rebuild and the kernel re-acquire when the key is unchanged (keeps the Ktop cache; direction-aware). - CCM: cache the CTR wrapper and re-init it per packet with IV-only parameters; same-key Init skips the engine init and kernel re-acquire. Decrypt now writes straight into the caller's buffer and zero-wipes it before raising on a tag failure, instead of bouncing the whole plaintext through a scratch copy; the no-unverified-plaintext guard test asserts the new wipe contract. - GCM: skip the key schedule, hash subkey, multiplier and H-power table rebuild and the kernel re-acquire on same-key re-Init (the CTR keystream is direction-independent, so key equality alone gates it). Kernel-level: - CTR (both arches): the counter staging no longer read-modify-writes counter memory per block. The counter lives in registers native-endian across the span (the constant prefix is store-only), with carry-outs rare-pathed; this removes a store-forward stall per staged block. - OCB (both arches): round-interleaved, double-buffered offset ladder - each iteration's AES rounds stage the next iteration's deltas into the opposite ping-pong bank, so the ladder's serial bsf/load/xor chain rides the AES latency instead of running as a prefix. i386 is 6-wide (min 12 blocks), x86_64 8-wide (min 16) with a branch-free 64-bit counter and key memory-operand rounds. - GCM (both arches): the kernel now owns an H^8..H^1 table pre-multiplied by x in the reflected representation, so the per-batch 256-bit product reduces with two carry-less multiplies by the folded field constant instead of a long shift/XOR ladder, and the running GHASH state stays reflected across batches (byte-reversal only at kernel entry/exit). x86_64 additionally holds all loop-carried state in registers (no per-batch context memory traffic) and feeds round keys as aligned memory operands. i386 gets a 16-aligned frame: aligned accumulator slots, a resident byte-reverse mask used as a shuffle memory operand, in-register cross-product combining and an in-register counter. --- CryptoLib.Tests/src/Crypto/CcmTests.pas | 4 +- .../CipherKernels/Block/ClpAesNiGcmKernel.pas | 38 +- .../CipherKernels/Block/ClpAesNiOcbKernel.pas | 4 +- .../src/Crypto/Modes/ClpCcmBlockCipher.pas | 84 +- .../src/Crypto/Modes/ClpGcmBlockCipher.pas | 9 +- .../src/Crypto/Modes/ClpOcbBlockCipher.pas | 82 +- .../Simd/Aes/Ctr/AesNiCtrEight_i386.inc | 508 +++--- .../Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc | 190 +- .../Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc | 870 ++++----- .../Gcm/AesGcmFusedCtrGhashEight_x86_64.inc | 912 +++++----- .../Simd/Aes/Ocb/AesOcbFusedWide_i386.inc | 1251 ++++++++++--- .../Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc | 1599 +++++++++++------ 12 files changed, 3386 insertions(+), 2165 deletions(-) diff --git a/CryptoLib.Tests/src/Crypto/CcmTests.pas b/CryptoLib.Tests/src/Crypto/CcmTests.pas index 3bf91d66..c0c405aa 100644 --- a/CryptoLib.Tests/src/Crypto/CcmTests.pas +++ b/CryptoLib.Tests/src/Crypto/CcmTests.pas @@ -668,9 +668,11 @@ procedure TTestCcm.DoTestNoUnverifiedPlaintextOnFailure; except on E: EInvalidCipherTextCryptoLibException do begin + // The tentative plaintext must be wiped before the exception leaves: + // every byte of the output region has to read back as zero. for LI := 0 to High(LOutput) do begin - if LOutput[LI] <> $55 then + if LOutput[LI] <> $00 then Fail('CCM left unverified plaintext in the output buffer on tag failure'); end; end; diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas index 56773b1d..217a164d 100644 --- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas +++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas @@ -22,6 +22,8 @@ interface uses SysUtils, + ClpCryptoLibTypes, + ClpBinaryPrimitives, ClpIBlockCipher, ClpIAesEngineX86, ClpCipherKernelTypes, @@ -55,6 +57,10 @@ TAesNiGcmKernel = class sealed(TInterfacedObject, IGcmKernel) FRounds: Int32; FHPow128: Pointer; FMask: Pointer; + // Kernel-owned copy of the H^8..H^1 table with every power pre-multiplied + // by x in the reflected representation, matching the carry-less-multiply + // folding reduction inside the fused kernel. FHPow128 points at it. + FHPowShifted: TCryptoLibByteArray; public constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; ARounds: Int32; AHPow128, AMask: Pointer); @@ -82,11 +88,12 @@ implementation // Natural pointer-sized alignment, no padding: the field offsets match the // [rcx + N] / [ebx + N] accesses in AesGcmFusedCtrGhashEight_x86_64.inc and // AesGcmFusedCtrGhashEight_i386.inc (they differ with pointer / NativeUInt - // width: 8-byte fields on x86_64, 4-byte on i386). The kernel advances - // PXorIn/POut/PPrevCipher and Counter32 in place across the batch run and + // width: 8-byte fields on x86_64, 4-byte on i386). The kernel holds the loop + // state in registers across the batch run, writing only Counter32 back, and // builds each batch's counter blocks in-register from Counter32 + PJ0Template // (both arches); the GHASH-from-output flag is 1 for encrypt (fold the output - // ciphertext) and 0 for decrypt (fold the input). + // ciphertext) and 0 for decrypt (fold the input). PHPow128 points at the + // kernel-owned x-pre-multiplied H-power table (see TAesNiGcmKernel.Create). TGcmFusedCtx = record PXorIn: Pointer; POut: Pointer; @@ -152,13 +159,36 @@ procedure GcmFusedAesEnc256GhashEight(PCtx: Pointer); constructor TAesNiGcmKernel.Create(const AEngine: IAesEngineX86; AKeys: Pointer; ARounds: Int32; AHPow128, AMask: Pointer); +var + LI: Int32; + LLo, LHi, LCarry: UInt64; begin inherited Create; FEngine := AEngine; FKeys := AKeys; FRounds := ARounds; - FHPow128 := AHPow128; FMask := AMask; + // Pre-multiply each H power by x in the reflected representation: shift the + // 128-bit value left by one with a conditional fold of the field polynomial + // (0xC2 in the top byte, 1 in the bottom). This lets the kernel reduce the + // batch product with two carry-less multiplies instead of a shift ladder. + System.SetLength(FHPowShifted, 128); + for LI := 0 to 7 do + begin + LLo := TBinaryPrimitives.ReadUInt64LittleEndian(PByte(AHPow128), LI * 16); + LHi := TBinaryPrimitives.ReadUInt64LittleEndian(PByte(AHPow128), LI * 16 + 8); + LCarry := LHi shr 63; + LHi := (LHi shl 1) or (LLo shr 63); + LLo := LLo shl 1; + if LCarry <> 0 then + begin + LHi := LHi xor UInt64($C200000000000000); + LLo := LLo xor 1; + end; + TBinaryPrimitives.WriteUInt64LittleEndian(FHPowShifted, LI * 16, LLo); + TBinaryPrimitives.WriteUInt64LittleEndian(FHPowShifted, LI * 16 + 8, LHi); + end; + FHPow128 := @FHPowShifted[0]; end; function TAesNiGcmKernel.MinimumBlockCount: Int32; diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas index 0aa6142f..5b300ffc 100644 --- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas +++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas @@ -49,9 +49,9 @@ TAesNiOcbKernel = class sealed(TInterfacedObject, IOcbKernel) strict private const {$IFDEF CRYPTOLIB_X86_64_ASM} - FUSED_OCB_MIN_BLOCKS = 8; + FUSED_OCB_MIN_BLOCKS = 16; {$ELSE} - FUSED_OCB_MIN_BLOCKS = 4; + FUSED_OCB_MIN_BLOCKS = 12; {$ENDIF} strict private FEngine: IAesEngineX86; diff --git a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas index 3def6d53..5695dad0 100644 --- a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas @@ -102,6 +102,11 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher, FMacSize: Int32; FKeyParam: ICipherParameters; FLastKey: TCryptoLibByteArray; + // CTR wrapper cached across packets (created once per keyed cipher); each + // packet re-inits it IV-only, so the AES schedule is never recomputed. + FCtrCipher: ISicBlockCipher; + FBulkCtr: IBulkBlockCipherMode; + FKernelForEnc: Boolean; FAssociatedText: TMemoryStream; // Accumulated body input (plaintext on encrypt, ciphertext+tag on decrypt). // CCM needs the total length before processing (B_0 encodes it), so input is @@ -115,7 +120,6 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher, // the plaintext copied to the caller's output (which stays untouched on // failure). Grown by doubling and reused across packets, and wiped after // every packet (in a finally) so no recovered plaintext lingers. - FPlainScratch: TCryptoLibByteArray; // Cached once per Init; non-nil when the registry resolved a fused // CCM kernel for the underlying cipher and current direction. FCcmKernel: ICcmKernel; @@ -189,7 +193,6 @@ constructor TCcmBlockCipher.Create(const ACipher: IBlockCipher); FAssociatedText := TMemoryStream.Create; FData := nil; FDataLen := 0; - FPlainScratch := nil; if (ACipher.GetBlockSize() <> BlockSize) then raise EArgumentCryptoLibException.CreateResFmt(@SCipherRequired, [BlockSize]); @@ -233,6 +236,8 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean; LChoice: TCipherAeadChoice; LRequestedMacSizeBits: Int32; LDirection: TCipherKernelDirection; + LPrevKey: TCryptoLibByteArray; + LSameKey: Boolean; begin FForEncryption := AForEncryption; @@ -245,6 +250,7 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean; raise EArgumentCryptoLibException.CreateResFmt(@SInvalidParameters, ['CCM']); CheckNonceReuse(FForEncryption, LChoice.Nonce, LChoice.KeyParameter); + LPrevKey := FLastKey; FNonce := LChoice.Nonce; FInitialAssociatedText := LChoice.AssociatedText; @@ -264,19 +270,35 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean; if (System.Length(FNonce) < 7) or (System.Length(FNonce) > 13) then raise EArgumentCryptoLibException.CreateRes(@SNonceLengthRange); - FCcmKernel := nil; + // Nonce-only rotation is the hot AEAD path: with the key and direction + // unchanged the AES schedule, the CTR wrapper and the fused-kernel binding + // all stay valid, so none of them are recomputed. + LSameKey := (LChoice.CipherKey = nil) or + ((LPrevKey <> nil) and (LChoice.KeyParameter <> nil) and + LChoice.KeyParameter.FixedTimeEquals(LPrevKey)); if FKeyParam <> nil then begin - // Key FCipher now so the factory can read the finalised AES - // schedule. ProcessPacket re-keys through the SIC wrapper later; the - // schedule contents are identical across those re-keys. - FCipher.Init(True, FKeyParam); - if AForEncryption then - LDirection := TCipherKernelDirection.Encrypt - else - LDirection := TCipherKernelDirection.Decrypt; - TCipherKernelRegistry.TryAcquireCcm(FCipher, LDirection, FCcmKernel); - end; + if not LSameKey then + FCipher.Init(True, FKeyParam); + if (FCcmKernel = nil) or (not LSameKey) or + (FKernelForEnc <> AForEncryption) then + begin + FCcmKernel := nil; + if AForEncryption then + LDirection := TCipherKernelDirection.Encrypt + else + LDirection := TCipherKernelDirection.Decrypt; + TCipherKernelRegistry.TryAcquireCcm(FCipher, LDirection, FCcmKernel); + FKernelForEnc := AForEncryption; + end; + if FCtrCipher = nil then + begin + FCtrCipher := TSicBlockCipher.Create(FCipher); + TBlockCipherBulkUtilities.TryResolveBulkCipherMode(FCtrCipher, FBulkCtr); + end; + end + else + FCcmKernel := nil; Reset(); end; @@ -420,9 +442,11 @@ function TCcmBlockCipher.ProcessPacket(const AInput: TCryptoLibByteArray; LIV[0] := Byte((LQ - 1) and $7); System.Move(FNonce[0], LIV[1], LN); - LCtrCipher := TSicBlockCipher.Create(FCipher); - LCtrCipher.Init(FForEncryption, TParametersWithIV.Create(FKeyParam, LIV) as IParametersWithIV); - TBlockCipherBulkUtilities.TryResolveBulkCipherMode(LCtrCipher, LBulkCtr); + // IV-only re-init of the cached CTR wrapper: the inner parameters are nil, + // so the (unchanged) AES schedule is not recomputed per packet. + LCtrCipher := FCtrCipher; + LCtrCipher.Init(FForEncryption, TParametersWithIV.Create(nil, LIV) as IParametersWithIV); + LBulkCtr := FBulkCtr; if FForEncryption then begin @@ -856,21 +880,21 @@ function TCcmBlockCipher.RunDecrypt(AUseFused: Boolean; var LOk: Boolean; begin - TArrayUtilities.EnsureCapacity(FPlainScratch, ACtx.OutputLen); - ACtx.Dest := FPlainScratch; - ACtx.DestOff := 0; - try - if AUseFused then - LOk := DecryptBodyFused(ACtx) - else - LOk := DecryptBodyScalar(ACtx); - if not LOk then - raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']); - System.Move(FPlainScratch[0], AOutput[AOutOff], ACtx.OutputLen); - Result := True; - finally - TArrayUtilities.Fill(FPlainScratch, 0, ACtx.OutputLen, Byte(0)); + // Decrypt straight into the caller's buffer (no scratch bounce); on a MAC + // failure the tentative plaintext is wiped before the exception leaves, so + // no unverified plaintext survives the call. + ACtx.Dest := AOutput; + ACtx.DestOff := AOutOff; + if AUseFused then + LOk := DecryptBodyFused(ACtx) + else + LOk := DecryptBodyScalar(ACtx); + if not LOk then + begin + TArrayUtilities.Fill(AOutput, AOutOff, AOutOff + ACtx.OutputLen, Byte(0)); + raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']); end; + Result := True; end; end. diff --git a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas index bfacaa53..db79cbb5 100644 --- a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas @@ -522,6 +522,7 @@ procedure TGcmBlockCipher.Init(AForEncryption: Boolean; LKeyParam: IKeyParameter; LNewNonce: TCryptoLibByteArray; LBufLength: Int32; + LSameKey: Boolean; begin FForEncryption := AForEncryption; FMacBlock := nil; @@ -551,8 +552,14 @@ procedure TGcmBlockCipher.Init(AForEncryption: Boolean; if LKeyParam <> nil then begin + // Same-key fast path: the key schedule, hash subkey, multiplier state, + // H-power table and fused kernel all depend only on the key, so a + // re-Init with the same key (fresh nonce per message) keeps them all. + LSameKey := (FH <> nil) and (FLastKey <> nil) and + LKeyParam.FixedTimeEquals(FLastKey); FLastKey := LKeyParam.GetKey(); - InitCipherAndHashSubKey(LKeyParam); + if not LSameKey then + InitCipherAndHashSubKey(LKeyParam); end else if FH = nil then begin diff --git a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas index 04240926..5def24aa 100644 --- a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas @@ -268,6 +268,8 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; LMacSizeBits, LBottom, LBits, LBytes, LI: Int32; LB1, LB2: UInt32; LFusedDirection: TCipherKernelDirection; + LPrevKey: TCryptoLibByteArray; + LSameKey, LSameKeyDir: Boolean; begin LOldForEncryption := FForEncryption; FForEncryption := AForEncryption; @@ -283,6 +285,7 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; LN := LChoice.Nonce; CheckNonceReuse(FForEncryption, LN, LChoice.KeyParameter); + LPrevKey := FLastKey; FLastN := System.Copy(LN); if LChoice.KeyParameter <> nil then FLastKey := LChoice.KeyParameter.GetKey(); @@ -290,6 +293,13 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; FInitialAssociatedText := LChoice.AssociatedText; LKeyParameter := LChoice.KeyParameter; + // Nonce-only rotation is the hot AEAD path: when the key (and, for the main + // cipher / fused kernel, the direction) is unchanged, the key schedules, the + // L table and the kernel binding all stay valid and are not recomputed. + LSameKey := (LKeyParameter = nil) or + ((LPrevKey <> nil) and LKeyParameter.FixedTimeEquals(LPrevKey)); + LSameKeyDir := LSameKey and (LOldForEncryption = AForEncryption); + if LChoice.IsAead then begin LMacSizeBits := LChoice.MacSizeBits; @@ -316,11 +326,14 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; if (System.Length(LN) > 15) then raise EArgumentCryptoLibException.CreateRes(@SIVTooLong); - if (LKeyParameter <> nil) then + if (LKeyParameter <> nil) and (not LSameKeyDir) then begin - FHashCipher.Init(True, LKeyParameter); + if not LSameKey then + begin + FHashCipher.Init(True, LKeyParameter); + FKTopInput := nil; + end; FMainCipher.Init(AForEncryption, LKeyParameter); - FKTopInput := nil; end else if (LOldForEncryption <> AForEncryption) then begin @@ -332,41 +345,48 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; // same capability set. Kept as a plain interface QI (no algorithm-name // assumption); any engine that advertises the contract in ClpIBulkBlockCipher // is eligible for the 8-wide path in ProcessEightBlocksBulk. - TBlockCipherBulkUtilities.TryResolveBulkCipher(FMainCipher, FMainBulk); + if not LSameKeyDir then + begin + TBlockCipherBulkUtilities.TryResolveBulkCipher(FMainCipher, FMainBulk); - // Resolve a fused OCB kernel via the open factory registry; the - // first factory whose TryCreate accepts the cipher / direction pair - // wins, and the result is cached for the whole Init cycle. - FOcbKernel := nil; - FOcbKernelMinBlocks := 0; - if FForEncryption then - LFusedDirection := TCipherKernelDirection.Encrypt - else - LFusedDirection := TCipherKernelDirection.Decrypt; - if TCipherKernelRegistry.TryAcquireOcb(FMainCipher, LFusedDirection, - FOcbKernel) and (FOcbKernel <> nil) then - begin - FOcbKernelMinBlocks := FOcbKernel.MinimumBlockCount; - // The fused batch buffer holds FUSED_BATCH_BLOCKS offsets; reject - // kernels whose stride does not divide that capacity so the mode - // can always present a full-stride batch. - if (FOcbKernelMinBlocks <= 0) or - (FUSED_BATCH_BLOCKS mod FOcbKernelMinBlocks <> 0) then + // Resolve a fused OCB kernel via the open factory registry; the + // first factory whose TryCreate accepts the cipher / direction pair + // wins, and the result stays bound until the key or direction changes + // (the engine schedule the kernel captured is untouched until then). + FOcbKernel := nil; + FOcbKernelMinBlocks := 0; + if FForEncryption then + LFusedDirection := TCipherKernelDirection.Encrypt + else + LFusedDirection := TCipherKernelDirection.Decrypt; + if TCipherKernelRegistry.TryAcquireOcb(FMainCipher, LFusedDirection, + FOcbKernel) and (FOcbKernel <> nil) then begin - FOcbKernel := nil; - FOcbKernelMinBlocks := 0; + FOcbKernelMinBlocks := FOcbKernel.MinimumBlockCount; + // The fused batch buffer holds FUSED_BATCH_BLOCKS offsets; reject + // kernels whose stride does not divide that capacity so the mode + // can always present a full-stride batch. + if (FOcbKernelMinBlocks <= 0) or + (FUSED_BATCH_BLOCKS mod FOcbKernelMinBlocks <> 0) then + begin + FOcbKernel := nil; + FOcbKernelMinBlocks := 0; + end; end; end; - System.SetLength(FL_Asterisk, 16); - TArrayUtilities.Fill(FL_Asterisk, 0, 16, Byte(0)); - FHashCipher.ProcessBlock(FL_Asterisk, 0, FL_Asterisk, 0); + if not LSameKey then + begin + System.SetLength(FL_Asterisk, 16); + TArrayUtilities.Fill(FL_Asterisk, 0, 16, Byte(0)); + FHashCipher.ProcessBlock(FL_Asterisk, 0, FL_Asterisk, 0); - FL_Dollar := OCB_double(FL_Asterisk); + FL_Dollar := OCB_double(FL_Asterisk); - FL.Clear; - FL.Add(OCB_double(FL_Dollar)); - FLTableFlatCount := 0; // FL rebuilt for the new key; drop the flattened cache + FL.Clear; + FL.Add(OCB_double(FL_Dollar)); + FLTableFlatCount := 0; // FL rebuilt for the new key; drop the flattened cache + end; LBottom := ProcessNonce(LN); diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc index fade6bc6..c22d356a 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc @@ -22,13 +22,15 @@ // direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must // NOT be defined when including this file. // -// Constraints: i386 has only xmm0..7 (all volatile) and, after unpacking the -// context, only edx as a free scratch GP register. The 128-bit counter lives in -// memory at [eax] and is bumped in place with a byte-wise big-endian carry (edx = -// byte index). Each staged block copies the current [eax] to the scratch (four -// dword moves through edx, since no xmm is free mid-round) then advances. The -// pipeline builds one batch ahead, so [eax] over-runs by exactly one batch (8) - -// undone before returning. The keystream^plaintext XOR has no free xmm for the +// Constraints: i386 has only xmm0..7 (all volatile). BlockCount parks in a frame +// slot so both edx and ecx serve the staging: edx holds the counter's low dword +// native-LE across the whole span (bswap/store/bswap/add per staged block - no +// read-modify-write of counter memory, so no store-forward stalls), while ecx +// copies the constant 12-byte prefix from [eax+0..11] (written only by the rare +// carry-out path, which bumps those bytes big-endian). The pipeline builds one +// batch ahead, so edx over-runs by exactly one batch (8) - undone (with a +// byte-wise borrow when it crosses back over a 2^32 boundary) before the low +// dword is stored back to [eax+12]. The keystream^plaintext XOR has no free xmm for the // plaintext (nor can it use a memory-operand pxor, which would require the input // to be 16-byte aligned), so it spills lane 7 to a scratch slot above the counter // area and reuses xmm7 for the movdqu plaintext load. AES-NI and register pxor @@ -42,81 +44,156 @@ mov ecx, [ebx + 16] // BlockCount mov ebx, [ebx + 0] // InPtr (overwrites PCtx) - sub esp, 144 // next-batch counters [esp+0..127]; ks7 spill [esp+128..143] + sub esp, 152 // next-batch counters [esp+0..127]; ks7 spill [esp+128..143]; countdown [esp+144] + mov [esp + 144], ecx // park BlockCount; ecx = staging scratch + mov edx, [eax + 12] // counter low dword, kept native-LE in edx + bswap edx // ---- stage batch 0 into the scratch (xmm0 is free here), then load it ---- - movdqu xmm0, [eax] - movdqu [esp + 0], xmm0 - mov edx, 15 -@@PreA0: - inc byte ptr [eax + edx] - jnz @@PreD0 - dec edx - jns @@PreA0 -@@PreD0: - movdqu xmm0, [eax] - movdqu [esp + 16], xmm0 - mov edx, 15 -@@PreA1: - inc byte ptr [eax + edx] - jnz @@PreD1 - dec edx - jns @@PreA1 -@@PreD1: - movdqu xmm0, [eax] - movdqu [esp + 32], xmm0 - mov edx, 15 -@@PreA2: - inc byte ptr [eax + edx] - jnz @@PreD2 - dec edx - jns @@PreA2 -@@PreD2: - movdqu xmm0, [eax] - movdqu [esp + 48], xmm0 - mov edx, 15 -@@PreA3: - inc byte ptr [eax + edx] - jnz @@PreD3 - dec edx - jns @@PreA3 -@@PreD3: - movdqu xmm0, [eax] - movdqu [esp + 64], xmm0 - mov edx, 15 -@@PreA4: - inc byte ptr [eax + edx] - jnz @@PreD4 - dec edx - jns @@PreA4 -@@PreD4: - movdqu xmm0, [eax] - movdqu [esp + 80], xmm0 - mov edx, 15 -@@PreA5: - inc byte ptr [eax + edx] - jnz @@PreD5 - dec edx - jns @@PreA5 -@@PreD5: - movdqu xmm0, [eax] - movdqu [esp + 96], xmm0 - mov edx, 15 -@@PreA6: - inc byte ptr [eax + edx] - jnz @@PreD6 - dec edx - jns @@PreA6 -@@PreD6: - movdqu xmm0, [eax] - movdqu [esp + 112], xmm0 - mov edx, 15 -@@PreA7: - inc byte ptr [eax + edx] - jnz @@PreD7 - dec edx - jns @@PreA7 -@@PreD7: + mov ecx, [eax] + mov [esp + 0], ecx + mov ecx, [eax + 4] + mov [esp + 4], ecx + mov ecx, [eax + 8] + mov [esp + 8], ecx + bswap edx + mov [esp + 12], edx + bswap edx + add edx, 1 + jnc @@StgOkP0 + mov ecx, 11 +@@StgCyP0: + inc byte ptr [eax + ecx] + jnz @@StgOkP0 + dec ecx + jns @@StgCyP0 +@@StgOkP0: + mov ecx, [eax] + mov [esp + 16], ecx + mov ecx, [eax + 4] + mov [esp + 20], ecx + mov ecx, [eax + 8] + mov [esp + 24], ecx + bswap edx + mov [esp + 28], edx + bswap edx + add edx, 1 + jnc @@StgOkP1 + mov ecx, 11 +@@StgCyP1: + inc byte ptr [eax + ecx] + jnz @@StgOkP1 + dec ecx + jns @@StgCyP1 +@@StgOkP1: + mov ecx, [eax] + mov [esp + 32], ecx + mov ecx, [eax + 4] + mov [esp + 36], ecx + mov ecx, [eax + 8] + mov [esp + 40], ecx + bswap edx + mov [esp + 44], edx + bswap edx + add edx, 1 + jnc @@StgOkP2 + mov ecx, 11 +@@StgCyP2: + inc byte ptr [eax + ecx] + jnz @@StgOkP2 + dec ecx + jns @@StgCyP2 +@@StgOkP2: + mov ecx, [eax] + mov [esp + 48], ecx + mov ecx, [eax + 4] + mov [esp + 52], ecx + mov ecx, [eax + 8] + mov [esp + 56], ecx + bswap edx + mov [esp + 60], edx + bswap edx + add edx, 1 + jnc @@StgOkP3 + mov ecx, 11 +@@StgCyP3: + inc byte ptr [eax + ecx] + jnz @@StgOkP3 + dec ecx + jns @@StgCyP3 +@@StgOkP3: + mov ecx, [eax] + mov [esp + 64], ecx + mov ecx, [eax + 4] + mov [esp + 68], ecx + mov ecx, [eax + 8] + mov [esp + 72], ecx + bswap edx + mov [esp + 76], edx + bswap edx + add edx, 1 + jnc @@StgOkP4 + mov ecx, 11 +@@StgCyP4: + inc byte ptr [eax + ecx] + jnz @@StgOkP4 + dec ecx + jns @@StgCyP4 +@@StgOkP4: + mov ecx, [eax] + mov [esp + 80], ecx + mov ecx, [eax + 4] + mov [esp + 84], ecx + mov ecx, [eax + 8] + mov [esp + 88], ecx + bswap edx + mov [esp + 92], edx + bswap edx + add edx, 1 + jnc @@StgOkP5 + mov ecx, 11 +@@StgCyP5: + inc byte ptr [eax + ecx] + jnz @@StgOkP5 + dec ecx + jns @@StgCyP5 +@@StgOkP5: + mov ecx, [eax] + mov [esp + 96], ecx + mov ecx, [eax + 4] + mov [esp + 100], ecx + mov ecx, [eax + 8] + mov [esp + 104], ecx + bswap edx + mov [esp + 108], edx + bswap edx + add edx, 1 + jnc @@StgOkP6 + mov ecx, 11 +@@StgCyP6: + inc byte ptr [eax + ecx] + jnz @@StgOkP6 + dec ecx + jns @@StgCyP6 +@@StgOkP6: + mov ecx, [eax] + mov [esp + 112], ecx + mov ecx, [eax + 4] + mov [esp + 116], ecx + mov ecx, [eax + 8] + mov [esp + 120], ecx + bswap edx + mov [esp + 124], edx + bswap edx + add edx, 1 + jnc @@StgOkP7 + mov ecx, 11 +@@StgCyP7: + inc byte ptr [eax + ecx] + jnz @@StgOkP7 + dec ecx + jns @@StgCyP7 +@@StgOkP7: movdqu xmm0, [esp + 0] movdqu xmm1, [esp + 16] @@ -147,21 +224,24 @@ db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16] db $66, $0F, $38, $DC, $77, $10 // aesenc xmm6, [edi+16] db $66, $0F, $38, $DC, $7F, $10 // aesenc xmm7, [edi+16] - mov edx, [eax] - mov [esp + 0], edx - mov edx, [eax + 4] - mov [esp + 4], edx - mov edx, [eax + 8] - mov [esp + 8], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 0], ecx + mov ecx, [eax + 4] + mov [esp + 4], ecx + mov ecx, [eax + 8] + mov [esp + 8], ecx + bswap edx mov [esp + 12], edx - mov edx, 15 -@@LpA0: - inc byte ptr [eax + edx] - jnz @@LpD0 - dec edx - jns @@LpA0 -@@LpD0: + bswap edx + add edx, 1 + jnc @@StgOkL0 + mov ecx, 11 +@@StgCyL0: + inc byte ptr [eax + ecx] + jnz @@StgOkL0 + dec ecx + jns @@StgCyL0 +@@StgOkL0: db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32] db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32] @@ -171,21 +251,24 @@ db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32] db $66, $0F, $38, $DC, $77, $20 // aesenc xmm6, [edi+32] db $66, $0F, $38, $DC, $7F, $20 // aesenc xmm7, [edi+32] - mov edx, [eax] - mov [esp + 16], edx - mov edx, [eax + 4] - mov [esp + 20], edx - mov edx, [eax + 8] - mov [esp + 24], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 16], ecx + mov ecx, [eax + 4] + mov [esp + 20], ecx + mov ecx, [eax + 8] + mov [esp + 24], ecx + bswap edx mov [esp + 28], edx - mov edx, 15 -@@LpA1: - inc byte ptr [eax + edx] - jnz @@LpD1 - dec edx - jns @@LpA1 -@@LpD1: + bswap edx + add edx, 1 + jnc @@StgOkL1 + mov ecx, 11 +@@StgCyL1: + inc byte ptr [eax + ecx] + jnz @@StgOkL1 + dec ecx + jns @@StgCyL1 +@@StgOkL1: db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48] db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48] @@ -195,21 +278,24 @@ db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48] db $66, $0F, $38, $DC, $77, $30 // aesenc xmm6, [edi+48] db $66, $0F, $38, $DC, $7F, $30 // aesenc xmm7, [edi+48] - mov edx, [eax] - mov [esp + 32], edx - mov edx, [eax + 4] - mov [esp + 36], edx - mov edx, [eax + 8] - mov [esp + 40], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 32], ecx + mov ecx, [eax + 4] + mov [esp + 36], ecx + mov ecx, [eax + 8] + mov [esp + 40], ecx + bswap edx mov [esp + 44], edx - mov edx, 15 -@@LpA2: - inc byte ptr [eax + edx] - jnz @@LpD2 - dec edx - jns @@LpA2 -@@LpD2: + bswap edx + add edx, 1 + jnc @@StgOkL2 + mov ecx, 11 +@@StgCyL2: + inc byte ptr [eax + ecx] + jnz @@StgOkL2 + dec ecx + jns @@StgCyL2 +@@StgOkL2: db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64] db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64] @@ -219,21 +305,24 @@ db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64] db $66, $0F, $38, $DC, $77, $40 // aesenc xmm6, [edi+64] db $66, $0F, $38, $DC, $7F, $40 // aesenc xmm7, [edi+64] - mov edx, [eax] - mov [esp + 48], edx - mov edx, [eax + 4] - mov [esp + 52], edx - mov edx, [eax + 8] - mov [esp + 56], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 48], ecx + mov ecx, [eax + 4] + mov [esp + 52], ecx + mov ecx, [eax + 8] + mov [esp + 56], ecx + bswap edx mov [esp + 60], edx - mov edx, 15 -@@LpA3: - inc byte ptr [eax + edx] - jnz @@LpD3 - dec edx - jns @@LpA3 -@@LpD3: + bswap edx + add edx, 1 + jnc @@StgOkL3 + mov ecx, 11 +@@StgCyL3: + inc byte ptr [eax + ecx] + jnz @@StgOkL3 + dec ecx + jns @@StgCyL3 +@@StgOkL3: db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80] db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80] @@ -243,21 +332,24 @@ db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80] db $66, $0F, $38, $DC, $77, $50 // aesenc xmm6, [edi+80] db $66, $0F, $38, $DC, $7F, $50 // aesenc xmm7, [edi+80] - mov edx, [eax] - mov [esp + 64], edx - mov edx, [eax + 4] - mov [esp + 68], edx - mov edx, [eax + 8] - mov [esp + 72], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 64], ecx + mov ecx, [eax + 4] + mov [esp + 68], ecx + mov ecx, [eax + 8] + mov [esp + 72], ecx + bswap edx mov [esp + 76], edx - mov edx, 15 -@@LpA4: - inc byte ptr [eax + edx] - jnz @@LpD4 - dec edx - jns @@LpA4 -@@LpD4: + bswap edx + add edx, 1 + jnc @@StgOkL4 + mov ecx, 11 +@@StgCyL4: + inc byte ptr [eax + ecx] + jnz @@StgOkL4 + dec ecx + jns @@StgCyL4 +@@StgOkL4: db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96] db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96] @@ -267,21 +359,24 @@ db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96] db $66, $0F, $38, $DC, $77, $60 // aesenc xmm6, [edi+96] db $66, $0F, $38, $DC, $7F, $60 // aesenc xmm7, [edi+96] - mov edx, [eax] - mov [esp + 80], edx - mov edx, [eax + 4] - mov [esp + 84], edx - mov edx, [eax + 8] - mov [esp + 88], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 80], ecx + mov ecx, [eax + 4] + mov [esp + 84], ecx + mov ecx, [eax + 8] + mov [esp + 88], ecx + bswap edx mov [esp + 92], edx - mov edx, 15 -@@LpA5: - inc byte ptr [eax + edx] - jnz @@LpD5 - dec edx - jns @@LpA5 -@@LpD5: + bswap edx + add edx, 1 + jnc @@StgOkL5 + mov ecx, 11 +@@StgCyL5: + inc byte ptr [eax + ecx] + jnz @@StgOkL5 + dec ecx + jns @@StgCyL5 +@@StgOkL5: db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112] db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112] @@ -291,21 +386,24 @@ db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112] db $66, $0F, $38, $DC, $77, $70 // aesenc xmm6, [edi+112] db $66, $0F, $38, $DC, $7F, $70 // aesenc xmm7, [edi+112] - mov edx, [eax] - mov [esp + 96], edx - mov edx, [eax + 4] - mov [esp + 100], edx - mov edx, [eax + 8] - mov [esp + 104], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 96], ecx + mov ecx, [eax + 4] + mov [esp + 100], ecx + mov ecx, [eax + 8] + mov [esp + 104], ecx + bswap edx mov [esp + 108], edx - mov edx, 15 -@@LpA6: - inc byte ptr [eax + edx] - jnz @@LpD6 - dec edx - jns @@LpA6 -@@LpD6: + bswap edx + add edx, 1 + jnc @@StgOkL6 + mov ecx, 11 +@@StgCyL6: + inc byte ptr [eax + ecx] + jnz @@StgOkL6 + dec ecx + jns @@StgCyL6 +@@StgOkL6: db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128] db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128] @@ -315,21 +413,24 @@ db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128] db $66, $0F, $38, $DC, $B7, $80, $00, $00, $00 // aesenc xmm6, [edi+128] db $66, $0F, $38, $DC, $BF, $80, $00, $00, $00 // aesenc xmm7, [edi+128] - mov edx, [eax] - mov [esp + 112], edx - mov edx, [eax + 4] - mov [esp + 116], edx - mov edx, [eax + 8] - mov [esp + 120], edx - mov edx, [eax + 12] + mov ecx, [eax] + mov [esp + 112], ecx + mov ecx, [eax + 4] + mov [esp + 116], ecx + mov ecx, [eax + 8] + mov [esp + 120], ecx + bswap edx mov [esp + 124], edx - mov edx, 15 -@@LpA7: - inc byte ptr [eax + edx] - jnz @@LpD7 - dec edx - jns @@LpA7 -@@LpD7: + bswap edx + add edx, 1 + jnc @@StgOkL7 + mov ecx, 11 +@@StgCyL7: + inc byte ptr [eax + ecx] + jnz @@StgOkL7 + dec ecx + jns @@StgCyL7 +@@StgOkL7: // ---- round 9 (all key sizes): aesenc, no more staging ---- db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144] @@ -453,21 +554,24 @@ add ebx, 128 add esi, 128 - sub ecx, 8 + sub dword ptr [esp + 144], 8 jnz @@CtrEightLoopI386 - // ---- undo the one-batch build-ahead: counter (at [eax]) -= 8, big-endian ---- - sub byte ptr [eax + 15], 8 - jnc @@CtrWbDone - mov edx, 14 + // ---- undo the one-batch build-ahead (edx is 8 blocks ahead); a borrow + // unwinds the prefix bytes, then the low dword stores big-endian ---- + sub edx, 8 + jnc @@CtrWbStore + mov ecx, 11 @@CtrWbBorrow: - sub byte ptr [eax + edx], 1 - jnc @@CtrWbDone - dec edx + sub byte ptr [eax + ecx], 1 + jnc @@CtrWbStore + dec ecx jns @@CtrWbBorrow -@@CtrWbDone: +@@CtrWbStore: + bswap edx + mov [eax + 12], edx - add esp, 144 + add esp, 152 pop edi pop esi pop ebx diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc index ad14023f..aecf45f7 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc @@ -24,11 +24,11 @@ // direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must // NOT be defined when including this file. // -// The 128-bit counter is held natively as r11 (high 64 bits) : rbx (low 64 bits) -// across the whole span; the low half is bumped with add/adc so carry propagates -// through the full 128 bits, matching SIC's big-endian whole-counter increment. -// Each staged block writes bswap(r11) || bswap(rbx) (big-endian) to the scratch, -// then advances. The pipeline builds one batch ahead, so on exit the counter has +// The 128-bit counter is held as r11 (high 64 bits, KEPT big-endian since the +// fast path only stores it) : rbx (low 64 bits, native-LE). Each staged block +// writes r11 || bswap(rbx) to the scratch and bumps rbx; a low-half carry-out +// (once per 2^64 blocks) increments r11 through a bswap round-trip, matching +// SIC's big-endian whole-counter increment. The pipeline builds one batch ahead, so on exit the counter has // over-run by exactly one batch (8) - undone before write-back. // // Register/stack notes: rbx is callee-saved (push/pop). xmm6-15 are saved once @@ -47,76 +47,91 @@ mov rdx, [rcx + 8] // OutPtr mov rcx, [rcx + 0] // InPtr (overwrites PCtx) - mov r11, [r9] // counter high 64 bits (big-endian) - bswap r11 - mov rbx, [r9 + 8] // counter low 64 bits (big-endian) + mov r11, [r9] // counter high 64 bits, KEPT big-endian (store-only) + mov rbx, [r9 + 8] // counter low 64 bits, native-LE in rbx bswap rbx // ---- stage batch 0 into the scratch, then load it into xmm0..7 ---- - mov rax, r11 - bswap rax - mov [rsp + 0], rax + mov [rsp + 0], r11 mov rax, rbx bswap rax mov [rsp + 8], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 16], rax + jnc @@StgOk0 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk0: + mov [rsp + 16], r11 mov rax, rbx bswap rax mov [rsp + 24], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 32], rax + jnc @@StgOk1 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk1: + mov [rsp + 32], r11 mov rax, rbx bswap rax mov [rsp + 40], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 48], rax + jnc @@StgOk2 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk2: + mov [rsp + 48], r11 mov rax, rbx bswap rax mov [rsp + 56], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 64], rax + jnc @@StgOk3 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk3: + mov [rsp + 64], r11 mov rax, rbx bswap rax mov [rsp + 72], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 80], rax + jnc @@StgOk4 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk4: + mov [rsp + 80], r11 mov rax, rbx bswap rax mov [rsp + 88], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 96], rax + jnc @@StgOk5 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk5: + mov [rsp + 96], r11 mov rax, rbx bswap rax mov [rsp + 104], rax add rbx, 1 - adc r11, 0 - mov rax, r11 - bswap rax - mov [rsp + 112], rax + jnc @@StgOk6 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk6: + mov [rsp + 112], r11 mov rax, rbx bswap rax mov [rsp + 120], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk7 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk7: movdqu xmm0, [rsp] movdqu xmm1, [rsp + 16] @@ -149,14 +164,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 0], rax + mov [rsp + 0], r11 mov rax, rbx bswap rax mov [rsp + 8], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk8 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk8: movdqa xmm8, [r8 + 32] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -167,14 +184,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 16], rax + mov [rsp + 16], r11 mov rax, rbx bswap rax mov [rsp + 24], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk9 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk9: movdqa xmm8, [r8 + 48] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -185,14 +204,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 32], rax + mov [rsp + 32], r11 mov rax, rbx bswap rax mov [rsp + 40], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk10 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk10: movdqa xmm8, [r8 + 64] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -203,14 +224,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 48], rax + mov [rsp + 48], r11 mov rax, rbx bswap rax mov [rsp + 56], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk11 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk11: movdqa xmm8, [r8 + 80] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -221,14 +244,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 64], rax + mov [rsp + 64], r11 mov rax, rbx bswap rax mov [rsp + 72], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk12 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk12: movdqa xmm8, [r8 + 96] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -239,14 +264,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 80], rax + mov [rsp + 80], r11 mov rax, rbx bswap rax mov [rsp + 88], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk13 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk13: movdqa xmm8, [r8 + 112] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -257,14 +284,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 96], rax + mov [rsp + 96], r11 mov rax, rbx bswap rax mov [rsp + 104], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk14 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk14: movdqa xmm8, [r8 + 128] db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 @@ -275,14 +304,16 @@ db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - mov rax, r11 - bswap rax - mov [rsp + 112], rax + mov [rsp + 112], r11 mov rax, rbx bswap rax mov [rsp + 120], rax add rbx, 1 - adc r11, 0 + jnc @@StgOk15 + bswap r11 + add r11, 1 + bswap r11 +@@StgOk15: // ---- round 9 (all key sizes): aesenc, no more staging ---- movdqa xmm8, [r8 + 144] @@ -399,13 +430,14 @@ // ---- undo the one-batch build-ahead, write the counter back (big-endian) ---- sub rbx, 8 - sbb r11, 0 - mov rax, r11 - bswap rax - mov [r9], rax - mov rax, rbx - bswap rax - mov [r9 + 8], rax + jnc @@CtrWbStore + bswap r11 + sub r11, 1 + bswap r11 +@@CtrWbStore: + mov [r9], r11 + bswap rbx + mov [r9 + 8], rbx add rsp, 128 {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc index 7b401a64..19d08026 100644 --- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc @@ -1,324 +1,280 @@ // Fused AES-NI CTR keystream + 8-way GHASH kernel (i386). // ------------------------------------------------------------------- // Technique: Shay Gueron, "Intel AES New Instructions" Rev 3.01 - AES -// and PCLMULQDQ instructions are interleaved so the AES port (p0 on -// Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding -// GHASH latency inside the AES critical path. +// and PCLMULQDQ instructions are interleaved so the AES port and the +// carry-less-multiply port operate in parallel, hiding GHASH latency +// inside the AES critical path. // // Loops over BatchCount pipelined batches. Each batch AES-encrypts eight -// counter blocks to produce a keystream, XORs it with PXorIn to produce POut, -// and concurrently GHASHes the PREVIOUS batch's 8 ciphertext blocks into the -// running authentication state (the pipeline lags by one batch). AES inside GCM -// is always CTR keystream construction; GhashFromOutput picks which buffer feeds -// GHASH (output on encrypt, input on decrypt). The 8-xmm budget does not fit a -// single-pass 8-wide layout, so the body processes the batch as two back-to-back -// 4-block halves sharing the same running Z0/Z1/Z2 accumulators; each half packs -// 4 GHASH iters into AES rounds 1..4 to keep both ports busy, rounds 5..N-1 are -// AES-only, and the 3-limb reduction runs once after half 2. Only 4 counter -// blocks are live per half, so each half builds its four in-register (eax/edx -// are free there) - no counter scratch buffer is needed. The mode primes batch 0 -// and drains the last. +// counter blocks (generated in-register from Counter32 + PJ0Template) to +// produce a keystream, XORs it with PXorIn to produce POut, and concurrently +// GHASHes the PREVIOUS batch's 8 ciphertext blocks into the running +// authentication state (the pipeline lags by one batch). AES inside GCM is +// always CTR keystream construction; GhashFromOutput picks which buffer feeds +// GHASH (output on encrypt, input on decrypt). The mode primes batch 0 and +// drains the last. +// +// PHPow128 holds the kernel-owned H^8..H^1 table PRE-MULTIPLIED BY x in the +// reflected representation, so the batch product reduces with two +// carry-less multiplies by 0xC2000000_00000000 instead of a shift/XOR +// ladder. The running GHASH state stays reflected between batches and is +// only byte-reversed at kernel entry/exit. // // Variant selection (set EXACTLY ONE): // GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes). // GCM_FUSED_AES_ROUNDS_12 -> AES-192 (schedule = 13 x 16 bytes). // GCM_FUSED_AES_ROUNDS_14 -> AES-256 (schedule = 15 x 16 bytes). // +// The 8-xmm budget does not fit a single-pass 8-wide layout, so the body +// processes the batch as two back-to-back 4-block halves sharing the same +// Z0/Z1/Z2 accumulator slots; each half packs 4 GHASH iters into AES +// rounds 1..4, rounds 5..N-1 are AES-only, and the reduction runs once +// after half 2. Only 4 counter blocks are live per half, so each half +// builds its four in-register right before that half runs. +// // On entry (after ClpSimdProc1Begin_i386.inc): ebx = PCtx. +// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 4-byte fields): +// [ebx + 0] PXorIn [ebx + 4] POut [ebx + 8] PPrevCipher +// [ebx + 12] PRoundKeys (16-byte aligned) [ebx + 16] PHPow128 +// [ebx + 20] PFS [ebx + 24] PMask [ebx + 28] Counter32 +// [ebx + 32] PJ0Template [ebx + 36] BatchCount +// [ebx + 40] GhashFromOutput (1 = encrypt) // -// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 4-byte fields). -// [ebx + 0] PXorIn : 128-byte input to XOR with keystream (advances) -// [ebx + 4] POut : 128-byte output (advances; may alias PXorIn) -// [ebx + 8] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch) -// [ebx + 12] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) -// [ebx + 16] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) -// [ebx + 20] PFS : 16-byte running GHASH state (in/out, canonical) -// [ebx + 24] PMask : 16-byte PSHUFB byte-reverse mask -// [ebx + 28] Counter32 : 32-bit GCM counter, last-used value (advances by 8) -// [ebx + 32] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block) -// [ebx + 36] BatchCount : number of 8-block batches to process -// [ebx + 40] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt) +// GPR: ebx=PCtx, edi=Keys (pinned), ebp=HPow (pinned), esi=PPrevCipher +// (rotates per batch), ecx=counter32 (in-register across the call), +// eax/edx=scratch. // -// XMM (8/8 used; i386 has only xmm0..xmm7): -// xmm0..xmm3 : 4 AES keystream lanes (per half; reloaded between halves) -// xmm4 : GHASH block cache (byte-reversed prev-cipher block, per iter) -// xmm5 : current H-power limb (per iter) -// xmm6 : pclmul product destination (re-inited from xmm4 per imm) -// xmm7 : transient scratch (byte-reverse mask, then Z-reload fold) +// XMM: xmm0..xmm3 = 4 AES keystream lanes (per half); xmm4 = GHASH block; +// xmm5 = H-power limb; xmm6/xmm7 = pclmul product scratch. // -// Stack scratch (64 bytes, base unaligned - movdqu only): -// [esp + 0..15] Z0 accumulator (updated 8x per batch) -// [esp + 16..31] Z1 accumulator (updated 16x per batch; imm=1 + imm=16) -// [esp + 32..47] Z2 accumulator (updated 8x per batch) -// [esp + 48..63] byte-reversed PFS (computed once, folded into iter 0) +// Stack scratch (base 16-aligned; caller esp parked inside): +// [esp + 0] Z0 [esp + 16] Z1 [esp + 32] Z2 (movdqa) +// [esp + 48] reflected running state (carried across batches) +// [esp + 64] byte-reverse mask copy (PSHUFB memory operand) +// [esp + 80] reduction constant [esp + 96] parked caller esp // -// AES-NI (aesenc / aesenclast), PCLMULQDQ, and PINSRD (counter build) -// instructions are db-encoded for broad assembler compatibility. +// AES-NI (aesenc / aesenclast), PCLMULQDQ, PINSRD, PSHUFD and the +// PSHUFB memory-operand form are db-encoded for broad assembler +// compatibility. // ------------------------------------------------------------------- - // --- Callee-save prologue. ClpSimdProc1Begin_i386.inc already pushed ebx - // and moved PCtx into ebx, so this block only pushes the other - // three callee-saved GPRs and reserves the Z / br(PFS) scratch. --- + // --- Callee-save prologue (ClpSimdProc1Begin_i386.inc already pushed + // ebx); park the caller esp inside the 16-aligned frame. --- push edi push esi push ebp - sub esp, 64 - - // --- Batch loop: process BatchCount batches, generating GCM counters and - // rotating the GHASH pipeline in-asm. i386 has no spare GP registers, so - // the loop state lives in the context: the 32-bit counter [ebx+28], the - // J0 template [ebx+32], the batch count [ebx+36], and the - // "GHASH-from-output" flag [ebx+40] (1=encrypt). Each batch's 8 counter - // blocks are built in-register, 4 per half, right before that half runs - // (see the per-half counter builds below). --- -@@GcmFusedLoop: + mov eax, esp + sub esp, 120 + and esp, -16 // align the frame + mov [esp + 96], eax // park the caller esp + + // --- Frame constants: reduction constant, mask copy, reflected state. --- + mov dword ptr [esp + 80], 0 + mov dword ptr [esp + 84], $C2000000 + mov dword ptr [esp + 88], 0 + mov dword ptr [esp + 92], 0 + mov eax, [ebx + 24] // PMask + movdqu xmm0, [eax] + movdqa [esp + 64], xmm0 + mov eax, [ebx + 20] // PFS + movdqu xmm1, [eax] + pshufb xmm1, xmm0 // canonical -> reflected + movdqa [esp + 48], xmm1 - // --- Load context pointers. --- - mov edi, [ebx + 12] // edi = PRoundKeys (pinned) - mov esi, [ebx + 8] // esi = PPrevCipher (pinned base) - mov ebp, [ebx + 16] // ebp = PHPow128 (pinned base) - mov ecx, [ebx + 24] // ecx = PMask (pinned) + // --- Pinned loop registers. --- + mov edi, [ebx + 12] // edi = PRoundKeys (aligned) + mov ebp, [ebx + 16] // ebp = PHPow128 (x-shifted) + mov esi, [ebx + 8] // esi = PPrevCipher + mov ecx, [ebx + 28] // ecx = counter32 - // --- Zero Z0 / Z1 / Z2 accumulator slots. --- +@@GcmFusedLoop: + // --- Zero the Z accumulator slots. --- pxor xmm0, xmm0 - movdqu [esp + 0], xmm0 // Z0 = 0 - movdqu [esp + 16], xmm0 // Z1 = 0 - movdqu [esp + 32], xmm0 // Z2 = 0 - - // --- Compute byte-reversed running state once, stash for iter-0 fold. --- - mov eax, [ebx + 20] // eax = PFS - movdqu xmm0, [eax] // load canonical PFS - movdqu xmm7, [ecx] // mask - pshufb xmm0, xmm7 // byte-reverse - movdqu [esp + 48], xmm0 // stash byte-reversed PFS + movdqa [esp + 0], xmm0 // Z0 = 0 + movdqa [esp + 16], xmm0 // Z1 = 0 + movdqa [esp + 32], xmm0 // Z2 = 0 // --- Build 4 counter blocks for HALF 1 (blocks 0..3): J0[0..11] || - // BE(counter32+1..+4). GCM's counter is the last-used value, so the batch - // based at counter32 spans counter32+1..counter32+8 (matching the scalar - // GetNextCtrBlocks8). Load J0 into each lane, then PINSRD (db-encoded) the - // big-endian counter word into dword 3; eax/edx are free here. --- - mov eax, [ebx + 32] // PJ0Template + // BE(counter32+1..+4); the counter holds the last-used value. --- + mov eax, [ebx + 32] // PJ0Template movdqu xmm0, [eax] movdqu xmm1, [eax] movdqu xmm2, [eax] movdqu xmm3, [eax] - mov eax, [ebx + 28] // counter32 - lea edx, [eax + 1] + lea edx, [ecx + 1] bswap edx - db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 - lea edx, [eax + 2] + db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 + lea edx, [ecx + 2] bswap edx - db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 - lea edx, [eax + 3] + db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 + lea edx, [ecx + 3] bswap edx - db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 - lea edx, [eax + 4] + db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 + lea edx, [ecx + 4] bswap edx - db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 + db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 // ====================================================================== - // HALF 1: blocks 0..3, GHASH iters 0..3 (prev_cipher[0..3], H^8..H^5) - // iter 0 additionally folds byte-reversed PFS into the block. + // HALF 1: blocks 0..3, GHASH iters 0..3 (prev_cipher[0..3], H^8..H^5); + // iter 0 additionally folds the carried reflected state. // ====================================================================== - // --- Round 0: AddRoundKey (pxor with rk[0]). Round keys are aligned. --- + // --- Round 0: AddRoundKey (aligned key memory operands). --- pxor xmm0, [edi + 0] pxor xmm1, [edi + 0] pxor xmm2, [edi + 0] pxor xmm3, [edi + 0] - // --- Round 1 + GHASH iter 0 (with state fold). --- - db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10] - db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10] - db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10] - db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10] - - // GHASH iter 0: block 0 byte-reverse + state fold - movdqu xmm4, [esi + 0] // prev cipher block 0 - movdqu xmm7, [ecx] // mask - pshufb xmm4, xmm7 // byte-reverse - movdqu xmm7, [esp + 48] // byte-reversed PFS - pxor xmm4, xmm7 // fold running state - movdqu xmm5, [ebp + 0] // H^8 - - // 4 pclmul -> Z0 (imm=0), Z1 (imm=1), Z1 (imm=16), Z2 (imm=17). - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 // Z0 += + // --- Round 1 + GHASH iter 0 (with carried-state fold). --- + db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10] + db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10] + db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10] + db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10] - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 // Z1 += + movdqu xmm4, [esi + 0] // prev cipher block 0 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + pxor xmm4, [esp + 48] // fold carried state + movdqu xmm5, [ebp + 0] // H^8 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 // Z1 += - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 // Z2 += + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += // --- Round 2 + GHASH iter 1. --- - db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20] - db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20] - db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20] - db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20] + db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20] + db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20] + db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20] + db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20] movdqu xmm4, [esi + 16] // prev cipher block 1 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 16] // H^7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 16] // H^7 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += // --- Round 3 + GHASH iter 2. --- - db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30] - db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30] - db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30] - db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30] + db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30] + db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30] + db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30] + db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30] movdqu xmm4, [esi + 32] // prev cipher block 2 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 32] // H^6 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 32] // H^6 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += - // --- Round 4 + GHASH iter 3 (last GHASH iter of half 1). --- - db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40] - db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40] - db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40] - db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40] + // --- Round 4 + GHASH iter 3. --- + db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40] + db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40] + db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40] + db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40] movdqu xmm4, [esi + 48] // prev cipher block 3 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 48] // H^5 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 48] // H^5 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += // --- Rounds 5..Nr-1 (variant-specific): AES-only. --- - - // Round 5 (disp8 = 80) - db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50] - db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50] - db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50] - db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50] - // Round 6 (disp8 = 96) - db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60] - db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60] - db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60] - db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60] - // Round 7 (disp8 = 112) - db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70] - db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70] - db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70] - db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70] - // Round 8 (disp32 = 128) + // Round 5 + db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50] + db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50] + db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50] + db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50] + // Round 6 + db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60] + db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60] + db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60] + db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60] + // Round 7 + db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70] + db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70] + db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70] + db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70] + // Round 8 db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+0x80] db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+0x80] db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+0x80] db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+0x80] - // Round 9 (disp32 = 144) + // Round 9 db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+0x90] db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+0x90] db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+0x90] db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+0x90] - {$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)} - // Round 10 (disp32 = 160) + // Round 10 db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+0xA0] db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+0xA0] db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+0xA0] db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+0xA0] - // Round 11 (disp32 = 176) + // Round 11 db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+0xB0] db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+0xB0] db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+0xB0] db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+0xB0] {$IFEND} - {$IFDEF GCM_FUSED_AES_ROUNDS_14} - // Round 12 (disp32 = 192) + // Round 12 db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+0xC0] db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+0xC0] db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+0xC0] db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+0xC0] - // Round 13 (disp32 = 208) + // Round 13 db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+0xD0] db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+0xD0] db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+0xD0] @@ -327,69 +283,63 @@ // --- Final AES round (aesenclast on rk[Nr]). --- {$IFDEF GCM_FUSED_AES_ROUNDS_10} - // AES-128: aesenclast at disp32 = 160 db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+0xA0] db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+0xA0] db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+0xA0] db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+0xA0] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_12} - // AES-192: aesenclast at disp32 = 192 db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+0xC0] db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+0xC0] db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+0xC0] db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+0xC0] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_14} - // AES-256: aesenclast at disp32 = 224 db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+0xE0] db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+0xE0] db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+0xE0] db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+0xE0] {$ENDIF} - // --- Post-AES (half 1): XOR with PXorIn[0..48] and store to POut[0..48]. --- - mov eax, [ebx + 0] // eax = PXorIn - mov edx, [ebx + 4] // edx = POut - - movdqu xmm7, [eax + 0] - pxor xmm0, xmm7 + // --- Post-AES (half 1): XOR with PXorIn[0..48], store POut. --- + mov eax, [ebx + 0] // eax = PXorIn + mov edx, [ebx + 4] // edx = POut + movdqu xmm4, [eax + 0] + pxor xmm0, xmm4 movdqu [edx + 0], xmm0 - movdqu xmm7, [eax + 16] - pxor xmm1, xmm7 + movdqu xmm4, [eax + 16] + pxor xmm1, xmm4 movdqu [edx + 16], xmm1 - movdqu xmm7, [eax + 32] - pxor xmm2, xmm7 + movdqu xmm4, [eax + 32] + pxor xmm2, xmm4 movdqu [edx + 32], xmm2 - movdqu xmm7, [eax + 48] - pxor xmm3, xmm7 + movdqu xmm4, [eax + 48] + pxor xmm3, xmm4 movdqu [edx + 48], xmm3 - // ====================================================================== - // HALF 2: blocks 4..7, GHASH iters 4..7 (prev_cipher[4..7], H^4..H^1) - // no state fold (already consumed by iter 0 of half 1). - // ====================================================================== - - // --- Build 4 counter blocks for half 2 (blocks 4..7): counter32+5..+8. The - // pinned regs (edi/esi/ebp/ecx) are untouched; eax/edx are free here. --- - mov eax, [ebx + 32] // PJ0Template + // --- Build 4 counter blocks for HALF 2 (blocks 4..7): counter32+5..+8. --- + mov eax, [ebx + 32] // PJ0Template movdqu xmm0, [eax] movdqu xmm1, [eax] movdqu xmm2, [eax] movdqu xmm3, [eax] - mov eax, [ebx + 28] // counter32 - lea edx, [eax + 5] + lea edx, [ecx + 5] bswap edx - db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 - lea edx, [eax + 6] + db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 + lea edx, [ecx + 6] bswap edx - db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 - lea edx, [eax + 7] + db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 + lea edx, [ecx + 7] bswap edx - db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 - lea edx, [eax + 8] + db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 + lea edx, [ecx + 8] bswap edx - db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 + db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 + + // ====================================================================== + // HALF 2: blocks 4..7, GHASH iters 4..7 (prev_cipher[4..7], H^4..H^1); + // no state fold (already consumed by iter 0 of half 1). + // ====================================================================== // --- Round 0: AddRoundKey. --- pxor xmm0, [edi + 0] @@ -398,337 +348,245 @@ pxor xmm3, [edi + 0] // --- Round 1 + GHASH iter 4. --- - db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10] - db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10] - db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10] - db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10] + db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10] + db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10] + db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10] + db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10] movdqu xmm4, [esi + 64] // prev cipher block 4 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 64] // H^4 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 64] // H^4 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += // --- Round 2 + GHASH iter 5. --- - db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20] - db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20] - db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20] - db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20] + db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20] + db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20] + db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20] + db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20] movdqu xmm4, [esi + 80] // prev cipher block 5 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 80] // H^3 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 80] // H^3 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += // --- Round 3 + GHASH iter 6. --- - db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30] - db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30] - db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30] - db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30] + db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30] + db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30] + db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30] + db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30] movdqu xmm4, [esi + 96] // prev cipher block 6 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 96] // H^2 - - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 96] // H^2 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 - - // --- Round 4 + GHASH iter 7 (last GHASH iter of batch). --- - db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40] - db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40] - db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40] - db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40] - - movdqu xmm4, [esi + 112] // prev cipher block 7 - movdqu xmm7, [ecx] - pshufb xmm4, xmm7 - movdqu xmm5, [ebp + 112] // H^1 + // --- Round 4 + GHASH iter 7. --- + db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40] + db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40] + db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40] + db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40] - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 - movdqu xmm7, [esp + 0] - pxor xmm7, xmm6 - movdqu [esp + 0], xmm7 + movdqu xmm4, [esi + 112] // prev cipher block 7 + db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse) + movdqu xmm5, [ebp + 112] // H^1 (x-shifted) movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 - + db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11 + pxor xmm6, [esp + 0] + movdqa [esp + 0], xmm6 // Z0 += + pxor xmm7, [esp + 32] + movdqa [esp + 32], xmm7 // Z2 += movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10 - movdqu xmm7, [esp + 16] - pxor xmm7, xmm6 - movdqu [esp + 16], xmm7 + db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01 + movdqa xmm7, xmm4 + db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10 + pxor xmm6, xmm7 // both cross products + pxor xmm6, [esp + 16] + movdqa [esp + 16], xmm6 // Z1 += - movdqa xmm6, xmm4 - db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11 - movdqu xmm7, [esp + 32] - pxor xmm7, xmm6 - movdqu [esp + 32], xmm7 - - // --- Rounds 5..Nr-1 AES-only (variant-specific). --- - // Round 5 (disp8 = 80) - db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50] - db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50] - db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50] - db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50] - // Round 6 (disp8 = 96) - db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60] - db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60] - db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60] - db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60] - // Round 7 (disp8 = 112) - db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70] - db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70] - db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70] - db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70] - // Round 8 (disp32 = 128) + // --- Rounds 5..Nr-1 (variant-specific): AES-only. --- + // Round 5 + db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50] + db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50] + db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50] + db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50] + // Round 6 + db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60] + db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60] + db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60] + db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60] + // Round 7 + db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70] + db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70] + db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70] + db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70] + // Round 8 db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+0x80] db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+0x80] db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+0x80] db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+0x80] - // Round 9 (disp32 = 144) + // Round 9 db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+0x90] db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+0x90] db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+0x90] db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+0x90] - {$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)} - // Round 10 (disp32 = 160) + // Round 10 db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+0xA0] db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+0xA0] db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+0xA0] db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+0xA0] - // Round 11 (disp32 = 176) + // Round 11 db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+0xB0] db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+0xB0] db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+0xB0] db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+0xB0] {$IFEND} - {$IFDEF GCM_FUSED_AES_ROUNDS_14} - // Round 12 (disp32 = 192) + // Round 12 db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+0xC0] db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+0xC0] db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+0xC0] db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+0xC0] - // Round 13 (disp32 = 208) + // Round 13 db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+0xD0] db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+0xD0] db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+0xD0] db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+0xD0] {$ENDIF} - // --- Final AES round (aesenclast) for half 2. --- + // --- Final AES round (aesenclast on rk[Nr]). --- {$IFDEF GCM_FUSED_AES_ROUNDS_10} - // AES-128: aesenclast at disp32 = 160 db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+0xA0] db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+0xA0] db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+0xA0] db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+0xA0] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_12} - // AES-192: aesenclast at disp32 = 192 db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+0xC0] db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+0xC0] db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+0xC0] db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+0xC0] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_14} - // AES-256: aesenclast at disp32 = 224 db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+0xE0] db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+0xE0] db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+0xE0] db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+0xE0] {$ENDIF} - // --- Post-AES (half 2): XOR with PXorIn[64..112] and store POut[64..112]. --- - mov eax, [ebx + 0] // eax = PXorIn - mov edx, [ebx + 4] // edx = POut - - movdqu xmm7, [eax + 64] - pxor xmm0, xmm7 + // --- Post-AES (half 2): XOR with PXorIn[64..112], store POut. --- + mov eax, [ebx + 0] // eax = PXorIn + mov edx, [ebx + 4] // edx = POut + movdqu xmm4, [eax + 64] + pxor xmm0, xmm4 movdqu [edx + 64], xmm0 - movdqu xmm7, [eax + 80] - pxor xmm1, xmm7 + movdqu xmm4, [eax + 80] + pxor xmm1, xmm4 movdqu [edx + 80], xmm1 - movdqu xmm7, [eax + 96] - pxor xmm2, xmm7 + movdqu xmm4, [eax + 96] + pxor xmm2, xmm4 movdqu [edx + 96], xmm2 - movdqu xmm7, [eax + 112] - pxor xmm3, xmm7 + movdqu xmm4, [eax + 112] + pxor xmm3, xmm4 movdqu [edx + 112], xmm3 - // ====================================================================== - // Inline 3-limb GHASH reduction on Z0 / Z1 / Z2 (copied from - // GcmGhashFull_i386.inc). Load the stack-resident accumulators into - // xmm0..xmm2 (keystream lanes are no longer needed after the stores - // above), then fold Z1 into (Z0, Z2) and reduce modulo - // x^128 + x^7 + x^2 + x + 1. - // ====================================================================== - movdqu xmm0, [esp + 0] // Z0 - movdqu xmm1, [esp + 16] // Z1 - movdqu xmm2, [esp + 32] // Z2 - + // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into + // the 256-bit product, then two carry-less multiplies by the + // folded constant 0xC2000000_00000000 (the H powers are + // pre-multiplied by x, so no bit-shift correction is needed). --- + movdqa xmm0, [esp + 0] // Z0 + movdqa xmm1, [esp + 16] // Z1 + movdqa xmm2, [esp + 32] // Z2 + movdqa xmm3, [esp + 80] movdqa xmm4, xmm1 - movdqa xmm5, xmm1 pslldq xmm4, 8 + pxor xmm0, xmm4 // ZLo ^= Z1 << 64 + movdqa xmm5, xmm1 psrldq xmm5, 8 - pxor xmm0, xmm4 - pxor xmm2, xmm5 - movq xmm3, xmm0 - movq xmm4, xmm2 - movdqa xmm5, xmm3 - psrlq xmm5, 1 - movdqa xmm6, xmm3 - psrlq xmm6, 2 - movdqa xmm7, xmm3 - psrlq xmm7, 7 - pxor xmm4, xmm3 - pxor xmm4, xmm5 - pxor xmm4, xmm6 - pxor xmm4, xmm7 - movdqa xmm1, xmm0 - psrldq xmm1, 8 - movdqa xmm5, xmm3 - psllq xmm5, 63 - movdqa xmm6, xmm3 - psllq xmm6, 62 - movdqa xmm7, xmm3 - psllq xmm7, 57 - pxor xmm1, xmm5 - pxor xmm1, xmm6 - pxor xmm1, xmm7 - movdqa xmm5, xmm2 - psrldq xmm5, 8 - movdqa xmm6, xmm5 - psllq xmm6, 1 - movdqa xmm7, xmm4 - psrlq xmm7, 63 - por xmm6, xmm7 - movdqa xmm7, xmm4 - psllq xmm7, 1 - movdqa xmm3, xmm1 - psrlq xmm3, 63 - por xmm7, xmm3 - movdqa xmm0, xmm1 - psllq xmm0, 1 - movdqa xmm3, xmm0 - psrlq xmm3, 1 - movdqa xmm2, xmm0 - psrlq xmm2, 2 - movdqa xmm5, xmm0 - psrlq xmm5, 7 - pxor xmm6, xmm0 - pxor xmm6, xmm3 - pxor xmm6, xmm2 - pxor xmm6, xmm5 - movdqa xmm3, xmm1 - psllq xmm3, 63 - movdqa xmm2, xmm1 - psllq xmm2, 58 - pxor xmm7, xmm3 - pxor xmm7, xmm2 - - // Pack and byte-reverse final GHASH state; store to PFS. - movq xmm0, xmm7 - pslldq xmm6, 8 - por xmm0, xmm6 - movdqu xmm7, [ecx] // mask reload - pshufb xmm0, xmm7 // byte-reverse -> canonical - mov eax, [ebx + 20] // PFS - movdqu [eax], xmm0 - - // --- Advance the context for the next batch, then loop. PPrevCipher becomes - // this batch's ciphertext (output on encrypt, input on decrypt); the - // counter advances by 8 in place so the caller reads the final value. --- - mov eax, [ebx + 4] // POut (encrypt: prev = output) - cmp dword ptr [ebx + 40], 0 // GHASH-from-output flag (1=encrypt) + pxor xmm2, xmm5 // ZHi ^= Z1 >> 64 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 2 + pxor xmm0, xmm2 // ^= ZHi -> reflected state + movdqa [esp + 48], xmm0 // carry state to the next batch + + // --- Advance for the next batch: PPrevCipher becomes this batch's + // ciphertext (output on encrypt, input on decrypt). --- + mov esi, [ebx + 4] // POut (encrypt: prev = output) + cmp dword ptr [ebx + 40], 0 // GHASH-from-output flag jne @@GcmPrevSet - mov eax, [ebx + 0] // PXorIn (decrypt: prev = input) + mov esi, [ebx + 0] // PXorIn (decrypt: prev = input) @@GcmPrevSet: - mov [ebx + 8], eax // PPrevCipher = this batch's ciphertext - add dword ptr [ebx + 0], 128 // PXorIn += 128 - add dword ptr [ebx + 4], 128 // POut += 128 - add dword ptr [ebx + 28], 8 // counter32 += 8 - dec dword ptr [ebx + 36] // batch count + add dword ptr [ebx + 0], 128 // PXorIn += 128 + add dword ptr [ebx + 4], 128 // POut += 128 + add ecx, 8 // counter32 += 8 + dec dword ptr [ebx + 36] // batch count jnz @@GcmFusedLoop - // --- Epilogue: unwind stack scratch and pop callee-saved GPRs. ebx - // is popped last because ClpSimdProc1Begin_i386.inc pushed it first. --- - add esp, 64 + // --- Epilogue: store counter, canonicalize + store the state, unwind. + // ebx is popped last (ClpSimdProc1Begin_i386.inc pushed it first). --- + mov [ebx + 28], ecx // store advanced counter32 + movdqa xmm0, [esp + 48] + db $66, $0F, $38, $00, $44, $24, $40 // pshufb xmm0, [esp+64] (-> canonical) + mov eax, [ebx + 20] // PFS + movdqu [eax], xmm0 + + mov esp, [esp + 96] pop ebp pop esi pop edi pop ebx + diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc index 7fa864b7..da216efc 100644 --- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc @@ -1,9 +1,9 @@ // Fused AES-NI CTR keystream + 8-way GHASH kernel (x86-64). // ------------------------------------------------------------------- // Technique: Shay Gueron, "Intel AES New Instructions" Rev 3.01 - AES -// and PCLMULQDQ instructions are interleaved so the AES port (p0 on -// Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding -// GHASH latency inside the AES critical path. +// and PCLMULQDQ instructions are interleaved so the AES port and the +// carry-less-multiply port operate in parallel, hiding GHASH latency +// inside the AES critical path. // // Loops over BatchCount pipelined batches. Each batch AES-encrypts eight // counter blocks (generated in-register from Counter32 + PJ0Template) to @@ -11,9 +11,14 @@ // GHASHes the PREVIOUS batch's 8 ciphertext blocks into the running // authentication state (the pipeline lags by one batch). AES inside GCM is // always CTR keystream construction; GhashFromOutput picks which buffer feeds -// GHASH (output on encrypt, input on decrypt). Rounds 0..8 (with all 8 GHASH -// iters) are shared across key sizes; only rounds 9..N-1 (extra aesenc) and -// round N (aesenclast) depend on N. The mode primes batch 0 and drains the last. +// GHASH (output on encrypt, input on decrypt). The mode primes batch 0 and +// drains the last. +// +// PHPow128 holds the kernel-owned H^8..H^1 table PRE-MULTIPLIED BY x in the +// reflected representation, so the batch product reduces with two +// carry-less multiplies by 0xC2000000_00000000 instead of a shift/XOR +// ladder. The running GHASH state stays reflected between batches and is +// only byte-reversed at kernel entry/exit. // // Variant selection (set EXACTLY ONE): // GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes). @@ -21,42 +26,64 @@ // GCM_FUSED_AES_ROUNDS_14 -> AES-256 (schedule = 15 x 16 bytes). // // On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx. +// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 8-byte fields): +// [rcx + 0] PXorIn [rcx + 8] POut [rcx + 16] PPrevCipher +// [rcx + 24] PRoundKeys (16-byte aligned) [rcx + 32] PHPow128 +// [rcx + 40] PFS [rcx + 48] PMask [rcx + 56] Counter32 +// [rcx + 64] PJ0Template [rcx + 72] BatchCount +// [rcx + 80] GhashFromOutput (1 = encrypt) // -// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 8-byte fields). -// [rcx + 0] PXorIn : 128-byte input to XOR with keystream (advances) -// [rcx + 8] POut : 128-byte output (advances; may alias PXorIn) -// [rcx + 16] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch) -// [rcx + 24] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) -// [rcx + 32] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) -// [rcx + 40] PFS : 16-byte running GHASH state (in/out, canonical) -// [rcx + 48] PMask : 16-byte PSHUFB byte-reverse mask -// [rcx + 56] Counter32 : 32-bit GCM counter, last-used value (advances by 8) -// [rcx + 64] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block) -// [rcx + 72] BatchCount : number of 8-block batches to process -// [rcx + 80] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt) +// All loop-carried state lives in registers - no context memory is +// touched inside the batch loop: +// rsi/rdi = PXorIn/POut (advance) r10 = PPrevCipher (rotates) +// rdx = PRoundKeys r11 = PHPow128 r8d = counter32 r9 = PJ0 +// rbx = batch count ebp = GHASH-from-output flag rax = scratch // // XMM (16/16 used): -// xmm0..xmm7 : 8 AES keystream lanes (persistent across rounds) -// xmm8 : current AES round key (reloaded per round) -// xmm9..xmm11 : Z0 / Z1 / Z2 GHASH accumulators (across all 8 iters) +// xmm0..xmm7 : 8 AES keystream lanes (round keys via memory operands) +// xmm8 : keystream-XOR scratch (free during the rounds) +// xmm9..xmm11 : Z0 / Z1 / Z2 GHASH accumulators; xmm9 doubles as the +// reflected running state carried across batches // xmm12 : GHASH input block (byte-reversed, per iter) // xmm13 : current H-power limb (per iter) // xmm14 : pclmul scratch destination // xmm15 : byte-reverse mask (held for the whole kernel) // +// Stack: 24 bytes below the nonvolatile-save area; [rsp] is 16-aligned +// and holds the reduction constant (low qword 0xC2000000_00000000). +// // AES-NI and PCLMULQDQ instructions, plus selected forms touching -// xmm8..xmm15 (pxor, movdqa), are db-encoded for broad assembler -// compatibility. +// xmm8..xmm15 (pxor) and PINSRD/PSHUFD, are db-encoded for broad +// assembler compatibility. // ------------------------------------------------------------------- + push rbx + push rsi + push rdi + push rbp {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - // --- Batch loop. The context (rcx) carries advancing PXorIn/POut/PPrevCipher, - // the 32-bit GCM counter [rcx+64], the J0 template [rcx+72], the batch - // count [rcx+80], and the "GHASH-from-output" flag [rcx+88] (1=encrypt). - // r8d = counter32, r9 = J0 template (persist across the loop; both are - // volatile and untouched by the shared body). --- - mov r8d, [rcx + 56] // counter32 - mov r9, [rcx + 64] // PJ0Template + sub rsp, 24 // [rsp] 16-aligned + mov rax, $C200000000000000 + mov [rsp], rax // reduction constant (low qword) + xor eax, eax + mov [rsp + 8], rax + + // --- Hoist the whole loop state into registers. --- + mov rdx, [rcx + 24] // rdx = PRoundKeys (aligned) + mov r11, [rcx + 32] // r11 = PHPow128 (x-shifted) + mov rax, [rcx + 48] + movdqu xmm15, [rax] // xmm15 = byte-reverse mask + mov rsi, [rcx + 0] // rsi = PXorIn + mov rdi, [rcx + 8] // rdi = POut + mov r10, [rcx + 16] // r10 = PPrevCipher + mov r8d, [rcx + 56] // r8d = counter32 + mov r9, [rcx + 64] // r9 = PJ0Template + mov rbx, [rcx + 72] // rbx = batch count + mov ebp, [rcx + 80] // ebp = GHASH-from-output flag + mov rax, [rcx + 40] // rax = PFS + movdqu xmm9, [rax] + pshufb xmm9, xmm15 // xmm9 = reflected running state + @@GcmFusedLoop: // Build 8 counter blocks into xmm0..7: block j = J0[0..11] || BE(counter32+j+1). // GCM's 32-bit counter holds the last-used value, so the batch based at @@ -65,527 +92,408 @@ movdqu xmm0, [r9] lea eax, [r8 + 1] bswap eax - db $66, $0F, $3A, $22, $C0, $03 // pinsrd xmm0, eax, 3 + db $66, $0F, $3A, $22, $C0, $03 // pinsrd xmm0, eax, 3 movdqu xmm1, [r9] lea eax, [r8 + 2] bswap eax - db $66, $0F, $3A, $22, $C8, $03 // pinsrd xmm1, eax, 3 + db $66, $0F, $3A, $22, $C8, $03 // pinsrd xmm1, eax, 3 movdqu xmm2, [r9] lea eax, [r8 + 3] bswap eax - db $66, $0F, $3A, $22, $D0, $03 // pinsrd xmm2, eax, 3 + db $66, $0F, $3A, $22, $D0, $03 // pinsrd xmm2, eax, 3 movdqu xmm3, [r9] lea eax, [r8 + 4] bswap eax - db $66, $0F, $3A, $22, $D8, $03 // pinsrd xmm3, eax, 3 + db $66, $0F, $3A, $22, $D8, $03 // pinsrd xmm3, eax, 3 movdqu xmm4, [r9] lea eax, [r8 + 5] bswap eax - db $66, $0F, $3A, $22, $E0, $03 // pinsrd xmm4, eax, 3 + db $66, $0F, $3A, $22, $E0, $03 // pinsrd xmm4, eax, 3 movdqu xmm5, [r9] lea eax, [r8 + 6] bswap eax - db $66, $0F, $3A, $22, $E8, $03 // pinsrd xmm5, eax, 3 + db $66, $0F, $3A, $22, $E8, $03 // pinsrd xmm5, eax, 3 movdqu xmm6, [r9] lea eax, [r8 + 7] bswap eax - db $66, $0F, $3A, $22, $F0, $03 // pinsrd xmm6, eax, 3 + db $66, $0F, $3A, $22, $F0, $03 // pinsrd xmm6, eax, 3 movdqu xmm7, [r9] lea eax, [r8 + 8] bswap eax - db $66, $0F, $3A, $22, $F8, $03 // pinsrd xmm7, eax, 3 - - mov rax, [rcx + 48] // rax = PMask - movdqu xmm15, [rax] // xmm15 = byte-reverse mask - - mov rdx, [rcx + 24] // rdx = PRoundKeys - mov r10, [rcx + 16] // r10 = PPrevCipher - mov r11, [rcx + 32] // r11 = PHPow128 - - // --- Round 0: AddRoundKey + zero Z0/Z1/Z2 ---------------------------- - movdqa xmm8, [rdx] // rk[0] - db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8 - db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8 - db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8 - db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8 - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - db $66, $45, $0F, $EF, $C9 // pxor xmm9, xmm9 (Z0=0) - db $66, $45, $0F, $EF, $D2 // pxor xmm10, xmm10 (Z1=0) - db $66, $45, $0F, $EF, $DB // pxor xmm11, xmm11 (Z2=0) - - // --- Round 1 + GHASH iter 0 (with state fold) ------------------------ - movdqa xmm8, [rdx + 16] // rk[1] - - // GHASH iter 0 setup: load block 0, byte-reverse, fold running state - movdqu xmm12, [r10] // prev cipher block 0 + db $66, $0F, $3A, $22, $F8, $03 // pinsrd xmm7, eax, 3 + + // --- Round 0: AddRoundKey (aligned key memory operands) + zero Z1/Z2. + // Z0 (xmm9) still holds the carried state; iter 0 folds and clears it. --- + pxor xmm0, [rdx] + pxor xmm1, [rdx] + pxor xmm2, [rdx] + pxor xmm3, [rdx] + pxor xmm4, [rdx] + pxor xmm5, [rdx] + pxor xmm6, [rdx] + pxor xmm7, [rdx] + db $66, $45, $0F, $EF, $D2 // pxor xmm10, xmm10 (Z1=0) + db $66, $45, $0F, $EF, $DB // pxor xmm11, xmm11 (Z2=0) + + // --- Round 1 + GHASH iter 0 (with carried-state fold). --- + movdqu xmm12, [r10 + 0] // prev cipher block 0 pshufb xmm12, xmm15 // byte-reverse - mov rax, [rcx + 40] // rax = PFS - movdqu xmm14, [rax] // load running state - pshufb xmm14, xmm15 // byte-reverse state - db $66, $45, $0F, $EF, $E6 // pxor xmm12, xmm14 (fold) - movdqu xmm13, [r11] // H^8 + db $66, $45, $0F, $EF, $E1 // pxor xmm12, xmm9 (fold carried state) + db $66, $45, $0F, $EF, $C9 // pxor xmm9, xmm9 (Z0=0) + movdqu xmm13, [r11 + 0] // H^8 (x-shifted) - // Interleave 8 aesenc with 4 pclmul + 4 xor-accumulate - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0+=) - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1+=) - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1+=) - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2+=) + db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 2 + GHASH iter 1 ------------------------------------------ - movdqa xmm8, [rdx + 32] // rk[2] - movdqu xmm12, [r10 + 16] // prev cipher block 1 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 16] // H^7 + // --- Round 2 + GHASH iter 1. --- + movdqu xmm12, [r10 + 16] // prev cipher block 1 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 16] // H^7 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 3 + GHASH iter 2 ------------------------------------------ - movdqa xmm8, [rdx + 48] // rk[3] - movdqu xmm12, [r10 + 32] // prev cipher block 2 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 32] // H^6 + // --- Round 3 + GHASH iter 2. --- + movdqu xmm12, [r10 + 32] // prev cipher block 2 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 32] // H^6 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 4 + GHASH iter 3 ------------------------------------------ - movdqa xmm8, [rdx + 64] // rk[4] - movdqu xmm12, [r10 + 48] // prev cipher block 3 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 48] // H^5 + // --- Round 4 + GHASH iter 3. --- + movdqu xmm12, [r10 + 48] // prev cipher block 3 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 48] // H^5 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 5 + GHASH iter 4 ------------------------------------------ - movdqa xmm8, [rdx + 80] // rk[5] - movdqu xmm12, [r10 + 64] // prev cipher block 4 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 64] // H^4 + // --- Round 5 + GHASH iter 4. --- + movdqu xmm12, [r10 + 64] // prev cipher block 4 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 64] // H^4 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 6 + GHASH iter 5 ------------------------------------------ - movdqa xmm8, [rdx + 96] // rk[6] - movdqu xmm12, [r10 + 80] // prev cipher block 5 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 80] // H^3 + // --- Round 6 + GHASH iter 5. --- + movdqu xmm12, [r10 + 80] // prev cipher block 5 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 80] // H^3 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 7 + GHASH iter 6 ------------------------------------------ - movdqa xmm8, [rdx + 112] // rk[7] - movdqu xmm12, [r10 + 96] // prev cipher block 6 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 96] // H^2 + // --- Round 7 + GHASH iter 6. --- + movdqu xmm12, [r10 + 96] // prev cipher block 6 + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 96] // H^2 (x-shifted) - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 + db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Round 8 + GHASH iter 7 (last GHASH) ----------------------------- - movdqa xmm8, [rdx + 128] // rk[8] + // --- Round 8 + GHASH iter 7. --- movdqu xmm12, [r10 + 112] // prev cipher block 7 - pshufb xmm12, xmm15 - movdqu xmm13, [r11 + 112] // H^1 - - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - movdqa xmm14, xmm12 - db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 - - // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast --- - // AES-128 (Nr=10): aesenc rk[9]; aesenclast on rk[10] (offset 160). - // AES-192 (Nr=12): aesenc rk[9..11]; aesenclast on rk[12] (offset 192). - // AES-256 (Nr=14): aesenc rk[9..13]; aesenclast on rk[14] (offset 224). - // All three variants consume rk[9] (offset 144) as their first tail key. - -{$IFDEF GCM_FUSED_AES_ROUNDS_10} - movdqa xmm8, [rdx + 144] // rk[9] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 -{$ENDIF} - -{$IFDEF GCM_FUSED_AES_ROUNDS_12} - movdqa xmm8, [rdx + 144] // rk[9] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 160] // rk[10] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 176] // rk[11] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 -{$ENDIF} - + pshufb xmm12, xmm15 // byte-reverse + movdqu xmm13, [r11 + 112] // H^1 (x-shifted) + + db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00 + db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128] + db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=) + db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01 + db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10 + db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128] + db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=) + db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128] + movdqa xmm14, xmm12 + db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11 + db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128] + db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) + + // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast. --- + db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144] + db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144] + db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144] + db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144] + db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144] + db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144] + db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144] + db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144] +{$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)} + db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160] + db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160] + db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160] + db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160] + db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160] + db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160] + db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160] + db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160] + db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176] + db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176] + db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176] + db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176] + db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176] + db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176] + db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176] + db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176] +{$IFEND} {$IFDEF GCM_FUSED_AES_ROUNDS_14} - movdqa xmm8, [rdx + 144] // rk[9] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 160] // rk[10] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 176] // rk[11] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 192] // rk[12] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 - - movdqa xmm8, [rdx + 208] // rk[13] - db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 - db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 - db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 - db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 - db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 - db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 - db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 - db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192] + db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192] + db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192] + db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192] + db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192] + db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192] + db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192] + db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192] + db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208] + db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208] + db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208] + db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208] + db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208] + db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208] + db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208] + db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208] {$ENDIF} - - // --- Final AES round: aesenclast on rk[Nr] --------------------------- {$IFDEF GCM_FUSED_AES_ROUNDS_10} - movdqa xmm8, [rdx + 160] // rk[10] (last for AES-128) + db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160] + db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160] + db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160] + db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160] + db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160] + db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160] + db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160] + db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_12} - movdqa xmm8, [rdx + 192] // rk[12] (last for AES-192) + db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192] + db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192] + db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192] + db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192] + db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192] + db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192] + db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192] + db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192] {$ENDIF} {$IFDEF GCM_FUSED_AES_ROUNDS_14} - movdqa xmm8, [rdx + 224] // rk[14] (last for AES-256) + db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224] + db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224] + db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224] + db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224] + db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224] + db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224] + db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224] + db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224] {$ENDIF} - db $66, $41, $0F, $38, $DD, $C0 // aesenclast xmm0, xmm8 - db $66, $41, $0F, $38, $DD, $C8 // aesenclast xmm1, xmm8 - db $66, $41, $0F, $38, $DD, $D0 // aesenclast xmm2, xmm8 - db $66, $41, $0F, $38, $DD, $D8 // aesenclast xmm3, xmm8 - db $66, $41, $0F, $38, $DD, $E0 // aesenclast xmm4, xmm8 - db $66, $41, $0F, $38, $DD, $E8 // aesenclast xmm5, xmm8 - db $66, $41, $0F, $38, $DD, $F0 // aesenclast xmm6, xmm8 - db $66, $41, $0F, $38, $DD, $F8 // aesenclast xmm7, xmm8 - - // --- Post-AES: XOR keystream with PXorIn, store to POut -------------- - mov rax, [rcx + 0] // rax = PXorIn - mov rdx, [rcx + 8] // rdx = POut (was PRoundKeys; no longer needed) - - movdqu xmm8, [rax] - db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8 - movdqu [rdx], xmm0 - movdqu xmm8, [rax + 16] - db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8 - movdqu [rdx + 16], xmm1 - movdqu xmm8, [rax + 32] - db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8 - movdqu [rdx + 32], xmm2 - movdqu xmm8, [rax + 48] - db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8 - movdqu [rdx + 48], xmm3 - movdqu xmm8, [rax + 64] - db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 - movdqu [rdx + 64], xmm4 - movdqu xmm8, [rax + 80] - db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 - movdqu [rdx + 80], xmm5 - movdqu xmm8, [rax + 96] - db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 - movdqu [rdx + 96], xmm6 - movdqu xmm8, [rax + 112] - db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 - movdqu [rdx + 112], xmm7 - - // --- Inline 3-limb GHASH reduction on Z0 (xmm9), Z1 (xmm10), Z2 (xmm11): - // fold Z1 into (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1. --- - // Migrate to xmm0/xmm1/xmm2 to mirror GcmReduce3FoldSse2 layout so the - // scalar shift/XOR sequence below reuses xmm0..xmm7 as scratch freely. - movdqa xmm0, xmm9 // xmm0 = Z0 - movdqa xmm1, xmm10 // xmm1 = Z1 - movdqa xmm2, xmm11 // xmm2 = Z2 - // Fold Z1 into Z0/Z2 + // --- Post-AES: XOR keystream with PXorIn, store to POut. --- + movdqu xmm8, [rsi] + db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8 + movdqu [rdi], xmm0 + movdqu xmm8, [rsi + 16] + db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8 + movdqu [rdi + 16], xmm1 + movdqu xmm8, [rsi + 32] + db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8 + movdqu [rdi + 32], xmm2 + movdqu xmm8, [rsi + 48] + db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8 + movdqu [rdi + 48], xmm3 + movdqu xmm8, [rsi + 64] + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + movdqu [rdi + 64], xmm4 + movdqu xmm8, [rsi + 80] + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + movdqu [rdi + 80], xmm5 + movdqu xmm8, [rsi + 96] + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + movdqu [rdi + 96], xmm6 + movdqu xmm8, [rsi + 112] + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + movdqu [rdi + 112], xmm7 + + // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into + // the 256-bit product, then two carry-less multiplies by the + // folded constant 0xC2000000_00000000 (the H powers are + // pre-multiplied by x, so no bit-shift correction is needed). --- + movdqa xmm0, xmm9 // Z0 + movdqa xmm1, xmm10 // Z1 + movdqa xmm2, xmm11 // Z2 + movdqa xmm3, [rsp] movdqa xmm4, xmm1 - movdqa xmm5, xmm1 pslldq xmm4, 8 + pxor xmm0, xmm4 // ZLo ^= Z1 << 64 + movdqa xmm5, xmm1 psrldq xmm5, 8 - pxor xmm0, xmm4 // Z0 ^= (Z1 << 64) - pxor xmm2, xmm5 // Z2 ^= (Z1 >> 64) - - // Extract T3, T1 (low 64s); T2, T0 (upper 64s). - movq xmm3, xmm0 // T3 = low64(Z0 folded) - movq xmm4, xmm2 // T1 = low64(Z2 folded) - movdqa xmm5, xmm3 - psrlq xmm5, 1 - movdqa xmm6, xmm3 - psrlq xmm6, 2 - movdqa xmm7, xmm3 - psrlq xmm7, 7 - pxor xmm4, xmm3 - pxor xmm4, xmm5 - pxor xmm4, xmm6 - pxor xmm4, xmm7 // T1 ^ T3 ^ (T3>>1)^(>>2)^(>>7) - - movdqa xmm1, xmm0 - psrldq xmm1, 8 // T2 = upper64(Z0 folded) - movdqa xmm5, xmm3 - psllq xmm5, 63 - movdqa xmm6, xmm3 - psllq xmm6, 62 - movdqa xmm7, xmm3 - psllq xmm7, 57 - pxor xmm1, xmm5 - pxor xmm1, xmm6 - pxor xmm1, xmm7 // T2 ^ (T3<<63)^(<<62)^(<<57) - - movdqa xmm5, xmm2 - psrldq xmm5, 8 // T0 = upper64(Z2 folded) - movdqa xmm6, xmm5 - psllq xmm6, 1 - movdqa xmm7, xmm4 - psrlq xmm7, 63 - por xmm6, xmm7 // LZ0 init = (T0<<1)|(T1>>63) - - movdqa xmm7, xmm4 - psllq xmm7, 1 - movdqa xmm3, xmm1 - psrlq xmm3, 63 - por xmm7, xmm3 // LZ1 init = (T1<<1)|(T2>>63) - - movdqa xmm0, xmm1 - psllq xmm0, 1 // LZ2 = T2<<1 - - movdqa xmm3, xmm0 - psrlq xmm3, 1 - movdqa xmm2, xmm0 - psrlq xmm2, 2 - movdqa xmm5, xmm0 - psrlq xmm5, 7 - pxor xmm6, xmm0 - pxor xmm6, xmm3 - pxor xmm6, xmm2 - pxor xmm6, xmm5 // xmm6.low = final LZ0 - - movdqa xmm3, xmm1 - psllq xmm3, 63 - movdqa xmm2, xmm1 - psllq xmm2, 58 - pxor xmm7, xmm3 - pxor xmm7, xmm2 // xmm7.low = final LZ1 - - // Pack and byte-reverse final GHASH state; store to PFS. - movq xmm0, xmm7 - pslldq xmm6, 8 - por xmm0, xmm6 // xmm0 = [LZ0 | LZ1] - pshufb xmm0, xmm15 // byte-reverse -> canonical - mov rax, [rcx + 40] // PFS - movdqu [rax], xmm0 - - // --- Advance the context for the next batch, then loop. PPrevCipher becomes - // this batch's ciphertext (output on encrypt, input on decrypt). --- - mov rax, [rcx + 8] // POut (out[i]); encrypt: prev = output - cmp qword ptr [rcx + 80], 0 // GHASH-from-output flag (1 = encrypt) - jne @@GcmPrevSet - mov rax, [rcx + 0] // PXorIn (in[i]); decrypt: prev = input + pxor xmm2, xmm5 // ZHi ^= Z1 >> 64 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 2 + pxor xmm0, xmm2 // ^= ZHi -> reflected state + movdqa xmm9, xmm0 // carry state to the next batch + + // --- Advance for the next batch: PPrevCipher becomes this batch's + // ciphertext (output on encrypt, input on decrypt). --- + mov r10, rdi + test ebp, ebp // GHASH-from-output flag + jnz @@GcmPrevSet + mov r10, rsi // decrypt: prev = input @@GcmPrevSet: - mov [rcx + 16], rax // PPrevCipher = this batch's ciphertext - add qword ptr [rcx + 0], 128 // PXorIn += 128 - add qword ptr [rcx + 8], 128 // POut += 128 - add r8d, 8 // counter32 += 8 - dec qword ptr [rcx + 72] // batch count + add rsi, 128 + add rdi, 128 + add r8d, 8 + dec rbx jnz @@GcmFusedLoop - mov [rcx + 56], r8d // store advanced counter32 + + // --- Epilogue: canonicalize the state, store counter + PFS, unwind. --- + mov [rcx + 56], r8d // store advanced counter32 + pshufb xmm9, xmm15 // reflected -> canonical + mov rax, [rcx + 40] // PFS + movdqu [rax], xmm9 + add rsp, 24 {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} + pop rbp + pop rdi + pop rsi + pop rbx + diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc index 01d10874..2f470206 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc @@ -1,446 +1,1135 @@ -// OCB fused 4-wide AES-NI kernel (i386) with in-kernel offset ladder, -// L-table lookup and checksum fold. +// OCB fused 6-wide AES-NI kernel (i386) with a round-interleaved offset +// ladder, L-table lookup and checksum fold. // ------------------------------------------------------------------- -// Processes ABlockCount consecutive 16-byte OCB body blocks in chunks -// of 4 blocks (64 bytes) per iteration. 4 is the widest fused lane -// count that fits the 8-xmm budget once xmm_offset + xmm_checksum + -// scratch are reserved; memory-form aesenc / aesdec is therefore -// used throughout against the 16-byte-aligned round-key schedule. -// The kernel owns all per-block OCB bookkeeping across the batch: -// on entry it seeds xmm_offset from OffsetPtr, xmm_checksum from -// ChecksumPtr and a running block counter from StartBlockCount; the -// loop derives ntz per block itself (bsf on the incremented counter), -// XORs the L-table entry into xmm_offset to derive each Delta, runs -// P_i XOR delta XOR K_0 through the AES rounds, XORs delta into -// the post-aesenclast state to produce C_i, and folds plaintext -// into xmm_checksum; on exit it writes xmm_offset and xmm_checksum -// back. The mode layer never touches per-block state. +// Processes ABlockCount consecutive 16-byte OCB body blocks, two 6-block +// iterations (192 bytes) per loop pass. All 8 xmm registers are +// productive: xmm0..5 data lanes, xmm6 offset accumulator, xmm7 +// checksum accumulator outside the rounds / L-table scratch inside them +// (the checksum spills to an aligned frame slot across the round chain, +// where it is never read). The offset ladder for the NEXT 6-block +// iteration is staged between the AES round groups of the current one +// into the opposite of two ping-pong delta banks, so the ladder's +// serial bsf/load/xor chain rides the AES latency instead of running +// as a prefix; the prologue pre-stages iteration 0, and the final +// pass over-stages one iteration, undone afterwards by re-XORing the +// same six L entries (XOR is self-inverse; the counter is not +// persisted, so only the offset accumulator needs the rollback). +// Whitening / unwhitening take the aligned stashes and the aligned key +// schedule as pxor memory operands. The kernel owns all per-block OCB +// bookkeeping across the batch: it seeds xmm_offset / xmm_checksum and +// the 64-bit block counter on entry, derives ntz per block (bsf; the +// counter-high dword is only touched on a low-dword wrap), and writes +// both accumulators back on exit. // // Variant selection: // CRYPTOLIB_AESNI_DECRYPT -> aesdec / aesdeclast; else aesenc / aesenclast. // Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 selects N_R. // // On entry (after ClpSimdProc1Begin_i386.inc): ebx = PCtx. -// -// Context layout (7 pointers + NativeUInt + UInt64 = 40 bytes, 4-byte aligned). -// See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. -// [ebx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned) -// [ebx + 4] InPtr : base of the main 16-byte-block stream -// [ebx + 8] OutPtr : base of BlockCount output blocks (may alias InPtr) -// [ebx + 12] OffsetPtr : 16-byte live FOffsetMAIN (r/w) -// [ebx + 16] ChecksumPtr : 16-byte live FChecksum (r/w) -// [ebx + 20] LTablePtr : flat L[0..LMax] * 16 bytes (read-only; cached on stack) -// [ebx + 24] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) -// [ebx + 28] BlockCount : NativeUInt; multiple of 4 -// [ebx + 32] StartBlockCount : UInt64; block count before this span (ntz counter seed). -// lo at +32, hi at +36. -// -// XMM (7/8 used; i386 has only xmm0..xmm7): -// xmm0..xmm3 : 4 data lanes -// xmm4 : xmm_offset accumulator (seeded/stored once per batch) -// xmm5 : xmm_checksum accumulator (seeded/stored once per batch) -// xmm6 : scratch (L-table loads; stack-offset reloads during whitening) -// xmm7 : free +// Context layout: see TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. +// BlockCount must be a positive multiple of 12. // // GPR: ebx=PCtx, edi=Keys, edx=InPtr, esi=OutPtr, ecx=BlockCount, -// ebp=block counter low dword (bsf source), eax=scratch. The counter -// high dword lives on the stack (only touched on a 2^32 low-dword wrap), -// so the full 64-bit ntz is exact yet the hot path stays register-only. +// ebp=block counter low dword (bsf source), eax=scratch. // -// Stack scratch (72 bytes, base unaligned - movdqu only): -// [esp + 0..63] 4 x 16-byte per-block offset stashes -// [esp + 64..67] LTablePtr cache (one dword; GPR budget cannot pin it) -// [esp + 68..71] block counter high dword +// Stack scratch (16-aligned; caller esp parked inside): +// [esp + 0..95] delta bank 0 (6 x 16) [esp + 96..191] delta bank 1 +// [esp + 192] LTablePtr cache [esp + 196] counter high dword +// [esp + 200] parked caller esp [esp + 204] block-0 source ptr +// [esp + 208..223] checksum spill during the round chain // // AES-NI instructions are db-encoded for broad assembler compatibility. // ------------------------------------------------------------------- - // --- Callee-save prologue. ClpSimdProc1Begin_i386.inc already pushed ebx - // and moved PCtx into ebx, so this block only pushes the other - // three callee-saved GPRs and reserves stack scratch. --- push edi push esi push ebp - sub esp, 72 + mov eax, esp + sub esp, 240 + and esp, -16 // align the stash frame + mov [esp + 200], eax // park the caller esp - // --- Load context pointers and seed accumulators. --- + // --- Load context, seed accumulators and the block counter. --- mov edi, [ebx + 0] // edi = Keys (pinned) mov edx, [ebx + 4] // edx = InPtr (advancing) mov esi, [ebx + 8] // esi = OutPtr (advancing) - mov ecx, [ebx + 28] // ecx = BlockCount (>= 4, multiple of 4) - - // Seed the running block counter (64-bit) from StartBlockCount: low - // dword in ebp (bsf source), high dword in the [esp + 68] stack slot. + mov ecx, [ebx + 28] // ecx = BlockCount (>= 12, multiple of 12) mov ebp, [ebx + 32] // ebp = StartBlockCount low - mov eax, [ebx + 36] // eax = StartBlockCount high - mov [esp + 68], eax - - // Stash LTablePtr into the stack slot at [esp + 64]; the inner loop - // re-adds it to ntz*16 to form each L-table entry address. - mov eax, [ebx + 20] // eax = LTablePtr - mov [esp + 64], eax - - // Seed xmm_offset from [OffsetPtr] and xmm_checksum from [ChecksumPtr]. - mov eax, [ebx + 12] // eax = OffsetPtr - movdqu xmm4, [eax] // xmm4 = FOffsetMAIN - mov eax, [ebx + 16] // eax = ChecksumPtr - movdqu xmm5, [eax] // xmm5 = FChecksum + mov eax, [ebx + 36] + mov [esp + 196], eax // counter high dword + mov eax, [ebx + 20] + mov [esp + 192], eax // LTablePtr cache + mov eax, [ebx + 24] + mov [esp + 204], eax // iteration-0 block-0 source + mov eax, [ebx + 12] + movdqu xmm6, [eax] // xmm6 = FOffsetMAIN + mov eax, [ebx + 16] + movdqu xmm7, [eax] // xmm7 = FChecksum - // Seed the block-0 pointer. eax is consumed by Phase 2 at the top - // of each iteration (before Phase 1 clobbers it), and refreshed to - // edx at the bottom of every iteration so iter >= 1 transparently - // reads block 0 from the advancing main stream at offset 0. - mov eax, [ebx + 24] // eax = Block0Ptr + // --- Pre-stage iteration 0's Deltas into bank 0 (lanes dead: xmm0 + // is the table scratch here). --- + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p0 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 0], xmm6 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p1 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p1: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 16], xmm6 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p2 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p2: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 32], xmm6 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p3 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p3: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 48], xmm6 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p4 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p4: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 64], xmm6 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_p5 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_p5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + movdqa [esp + 80], xmm6 @@ocb_i386_loop: - // --- Phase 2: load 4 input blocks. Must run BEFORE the ladder - // because Phase 1 below clobbers eax (ntz scratch / L-table - // index). Block 0 comes from eax (iter 0: Block0Ptr; iter >= 1: - // main stream via the mov eax, edx at the bottom of iter k-1). - // Blocks 1..3 always come from the advancing main stream edx - // at [edx + 16..48]. --- + // --- Iteration A: load 6 blocks (block 0 via the staged source). --- + mov eax, [esp + 204] movdqu xmm0, [eax] movdqu xmm1, [edx + 16] movdqu xmm2, [edx + 32] movdqu xmm3, [edx + 48] + movdqu xmm4, [edx + 64] + movdqu xmm5, [edx + 80] + +{$IFNDEF CRYPTOLIB_AESNI_DECRYPT} + // Fold plaintext into the checksum (still live in xmm7). + pxor xmm7, xmm0 + pxor xmm7, xmm1 + pxor xmm7, xmm2 + pxor xmm7, xmm3 + pxor xmm7, xmm4 + pxor xmm7, xmm5 +{$ENDIF} + movdqa [esp + 208], xmm7 // spill checksum; xmm7 = ladder scratch + + // Pre-whiten with K_0 and this iteration's stashed Deltas. + pxor xmm0, [edi + 0] + pxor xmm1, [edi + 0] + pxor xmm2, [edi + 0] + pxor xmm3, [edi + 0] + pxor xmm4, [edi + 0] + pxor xmm5, [edi + 0] + pxor xmm0, [esp + 0] + pxor xmm1, [esp + 16] + pxor xmm2, [esp + 32] + pxor xmm3, [esp + 48] + pxor xmm4, [esp + 64] + pxor xmm5, [esp + 80] - // ------------------------------------------------------------------ - // Phase 1: offset ladder. For each block i in 0..3: - // advance the 64-bit block counter (ebp low + [esp+68] high) by one - // eax = ntz(counter) (bsf ebp; on a low-dword wrap, 32 + bsf hi) - // eax = eax * 16 (shl eax, 4) - // eax = LTablePtr + eax (add from stashed slot) - // xmm6 = L[ntz] (movdqu from [eax]) - // xmm4 ^= xmm6 (pxor) - // [esp + 16*i] <- xmm4 (movdqu stash of offset_i) - // After this phase xmm4 holds offset_3 which is the Delta carried - // into the next iteration, and [esp + 0..63] holds offsets_0..3. - // `add ebp, 1` (not inc) sets CF so the rare high-dword carry - // propagates; the bsf-high branch is essentially never taken. - // ------------------------------------------------------------------ - // Block 0 +{$IFDEF CRYPTOLIB_AESNI_DECRYPT} + // Round 1 + db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+16] + db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16] + db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16] + db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16] + db $66, $0F, $38, $DE, $67, $10 // aesdec xmm4, [edi+16] + db $66, $0F, $38, $DE, $6F, $10 // aesdec xmm5, [edi+16] + // stage next-iteration Delta 0 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA0 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 96], xmm6 + // Round 2 + db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32] + db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32] + db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32] + db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32] + db $66, $0F, $38, $DE, $67, $20 // aesdec xmm4, [edi+32] + db $66, $0F, $38, $DE, $6F, $20 // aesdec xmm5, [edi+32] + // stage next-iteration Delta 1 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA1 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA1: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 112], xmm6 + // Round 3 + db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48] + db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48] + db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48] + db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48] + db $66, $0F, $38, $DE, $67, $30 // aesdec xmm4, [edi+48] + db $66, $0F, $38, $DE, $6F, $30 // aesdec xmm5, [edi+48] + // stage next-iteration Delta 2 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA2 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA2: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 128], xmm6 + // Round 4 + db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64] + db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64] + db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64] + db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64] + db $66, $0F, $38, $DE, $67, $40 // aesdec xmm4, [edi+64] + db $66, $0F, $38, $DE, $6F, $40 // aesdec xmm5, [edi+64] + // stage next-iteration Delta 3 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA3 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA3: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 144], xmm6 + // Round 5 + db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80] + db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80] + db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80] + db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80] + db $66, $0F, $38, $DE, $67, $50 // aesdec xmm4, [edi+80] + db $66, $0F, $38, $DE, $6F, $50 // aesdec xmm5, [edi+80] + // stage next-iteration Delta 4 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA4 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA4: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 160], xmm6 + // Round 6 + db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96] + db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96] + db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96] + db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96] + db $66, $0F, $38, $DE, $67, $60 // aesdec xmm4, [edi+96] + db $66, $0F, $38, $DE, $6F, $60 // aesdec xmm5, [edi+96] + // stage next-iteration Delta 5 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA5 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 176], xmm6 + // Round 7 + db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112] + db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112] + db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112] + db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112] + db $66, $0F, $38, $DE, $67, $70 // aesdec xmm4, [edi+112] + db $66, $0F, $38, $DE, $6F, $70 // aesdec xmm5, [edi+112] + // Round 8 + db $66, $0F, $38, $DE, $87, $80, $00, $00, $00 // aesdec xmm0, [edi+128] + db $66, $0F, $38, $DE, $8F, $80, $00, $00, $00 // aesdec xmm1, [edi+128] + db $66, $0F, $38, $DE, $97, $80, $00, $00, $00 // aesdec xmm2, [edi+128] + db $66, $0F, $38, $DE, $9F, $80, $00, $00, $00 // aesdec xmm3, [edi+128] + db $66, $0F, $38, $DE, $A7, $80, $00, $00, $00 // aesdec xmm4, [edi+128] + db $66, $0F, $38, $DE, $AF, $80, $00, $00, $00 // aesdec xmm5, [edi+128] + // Round 9 + db $66, $0F, $38, $DE, $87, $90, $00, $00, $00 // aesdec xmm0, [edi+144] + db $66, $0F, $38, $DE, $8F, $90, $00, $00, $00 // aesdec xmm1, [edi+144] + db $66, $0F, $38, $DE, $97, $90, $00, $00, $00 // aesdec xmm2, [edi+144] + db $66, $0F, $38, $DE, $9F, $90, $00, $00, $00 // aesdec xmm3, [edi+144] + db $66, $0F, $38, $DE, $A7, $90, $00, $00, $00 // aesdec xmm4, [edi+144] + db $66, $0F, $38, $DE, $AF, $90, $00, $00, $00 // aesdec xmm5, [edi+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DE, $87, $A0, $00, $00, $00 // aesdec xmm0, [edi+160] + db $66, $0F, $38, $DE, $8F, $A0, $00, $00, $00 // aesdec xmm1, [edi+160] + db $66, $0F, $38, $DE, $97, $A0, $00, $00, $00 // aesdec xmm2, [edi+160] + db $66, $0F, $38, $DE, $9F, $A0, $00, $00, $00 // aesdec xmm3, [edi+160] + db $66, $0F, $38, $DE, $A7, $A0, $00, $00, $00 // aesdec xmm4, [edi+160] + db $66, $0F, $38, $DE, $AF, $A0, $00, $00, $00 // aesdec xmm5, [edi+160] + db $66, $0F, $38, $DE, $87, $B0, $00, $00, $00 // aesdec xmm0, [edi+176] + db $66, $0F, $38, $DE, $8F, $B0, $00, $00, $00 // aesdec xmm1, [edi+176] + db $66, $0F, $38, $DE, $97, $B0, $00, $00, $00 // aesdec xmm2, [edi+176] + db $66, $0F, $38, $DE, $9F, $B0, $00, $00, $00 // aesdec xmm3, [edi+176] + db $66, $0F, $38, $DE, $A7, $B0, $00, $00, $00 // aesdec xmm4, [edi+176] + db $66, $0F, $38, $DE, $AF, $B0, $00, $00, $00 // aesdec xmm5, [edi+176] +{$IFEND} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DE, $87, $C0, $00, $00, $00 // aesdec xmm0, [edi+192] + db $66, $0F, $38, $DE, $8F, $C0, $00, $00, $00 // aesdec xmm1, [edi+192] + db $66, $0F, $38, $DE, $97, $C0, $00, $00, $00 // aesdec xmm2, [edi+192] + db $66, $0F, $38, $DE, $9F, $C0, $00, $00, $00 // aesdec xmm3, [edi+192] + db $66, $0F, $38, $DE, $A7, $C0, $00, $00, $00 // aesdec xmm4, [edi+192] + db $66, $0F, $38, $DE, $AF, $C0, $00, $00, $00 // aesdec xmm5, [edi+192] + db $66, $0F, $38, $DE, $87, $D0, $00, $00, $00 // aesdec xmm0, [edi+208] + db $66, $0F, $38, $DE, $8F, $D0, $00, $00, $00 // aesdec xmm1, [edi+208] + db $66, $0F, $38, $DE, $97, $D0, $00, $00, $00 // aesdec xmm2, [edi+208] + db $66, $0F, $38, $DE, $9F, $D0, $00, $00, $00 // aesdec xmm3, [edi+208] + db $66, $0F, $38, $DE, $A7, $D0, $00, $00, $00 // aesdec xmm4, [edi+208] + db $66, $0F, $38, $DE, $AF, $D0, $00, $00, $00 // aesdec xmm5, [edi+208] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY128} + db $66, $0F, $38, $DF, $87, $A0, $00, $00, $00 // aesdeclast xmm0, [edi+160] + db $66, $0F, $38, $DF, $8F, $A0, $00, $00, $00 // aesdeclast xmm1, [edi+160] + db $66, $0F, $38, $DF, $97, $A0, $00, $00, $00 // aesdeclast xmm2, [edi+160] + db $66, $0F, $38, $DF, $9F, $A0, $00, $00, $00 // aesdeclast xmm3, [edi+160] + db $66, $0F, $38, $DF, $A7, $A0, $00, $00, $00 // aesdeclast xmm4, [edi+160] + db $66, $0F, $38, $DF, $AF, $A0, $00, $00, $00 // aesdeclast xmm5, [edi+160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + db $66, $0F, $38, $DF, $87, $C0, $00, $00, $00 // aesdeclast xmm0, [edi+192] + db $66, $0F, $38, $DF, $8F, $C0, $00, $00, $00 // aesdeclast xmm1, [edi+192] + db $66, $0F, $38, $DF, $97, $C0, $00, $00, $00 // aesdeclast xmm2, [edi+192] + db $66, $0F, $38, $DF, $9F, $C0, $00, $00, $00 // aesdeclast xmm3, [edi+192] + db $66, $0F, $38, $DF, $A7, $C0, $00, $00, $00 // aesdeclast xmm4, [edi+192] + db $66, $0F, $38, $DF, $AF, $C0, $00, $00, $00 // aesdeclast xmm5, [edi+192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DF, $87, $E0, $00, $00, $00 // aesdeclast xmm0, [edi+224] + db $66, $0F, $38, $DF, $8F, $E0, $00, $00, $00 // aesdeclast xmm1, [edi+224] + db $66, $0F, $38, $DF, $97, $E0, $00, $00, $00 // aesdeclast xmm2, [edi+224] + db $66, $0F, $38, $DF, $9F, $E0, $00, $00, $00 // aesdeclast xmm3, [edi+224] + db $66, $0F, $38, $DF, $A7, $E0, $00, $00, $00 // aesdeclast xmm4, [edi+224] + db $66, $0F, $38, $DF, $AF, $E0, $00, $00, $00 // aesdeclast xmm5, [edi+224] +{$ENDIF} +{$ELSE} + // Round 1 + db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16] + db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16] + db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16] + db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16] + db $66, $0F, $38, $DC, $67, $10 // aesenc xmm4, [edi+16] + db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16] + // stage next-iteration Delta 0 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA0 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 96], xmm6 + // Round 2 + db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32] + db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32] + db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32] + db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32] + db $66, $0F, $38, $DC, $67, $20 // aesenc xmm4, [edi+32] + db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32] + // stage next-iteration Delta 1 -> bank 1 add ebp, 1 - adc dword [esp + 68], 0 bsf eax, ebp - jnz @@ocb_i386_ntz0 - bsf eax, [esp + 68] + jnz @@ocb_i386_sA1 + inc dword [esp + 196] + bsf eax, [esp + 196] add eax, 32 -@@ocb_i386_ntz0: +@@ocb_i386_sA1: shl eax, 4 - add eax, [esp + 64] - movdqu xmm6, [eax] - pxor xmm4, xmm6 - movdqu [esp + 0], xmm4 - // Block 1 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 112], xmm6 + // Round 3 + db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48] + db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48] + db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48] + db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48] + db $66, $0F, $38, $DC, $67, $30 // aesenc xmm4, [edi+48] + db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48] + // stage next-iteration Delta 2 -> bank 1 add ebp, 1 - adc dword [esp + 68], 0 bsf eax, ebp - jnz @@ocb_i386_ntz1 - bsf eax, [esp + 68] + jnz @@ocb_i386_sA2 + inc dword [esp + 196] + bsf eax, [esp + 196] add eax, 32 -@@ocb_i386_ntz1: +@@ocb_i386_sA2: shl eax, 4 - add eax, [esp + 64] - movdqu xmm6, [eax] - pxor xmm4, xmm6 - movdqu [esp + 16], xmm4 - // Block 2 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 128], xmm6 + // Round 4 + db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64] + db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64] + db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64] + db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64] + db $66, $0F, $38, $DC, $67, $40 // aesenc xmm4, [edi+64] + db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64] + // stage next-iteration Delta 3 -> bank 1 add ebp, 1 - adc dword [esp + 68], 0 bsf eax, ebp - jnz @@ocb_i386_ntz2 - bsf eax, [esp + 68] + jnz @@ocb_i386_sA3 + inc dword [esp + 196] + bsf eax, [esp + 196] add eax, 32 -@@ocb_i386_ntz2: +@@ocb_i386_sA3: shl eax, 4 - add eax, [esp + 64] - movdqu xmm6, [eax] - pxor xmm4, xmm6 - movdqu [esp + 32], xmm4 - // Block 3 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 144], xmm6 + // Round 5 + db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80] + db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80] + db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80] + db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80] + db $66, $0F, $38, $DC, $67, $50 // aesenc xmm4, [edi+80] + db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80] + // stage next-iteration Delta 4 -> bank 1 add ebp, 1 - adc dword [esp + 68], 0 bsf eax, ebp - jnz @@ocb_i386_ntz3 - bsf eax, [esp + 68] + jnz @@ocb_i386_sA4 + inc dword [esp + 196] + bsf eax, [esp + 196] add eax, 32 -@@ocb_i386_ntz3: +@@ocb_i386_sA4: shl eax, 4 - add eax, [esp + 64] - movdqu xmm6, [eax] - pxor xmm4, xmm6 - movdqu [esp + 48], xmm4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 160], xmm6 + // Round 6 + db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96] + db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96] + db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96] + db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96] + db $66, $0F, $38, $DC, $67, $60 // aesenc xmm4, [edi+96] + db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96] + // stage next-iteration Delta 5 -> bank 1 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sA5 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sA5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 176], xmm6 + // Round 7 + db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112] + db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112] + db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112] + db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112] + db $66, $0F, $38, $DC, $67, $70 // aesenc xmm4, [edi+112] + db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112] + // Round 8 + db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128] + db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128] + db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+128] + db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+128] + db $66, $0F, $38, $DC, $A7, $80, $00, $00, $00 // aesenc xmm4, [edi+128] + db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128] + // Round 9 + db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144] + db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+144] + db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+144] + db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+144] + db $66, $0F, $38, $DC, $A7, $90, $00, $00, $00 // aesenc xmm4, [edi+144] + db $66, $0F, $38, $DC, $AF, $90, $00, $00, $00 // aesenc xmm5, [edi+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+160] + db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+160] + db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+160] + db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+160] + db $66, $0F, $38, $DC, $A7, $A0, $00, $00, $00 // aesenc xmm4, [edi+160] + db $66, $0F, $38, $DC, $AF, $A0, $00, $00, $00 // aesenc xmm5, [edi+160] + db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+176] + db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+176] + db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+176] + db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+176] + db $66, $0F, $38, $DC, $A7, $B0, $00, $00, $00 // aesenc xmm4, [edi+176] + db $66, $0F, $38, $DC, $AF, $B0, $00, $00, $00 // aesenc xmm5, [edi+176] +{$IFEND} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+192] + db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+192] + db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+192] + db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+192] + db $66, $0F, $38, $DC, $A7, $C0, $00, $00, $00 // aesenc xmm4, [edi+192] + db $66, $0F, $38, $DC, $AF, $C0, $00, $00, $00 // aesenc xmm5, [edi+192] + db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+208] + db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+208] + db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+208] + db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+208] + db $66, $0F, $38, $DC, $A7, $D0, $00, $00, $00 // aesenc xmm4, [edi+208] + db $66, $0F, $38, $DC, $AF, $D0, $00, $00, $00 // aesenc xmm5, [edi+208] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY128} + db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+160] + db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+160] + db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+160] + db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+160] + db $66, $0F, $38, $DD, $A7, $A0, $00, $00, $00 // aesenclast xmm4, [edi+160] + db $66, $0F, $38, $DD, $AF, $A0, $00, $00, $00 // aesenclast xmm5, [edi+160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+192] + db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+192] + db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+192] + db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+192] + db $66, $0F, $38, $DD, $A7, $C0, $00, $00, $00 // aesenclast xmm4, [edi+192] + db $66, $0F, $38, $DD, $AF, $C0, $00, $00, $00 // aesenclast xmm5, [edi+192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+224] + db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+224] + db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+224] + db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+224] + db $66, $0F, $38, $DD, $A7, $E0, $00, $00, $00 // aesenclast xmm4, [edi+224] + db $66, $0F, $38, $DD, $AF, $E0, $00, $00, $00 // aesenclast xmm5, [edi+224] +{$ENDIF} +{$ENDIF} + + movdqa xmm7, [esp + 208] // reload checksum + // Post-whiten with this iteration's Deltas. + pxor xmm0, [esp + 0] + pxor xmm1, [esp + 16] + pxor xmm2, [esp + 32] + pxor xmm3, [esp + 48] + pxor xmm4, [esp + 64] + pxor xmm5, [esp + 80] + +{$IFDEF CRYPTOLIB_AESNI_DECRYPT} + // Fold recovered plaintext into the checksum. + pxor xmm7, xmm0 + pxor xmm7, xmm1 + pxor xmm7, xmm2 + pxor xmm7, xmm3 + pxor xmm7, xmm4 + pxor xmm7, xmm5 +{$ENDIF} + + movdqu [esi + 0], xmm0 + movdqu [esi + 16], xmm1 + movdqu [esi + 32], xmm2 + movdqu [esi + 48], xmm3 + movdqu [esi + 64], xmm4 + movdqu [esi + 80], xmm5 + add edx, 96 + add esi, 96 + mov [esp + 204], edx // next iteration's block-0 source + + // --- Iteration B: load 6 blocks (block 0 via the staged source). --- + mov eax, [esp + 204] + movdqu xmm0, [eax] + movdqu xmm1, [edx + 16] + movdqu xmm2, [edx + 32] + movdqu xmm3, [edx + 48] + movdqu xmm4, [edx + 64] + movdqu xmm5, [edx + 80] {$IFNDEF CRYPTOLIB_AESNI_DECRYPT} - // --- Phase 3 (encrypt): fold plaintext into xmm_checksum. --- - pxor xmm5, xmm0 - pxor xmm5, xmm1 - pxor xmm5, xmm2 - pxor xmm5, xmm3 + // Fold plaintext into the checksum (still live in xmm7). + pxor xmm7, xmm0 + pxor xmm7, xmm1 + pxor xmm7, xmm2 + pxor xmm7, xmm3 + pxor xmm7, xmm4 + pxor xmm7, xmm5 {$ENDIF} + movdqa [esp + 208], xmm7 // spill checksum; xmm7 = ladder scratch - // --- Phase 4: pre-whiten with K_0 (memory-form, aligned schedule). --- + // Pre-whiten with K_0 and this iteration's stashed Deltas. pxor xmm0, [edi + 0] pxor xmm1, [edi + 0] pxor xmm2, [edi + 0] pxor xmm3, [edi + 0] + pxor xmm4, [edi + 0] + pxor xmm5, [edi + 0] + pxor xmm0, [esp + 96] + pxor xmm1, [esp + 112] + pxor xmm2, [esp + 128] + pxor xmm3, [esp + 144] + pxor xmm4, [esp + 160] + pxor xmm5, [esp + 176] - // --- Phase 5: pre-whiten with stashed per-block offsets (unaligned - // stack -> movdqu + reg-reg pxor). --- - movdqu xmm6, [esp + 0] - pxor xmm0, xmm6 - movdqu xmm6, [esp + 16] - pxor xmm1, xmm6 - movdqu xmm6, [esp + 32] - pxor xmm2, xmm6 - movdqu xmm6, [esp + 48] - pxor xmm3, xmm6 - - // ------------------------------------------------------------------ - // Phase 6: AES middle rounds 1..N-1 via memory-form aesenc / aesdec. - // Key size drives the number of rounds; shared through the same - // pattern used by the previous 6-wide kernel. - // ------------------------------------------------------------------ {$IFDEF CRYPTOLIB_AESNI_DECRYPT} - - // Round 1 (disp8 = 16) - db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+0x10] - db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16] - db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16] - db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16] - // Round 2 (disp8 = 32) - db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32] - db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32] - db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32] - db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32] - // Round 3 (disp8 = 48) - db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48] - db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48] - db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48] - db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48] - // Round 4 (disp8 = 64) - db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64] - db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64] - db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64] - db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64] - // Round 5 (disp8 = 80) - db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80] - db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80] - db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80] - db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80] - // Round 6 (disp8 = 96) - db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96] - db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96] - db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96] - db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96] - // Round 7 (disp8 = 112) - db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112] - db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112] - db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112] - db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112] - // Round 8 (disp32 = 128) + // Round 1 + db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+16] + db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16] + db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16] + db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16] + db $66, $0F, $38, $DE, $67, $10 // aesdec xmm4, [edi+16] + db $66, $0F, $38, $DE, $6F, $10 // aesdec xmm5, [edi+16] + // stage next-iteration Delta 0 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB0 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 0], xmm6 + // Round 2 + db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32] + db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32] + db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32] + db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32] + db $66, $0F, $38, $DE, $67, $20 // aesdec xmm4, [edi+32] + db $66, $0F, $38, $DE, $6F, $20 // aesdec xmm5, [edi+32] + // stage next-iteration Delta 1 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB1 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB1: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 16], xmm6 + // Round 3 + db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48] + db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48] + db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48] + db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48] + db $66, $0F, $38, $DE, $67, $30 // aesdec xmm4, [edi+48] + db $66, $0F, $38, $DE, $6F, $30 // aesdec xmm5, [edi+48] + // stage next-iteration Delta 2 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB2 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB2: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 32], xmm6 + // Round 4 + db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64] + db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64] + db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64] + db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64] + db $66, $0F, $38, $DE, $67, $40 // aesdec xmm4, [edi+64] + db $66, $0F, $38, $DE, $6F, $40 // aesdec xmm5, [edi+64] + // stage next-iteration Delta 3 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB3 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB3: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 48], xmm6 + // Round 5 + db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80] + db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80] + db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80] + db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80] + db $66, $0F, $38, $DE, $67, $50 // aesdec xmm4, [edi+80] + db $66, $0F, $38, $DE, $6F, $50 // aesdec xmm5, [edi+80] + // stage next-iteration Delta 4 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB4 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB4: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 64], xmm6 + // Round 6 + db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96] + db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96] + db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96] + db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96] + db $66, $0F, $38, $DE, $67, $60 // aesdec xmm4, [edi+96] + db $66, $0F, $38, $DE, $6F, $60 // aesdec xmm5, [edi+96] + // stage next-iteration Delta 5 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB5 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 80], xmm6 + // Round 7 + db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112] + db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112] + db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112] + db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112] + db $66, $0F, $38, $DE, $67, $70 // aesdec xmm4, [edi+112] + db $66, $0F, $38, $DE, $6F, $70 // aesdec xmm5, [edi+112] + // Round 8 db $66, $0F, $38, $DE, $87, $80, $00, $00, $00 // aesdec xmm0, [edi+128] db $66, $0F, $38, $DE, $8F, $80, $00, $00, $00 // aesdec xmm1, [edi+128] db $66, $0F, $38, $DE, $97, $80, $00, $00, $00 // aesdec xmm2, [edi+128] db $66, $0F, $38, $DE, $9F, $80, $00, $00, $00 // aesdec xmm3, [edi+128] - // Round 9 (disp32 = 144) + db $66, $0F, $38, $DE, $A7, $80, $00, $00, $00 // aesdec xmm4, [edi+128] + db $66, $0F, $38, $DE, $AF, $80, $00, $00, $00 // aesdec xmm5, [edi+128] + // Round 9 db $66, $0F, $38, $DE, $87, $90, $00, $00, $00 // aesdec xmm0, [edi+144] db $66, $0F, $38, $DE, $8F, $90, $00, $00, $00 // aesdec xmm1, [edi+144] db $66, $0F, $38, $DE, $97, $90, $00, $00, $00 // aesdec xmm2, [edi+144] db $66, $0F, $38, $DE, $9F, $90, $00, $00, $00 // aesdec xmm3, [edi+144] - + db $66, $0F, $38, $DE, $A7, $90, $00, $00, $00 // aesdec xmm4, [edi+144] + db $66, $0F, $38, $DE, $AF, $90, $00, $00, $00 // aesdec xmm5, [edi+144] {$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} - // Round 10 (disp32 = 160) db $66, $0F, $38, $DE, $87, $A0, $00, $00, $00 // aesdec xmm0, [edi+160] db $66, $0F, $38, $DE, $8F, $A0, $00, $00, $00 // aesdec xmm1, [edi+160] db $66, $0F, $38, $DE, $97, $A0, $00, $00, $00 // aesdec xmm2, [edi+160] db $66, $0F, $38, $DE, $9F, $A0, $00, $00, $00 // aesdec xmm3, [edi+160] - // Round 11 (disp32 = 176) + db $66, $0F, $38, $DE, $A7, $A0, $00, $00, $00 // aesdec xmm4, [edi+160] + db $66, $0F, $38, $DE, $AF, $A0, $00, $00, $00 // aesdec xmm5, [edi+160] db $66, $0F, $38, $DE, $87, $B0, $00, $00, $00 // aesdec xmm0, [edi+176] db $66, $0F, $38, $DE, $8F, $B0, $00, $00, $00 // aesdec xmm1, [edi+176] db $66, $0F, $38, $DE, $97, $B0, $00, $00, $00 // aesdec xmm2, [edi+176] db $66, $0F, $38, $DE, $9F, $B0, $00, $00, $00 // aesdec xmm3, [edi+176] + db $66, $0F, $38, $DE, $A7, $B0, $00, $00, $00 // aesdec xmm4, [edi+176] + db $66, $0F, $38, $DE, $AF, $B0, $00, $00, $00 // aesdec xmm5, [edi+176] {$IFEND} - {$IFDEF CRYPTOLIB_AESNI_KEY256} - // Round 12 (disp32 = 192) db $66, $0F, $38, $DE, $87, $C0, $00, $00, $00 // aesdec xmm0, [edi+192] db $66, $0F, $38, $DE, $8F, $C0, $00, $00, $00 // aesdec xmm1, [edi+192] db $66, $0F, $38, $DE, $97, $C0, $00, $00, $00 // aesdec xmm2, [edi+192] db $66, $0F, $38, $DE, $9F, $C0, $00, $00, $00 // aesdec xmm3, [edi+192] - // Round 13 (disp32 = 208) + db $66, $0F, $38, $DE, $A7, $C0, $00, $00, $00 // aesdec xmm4, [edi+192] + db $66, $0F, $38, $DE, $AF, $C0, $00, $00, $00 // aesdec xmm5, [edi+192] db $66, $0F, $38, $DE, $87, $D0, $00, $00, $00 // aesdec xmm0, [edi+208] db $66, $0F, $38, $DE, $8F, $D0, $00, $00, $00 // aesdec xmm1, [edi+208] db $66, $0F, $38, $DE, $97, $D0, $00, $00, $00 // aesdec xmm2, [edi+208] db $66, $0F, $38, $DE, $9F, $D0, $00, $00, $00 // aesdec xmm3, [edi+208] + db $66, $0F, $38, $DE, $A7, $D0, $00, $00, $00 // aesdec xmm4, [edi+208] + db $66, $0F, $38, $DE, $AF, $D0, $00, $00, $00 // aesdec xmm5, [edi+208] {$ENDIF} - - // --- Phase 7: final round (aesdeclast). --- {$IFDEF CRYPTOLIB_AESNI_KEY128} - // AES-128 final at disp32 = 160 db $66, $0F, $38, $DF, $87, $A0, $00, $00, $00 // aesdeclast xmm0, [edi+160] db $66, $0F, $38, $DF, $8F, $A0, $00, $00, $00 // aesdeclast xmm1, [edi+160] db $66, $0F, $38, $DF, $97, $A0, $00, $00, $00 // aesdeclast xmm2, [edi+160] db $66, $0F, $38, $DF, $9F, $A0, $00, $00, $00 // aesdeclast xmm3, [edi+160] + db $66, $0F, $38, $DF, $A7, $A0, $00, $00, $00 // aesdeclast xmm4, [edi+160] + db $66, $0F, $38, $DF, $AF, $A0, $00, $00, $00 // aesdeclast xmm5, [edi+160] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY192} - // AES-192 final at disp32 = 192 db $66, $0F, $38, $DF, $87, $C0, $00, $00, $00 // aesdeclast xmm0, [edi+192] db $66, $0F, $38, $DF, $8F, $C0, $00, $00, $00 // aesdeclast xmm1, [edi+192] db $66, $0F, $38, $DF, $97, $C0, $00, $00, $00 // aesdeclast xmm2, [edi+192] db $66, $0F, $38, $DF, $9F, $C0, $00, $00, $00 // aesdeclast xmm3, [edi+192] + db $66, $0F, $38, $DF, $A7, $C0, $00, $00, $00 // aesdeclast xmm4, [edi+192] + db $66, $0F, $38, $DF, $AF, $C0, $00, $00, $00 // aesdeclast xmm5, [edi+192] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY256} - // AES-256 final at disp32 = 224 db $66, $0F, $38, $DF, $87, $E0, $00, $00, $00 // aesdeclast xmm0, [edi+224] db $66, $0F, $38, $DF, $8F, $E0, $00, $00, $00 // aesdeclast xmm1, [edi+224] db $66, $0F, $38, $DF, $97, $E0, $00, $00, $00 // aesdeclast xmm2, [edi+224] db $66, $0F, $38, $DF, $9F, $E0, $00, $00, $00 // aesdeclast xmm3, [edi+224] + db $66, $0F, $38, $DF, $A7, $E0, $00, $00, $00 // aesdeclast xmm4, [edi+224] + db $66, $0F, $38, $DF, $AF, $E0, $00, $00, $00 // aesdeclast xmm5, [edi+224] {$ENDIF} - -{$ELSE} // ====================== ENCRYPT variant ====================== - - // Rounds 1..9 : aesenc xmm_i, [edi + 16*r] (opcode $DC) - // Round 1 (disp8 = 16) - db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16] - db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16] - db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16] - db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16] - // Round 2 (disp8 = 32) - db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32] - db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32] - db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32] - db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32] - // Round 3 (disp8 = 48) - db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48] - db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48] - db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48] - db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48] - // Round 4 (disp8 = 64) - db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64] - db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64] - db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64] - db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64] - // Round 5 (disp8 = 80) - db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80] - db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80] - db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80] - db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80] - // Round 6 (disp8 = 96) - db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96] - db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96] - db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96] - db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96] - // Round 7 (disp8 = 112) - db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112] - db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112] - db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112] - db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112] - // Round 8 (disp32 = 128) +{$ELSE} + // Round 1 + db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16] + db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16] + db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16] + db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16] + db $66, $0F, $38, $DC, $67, $10 // aesenc xmm4, [edi+16] + db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16] + // stage next-iteration Delta 0 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB0 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 0], xmm6 + // Round 2 + db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32] + db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32] + db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32] + db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32] + db $66, $0F, $38, $DC, $67, $20 // aesenc xmm4, [edi+32] + db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32] + // stage next-iteration Delta 1 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB1 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB1: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 16], xmm6 + // Round 3 + db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48] + db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48] + db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48] + db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48] + db $66, $0F, $38, $DC, $67, $30 // aesenc xmm4, [edi+48] + db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48] + // stage next-iteration Delta 2 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB2 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB2: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 32], xmm6 + // Round 4 + db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64] + db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64] + db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64] + db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64] + db $66, $0F, $38, $DC, $67, $40 // aesenc xmm4, [edi+64] + db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64] + // stage next-iteration Delta 3 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB3 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB3: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 48], xmm6 + // Round 5 + db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80] + db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80] + db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80] + db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80] + db $66, $0F, $38, $DC, $67, $50 // aesenc xmm4, [edi+80] + db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80] + // stage next-iteration Delta 4 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB4 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB4: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 64], xmm6 + // Round 6 + db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96] + db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96] + db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96] + db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96] + db $66, $0F, $38, $DC, $67, $60 // aesenc xmm4, [edi+96] + db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96] + // stage next-iteration Delta 5 -> bank 0 + add ebp, 1 + bsf eax, ebp + jnz @@ocb_i386_sB5 + inc dword [esp + 196] + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_sB5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm7, [eax] + pxor xmm6, xmm7 + movdqa [esp + 80], xmm6 + // Round 7 + db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112] + db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112] + db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112] + db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112] + db $66, $0F, $38, $DC, $67, $70 // aesenc xmm4, [edi+112] + db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112] + // Round 8 db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128] db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128] db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+128] db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+128] - // Round 9 (disp32 = 144) + db $66, $0F, $38, $DC, $A7, $80, $00, $00, $00 // aesenc xmm4, [edi+128] + db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128] + // Round 9 db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144] db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+144] db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+144] db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+144] - + db $66, $0F, $38, $DC, $A7, $90, $00, $00, $00 // aesenc xmm4, [edi+144] + db $66, $0F, $38, $DC, $AF, $90, $00, $00, $00 // aesenc xmm5, [edi+144] {$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} - // Round 10 (disp32 = 160) db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+160] db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+160] db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+160] db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+160] - // Round 11 (disp32 = 176) + db $66, $0F, $38, $DC, $A7, $A0, $00, $00, $00 // aesenc xmm4, [edi+160] + db $66, $0F, $38, $DC, $AF, $A0, $00, $00, $00 // aesenc xmm5, [edi+160] db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+176] db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+176] db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+176] db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+176] + db $66, $0F, $38, $DC, $A7, $B0, $00, $00, $00 // aesenc xmm4, [edi+176] + db $66, $0F, $38, $DC, $AF, $B0, $00, $00, $00 // aesenc xmm5, [edi+176] {$IFEND} - {$IFDEF CRYPTOLIB_AESNI_KEY256} - // Round 12 (disp32 = 192) db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+192] db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+192] db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+192] db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+192] - // Round 13 (disp32 = 208) + db $66, $0F, $38, $DC, $A7, $C0, $00, $00, $00 // aesenc xmm4, [edi+192] + db $66, $0F, $38, $DC, $AF, $C0, $00, $00, $00 // aesenc xmm5, [edi+192] db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+208] db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+208] db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+208] db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+208] + db $66, $0F, $38, $DC, $A7, $D0, $00, $00, $00 // aesenc xmm4, [edi+208] + db $66, $0F, $38, $DC, $AF, $D0, $00, $00, $00 // aesenc xmm5, [edi+208] {$ENDIF} - - // --- Phase 7: final round (aesenclast). --- {$IFDEF CRYPTOLIB_AESNI_KEY128} - // AES-128 final at disp32 = 160 db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+160] db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+160] db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+160] db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+160] + db $66, $0F, $38, $DD, $A7, $A0, $00, $00, $00 // aesenclast xmm4, [edi+160] + db $66, $0F, $38, $DD, $AF, $A0, $00, $00, $00 // aesenclast xmm5, [edi+160] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY192} - // AES-192 final at disp32 = 192 db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+192] db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+192] db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+192] db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+192] + db $66, $0F, $38, $DD, $A7, $C0, $00, $00, $00 // aesenclast xmm4, [edi+192] + db $66, $0F, $38, $DD, $AF, $C0, $00, $00, $00 // aesenclast xmm5, [edi+192] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY256} - // AES-256 final at disp32 = 224 db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+224] db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+224] db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+224] db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+224] + db $66, $0F, $38, $DD, $A7, $E0, $00, $00, $00 // aesenclast xmm4, [edi+224] + db $66, $0F, $38, $DD, $AF, $E0, $00, $00, $00 // aesenclast xmm5, [edi+224] +{$ENDIF} {$ENDIF} -{$ENDIF} // CRYPTOLIB_AESNI_DECRYPT - - // --- Phase 8: post-whiten with stashed per-block offsets. --- - movdqu xmm6, [esp + 0] - pxor xmm0, xmm6 - movdqu xmm6, [esp + 16] - pxor xmm1, xmm6 - movdqu xmm6, [esp + 32] - pxor xmm2, xmm6 - movdqu xmm6, [esp + 48] - pxor xmm3, xmm6 + movdqa xmm7, [esp + 208] // reload checksum + // Post-whiten with this iteration's Deltas. + pxor xmm0, [esp + 96] + pxor xmm1, [esp + 112] + pxor xmm2, [esp + 128] + pxor xmm3, [esp + 144] + pxor xmm4, [esp + 160] + pxor xmm5, [esp + 176] {$IFDEF CRYPTOLIB_AESNI_DECRYPT} - // --- Phase 9 (decrypt): fold recovered plaintext into checksum. --- - pxor xmm5, xmm0 - pxor xmm5, xmm1 - pxor xmm5, xmm2 - pxor xmm5, xmm3 + // Fold recovered plaintext into the checksum. + pxor xmm7, xmm0 + pxor xmm7, xmm1 + pxor xmm7, xmm2 + pxor xmm7, xmm3 + pxor xmm7, xmm4 + pxor xmm7, xmm5 {$ENDIF} - // --- Phase 10: store 4 output blocks. --- movdqu [esi + 0], xmm0 movdqu [esi + 16], xmm1 movdqu [esi + 32], xmm2 movdqu [esi + 48], xmm3 + movdqu [esi + 64], xmm4 + movdqu [esi + 80], xmm5 + add edx, 96 + add esi, 96 + mov [esp + 204], edx // next iteration's block-0 source - // --- Advance pointers and decrement loop counter. The block counter - // (ebp + [esp+68]) is advanced per block inside the ladder above. --- - add edx, 64 - add esi, 64 - mov eax, edx // next iter's block-0 source - // tracks the main stream - sub ecx, 4 + sub ecx, 12 jnz @@ocb_i386_loop - // --- Epilogue: flush accumulators back to memory, then unwind the - // stack scratch and pop the callee-saved GPRs. ebx is popped - // last because ClpSimdProc1Begin_i386.inc pushed it first. --- - mov eax, [ebx + 12] // eax = OffsetPtr - movdqu [eax], xmm4 - mov eax, [ebx + 16] // eax = ChecksumPtr - movdqu [eax], xmm5 + // --- Undo the one-iteration ladder over-stage: re-XOR the same six + // L entries while walking the counter back (XOR is self-inverse; + // only xmm_offset needs this - the counter is not persisted). --- + bsf eax, ebp + jnz @@ocb_i386_u0 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u0: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + bsf eax, ebp + jnz @@ocb_i386_u1 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u1: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + bsf eax, ebp + jnz @@ocb_i386_u2 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u2: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + bsf eax, ebp + jnz @@ocb_i386_u3 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u3: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + bsf eax, ebp + jnz @@ocb_i386_u4 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u4: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + bsf eax, ebp + jnz @@ocb_i386_u5 + bsf eax, [esp + 196] + add eax, 32 +@@ocb_i386_u5: + shl eax, 4 + add eax, [esp + 192] + movdqu xmm0, [eax] + pxor xmm6, xmm0 + sub ebp, 1 + sbb dword [esp + 196], 0 + + // --- Epilogue: flush accumulators, unwind, pop callee-saves + // (ebx last: ClpSimdProc1Begin_i386.inc pushed it first). --- + mov eax, [ebx + 12] + movdqu [eax], xmm6 + mov eax, [ebx + 16] + movdqu [eax], xmm7 - add esp, 72 + mov esp, [esp + 200] pop ebp pop esi pop edi diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc index c414e16a..638ad1bc 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc @@ -1,207 +1,149 @@ -// OCB fused 8-wide AES-NI kernel (x86-64) with in-kernel offset ladder, -// L-table lookup and checksum fold. +// OCB fused 8-wide AES-NI kernel (x86-64) with a round-interleaved offset +// ladder, L-table lookup and checksum fold. // ------------------------------------------------------------------- -// Processes ABlockCount consecutive 16-byte OCB body blocks in chunks -// of 8 blocks (128 bytes) per iteration. The kernel owns all per-block -// OCB bookkeeping across the batch: on entry it seeds xmm_offset from -// OffsetPtr and xmm_checksum from ChecksumPtr and a running block counter -// from StartBlockCount; the loop derives ntz per block in-register (bsf on -// the incremented counter), XORs the L-table entry into xmm_offset -// to derive each Delta, runs P_i XOR delta XOR K_0 through the AES -// rounds, XORs delta into the post-aesenclast state to produce C_i, -// and folds plaintext into xmm_checksum; on exit it writes xmm_offset -// and xmm_checksum back. The mode layer never touches per-block state. +// Processes ABlockCount consecutive 16-byte OCB body blocks, two 8-block +// iterations (256 bytes) per loop pass. xmm0..7 data lanes, xmm8 offset +// accumulator, xmm9 checksum accumulator, xmm11 L-table scratch. Round +// keys are aesenc/aesdec memory operands against the 16-aligned +// schedule. The offset ladder for the NEXT 8-block iteration is staged +// between the AES round groups of the current one into the opposite of +// two ping-pong delta banks, so the ladder's serial bsf/load/xor chain +// rides the AES latency instead of running as a prefix; the prologue +// pre-stages iteration 0, and the final pass over-stages one iteration, +// undone afterwards by re-XORing the same eight L entries (XOR is +// self-inverse; the counter is not persisted, so only the offset +// accumulator needs the rollback). The 64-bit block counter lives whole +// in r11, so the ladder is branch-free (bsf sees zero only at the 2^64 +// wrap, unreachable under OCB limits). // // Variant selection: // CRYPTOLIB_AESNI_DECRYPT -> aesdec / aesdeclast; else aesenc / aesenclast. // Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 selects N_R. // -// On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx. +// On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx (pinned). +// Context layout: see TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. +// BlockCount must be a positive multiple of 16. // -// Context layout (7 pointers + NativeUInt + UInt64 = 72 bytes, 8-byte aligned). -// See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. -// [rcx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned) -// [rcx + 8] InPtr : base of the main 16-byte-block stream -// [rcx + 16] OutPtr : base of BlockCount output blocks (may alias InPtr) -// [rcx + 24] OffsetPtr : 16-byte live FOffsetMAIN (r/w) -// [rcx + 32] ChecksumPtr : 16-byte live FChecksum (r/w) -// [rcx + 40] LTablePtr : flat L[0..LMax] * 16 bytes (read-only) -// [rcx + 48] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) -// [rcx + 56] BlockCount : NativeUInt; multiple of 8 -// [rcx + 64] StartBlockCount : UInt64; block count before this span (ntz counter seed) +// GPR: rcx=PCtx, rdx=Keys, rbx=InPtr, r8=OutPtr, r9=LTablePtr, +// r10=pass countdown, r11=block counter, rax=scratch/block-0 source. // -// XMM (12/16 used): -// xmm0..xmm7 : 8 data lanes -// xmm8 : xmm_offset accumulator (seeded/stored once per batch) -// xmm9 : xmm_checksum accumulator (seeded/stored once per batch) -// xmm10 : AES round-key scratch (movdqa reload per round) -// xmm11 : L-table load scratch -// xmm12..xmm15 : free +// Stack scratch (16-aligned): [rsp+0..127] delta bank 0, [rsp+128..255] +// delta bank 1. // -// Stack scratch (128 bytes, 16-aligned): -// [rsp + 0 .. 127] 8 x 16-byte per-block offset stashes (movdqa-safe) -// -// AES-NI instructions and selected forms touching xmm8..xmm15 (pxor, -// movdqa) are db-encoded for broad assembler compatibility. +// AES-NI instructions and forms touching xmm8..xmm15 are db-encoded for +// broad assembler compatibility. // ------------------------------------------------------------------- {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - // --- Prologue: preserve rbx, reserve 128B aligned stack scratch. --- + // --- Prologue: preserve rbx, reserve the aligned delta banks. --- push rbx // rsp: 8 -> 0 mod 16 - sub rsp, 128 // rsp stays 0 mod 16 + sub rsp, 256 - // --- Load context pointers. rcx (PCtx) stays live for the - // epilogue store-backs; everything else derives from rcx now. --- mov rdx, [rcx + 0] // rdx = Keys mov rbx, [rcx + 8] // rbx = InPtr mov r8, [rcx + 16] // r8 = OutPtr mov r9, [rcx + 40] // r9 = LTablePtr mov r11, [rcx + 64] // r11 = running block counter - // (seeded from StartBlockCount); - // ntz derived per block via bsf mov r10, [rcx + 56] // r10 = BlockCount - shr r10, 3 // r10 /= 8 (>= 1) + shr r10, 4 // r10 = pass count (BlockCount/16) - // --- Seed the persistent accumulators from memory (one-shot). - // rax is used as a scratch for the two-hop ptr-of-ptr loads - // and is then overwritten with Block0Ptr for the loop. --- - mov rax, [rcx + 24] // rax = OffsetPtr - movdqu xmm8, [rax] // xmm8 = FOffsetMAIN - mov rax, [rcx + 32] // rax = ChecksumPtr - movdqu xmm9, [rax] // xmm9 = FChecksum + mov rax, [rcx + 24] + db $F3, $44, $0F, $6F, $00 // movdqu xmm8, [rax] (FOffsetMAIN) + mov rax, [rcx + 32] + db $F3, $44, $0F, $6F, $08 // movdqu xmm9, [rax] (FChecksum) - // --- Seed the block-0 pointer. rax persists across loop - // iterations: iter 0 consumes it via Phase 2 below, and - // every iteration's bottom refreshes it to rbx (the main - // stream advance) so iter >= 1 loads block 0 from the - // normal +0 offset transparently. --- - mov rax, [rcx + 48] // rax = Block0Ptr + // --- Pre-stage iteration 0's Deltas into bank 0. --- + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8 + mov rax, [rcx + 48] // iteration-0 block-0 source @@ocb_x64_loop: - // ------------------------------------------------------------------ - // Phase 2: load 8 input blocks into xmm0..xmm7. Must run BEFORE the - // ladder because Phase 1 below clobbers rax (ntz scratch and - // L-table index). On iter 0 rax points at Block0Ptr (which may be - // unrelated to rbx, e.g. FMainBlock for full-MAC decrypt); on every - // subsequent iter the end-of-iter epilogue sets rax := rbx (the - // freshly advanced main stream), so `movdqu xmm0, [rax]` becomes - // effectively identical to `movdqu xmm0, [rbx + 0]`. Blocks 1..7 - // always come from the advancing main stream at rbx + 16..112. - // ------------------------------------------------------------------ + // --- Iteration A: load 8 blocks (block 0 via the staged source). --- movdqu xmm0, [rax] - movdqu xmm1, [rbx + 16] - movdqu xmm2, [rbx + 32] - movdqu xmm3, [rbx + 48] - movdqu xmm4, [rbx + 64] - movdqu xmm5, [rbx + 80] - movdqu xmm6, [rbx + 96] + movdqu xmm1, [rbx + 16] + movdqu xmm2, [rbx + 32] + movdqu xmm3, [rbx + 48] + movdqu xmm4, [rbx + 64] + movdqu xmm5, [rbx + 80] + movdqu xmm6, [rbx + 96] movdqu xmm7, [rbx + 112] - // ------------------------------------------------------------------ - // Phase 1: offset ladder. For each block i in 0..7 read ntz[i], - // scale to a 16-byte L-table offset, XOR the entry into xmm_offset - // (xmm8) and stash the post-XOR value into [rsp + 16*i]. After this - // phase [rsp + 0..127] holds eight consecutive 16-byte Delta values - // and xmm8 holds the Delta after the final block of this chunk - // (which is also the starting Delta for the next chunk). - // - // Each step emits: - // movzx eax, byte [r11 + i] ; eax = ntz[i] - // shl rax, 4 ; rax = ntz[i] * 16 - // movdqu xmm11, [r9 + rax] ; xmm11 = L[ntz[i]] - // pxor xmm8, xmm11 ; xmm8 ^= L[ntz[i]] (db) - // movdqa [rsp + 16*i], xmm8 ; stash offset_i (db) - // ------------------------------------------------------------------ - // Block 0 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $00 // movdqa [rsp+0], xmm8 - // Block 1 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8 - // Block 2 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8 - // Block 3 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8 - // Block 4 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8 - // Block 5 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8 - // Block 6 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8 - // Block 7 - inc r11 - bsf rax, r11 // rax = ntz(block counter) - shl rax, 4 - movdqu xmm11, [r9 + rax] - db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 - db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8 - {$IFNDEF CRYPTOLIB_AESNI_DECRYPT} - // --- Phase 3 (encrypt): fold plaintext into xmm_checksum BEFORE - // any data XOR. --- - // pxor xmm9, xmmX : REX.R only ($44), ModR/M = 11 001 rrr (reg=xmm9, - // r/m=xmmX). bytes: $C8 $C9 $CA $CB $CC $CD $CE $CF for xmm0..xmm7. - db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 - db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 - db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 - db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 - db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 - db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 - db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 - db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 + // Fold plaintext into the checksum. + db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 + db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 + db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 + db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 + db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 + db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 {$ENDIF} - // --- Phase 4: pre-whiten with K_0 (aligned reg reload into xmm10 - // broadcast across all 8 data lanes via db-encoded pxor). --- - movdqa xmm10, [rdx] // xmm10 = K_0 - // pxor xmmX, xmm10 : REX.B ($41), ModR/M = 11 rrr 010 (r/m=xmm10, - // low-3 bits = 010). bytes for reg=xmm0..xmm7: - // $C2 $CA $D2 $DA $E2 $EA $F2 $FA - db $66, $41, $0F, $EF, $C2 // pxor xmm0, xmm10 - db $66, $41, $0F, $EF, $CA // pxor xmm1, xmm10 - db $66, $41, $0F, $EF, $D2 // pxor xmm2, xmm10 - db $66, $41, $0F, $EF, $DA // pxor xmm3, xmm10 - db $66, $41, $0F, $EF, $E2 // pxor xmm4, xmm10 - db $66, $41, $0F, $EF, $EA // pxor xmm5, xmm10 - db $66, $41, $0F, $EF, $F2 // pxor xmm6, xmm10 - db $66, $41, $0F, $EF, $FA // pxor xmm7, xmm10 - - // --- Phase 5: pre-whiten with stashed per-block offsets. Stack - // scratch is 16-aligned so memory-form pxor is safe; no REX. --- + // Pre-whiten with K_0 and this iteration's stashed Deltas. + pxor xmm0, [rdx + 0] + pxor xmm1, [rdx + 0] + pxor xmm2, [rdx + 0] + pxor xmm3, [rdx + 0] + pxor xmm4, [rdx + 0] + pxor xmm5, [rdx + 0] + pxor xmm6, [rdx + 0] + pxor xmm7, [rdx + 0] pxor xmm0, [rsp + 0] pxor xmm1, [rsp + 16] pxor xmm2, [rsp + 32] @@ -211,373 +153,433 @@ pxor xmm6, [rsp + 96] pxor xmm7, [rsp + 112] - // ------------------------------------------------------------------ - // Phase 6: AES middle rounds 1..N-1. xmm10 is reloaded per round - // with the round key and broadcast-XORed/AES'd across all 8 data - // lanes. Opcode byte selects direction: - // $DC = aesenc $DE = aesdec - // ModR/M bytes with r/m=xmm10 follow the same $C2..$FA sequence - // used in Phase 4. - // ------------------------------------------------------------------ {$IFDEF CRYPTOLIB_AESNI_DECRYPT} - // Round 1 - movdqa xmm10, [rdx + 16] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $10 // aesdec xmm0, [rdx+16] + db $66, $0F, $38, $DE, $4A, $10 // aesdec xmm1, [rdx+16] + db $66, $0F, $38, $DE, $52, $10 // aesdec xmm2, [rdx+16] + db $66, $0F, $38, $DE, $5A, $10 // aesdec xmm3, [rdx+16] + db $66, $0F, $38, $DE, $62, $10 // aesdec xmm4, [rdx+16] + db $66, $0F, $38, $DE, $6A, $10 // aesdec xmm5, [rdx+16] + db $66, $0F, $38, $DE, $72, $10 // aesdec xmm6, [rdx+16] + db $66, $0F, $38, $DE, $7A, $10 // aesdec xmm7, [rdx+16] + // stage next-iteration Delta 0 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqa [rsp+128], xmm8 // Round 2 - movdqa xmm10, [rdx + 32] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $20 // aesdec xmm0, [rdx+32] + db $66, $0F, $38, $DE, $4A, $20 // aesdec xmm1, [rdx+32] + db $66, $0F, $38, $DE, $52, $20 // aesdec xmm2, [rdx+32] + db $66, $0F, $38, $DE, $5A, $20 // aesdec xmm3, [rdx+32] + db $66, $0F, $38, $DE, $62, $20 // aesdec xmm4, [rdx+32] + db $66, $0F, $38, $DE, $6A, $20 // aesdec xmm5, [rdx+32] + db $66, $0F, $38, $DE, $72, $20 // aesdec xmm6, [rdx+32] + db $66, $0F, $38, $DE, $7A, $20 // aesdec xmm7, [rdx+32] + // stage next-iteration Delta 1 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqa [rsp+144], xmm8 // Round 3 - movdqa xmm10, [rdx + 48] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $30 // aesdec xmm0, [rdx+48] + db $66, $0F, $38, $DE, $4A, $30 // aesdec xmm1, [rdx+48] + db $66, $0F, $38, $DE, $52, $30 // aesdec xmm2, [rdx+48] + db $66, $0F, $38, $DE, $5A, $30 // aesdec xmm3, [rdx+48] + db $66, $0F, $38, $DE, $62, $30 // aesdec xmm4, [rdx+48] + db $66, $0F, $38, $DE, $6A, $30 // aesdec xmm5, [rdx+48] + db $66, $0F, $38, $DE, $72, $30 // aesdec xmm6, [rdx+48] + db $66, $0F, $38, $DE, $7A, $30 // aesdec xmm7, [rdx+48] + // stage next-iteration Delta 2 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $A0, $00, $00, $00 // movdqa [rsp+160], xmm8 // Round 4 - movdqa xmm10, [rdx + 64] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $40 // aesdec xmm0, [rdx+64] + db $66, $0F, $38, $DE, $4A, $40 // aesdec xmm1, [rdx+64] + db $66, $0F, $38, $DE, $52, $40 // aesdec xmm2, [rdx+64] + db $66, $0F, $38, $DE, $5A, $40 // aesdec xmm3, [rdx+64] + db $66, $0F, $38, $DE, $62, $40 // aesdec xmm4, [rdx+64] + db $66, $0F, $38, $DE, $6A, $40 // aesdec xmm5, [rdx+64] + db $66, $0F, $38, $DE, $72, $40 // aesdec xmm6, [rdx+64] + db $66, $0F, $38, $DE, $7A, $40 // aesdec xmm7, [rdx+64] + // stage next-iteration Delta 3 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $B0, $00, $00, $00 // movdqa [rsp+176], xmm8 // Round 5 - movdqa xmm10, [rdx + 80] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $50 // aesdec xmm0, [rdx+80] + db $66, $0F, $38, $DE, $4A, $50 // aesdec xmm1, [rdx+80] + db $66, $0F, $38, $DE, $52, $50 // aesdec xmm2, [rdx+80] + db $66, $0F, $38, $DE, $5A, $50 // aesdec xmm3, [rdx+80] + db $66, $0F, $38, $DE, $62, $50 // aesdec xmm4, [rdx+80] + db $66, $0F, $38, $DE, $6A, $50 // aesdec xmm5, [rdx+80] + db $66, $0F, $38, $DE, $72, $50 // aesdec xmm6, [rdx+80] + db $66, $0F, $38, $DE, $7A, $50 // aesdec xmm7, [rdx+80] + // stage next-iteration Delta 4 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $C0, $00, $00, $00 // movdqa [rsp+192], xmm8 // Round 6 - movdqa xmm10, [rdx + 96] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $60 // aesdec xmm0, [rdx+96] + db $66, $0F, $38, $DE, $4A, $60 // aesdec xmm1, [rdx+96] + db $66, $0F, $38, $DE, $52, $60 // aesdec xmm2, [rdx+96] + db $66, $0F, $38, $DE, $5A, $60 // aesdec xmm3, [rdx+96] + db $66, $0F, $38, $DE, $62, $60 // aesdec xmm4, [rdx+96] + db $66, $0F, $38, $DE, $6A, $60 // aesdec xmm5, [rdx+96] + db $66, $0F, $38, $DE, $72, $60 // aesdec xmm6, [rdx+96] + db $66, $0F, $38, $DE, $7A, $60 // aesdec xmm7, [rdx+96] + // stage next-iteration Delta 5 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $D0, $00, $00, $00 // movdqa [rsp+208], xmm8 // Round 7 - movdqa xmm10, [rdx + 112] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $42, $70 // aesdec xmm0, [rdx+112] + db $66, $0F, $38, $DE, $4A, $70 // aesdec xmm1, [rdx+112] + db $66, $0F, $38, $DE, $52, $70 // aesdec xmm2, [rdx+112] + db $66, $0F, $38, $DE, $5A, $70 // aesdec xmm3, [rdx+112] + db $66, $0F, $38, $DE, $62, $70 // aesdec xmm4, [rdx+112] + db $66, $0F, $38, $DE, $6A, $70 // aesdec xmm5, [rdx+112] + db $66, $0F, $38, $DE, $72, $70 // aesdec xmm6, [rdx+112] + db $66, $0F, $38, $DE, $7A, $70 // aesdec xmm7, [rdx+112] + // stage next-iteration Delta 6 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $E0, $00, $00, $00 // movdqa [rsp+224], xmm8 // Round 8 - movdqa xmm10, [rdx + 128] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $82, $80, $00, $00, $00 // aesdec xmm0, [rdx+128] + db $66, $0F, $38, $DE, $8A, $80, $00, $00, $00 // aesdec xmm1, [rdx+128] + db $66, $0F, $38, $DE, $92, $80, $00, $00, $00 // aesdec xmm2, [rdx+128] + db $66, $0F, $38, $DE, $9A, $80, $00, $00, $00 // aesdec xmm3, [rdx+128] + db $66, $0F, $38, $DE, $A2, $80, $00, $00, $00 // aesdec xmm4, [rdx+128] + db $66, $0F, $38, $DE, $AA, $80, $00, $00, $00 // aesdec xmm5, [rdx+128] + db $66, $0F, $38, $DE, $B2, $80, $00, $00, $00 // aesdec xmm6, [rdx+128] + db $66, $0F, $38, $DE, $BA, $80, $00, $00, $00 // aesdec xmm7, [rdx+128] + // stage next-iteration Delta 7 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $F0, $00, $00, $00 // movdqa [rsp+240], xmm8 // Round 9 - movdqa xmm10, [rdx + 144] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 - -{$IFDEF CRYPTOLIB_AESNI_KEY192} - // Round 10 - movdqa xmm10, [rdx + 160] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 - // Round 11 - movdqa xmm10, [rdx + 176] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 -{$ENDIF} - + db $66, $0F, $38, $DE, $82, $90, $00, $00, $00 // aesdec xmm0, [rdx+144] + db $66, $0F, $38, $DE, $8A, $90, $00, $00, $00 // aesdec xmm1, [rdx+144] + db $66, $0F, $38, $DE, $92, $90, $00, $00, $00 // aesdec xmm2, [rdx+144] + db $66, $0F, $38, $DE, $9A, $90, $00, $00, $00 // aesdec xmm3, [rdx+144] + db $66, $0F, $38, $DE, $A2, $90, $00, $00, $00 // aesdec xmm4, [rdx+144] + db $66, $0F, $38, $DE, $AA, $90, $00, $00, $00 // aesdec xmm5, [rdx+144] + db $66, $0F, $38, $DE, $B2, $90, $00, $00, $00 // aesdec xmm6, [rdx+144] + db $66, $0F, $38, $DE, $BA, $90, $00, $00, $00 // aesdec xmm7, [rdx+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DE, $82, $A0, $00, $00, $00 // aesdec xmm0, [rdx+160] + db $66, $0F, $38, $DE, $8A, $A0, $00, $00, $00 // aesdec xmm1, [rdx+160] + db $66, $0F, $38, $DE, $92, $A0, $00, $00, $00 // aesdec xmm2, [rdx+160] + db $66, $0F, $38, $DE, $9A, $A0, $00, $00, $00 // aesdec xmm3, [rdx+160] + db $66, $0F, $38, $DE, $A2, $A0, $00, $00, $00 // aesdec xmm4, [rdx+160] + db $66, $0F, $38, $DE, $AA, $A0, $00, $00, $00 // aesdec xmm5, [rdx+160] + db $66, $0F, $38, $DE, $B2, $A0, $00, $00, $00 // aesdec xmm6, [rdx+160] + db $66, $0F, $38, $DE, $BA, $A0, $00, $00, $00 // aesdec xmm7, [rdx+160] + db $66, $0F, $38, $DE, $82, $B0, $00, $00, $00 // aesdec xmm0, [rdx+176] + db $66, $0F, $38, $DE, $8A, $B0, $00, $00, $00 // aesdec xmm1, [rdx+176] + db $66, $0F, $38, $DE, $92, $B0, $00, $00, $00 // aesdec xmm2, [rdx+176] + db $66, $0F, $38, $DE, $9A, $B0, $00, $00, $00 // aesdec xmm3, [rdx+176] + db $66, $0F, $38, $DE, $A2, $B0, $00, $00, $00 // aesdec xmm4, [rdx+176] + db $66, $0F, $38, $DE, $AA, $B0, $00, $00, $00 // aesdec xmm5, [rdx+176] + db $66, $0F, $38, $DE, $B2, $B0, $00, $00, $00 // aesdec xmm6, [rdx+176] + db $66, $0F, $38, $DE, $BA, $B0, $00, $00, $00 // aesdec xmm7, [rdx+176] +{$IFEND} {$IFDEF CRYPTOLIB_AESNI_KEY256} - // Round 10 - movdqa xmm10, [rdx + 160] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 - // Round 11 - movdqa xmm10, [rdx + 176] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 - // Round 12 - movdqa xmm10, [rdx + 192] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 - // Round 13 - movdqa xmm10, [rdx + 208] - db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10 - db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10 - db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10 - db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10 - db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10 - db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10 - db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10 - db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10 + db $66, $0F, $38, $DE, $82, $C0, $00, $00, $00 // aesdec xmm0, [rdx+192] + db $66, $0F, $38, $DE, $8A, $C0, $00, $00, $00 // aesdec xmm1, [rdx+192] + db $66, $0F, $38, $DE, $92, $C0, $00, $00, $00 // aesdec xmm2, [rdx+192] + db $66, $0F, $38, $DE, $9A, $C0, $00, $00, $00 // aesdec xmm3, [rdx+192] + db $66, $0F, $38, $DE, $A2, $C0, $00, $00, $00 // aesdec xmm4, [rdx+192] + db $66, $0F, $38, $DE, $AA, $C0, $00, $00, $00 // aesdec xmm5, [rdx+192] + db $66, $0F, $38, $DE, $B2, $C0, $00, $00, $00 // aesdec xmm6, [rdx+192] + db $66, $0F, $38, $DE, $BA, $C0, $00, $00, $00 // aesdec xmm7, [rdx+192] + db $66, $0F, $38, $DE, $82, $D0, $00, $00, $00 // aesdec xmm0, [rdx+208] + db $66, $0F, $38, $DE, $8A, $D0, $00, $00, $00 // aesdec xmm1, [rdx+208] + db $66, $0F, $38, $DE, $92, $D0, $00, $00, $00 // aesdec xmm2, [rdx+208] + db $66, $0F, $38, $DE, $9A, $D0, $00, $00, $00 // aesdec xmm3, [rdx+208] + db $66, $0F, $38, $DE, $A2, $D0, $00, $00, $00 // aesdec xmm4, [rdx+208] + db $66, $0F, $38, $DE, $AA, $D0, $00, $00, $00 // aesdec xmm5, [rdx+208] + db $66, $0F, $38, $DE, $B2, $D0, $00, $00, $00 // aesdec xmm6, [rdx+208] + db $66, $0F, $38, $DE, $BA, $D0, $00, $00, $00 // aesdec xmm7, [rdx+208] {$ENDIF} - - // --- Phase 7: final round. Opcode $DF = aesdeclast. --- {$IFDEF CRYPTOLIB_AESNI_KEY128} - movdqa xmm10, [rdx + 160] + db $66, $0F, $38, $DF, $82, $A0, $00, $00, $00 // aesdeclast xmm0, [rdx+160] + db $66, $0F, $38, $DF, $8A, $A0, $00, $00, $00 // aesdeclast xmm1, [rdx+160] + db $66, $0F, $38, $DF, $92, $A0, $00, $00, $00 // aesdeclast xmm2, [rdx+160] + db $66, $0F, $38, $DF, $9A, $A0, $00, $00, $00 // aesdeclast xmm3, [rdx+160] + db $66, $0F, $38, $DF, $A2, $A0, $00, $00, $00 // aesdeclast xmm4, [rdx+160] + db $66, $0F, $38, $DF, $AA, $A0, $00, $00, $00 // aesdeclast xmm5, [rdx+160] + db $66, $0F, $38, $DF, $B2, $A0, $00, $00, $00 // aesdeclast xmm6, [rdx+160] + db $66, $0F, $38, $DF, $BA, $A0, $00, $00, $00 // aesdeclast xmm7, [rdx+160] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY192} - movdqa xmm10, [rdx + 192] + db $66, $0F, $38, $DF, $82, $C0, $00, $00, $00 // aesdeclast xmm0, [rdx+192] + db $66, $0F, $38, $DF, $8A, $C0, $00, $00, $00 // aesdeclast xmm1, [rdx+192] + db $66, $0F, $38, $DF, $92, $C0, $00, $00, $00 // aesdeclast xmm2, [rdx+192] + db $66, $0F, $38, $DF, $9A, $C0, $00, $00, $00 // aesdeclast xmm3, [rdx+192] + db $66, $0F, $38, $DF, $A2, $C0, $00, $00, $00 // aesdeclast xmm4, [rdx+192] + db $66, $0F, $38, $DF, $AA, $C0, $00, $00, $00 // aesdeclast xmm5, [rdx+192] + db $66, $0F, $38, $DF, $B2, $C0, $00, $00, $00 // aesdeclast xmm6, [rdx+192] + db $66, $0F, $38, $DF, $BA, $C0, $00, $00, $00 // aesdeclast xmm7, [rdx+192] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY256} - movdqa xmm10, [rdx + 224] + db $66, $0F, $38, $DF, $82, $E0, $00, $00, $00 // aesdeclast xmm0, [rdx+224] + db $66, $0F, $38, $DF, $8A, $E0, $00, $00, $00 // aesdeclast xmm1, [rdx+224] + db $66, $0F, $38, $DF, $92, $E0, $00, $00, $00 // aesdeclast xmm2, [rdx+224] + db $66, $0F, $38, $DF, $9A, $E0, $00, $00, $00 // aesdeclast xmm3, [rdx+224] + db $66, $0F, $38, $DF, $A2, $E0, $00, $00, $00 // aesdeclast xmm4, [rdx+224] + db $66, $0F, $38, $DF, $AA, $E0, $00, $00, $00 // aesdeclast xmm5, [rdx+224] + db $66, $0F, $38, $DF, $B2, $E0, $00, $00, $00 // aesdeclast xmm6, [rdx+224] + db $66, $0F, $38, $DF, $BA, $E0, $00, $00, $00 // aesdeclast xmm7, [rdx+224] {$ENDIF} - db $66, $41, $0F, $38, $DF, $C2 // aesdeclast xmm0, xmm10 - db $66, $41, $0F, $38, $DF, $CA // aesdeclast xmm1, xmm10 - db $66, $41, $0F, $38, $DF, $D2 // aesdeclast xmm2, xmm10 - db $66, $41, $0F, $38, $DF, $DA // aesdeclast xmm3, xmm10 - db $66, $41, $0F, $38, $DF, $E2 // aesdeclast xmm4, xmm10 - db $66, $41, $0F, $38, $DF, $EA // aesdeclast xmm5, xmm10 - db $66, $41, $0F, $38, $DF, $F2 // aesdeclast xmm6, xmm10 - db $66, $41, $0F, $38, $DF, $FA // aesdeclast xmm7, xmm10 - -{$ELSE} // ===================== ENCRYPT variant ===================== - +{$ELSE} // Round 1 - movdqa xmm10, [rdx + 16] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16] + db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16] + db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16] + db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16] + db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16] + db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16] + db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16] + db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16] + // stage next-iteration Delta 0 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqa [rsp+128], xmm8 // Round 2 - movdqa xmm10, [rdx + 32] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32] + db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32] + db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32] + db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32] + db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32] + db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32] + db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32] + db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32] + // stage next-iteration Delta 1 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqa [rsp+144], xmm8 // Round 3 - movdqa xmm10, [rdx + 48] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48] + db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48] + db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48] + db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48] + db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48] + db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48] + db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48] + db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48] + // stage next-iteration Delta 2 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $A0, $00, $00, $00 // movdqa [rsp+160], xmm8 // Round 4 - movdqa xmm10, [rdx + 64] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64] + db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64] + db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64] + db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64] + db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64] + db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64] + db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64] + db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64] + // stage next-iteration Delta 3 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $B0, $00, $00, $00 // movdqa [rsp+176], xmm8 // Round 5 - movdqa xmm10, [rdx + 80] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80] + db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80] + db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80] + db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80] + db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80] + db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80] + db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80] + db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80] + // stage next-iteration Delta 4 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $C0, $00, $00, $00 // movdqa [rsp+192], xmm8 // Round 6 - movdqa xmm10, [rdx + 96] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96] + db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96] + db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96] + db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96] + db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96] + db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96] + db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96] + db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96] + // stage next-iteration Delta 5 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $D0, $00, $00, $00 // movdqa [rsp+208], xmm8 // Round 7 - movdqa xmm10, [rdx + 112] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112] + db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112] + db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112] + db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112] + db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112] + db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112] + db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112] + db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112] + // stage next-iteration Delta 6 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $E0, $00, $00, $00 // movdqa [rsp+224], xmm8 // Round 8 - movdqa xmm10, [rdx + 128] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128] + db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128] + db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128] + db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128] + db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128] + db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128] + db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128] + db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128] + // stage next-iteration Delta 7 -> bank 1 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $84, $24, $F0, $00, $00, $00 // movdqa [rsp+240], xmm8 // Round 9 - movdqa xmm10, [rdx + 144] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 - -{$IFDEF CRYPTOLIB_AESNI_KEY192} - // Round 10 - movdqa xmm10, [rdx + 160] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 - // Round 11 - movdqa xmm10, [rdx + 176] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 -{$ENDIF} - + db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144] + db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144] + db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144] + db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144] + db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144] + db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144] + db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144] + db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160] + db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160] + db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160] + db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160] + db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160] + db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160] + db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160] + db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160] + db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176] + db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176] + db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176] + db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176] + db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176] + db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176] + db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176] + db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176] +{$IFEND} {$IFDEF CRYPTOLIB_AESNI_KEY256} - // Round 10 - movdqa xmm10, [rdx + 160] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 - // Round 11 - movdqa xmm10, [rdx + 176] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 - // Round 12 - movdqa xmm10, [rdx + 192] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 - // Round 13 - movdqa xmm10, [rdx + 208] - db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10 - db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10 - db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10 - db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10 - db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10 - db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10 - db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10 - db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10 + db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192] + db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192] + db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192] + db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192] + db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192] + db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192] + db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192] + db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192] + db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208] + db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208] + db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208] + db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208] + db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208] + db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208] + db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208] + db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208] {$ENDIF} - - // --- Phase 7: final round. Opcode $DD = aesenclast. --- {$IFDEF CRYPTOLIB_AESNI_KEY128} - movdqa xmm10, [rdx + 160] + db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160] + db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160] + db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160] + db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160] + db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160] + db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160] + db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160] + db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY192} - movdqa xmm10, [rdx + 192] + db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192] + db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192] + db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192] + db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192] + db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192] + db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192] + db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192] + db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192] {$ENDIF} {$IFDEF CRYPTOLIB_AESNI_KEY256} - movdqa xmm10, [rdx + 224] + db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224] + db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224] + db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224] + db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224] + db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224] + db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224] + db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224] + db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224] {$ENDIF} - db $66, $41, $0F, $38, $DD, $C2 // aesenclast xmm0, xmm10 - db $66, $41, $0F, $38, $DD, $CA // aesenclast xmm1, xmm10 - db $66, $41, $0F, $38, $DD, $D2 // aesenclast xmm2, xmm10 - db $66, $41, $0F, $38, $DD, $DA // aesenclast xmm3, xmm10 - db $66, $41, $0F, $38, $DD, $E2 // aesenclast xmm4, xmm10 - db $66, $41, $0F, $38, $DD, $EA // aesenclast xmm5, xmm10 - db $66, $41, $0F, $38, $DD, $F2 // aesenclast xmm6, xmm10 - db $66, $41, $0F, $38, $DD, $FA // aesenclast xmm7, xmm10 - {$ENDIF} - // --- Phase 8: post-whiten with stashed per-block offsets. --- + // Post-whiten with this iteration's Deltas. pxor xmm0, [rsp + 0] pxor xmm1, [rsp + 16] pxor xmm2, [rsp + 32] @@ -588,19 +590,17 @@ pxor xmm7, [rsp + 112] {$IFDEF CRYPTOLIB_AESNI_DECRYPT} - // --- Phase 9 (decrypt): fold recovered plaintext into xmm_checksum - // AFTER post-whiten. --- - db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 - db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 - db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 - db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 - db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 - db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 - db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 - db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 + // Fold recovered plaintext into the checksum. + db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 + db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 + db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 + db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 + db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 + db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 {$ENDIF} - // --- Phase 10: store 8 output blocks. --- movdqu [r8 + 0], xmm0 movdqu [r8 + 16], xmm1 movdqu [r8 + 32], xmm2 @@ -609,23 +609,570 @@ movdqu [r8 + 80], xmm5 movdqu [r8 + 96], xmm6 movdqu [r8 + 112], xmm7 + add rbx, 128 + add r8, 128 + mov rax, rbx // next iteration's block-0 source + + // --- Iteration B: load 8 blocks (block 0 via the staged source). --- + movdqu xmm0, [rax] + movdqu xmm1, [rbx + 16] + movdqu xmm2, [rbx + 32] + movdqu xmm3, [rbx + 48] + movdqu xmm4, [rbx + 64] + movdqu xmm5, [rbx + 80] + movdqu xmm6, [rbx + 96] + movdqu xmm7, [rbx + 112] + +{$IFNDEF CRYPTOLIB_AESNI_DECRYPT} + // Fold plaintext into the checksum. + db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 + db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 + db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 + db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 + db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 + db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 +{$ENDIF} + + // Pre-whiten with K_0 and this iteration's stashed Deltas. + pxor xmm0, [rdx + 0] + pxor xmm1, [rdx + 0] + pxor xmm2, [rdx + 0] + pxor xmm3, [rdx + 0] + pxor xmm4, [rdx + 0] + pxor xmm5, [rdx + 0] + pxor xmm6, [rdx + 0] + pxor xmm7, [rdx + 0] + pxor xmm0, [rsp + 128] + pxor xmm1, [rsp + 144] + pxor xmm2, [rsp + 160] + pxor xmm3, [rsp + 176] + pxor xmm4, [rsp + 192] + pxor xmm5, [rsp + 208] + pxor xmm6, [rsp + 224] + pxor xmm7, [rsp + 240] + +{$IFDEF CRYPTOLIB_AESNI_DECRYPT} + // Round 1 + db $66, $0F, $38, $DE, $42, $10 // aesdec xmm0, [rdx+16] + db $66, $0F, $38, $DE, $4A, $10 // aesdec xmm1, [rdx+16] + db $66, $0F, $38, $DE, $52, $10 // aesdec xmm2, [rdx+16] + db $66, $0F, $38, $DE, $5A, $10 // aesdec xmm3, [rdx+16] + db $66, $0F, $38, $DE, $62, $10 // aesdec xmm4, [rdx+16] + db $66, $0F, $38, $DE, $6A, $10 // aesdec xmm5, [rdx+16] + db $66, $0F, $38, $DE, $72, $10 // aesdec xmm6, [rdx+16] + db $66, $0F, $38, $DE, $7A, $10 // aesdec xmm7, [rdx+16] + // stage next-iteration Delta 0 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8 + // Round 2 + db $66, $0F, $38, $DE, $42, $20 // aesdec xmm0, [rdx+32] + db $66, $0F, $38, $DE, $4A, $20 // aesdec xmm1, [rdx+32] + db $66, $0F, $38, $DE, $52, $20 // aesdec xmm2, [rdx+32] + db $66, $0F, $38, $DE, $5A, $20 // aesdec xmm3, [rdx+32] + db $66, $0F, $38, $DE, $62, $20 // aesdec xmm4, [rdx+32] + db $66, $0F, $38, $DE, $6A, $20 // aesdec xmm5, [rdx+32] + db $66, $0F, $38, $DE, $72, $20 // aesdec xmm6, [rdx+32] + db $66, $0F, $38, $DE, $7A, $20 // aesdec xmm7, [rdx+32] + // stage next-iteration Delta 1 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8 + // Round 3 + db $66, $0F, $38, $DE, $42, $30 // aesdec xmm0, [rdx+48] + db $66, $0F, $38, $DE, $4A, $30 // aesdec xmm1, [rdx+48] + db $66, $0F, $38, $DE, $52, $30 // aesdec xmm2, [rdx+48] + db $66, $0F, $38, $DE, $5A, $30 // aesdec xmm3, [rdx+48] + db $66, $0F, $38, $DE, $62, $30 // aesdec xmm4, [rdx+48] + db $66, $0F, $38, $DE, $6A, $30 // aesdec xmm5, [rdx+48] + db $66, $0F, $38, $DE, $72, $30 // aesdec xmm6, [rdx+48] + db $66, $0F, $38, $DE, $7A, $30 // aesdec xmm7, [rdx+48] + // stage next-iteration Delta 2 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8 + // Round 4 + db $66, $0F, $38, $DE, $42, $40 // aesdec xmm0, [rdx+64] + db $66, $0F, $38, $DE, $4A, $40 // aesdec xmm1, [rdx+64] + db $66, $0F, $38, $DE, $52, $40 // aesdec xmm2, [rdx+64] + db $66, $0F, $38, $DE, $5A, $40 // aesdec xmm3, [rdx+64] + db $66, $0F, $38, $DE, $62, $40 // aesdec xmm4, [rdx+64] + db $66, $0F, $38, $DE, $6A, $40 // aesdec xmm5, [rdx+64] + db $66, $0F, $38, $DE, $72, $40 // aesdec xmm6, [rdx+64] + db $66, $0F, $38, $DE, $7A, $40 // aesdec xmm7, [rdx+64] + // stage next-iteration Delta 3 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8 + // Round 5 + db $66, $0F, $38, $DE, $42, $50 // aesdec xmm0, [rdx+80] + db $66, $0F, $38, $DE, $4A, $50 // aesdec xmm1, [rdx+80] + db $66, $0F, $38, $DE, $52, $50 // aesdec xmm2, [rdx+80] + db $66, $0F, $38, $DE, $5A, $50 // aesdec xmm3, [rdx+80] + db $66, $0F, $38, $DE, $62, $50 // aesdec xmm4, [rdx+80] + db $66, $0F, $38, $DE, $6A, $50 // aesdec xmm5, [rdx+80] + db $66, $0F, $38, $DE, $72, $50 // aesdec xmm6, [rdx+80] + db $66, $0F, $38, $DE, $7A, $50 // aesdec xmm7, [rdx+80] + // stage next-iteration Delta 4 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8 + // Round 6 + db $66, $0F, $38, $DE, $42, $60 // aesdec xmm0, [rdx+96] + db $66, $0F, $38, $DE, $4A, $60 // aesdec xmm1, [rdx+96] + db $66, $0F, $38, $DE, $52, $60 // aesdec xmm2, [rdx+96] + db $66, $0F, $38, $DE, $5A, $60 // aesdec xmm3, [rdx+96] + db $66, $0F, $38, $DE, $62, $60 // aesdec xmm4, [rdx+96] + db $66, $0F, $38, $DE, $6A, $60 // aesdec xmm5, [rdx+96] + db $66, $0F, $38, $DE, $72, $60 // aesdec xmm6, [rdx+96] + db $66, $0F, $38, $DE, $7A, $60 // aesdec xmm7, [rdx+96] + // stage next-iteration Delta 5 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8 + // Round 7 + db $66, $0F, $38, $DE, $42, $70 // aesdec xmm0, [rdx+112] + db $66, $0F, $38, $DE, $4A, $70 // aesdec xmm1, [rdx+112] + db $66, $0F, $38, $DE, $52, $70 // aesdec xmm2, [rdx+112] + db $66, $0F, $38, $DE, $5A, $70 // aesdec xmm3, [rdx+112] + db $66, $0F, $38, $DE, $62, $70 // aesdec xmm4, [rdx+112] + db $66, $0F, $38, $DE, $6A, $70 // aesdec xmm5, [rdx+112] + db $66, $0F, $38, $DE, $72, $70 // aesdec xmm6, [rdx+112] + db $66, $0F, $38, $DE, $7A, $70 // aesdec xmm7, [rdx+112] + // stage next-iteration Delta 6 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8 + // Round 8 + db $66, $0F, $38, $DE, $82, $80, $00, $00, $00 // aesdec xmm0, [rdx+128] + db $66, $0F, $38, $DE, $8A, $80, $00, $00, $00 // aesdec xmm1, [rdx+128] + db $66, $0F, $38, $DE, $92, $80, $00, $00, $00 // aesdec xmm2, [rdx+128] + db $66, $0F, $38, $DE, $9A, $80, $00, $00, $00 // aesdec xmm3, [rdx+128] + db $66, $0F, $38, $DE, $A2, $80, $00, $00, $00 // aesdec xmm4, [rdx+128] + db $66, $0F, $38, $DE, $AA, $80, $00, $00, $00 // aesdec xmm5, [rdx+128] + db $66, $0F, $38, $DE, $B2, $80, $00, $00, $00 // aesdec xmm6, [rdx+128] + db $66, $0F, $38, $DE, $BA, $80, $00, $00, $00 // aesdec xmm7, [rdx+128] + // stage next-iteration Delta 7 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8 + // Round 9 + db $66, $0F, $38, $DE, $82, $90, $00, $00, $00 // aesdec xmm0, [rdx+144] + db $66, $0F, $38, $DE, $8A, $90, $00, $00, $00 // aesdec xmm1, [rdx+144] + db $66, $0F, $38, $DE, $92, $90, $00, $00, $00 // aesdec xmm2, [rdx+144] + db $66, $0F, $38, $DE, $9A, $90, $00, $00, $00 // aesdec xmm3, [rdx+144] + db $66, $0F, $38, $DE, $A2, $90, $00, $00, $00 // aesdec xmm4, [rdx+144] + db $66, $0F, $38, $DE, $AA, $90, $00, $00, $00 // aesdec xmm5, [rdx+144] + db $66, $0F, $38, $DE, $B2, $90, $00, $00, $00 // aesdec xmm6, [rdx+144] + db $66, $0F, $38, $DE, $BA, $90, $00, $00, $00 // aesdec xmm7, [rdx+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DE, $82, $A0, $00, $00, $00 // aesdec xmm0, [rdx+160] + db $66, $0F, $38, $DE, $8A, $A0, $00, $00, $00 // aesdec xmm1, [rdx+160] + db $66, $0F, $38, $DE, $92, $A0, $00, $00, $00 // aesdec xmm2, [rdx+160] + db $66, $0F, $38, $DE, $9A, $A0, $00, $00, $00 // aesdec xmm3, [rdx+160] + db $66, $0F, $38, $DE, $A2, $A0, $00, $00, $00 // aesdec xmm4, [rdx+160] + db $66, $0F, $38, $DE, $AA, $A0, $00, $00, $00 // aesdec xmm5, [rdx+160] + db $66, $0F, $38, $DE, $B2, $A0, $00, $00, $00 // aesdec xmm6, [rdx+160] + db $66, $0F, $38, $DE, $BA, $A0, $00, $00, $00 // aesdec xmm7, [rdx+160] + db $66, $0F, $38, $DE, $82, $B0, $00, $00, $00 // aesdec xmm0, [rdx+176] + db $66, $0F, $38, $DE, $8A, $B0, $00, $00, $00 // aesdec xmm1, [rdx+176] + db $66, $0F, $38, $DE, $92, $B0, $00, $00, $00 // aesdec xmm2, [rdx+176] + db $66, $0F, $38, $DE, $9A, $B0, $00, $00, $00 // aesdec xmm3, [rdx+176] + db $66, $0F, $38, $DE, $A2, $B0, $00, $00, $00 // aesdec xmm4, [rdx+176] + db $66, $0F, $38, $DE, $AA, $B0, $00, $00, $00 // aesdec xmm5, [rdx+176] + db $66, $0F, $38, $DE, $B2, $B0, $00, $00, $00 // aesdec xmm6, [rdx+176] + db $66, $0F, $38, $DE, $BA, $B0, $00, $00, $00 // aesdec xmm7, [rdx+176] +{$IFEND} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DE, $82, $C0, $00, $00, $00 // aesdec xmm0, [rdx+192] + db $66, $0F, $38, $DE, $8A, $C0, $00, $00, $00 // aesdec xmm1, [rdx+192] + db $66, $0F, $38, $DE, $92, $C0, $00, $00, $00 // aesdec xmm2, [rdx+192] + db $66, $0F, $38, $DE, $9A, $C0, $00, $00, $00 // aesdec xmm3, [rdx+192] + db $66, $0F, $38, $DE, $A2, $C0, $00, $00, $00 // aesdec xmm4, [rdx+192] + db $66, $0F, $38, $DE, $AA, $C0, $00, $00, $00 // aesdec xmm5, [rdx+192] + db $66, $0F, $38, $DE, $B2, $C0, $00, $00, $00 // aesdec xmm6, [rdx+192] + db $66, $0F, $38, $DE, $BA, $C0, $00, $00, $00 // aesdec xmm7, [rdx+192] + db $66, $0F, $38, $DE, $82, $D0, $00, $00, $00 // aesdec xmm0, [rdx+208] + db $66, $0F, $38, $DE, $8A, $D0, $00, $00, $00 // aesdec xmm1, [rdx+208] + db $66, $0F, $38, $DE, $92, $D0, $00, $00, $00 // aesdec xmm2, [rdx+208] + db $66, $0F, $38, $DE, $9A, $D0, $00, $00, $00 // aesdec xmm3, [rdx+208] + db $66, $0F, $38, $DE, $A2, $D0, $00, $00, $00 // aesdec xmm4, [rdx+208] + db $66, $0F, $38, $DE, $AA, $D0, $00, $00, $00 // aesdec xmm5, [rdx+208] + db $66, $0F, $38, $DE, $B2, $D0, $00, $00, $00 // aesdec xmm6, [rdx+208] + db $66, $0F, $38, $DE, $BA, $D0, $00, $00, $00 // aesdec xmm7, [rdx+208] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY128} + db $66, $0F, $38, $DF, $82, $A0, $00, $00, $00 // aesdeclast xmm0, [rdx+160] + db $66, $0F, $38, $DF, $8A, $A0, $00, $00, $00 // aesdeclast xmm1, [rdx+160] + db $66, $0F, $38, $DF, $92, $A0, $00, $00, $00 // aesdeclast xmm2, [rdx+160] + db $66, $0F, $38, $DF, $9A, $A0, $00, $00, $00 // aesdeclast xmm3, [rdx+160] + db $66, $0F, $38, $DF, $A2, $A0, $00, $00, $00 // aesdeclast xmm4, [rdx+160] + db $66, $0F, $38, $DF, $AA, $A0, $00, $00, $00 // aesdeclast xmm5, [rdx+160] + db $66, $0F, $38, $DF, $B2, $A0, $00, $00, $00 // aesdeclast xmm6, [rdx+160] + db $66, $0F, $38, $DF, $BA, $A0, $00, $00, $00 // aesdeclast xmm7, [rdx+160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + db $66, $0F, $38, $DF, $82, $C0, $00, $00, $00 // aesdeclast xmm0, [rdx+192] + db $66, $0F, $38, $DF, $8A, $C0, $00, $00, $00 // aesdeclast xmm1, [rdx+192] + db $66, $0F, $38, $DF, $92, $C0, $00, $00, $00 // aesdeclast xmm2, [rdx+192] + db $66, $0F, $38, $DF, $9A, $C0, $00, $00, $00 // aesdeclast xmm3, [rdx+192] + db $66, $0F, $38, $DF, $A2, $C0, $00, $00, $00 // aesdeclast xmm4, [rdx+192] + db $66, $0F, $38, $DF, $AA, $C0, $00, $00, $00 // aesdeclast xmm5, [rdx+192] + db $66, $0F, $38, $DF, $B2, $C0, $00, $00, $00 // aesdeclast xmm6, [rdx+192] + db $66, $0F, $38, $DF, $BA, $C0, $00, $00, $00 // aesdeclast xmm7, [rdx+192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DF, $82, $E0, $00, $00, $00 // aesdeclast xmm0, [rdx+224] + db $66, $0F, $38, $DF, $8A, $E0, $00, $00, $00 // aesdeclast xmm1, [rdx+224] + db $66, $0F, $38, $DF, $92, $E0, $00, $00, $00 // aesdeclast xmm2, [rdx+224] + db $66, $0F, $38, $DF, $9A, $E0, $00, $00, $00 // aesdeclast xmm3, [rdx+224] + db $66, $0F, $38, $DF, $A2, $E0, $00, $00, $00 // aesdeclast xmm4, [rdx+224] + db $66, $0F, $38, $DF, $AA, $E0, $00, $00, $00 // aesdeclast xmm5, [rdx+224] + db $66, $0F, $38, $DF, $B2, $E0, $00, $00, $00 // aesdeclast xmm6, [rdx+224] + db $66, $0F, $38, $DF, $BA, $E0, $00, $00, $00 // aesdeclast xmm7, [rdx+224] +{$ENDIF} +{$ELSE} + // Round 1 + db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16] + db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16] + db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16] + db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16] + db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16] + db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16] + db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16] + db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16] + // stage next-iteration Delta 0 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8 + // Round 2 + db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32] + db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32] + db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32] + db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32] + db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32] + db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32] + db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32] + db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32] + // stage next-iteration Delta 1 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8 + // Round 3 + db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48] + db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48] + db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48] + db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48] + db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48] + db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48] + db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48] + db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48] + // stage next-iteration Delta 2 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8 + // Round 4 + db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64] + db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64] + db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64] + db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64] + db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64] + db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64] + db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64] + db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64] + // stage next-iteration Delta 3 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8 + // Round 5 + db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80] + db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80] + db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80] + db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80] + db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80] + db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80] + db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80] + db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80] + // stage next-iteration Delta 4 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8 + // Round 6 + db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96] + db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96] + db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96] + db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96] + db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96] + db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96] + db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96] + db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96] + // stage next-iteration Delta 5 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8 + // Round 7 + db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112] + db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112] + db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112] + db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112] + db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112] + db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112] + db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112] + db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112] + // stage next-iteration Delta 6 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8 + // Round 8 + db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128] + db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128] + db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128] + db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128] + db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128] + db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128] + db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128] + db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128] + // stage next-iteration Delta 7 -> bank 0 + add r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8 + // Round 9 + db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144] + db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144] + db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144] + db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144] + db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144] + db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144] + db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144] + db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144] +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160] + db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160] + db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160] + db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160] + db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160] + db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160] + db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160] + db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160] + db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176] + db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176] + db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176] + db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176] + db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176] + db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176] + db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176] + db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176] +{$IFEND} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192] + db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192] + db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192] + db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192] + db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192] + db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192] + db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192] + db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192] + db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208] + db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208] + db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208] + db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208] + db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208] + db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208] + db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208] + db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY128} + db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160] + db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160] + db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160] + db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160] + db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160] + db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160] + db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160] + db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192] + db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192] + db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192] + db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192] + db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192] + db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192] + db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192] + db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224] + db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224] + db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224] + db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224] + db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224] + db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224] + db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224] + db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224] +{$ENDIF} +{$ENDIF} + + // Post-whiten with this iteration's Deltas. + pxor xmm0, [rsp + 128] + pxor xmm1, [rsp + 144] + pxor xmm2, [rsp + 160] + pxor xmm3, [rsp + 176] + pxor xmm4, [rsp + 192] + pxor xmm5, [rsp + 208] + pxor xmm6, [rsp + 224] + pxor xmm7, [rsp + 240] + +{$IFDEF CRYPTOLIB_AESNI_DECRYPT} + // Fold recovered plaintext into the checksum. + db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0 + db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1 + db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2 + db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3 + db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4 + db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5 + db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6 + db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7 +{$ENDIF} - // --- Advance pointers and decrement loop counter. --- + movdqu [r8 + 0], xmm0 + movdqu [r8 + 16], xmm1 + movdqu [r8 + 32], xmm2 + movdqu [r8 + 48], xmm3 + movdqu [r8 + 64], xmm4 + movdqu [r8 + 80], xmm5 + movdqu [r8 + 96], xmm6 + movdqu [r8 + 112], xmm7 add rbx, 128 - add r8, 128 - mov rax, rbx // next iter's block-0 source - // tracks the main stream - dec r10 + add r8, 128 + mov rax, rbx // next iteration's block-0 source + + sub r10, 1 jnz @@ocb_x64_loop - // --- Epilogue: flush xmm_offset and xmm_checksum back to memory, - // release stack scratch, restore rbx. --- - mov rax, [rcx + 24] // rax = OffsetPtr - movdqu [rax], xmm8 - mov rax, [rcx + 32] // rax = ChecksumPtr - movdqu [rax], xmm9 + // --- Undo the one-iteration ladder over-stage (XOR self-inverse). --- + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + bsf rax, r11 + shl rax, 4 + add rax, r9 + db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax] + db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 + sub r11, 1 + + // --- Epilogue: flush accumulators, unwind. --- + mov rax, [rcx + 24] + db $F3, $44, $0F, $7F, $00 // movdqu [rax], xmm8 + mov rax, [rcx + 32] + db $F3, $44, $0F, $7F, $08 // movdqu [rax], xmm9 - add rsp, 128 + add rsp, 256 pop rbx {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} From 95aa97c89384bf74d5d6052cdb8e3184b8f82232 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Sat, 11 Jul 2026 15:20:22 +0100 Subject: [PATCH 3/4] Stage GCM counter blocks ahead through GPR stores on x86-64 Replace the per-block shuffle-port register inserts in the fused GCM kernel's counter build with aligned stack slots: the fixed 12-byte J0 prefix is staged once in the prologue, and each batch's AES-only round 9 stages the NEXT batch's four-byte counter words, so the loop-top movdqa loads never read a freshly written slot (a naive store-then-load form forwards-fails on the partial overlap and serializes the loop top - it measured 40% slower). The prologue pre-stages batch 0; the final batch over-stages into scratch. This removes sixteen shuffle-port uops per 128-byte batch and is worth 3-7% on x86-64. The i386 kernel keeps its insert-based build: the staged form measured neutral-to-negative there, with both placements tried. A Karatsuba variant of the batch GHASH (three carry-less multiplies per block plus a precomputed halves-XOR table column) was also implemented, verified and benchmarked: it is throughput-neutral on x86-64 and a 4-7% regression on i386 - the saved multiply is repaid on the same execution port by the added shuffle, plus an extra table load per iteration - so the schoolbook four-multiply form stays. --- .../Gcm/AesGcmFusedCtrGhashEight_x86_64.inc | 114 +++++++++++++----- 1 file changed, 84 insertions(+), 30 deletions(-) diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc index da216efc..6abc36a2 100644 --- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc @@ -36,7 +36,7 @@ // All loop-carried state lives in registers - no context memory is // touched inside the batch loop: // rsi/rdi = PXorIn/POut (advance) r10 = PPrevCipher (rotates) -// rdx = PRoundKeys r11 = PHPow128 r8d = counter32 r9 = PJ0 +// rdx = PRoundKeys r11 = PHPow128 r8d = counter32 // rbx = batch count ebp = GHASH-from-output flag rax = scratch // // XMM (16/16 used): @@ -49,12 +49,18 @@ // xmm14 : pclmul scratch destination // xmm15 : byte-reverse mask (held for the whole kernel) // -// Stack: 24 bytes below the nonvolatile-save area; [rsp] is 16-aligned -// and holds the reduction constant (low qword 0xC2000000_00000000). +// Stack (below the nonvolatile-save area; [rsp] is 16-aligned): +// [rsp + 0] reduction constant (low qword 0xC2000000_00000000) +// [rsp + 16..143] 8 aligned counter-block slots. The fixed 12-byte J0 +// prefix is staged once in the prologue, and each batch's AES-only +// tail rounds stage the NEXT batch's counter words (GPR stores, no +// shuffle-port inserts), so the loop-top movdqa loads never read a +// freshly written slot (no store-forward stalls). The prologue +// pre-stages batch 0; the final batch over-stages into scratch. // // AES-NI and PCLMULQDQ instructions, plus selected forms touching -// xmm8..xmm15 (pxor) and PINSRD/PSHUFD, are db-encoded for broad -// assembler compatibility. +// xmm8..xmm15 (pxor) and PSHUFD, are db-encoded for broad assembler +// compatibility. // ------------------------------------------------------------------- push rbx @@ -62,12 +68,34 @@ push rdi push rbp {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - sub rsp, 24 // [rsp] 16-aligned + sub rsp, 152 // [rsp] 16-aligned mov rax, $C200000000000000 mov [rsp], rax // reduction constant (low qword) xor eax, eax mov [rsp + 8], rax + // --- Stage the fixed J0 prefix into the 8 counter slots once; only + // bytes 12..15 (the counter word) change per batch. --- + mov rax, [rcx + 64] // PJ0Template + mov rdx, [rax] // J0 bytes 0..7 + mov eax, [rax + 8] // J0 bytes 8..11 + mov [rsp + 16], rdx + mov [rsp + 24], eax + mov [rsp + 32], rdx + mov [rsp + 40], eax + mov [rsp + 48], rdx + mov [rsp + 56], eax + mov [rsp + 64], rdx + mov [rsp + 72], eax + mov [rsp + 80], rdx + mov [rsp + 88], eax + mov [rsp + 96], rdx + mov [rsp + 104], eax + mov [rsp + 112], rdx + mov [rsp + 120], eax + mov [rsp + 128], rdx + mov [rsp + 136], eax + // --- Hoist the whole loop state into registers. --- mov rdx, [rcx + 24] // rdx = PRoundKeys (aligned) mov r11, [rcx + 32] // r11 = PHPow128 (x-shifted) @@ -77,50 +105,50 @@ mov rdi, [rcx + 8] // rdi = POut mov r10, [rcx + 16] // r10 = PPrevCipher mov r8d, [rcx + 56] // r8d = counter32 - mov r9, [rcx + 64] // r9 = PJ0Template mov rbx, [rcx + 72] // rbx = batch count mov ebp, [rcx + 80] // ebp = GHASH-from-output flag mov rax, [rcx + 40] // rax = PFS movdqu xmm9, [rax] pshufb xmm9, xmm15 // xmm9 = reflected running state -@@GcmFusedLoop: - // Build 8 counter blocks into xmm0..7: block j = J0[0..11] || BE(counter32+j+1). - // GCM's 32-bit counter holds the last-used value, so the batch based at - // counter32 spans counter32+1..counter32+8 (matching FillCtr8BlocksRaw); only - // bytes 12-15 vary. PINSRD is db-encoded. - movdqu xmm0, [r9] + // --- Pre-stage batch 0's counter words (counter32+1..+8, big-endian). --- lea eax, [r8 + 1] bswap eax - db $66, $0F, $3A, $22, $C0, $03 // pinsrd xmm0, eax, 3 - movdqu xmm1, [r9] + mov [rsp + 28], eax lea eax, [r8 + 2] bswap eax - db $66, $0F, $3A, $22, $C8, $03 // pinsrd xmm1, eax, 3 - movdqu xmm2, [r9] + mov [rsp + 44], eax lea eax, [r8 + 3] bswap eax - db $66, $0F, $3A, $22, $D0, $03 // pinsrd xmm2, eax, 3 - movdqu xmm3, [r9] + mov [rsp + 60], eax lea eax, [r8 + 4] bswap eax - db $66, $0F, $3A, $22, $D8, $03 // pinsrd xmm3, eax, 3 - movdqu xmm4, [r9] + mov [rsp + 76], eax lea eax, [r8 + 5] bswap eax - db $66, $0F, $3A, $22, $E0, $03 // pinsrd xmm4, eax, 3 - movdqu xmm5, [r9] + mov [rsp + 92], eax lea eax, [r8 + 6] bswap eax - db $66, $0F, $3A, $22, $E8, $03 // pinsrd xmm5, eax, 3 - movdqu xmm6, [r9] + mov [rsp + 108], eax lea eax, [r8 + 7] bswap eax - db $66, $0F, $3A, $22, $F0, $03 // pinsrd xmm6, eax, 3 - movdqu xmm7, [r9] + mov [rsp + 124], eax lea eax, [r8 + 8] bswap eax - db $66, $0F, $3A, $22, $F8, $03 // pinsrd xmm7, eax, 3 + mov [rsp + 140], eax + +@@GcmFusedLoop: + // Load the 8 pre-staged counter blocks: block j = J0[0..11] || + // BE(counter32+j+1) (GCM's counter holds the last-used value, matching + // FillCtr8BlocksRaw). The words were staged a full round-chain ago. + movdqa xmm0, [rsp + 16] + movdqa xmm1, [rsp + 32] + movdqa xmm2, [rsp + 48] + movdqa xmm3, [rsp + 64] + movdqa xmm4, [rsp + 80] + movdqa xmm5, [rsp + 96] + movdqa xmm6, [rsp + 112] + movdqa xmm7, [rsp + 128] // --- Round 0: AddRoundKey (aligned key memory operands) + zero Z1/Z2. // Z0 (xmm9) still holds the carried state; iter 0 folds and clears it. --- @@ -345,15 +373,41 @@ db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128] db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=) - // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast. --- + // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast. + // Round 9 interleaves the staging of the NEXT batch's counter + // words (counter32+9..+16) into the slots. --- db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144] + lea eax, [r8 + 9] + bswap eax + mov [rsp + 28], eax db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144] + lea eax, [r8 + 10] + bswap eax + mov [rsp + 44], eax db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144] + lea eax, [r8 + 11] + bswap eax + mov [rsp + 60], eax db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144] + lea eax, [r8 + 12] + bswap eax + mov [rsp + 76], eax db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144] + lea eax, [r8 + 13] + bswap eax + mov [rsp + 92], eax db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144] + lea eax, [r8 + 14] + bswap eax + mov [rsp + 108], eax db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144] + lea eax, [r8 + 15] + bswap eax + mov [rsp + 124], eax db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144] + lea eax, [r8 + 16] + bswap eax + mov [rsp + 140], eax {$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)} db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160] db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160] @@ -490,7 +544,7 @@ pshufb xmm9, xmm15 // reflected -> canonical mov rax, [rcx + 40] // PFS movdqu [rax], xmm9 - add rsp, 24 + add rsp, 152 {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} pop rbp pop rdi From 679a6f6be491e97396dee61d9de1e3426edd2e48 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Sat, 11 Jul 2026 16:40:00 +0100 Subject: [PATCH 4/4] Rework the standalone GHASH batch kernels and give AAD processing a bulk path The 4-/8-way GHASH batch kernels (GcmGhashFull) previously paid their whole setup on every 64/128-byte call: a ten-register nonvolatile save/restore, the byte-reversal round-trip of the running state, a long shift/XOR reduction and a Pascal call boundary. Worse, the GCM mode's AAD path never used them at all - ProcessAadBytes fed one block at a time through the interface-dispatched multiplier, running at ~110 MB/s. Kernels: the batch body now loops over an ABatchCount parameter inside the asm, so the save/restore, mask load, reduction constant and state byte-reversal are paid once per call; the running state stays reflected across batches (a register on x86-64, an aligned frame slot on i386, which also serves the byte-reverse mask as shuffle memory operands). The per-batch reduction is two carry-less multiplies by the folded field constant instead of the shift ladder. Table: TGcmUtilities.InitEightWayHPowFromH now emits every H power pre-multiplied by x (DivideP), which is what the folding reduction requires. Every consumer funnels through the same table and kernel body - the GCM 4-/8-way paths, the new bulk-AAD path, the GCM-SIV POLYVAL Horner batch and the fused CTR+GHASH kernel (whose constructor now takes a plain copy instead of shifting its own) - so the change is consistent across the board. The mode's post-dispatch scalar fallbacks (unreachable in practice: they share their gate with the table precompute) were rewritten as per-block multiplier folds, keeping them correct independent of the table layout. Dispatch: TryFusedFour/EightShuffledGhash and the POLYVAL batch chain gained the batch count; ProcessAadBytes hashes whole 128-byte spans in a single kernel call and leaves the remainder to the per-block loop. --- .../CipherKernels/Block/ClpAesNiGcmKernel.pas | 33 +- .../Block/ClpPclmulGcmSivKernel.pas | 6 +- .../src/Crypto/Modes/ClpGcmBlockCipher.pas | 68 ++-- .../src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas | 8 +- .../Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas | 17 +- .../src/Crypto/Modes/Gcm/ClpGcmUtilities.pas | 16 +- .../src/Crypto/Modes/Gcm/ClpGhashSimd.pas | 20 +- .../Crypto/Modes/Gcm/ClpGhashX86Backend.pas | 42 ++- .../Include/Simd/Gcm/GcmGhashFull_i386.inc | 327 ++++++++---------- .../Include/Simd/Gcm/GcmGhashFull_x86_64.inc | 295 +++++++--------- 10 files changed, 389 insertions(+), 443 deletions(-) diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas index 217a164d..8d841fea 100644 --- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas +++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas @@ -23,7 +23,6 @@ interface uses SysUtils, ClpCryptoLibTypes, - ClpBinaryPrimitives, ClpIBlockCipher, ClpIAesEngineX86, ClpCipherKernelTypes, @@ -57,9 +56,9 @@ TAesNiGcmKernel = class sealed(TInterfacedObject, IGcmKernel) FRounds: Int32; FHPow128: Pointer; FMask: Pointer; - // Kernel-owned copy of the H^8..H^1 table with every power pre-multiplied - // by x in the reflected representation, matching the carry-less-multiply - // folding reduction inside the fused kernel. FHPow128 points at it. + // Kernel-owned copy of the mode's H^8..H^1 table (already x-pre-multiplied + // by TGcmUtilities.InitEightWayHPowFromH, matching the carry-less-multiply + // folding reduction inside the fused kernel). FHPow128 points at it. FHPowShifted: TCryptoLibByteArray; public constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; @@ -159,35 +158,17 @@ procedure GcmFusedAesEnc256GhashEight(PCtx: Pointer); constructor TAesNiGcmKernel.Create(const AEngine: IAesEngineX86; AKeys: Pointer; ARounds: Int32; AHPow128, AMask: Pointer); -var - LI: Int32; - LLo, LHi, LCarry: UInt64; begin inherited Create; FEngine := AEngine; FKeys := AKeys; FRounds := ARounds; FMask := AMask; - // Pre-multiply each H power by x in the reflected representation: shift the - // 128-bit value left by one with a conditional fold of the field polynomial - // (0xC2 in the top byte, 1 in the bottom). This lets the kernel reduce the - // batch product with two carry-less multiplies instead of a shift ladder. + // The mode's table already holds the powers pre-multiplied by x (see + // TGcmUtilities.InitEightWayHPowFromH); keep a kernel-owned copy so the + // pointer stays valid for the kernel's lifetime. System.SetLength(FHPowShifted, 128); - for LI := 0 to 7 do - begin - LLo := TBinaryPrimitives.ReadUInt64LittleEndian(PByte(AHPow128), LI * 16); - LHi := TBinaryPrimitives.ReadUInt64LittleEndian(PByte(AHPow128), LI * 16 + 8); - LCarry := LHi shr 63; - LHi := (LHi shl 1) or (LLo shr 63); - LLo := LLo shl 1; - if LCarry <> 0 then - begin - LHi := LHi xor UInt64($C200000000000000); - LLo := LLo xor 1; - end; - TBinaryPrimitives.WriteUInt64LittleEndian(FHPowShifted, LI * 16, LLo); - TBinaryPrimitives.WriteUInt64LittleEndian(FHPowShifted, LI * 16 + 8, LHi); - end; + System.Move(AHPow128^, FHPowShifted[0], 128); FHPow128 := @FHPowShifted[0]; end; diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas index c99b1ce1..d9238560 100644 --- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas +++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas @@ -85,9 +85,11 @@ function TPclmulGcmSivKernel.MinimumBlockCount: Int32; procedure TPclmulGcmSivKernel.ProcessPolyvalBatch(AInPtr, AAccumulator: Pointer; ABlockCount: Int32); begin - if ABlockCount <> FUSED_POLYVAL_MIN_BLOCKS then + if (ABlockCount < FUSED_POLYVAL_MIN_BLOCKS) or + (ABlockCount mod FUSED_POLYVAL_MIN_BLOCKS <> 0) then Exit; - TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask); + TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask, + ABlockCount div FUSED_POLYVAL_MIN_BLOCKS); end; { TPclmulGcmSivKernelFactory } diff --git a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas index db79cbb5..0784cfd5 100644 --- a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas @@ -636,7 +636,7 @@ procedure TGcmBlockCipher.ProcessAadByte(AInput: Byte); procedure TGcmBlockCipher.ProcessAadBytes(const AInput: TCryptoLibByteArray; AInOff, ALen: Int32); var - LAvailable, LInLimit: Int32; + LAvailable, LInLimit, LBatches: Int32; begin CheckStatus(); @@ -657,6 +657,20 @@ procedure TGcmBlockCipher.ProcessAadBytes(const AInput: TCryptoLibByteArray; ALen := ALen - LAvailable; end; + // Bulk fast path: fold whole 128-byte spans through the fused 8-way GHASH + // kernel in one call; the per-block loop below handles the remainder. + if (FHPow <> nil) and (ALen >= 128) then + begin + LBatches := ALen div 128; + if TGhashSimd.TryFusedEightShuffledGhash(@FS_at[0], @AInput[AInOff], + @FHPow[0], LBatches) then + begin + AInOff := AInOff + LBatches * 128; + ALen := ALen - LBatches * 128; + FAtLength := FAtLength + UInt64(LBatches) * 128; + end; + end; + LInLimit := AInOff + ALen - BlockSize; while AInOff <= LInLimit do @@ -1104,19 +1118,14 @@ class procedure TGcmBlockCipher.GcmReverse16(const ASrc, ADst: PByte); procedure TGcmBlockCipher.GhashFourShuffledBlocks(PC0, PC16, PC32, PC48: PByte); var - LB, LI: Int32; - LDblk: array[0..15] of Byte; - LBuf48: array[0..47] of Byte; - LU0, LU1, LU2: array[0..15] of Byte; - LSRev: array[0..15] of Byte; + LB: Int32; LPCiph: PByte; begin - if TGhashSimd.TryFusedFourShuffledGhash(@FS[0], PC0, @FHPow[64]) then + if TGhashSimd.TryFusedFourShuffledGhash(@FS[0], PC0, @FHPow[64], 1) then Exit; - GcmReverse16(@FS[0], @LSRev[0]); - FillChar(LU0, 16, 0); - FillChar(LU1, 16, 0); - FillChar(LU2, 16, 0); + // Unreachable in practice (the caller gates on the same predicate as the + // kernel); fold per block through the multiplier, which is independent of + // the x-pre-multiplied FHPow layout. for LB := 0 to 3 do begin case LB of @@ -1129,15 +1138,9 @@ procedure TGcmBlockCipher.GhashFourShuffledBlocks(PC0, PC16, PC32, PC48: PByte); else LPCiph := PC48; end; - GcmReverse16(LPCiph, @LDblk[0]); - if LB = 0 then - for LI := 0 to 15 do - LDblk[LI] := LDblk[LI] xor LSRev[LI]; - TGcmUtilities.MultiplyExt(@LDblk[0], @FHPow[64 + (LB * 16)], @LBuf48[0]); - TGcmUtilities.XorMultiplyExtLimbs48(@LU0[0], @LU1[0], @LU2[0], @LBuf48[0]); + TByteUtilities.&Xor(BlockSize, @FS[0], LPCiph, @FS[0]); + FMultiplier.MultiplyH(FS); end; - TGcmUtilities.Reduce3(@LU0[0], @LU1[0], @LU2[0], @LSRev[0]); - GcmReverse16(@LSRev[0], @FS[0]); end; // ======================================================================= @@ -1175,31 +1178,18 @@ procedure TGcmBlockCipher.ProcessBlocks4Fused(const AInBuf: TCryptoLibByteArray; procedure TGcmBlockCipher.GhashEightShuffledBlocks(PBase: PByte); var - LB, LI: Int32; - LDblk: array [0 .. 15] of Byte; - LBuf48: array [0 .. 47] of Byte; - LU0, LU1, LU2: array [0 .. 15] of Byte; - LSRev: array [0 .. 15] of Byte; - LPCiph: PByte; + LB: Int32; begin - if TGhashSimd.TryFusedEightShuffledGhash(@FS[0], PBase, @FHPow[0]) then + if TGhashSimd.TryFusedEightShuffledGhash(@FS[0], PBase, @FHPow[0], 1) then Exit; - GcmReverse16(@FS[0], @LSRev[0]); - FillChar(LU0, 16, 0); - FillChar(LU1, 16, 0); - FillChar(LU2, 16, 0); + // Unreachable in practice (the caller gates on the same predicate as the + // kernel); fold per block through the multiplier, which is independent of + // the x-pre-multiplied FHPow layout. for LB := 0 to 7 do begin - LPCiph := PBase + (LB * 16); - GcmReverse16(LPCiph, @LDblk[0]); - if LB = 0 then - for LI := 0 to 15 do - LDblk[LI] := LDblk[LI] xor LSRev[LI]; - TGcmUtilities.MultiplyExt(@LDblk[0], @FHPow[LB * 16], @LBuf48[0]); - TGcmUtilities.XorMultiplyExtLimbs48(@LU0[0], @LU1[0], @LU2[0], @LBuf48[0]); + TByteUtilities.&Xor(BlockSize, @FS[0], PBase + (LB * BlockSize), @FS[0]); + FMultiplier.MultiplyH(FS); end; - TGcmUtilities.Reduce3(@LU0[0], @LU1[0], @LU2[0], @LSRev[0]); - GcmReverse16(@LSRev[0], @FS[0]); end; // Single-batch fused 8-way GCM step. See ProcessBlocks4Fused for direction semantics. diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas index 0bc82c8c..d5e1ab38 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas @@ -39,7 +39,8 @@ TGcmSivSimd = class sealed /// True when a POLYVAL batch kernel is usable on this CPU. class function IsSupported: Boolean; static; /// Eight-block POLYVAL Horner batch. Precondition: IsSupported. - class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); static; + class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer; + ABatchCount: NativeInt); static; end; implementation @@ -55,10 +56,11 @@ class function TGcmSivSimd.IsSupported: Boolean; {$IFEND} end; -class procedure TGcmSivSimd.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); +class procedure TGcmSivSimd.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer; + ABatchCount: NativeInt); begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask); + TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask, ABatchCount); {$IFEND} end; diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas index 1b2f84e8..d9c584e2 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas @@ -36,20 +36,22 @@ TGcmSivX86Backend = class sealed public /// True when the PCLMULQDQ POLYVAL kernel is usable on this CPU. class function IsSupported: Boolean; static; - /// Eight-block POLYVAL Horner batch. Precondition: IsSupported. - class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); static; + /// ABatchCount eight-block POLYVAL Horner batches. Precondition: IsSupported. + class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer; + ABatchCount: NativeInt); static; end; implementation {$IFDEF CRYPTOLIB_X86_SIMD} -procedure GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask: Pointer); +procedure GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask: Pointer; + ABatchCount: NativeInt); {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\..\Include\Simd\GcmSiv\PolyvalHornerEight_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$I ..\..\..\Include\Simd\GcmSiv\PolyvalHornerEight_i386.inc} {$ENDIF} end; @@ -67,10 +69,11 @@ class function TGcmSivX86Backend.IsSupported: Boolean; {$ENDIF} end; -class procedure TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); +class procedure TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer; + ABatchCount: NativeInt); begin {$IFDEF CRYPTOLIB_X86_SIMD} - GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask); + GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask, ABatchCount); {$ENDIF} end; diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas index bc574749..a434ebff 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas @@ -69,7 +69,10 @@ TGcmUtilities = class sealed(TObject) class procedure Reduce3(PZ0, PZ1, PZ2, PSVector16: PByte); static; /// Xor three 16-byte limbs with three 16-byte slices from a 48-byte MultiplyExt output. class procedure XorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte); static; - /// HPow[0..7] = H^8..H^1 as 16-byte limbs at offsets 0,16,...,112 (index 0 = H^8). Four-way fused GHASH uses offsets 64..112 (H^4..H^1). + /// HPow[0..7] = H^8..H^1 as 16-byte limbs at offsets 0,16,...,112 (index 0 = H^8), + /// each PRE-MULTIPLIED BY x in the reflected representation (DivideP) to match the + /// carry-less-multiply folding reduction in the fused GHASH / POLYVAL kernels. + /// Four-way fused GHASH uses offsets 64..112 (H^4..H^1). class procedure InitEightWayHPowFromH(const AH: TCryptoLibByteArray; const AHPow128: TCryptoLibByteArray); static; class procedure &Xor(const AX, AY: TCryptoLibByteArray); overload; static; @@ -387,6 +390,17 @@ class procedure TGcmUtilities.InitEightWayHPowFromH(const AH: TCryptoLibByteArra LY := LF4; Multiply(LAcc, LY); LF8 := LAcc; + // Pre-multiply every power by x (DivideP in this field convention) so the + // fused kernels can reduce the batch product with two carry-less multiplies + // by 0xC2000000_00000000 instead of a bit-shift ladder. + LY := LF1; DivideP(LY, LF1); + LY := LF2; DivideP(LY, LF2); + LY := LF3; DivideP(LY, LF3); + LY := LF4; DivideP(LY, LF4); + LY := LF5; DivideP(LY, LF5); + LY := LF6; DivideP(LY, LF6); + LY := LF7; DivideP(LY, LF7); + LY := LF8; DivideP(LY, LF8); TPack.UInt64_To_LE(LF8.N1, AHPow128, 0); TPack.UInt64_To_LE(LF8.N0, AHPow128, 8); TPack.UInt64_To_LE(LF7.N1, AHPow128, 16); diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas index 6adee681..ffc204e1 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas @@ -46,10 +46,12 @@ TGhashSimd = class sealed class function TryReduce3(PZ0, PZ1, PZ2, PSVector16: PByte): Boolean; static; /// Xor three 16-byte limbs with three slices of a 48-byte MultiplyExt output. class function TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte): Boolean; static; - /// Fused 4-way GHASH over 64 contiguous ciphertext bytes. - class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; static; - /// Fused 8-way GHASH over 128 contiguous ciphertext bytes. - class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; static; + /// Fused 4-way GHASH over ABatchCount contiguous 64-byte batches. + class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte; + ABatchCount: NativeInt): Boolean; static; + /// Fused 8-way GHASH over ABatchCount contiguous 128-byte batches. + class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte; + ABatchCount: NativeInt): Boolean; static; /// True when the shuffled/fused GHASH path (4-/8-way) is usable on this build/CPU. Gates the batch dispatch and the H-power precompute. class function IsShuffledGhashSupported: Boolean; static; @@ -99,19 +101,21 @@ class function TGhashSimd.TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte) {$IFEND} end; -class function TGhashSimd.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; +class function TGhashSimd.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte; + ABatchCount: NativeInt): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64); + Result := TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64, ABatchCount); {$ELSE} Result := False; {$IFEND} end; -class function TGhashSimd.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; +class function TGhashSimd.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte; + ABatchCount: NativeInt): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128); + Result := TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128, ABatchCount); {$ELSE} Result := False; {$IFEND} diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas index 60b67233..c1fa43e4 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas @@ -45,10 +45,12 @@ TGhashX86Backend = class sealed class function TryReduce3(PZ0, PZ1, PZ2, PSVector16: PByte): Boolean; static; /// SIMD xor of three 16-byte limbs with three slices of a 48-byte MultiplyExt output. class function TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte): Boolean; static; - /// PCLMULQDQ fused 4-way GHASH (requires packed vector layout). Uses the backend's own byte-reverse mask. - class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; static; - /// PCLMULQDQ fused 8-way GHASH (requires packed vector layout). Uses the backend's own byte-reverse mask. - class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; static; + /// PCLMULQDQ fused 4-way GHASH over ABatchCount 64-byte batches (requires packed vector layout). Uses the backend's own byte-reverse mask. + class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte; + ABatchCount: NativeInt): Boolean; static; + /// PCLMULQDQ fused 8-way GHASH over ABatchCount 128-byte batches (requires packed vector layout). Uses the backend's own byte-reverse mask. + class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte; + ABatchCount: NativeInt): Boolean; static; /// True when the fused shuffled-GHASH path is usable on this CPU (needs packed vector layout). Gates the 4-/8-way batch dispatch and the H-power precompute. class function IsShuffledGhashSupported: Boolean; static; @@ -119,27 +121,27 @@ procedure GcmXorMultiplyExtLimbs48Sse2(PA0, PA1, PA2, PSrc48: Pointer); {$ENDIF} end; -procedure GcmGhashFourFull(PFS, PC0, PHPow64, PMask: Pointer); +procedure GcmGhashFourFull(PFS, PC0, PHPow64, PMask: Pointer; ABatchCount: NativeInt); {$DEFINE GCM_GHASH_FULL_BLOCKS_4} {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_i386.inc} {$ENDIF} {$UNDEF GCM_GHASH_FULL_BLOCKS_4} end; -procedure GcmGhashEightFull(PFS, PC0, PHPow128, PMask: Pointer); +procedure GcmGhashEightFull(PFS, PC0, PHPow128, PMask: Pointer; ABatchCount: NativeInt); {$DEFINE GCM_GHASH_FULL_BLOCKS_8} {$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_i386.inc} {$ENDIF} {$UNDEF GCM_GHASH_FULL_BLOCKS_8} @@ -242,28 +244,32 @@ class function TGhashX86Backend.TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: Result := False; end; -class function TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; +class function TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte; + ABatchCount: NativeInt): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} if TCpuFeatures.X86.HasSSSE3 and TCpuFeatures.X86.HasPCLMULQDQ and TIntrinsicsVector.IsPacked then begin - // Monolithic kernel: byte-reverse + state fold + 4-way multiply-accumulate + - // Reduce3 + byte-reverse back, all in a single assembly body (one call boundary). - GcmGhashFourFull(PFS, PC0, PHPow64, @ReverseBytesMask[0]); + // Monolithic kernel: the whole ABatchCount-batch run - byte-reverse, state + // fold, 4-way multiply-accumulate and folding reduction per batch - in a + // single assembly body (one call boundary). + GcmGhashFourFull(PFS, PC0, PHPow64, @ReverseBytesMask[0], ABatchCount); Exit(True); end; {$ENDIF} Result := False; end; -class function TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; +class function TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte; + ABatchCount: NativeInt): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} if TCpuFeatures.X86.HasSSSE3 and TCpuFeatures.X86.HasPCLMULQDQ and TIntrinsicsVector.IsPacked then begin - // Monolithic kernel: byte-reverse + state fold + 8-way multiply-accumulate + - // Reduce3 + byte-reverse back, all in a single assembly body (one call boundary). - GcmGhashEightFull(PFS, PC0, PHPow128, @ReverseBytesMask[0]); + // Monolithic kernel: the whole ABatchCount-batch run - byte-reverse, state + // fold, 8-way multiply-accumulate and folding reduction per batch - in a + // single assembly body (one call boundary). + GcmGhashEightFull(PFS, PC0, PHPow128, @ReverseBytesMask[0], ABatchCount); Exit(True); end; {$ENDIF} diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc index fa5b49ed..5fd64011 100644 --- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc +++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc @@ -1,18 +1,20 @@ -// SSE2 + PCLMULQDQ fused N-way GHASH batch kernel (i386). +// SSE2 + PCLMULQDQ N-way GHASH batch kernel (i386). // ------------------------------------------------------------------- -// One monolithic body that performs a full GHASH batch fold: for each -// block loads the (byte-reversed) ciphertext, XORs the running state -// into block 0, multiplies by the matching H power via four PCLMULQDQ -// products (Karatsuba), accumulates across blocks into three 128-bit -// limbs Z0/Z1/Z2, then folds Z1 and reduces (Z0, Z2) modulo the GCM -// polynomial x^128 + x^7 + x^2 + x + 1 to a single field element, -// byte-reverses, and stores back to PFS. i386 has only 8 XMM -// registers so the PSHUFB mask cannot stay pinned; it is reloaded -// from [eax] on every use. +// Loops over ABatchCount batches inside the asm body. Per batch: for each +// block loads the (byte-reversed) data, XORs the running state into +// block 0, multiplies by the matching H power via four PCLMULQDQ products, +// accumulates across blocks into three 128-bit limbs Z0/Z1/Z2, then folds +// Z1 and reduces with two carry-less multiplies by the folded constant +// 0xC2000000_00000000 (PHPow holds the powers PRE-MULTIPLIED BY x in the +// reflected representation - see TGcmUtilities.InitEightWayHPowFromH - so +// no bit-shift correction is needed). The running state stays reflected +// across batches; the canonical byte order is only touched at entry/exit, +// and the nonvolatile save, mask load and reduction constant are paid +// once per call instead of once per batch. // // Variant selection (set EXACTLY ONE of the block counts): -// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1. -// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1. +// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batches, H^4..H^1. +// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batches, H^8..H^1. // Optional: // GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so // skip the per-block input byte-reverse. Used by the @@ -21,261 +23,232 @@ // still byte-reversed. Unset for GCM (canonical // input), leaving that path byte-identical. // -// On entry (after ClpSimdProc4Begin_i386.inc): +// On entry (after ClpSimdProc5Begin_i386.inc): // ebx = PFS (16-byte running GHASH state, canonical; in/out) -// esi = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT) -// edi = PHPow (H^N..H^1 as 16-byte limbs, field form) +// esi = PC0 (data blocks; canonical, or field form with RAW_INPUT; advances) +// edi = PHPow (H^N..H^1 as 16-byte limbs, x-pre-multiplied) // eax = PMask (16-byte PSHUFB byte-reverse control) +// ecx = ABatchCount (> 0) // -// XMM (8/8 used): -// xmm0..xmm2 : Z0 / Z1 / Z2 accumulators (persistent across blocks) -// xmm3 : current byte-reversed data block -// xmm4 : current H-power limb -// xmm5..xmm7 : pclmul product destinations (per-block); xmm7 is also -// reused as the byte-reverse mask (reloaded from [eax] -// before each PSHUFB) +// XMM (8/8 used): xmm0..xmm2 = Z0/Z1/Z2, xmm3 = data block / reduction +// constant, xmm4..xmm7 = scratch. The byte-reverse mask, the reflected +// running state and the reduction constant live in a 16-aligned frame +// (PSHUFB memory operands); the caller esp is parked inside it. // -// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. +// PCLMULQDQ / PSHUFD and the PSHUFB memory-operand form are db-encoded +// for broad assembler compatibility. // - pxor xmm0, xmm0 - pxor xmm1, xmm1 - pxor xmm2, xmm2 + mov edx, esp + sub esp, 72 + and esp, -16 // align the frame + mov [esp + 48], edx // park the caller esp - // Block 0: byte-reverse + fold running state + mov dword ptr [esp + 32], 0 + mov dword ptr [esp + 36], $C2000000 // reduction constant (low qword) + mov dword ptr [esp + 40], 0 + mov dword ptr [esp + 44], 0 + movdqu xmm0, [eax] // byte-reverse mask + movdqa [esp + 16], xmm0 + movdqu xmm1, [ebx] + pshufb xmm1, xmm0 + movdqa [esp + 0], xmm1 // reflected running state + +@@GhashFullLoop: + pxor xmm0, xmm0 // Z0 = 0 + pxor xmm1, xmm1 // Z1 = 0 + pxor xmm2, xmm2 // Z2 = 0 + + // Block 0 (folds the carried state) movdqu xmm3, [esi] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} - movdqu xmm5, [ebx] - pshufb xmm5, xmm7 - pxor xmm3, xmm5 + pxor xmm3, [esp + 0] // fold carried state movdqu xmm4, [edi] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 1 + // Block 1 (folds the carried state) movdqu xmm3, [esi + 16] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 16] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 2 + // Block 2 (folds the carried state) movdqu xmm3, [esi + 32] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 32] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 3 + // Block 3 (folds the carried state) movdqu xmm3, [esi + 48] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 48] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 - + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += {$IFDEF GCM_GHASH_FULL_BLOCKS_8} - // Block 4 + + // Block 4 (folds the carried state) movdqu xmm3, [esi + 64] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 64] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 5 + // Block 5 (folds the carried state) movdqu xmm3, [esi + 80] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 80] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 6 + // Block 6 (folds the carried state) movdqu xmm3, [esi + 96] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 96] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 7 + // Block 7 (folds the carried state) movdqu xmm3, [esi + 112] - movdqu xmm7, [eax] {$IFNDEF GCM_GHASH_RAW_INPUT} - pshufb xmm3, xmm7 + db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse) {$ENDIF} movdqu xmm4, [edi + 112] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += {$ENDIF GCM_GHASH_FULL_BLOCKS_8} - // Inline 3-limb GHASH reduction (Z0=xmm0, Z1=xmm1, Z2=xmm2): fold Z1 into - // (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1. + // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into + // the 256-bit product, then two carry-less multiplies by the + // folded constant 0xC2000000_00000000 (the H powers are + // pre-multiplied by x, so no bit-shift correction is needed). --- + movdqa xmm3, [esp + 32] movdqa xmm4, xmm1 - movdqa xmm5, xmm1 pslldq xmm4, 8 + pxor xmm0, xmm4 // ZLo ^= Z1 << 64 + movdqa xmm5, xmm1 psrldq xmm5, 8 - pxor xmm0, xmm4 - pxor xmm2, xmm5 - movq xmm3, xmm0 - movq xmm4, xmm2 - movdqa xmm5, xmm3 - psrlq xmm5, 1 - movdqa xmm6, xmm3 - psrlq xmm6, 2 - movdqa xmm7, xmm3 - psrlq xmm7, 7 - pxor xmm4, xmm3 - pxor xmm4, xmm5 - pxor xmm4, xmm6 - pxor xmm4, xmm7 - movdqa xmm1, xmm0 - psrldq xmm1, 8 - movdqa xmm5, xmm3 - psllq xmm5, 63 - movdqa xmm6, xmm3 - psllq xmm6, 62 - movdqa xmm7, xmm3 - psllq xmm7, 57 - pxor xmm1, xmm5 - pxor xmm1, xmm6 - pxor xmm1, xmm7 - movdqa xmm5, xmm2 - psrldq xmm5, 8 - movdqa xmm6, xmm5 - psllq xmm6, 1 - movdqa xmm7, xmm4 - psrlq xmm7, 63 - por xmm6, xmm7 - movdqa xmm7, xmm4 - psllq xmm7, 1 - movdqa xmm3, xmm1 - psrlq xmm3, 63 - por xmm7, xmm3 - movdqa xmm0, xmm1 - psllq xmm0, 1 - movdqa xmm3, xmm0 - psrlq xmm3, 1 - movdqa xmm2, xmm0 - psrlq xmm2, 2 - movdqa xmm5, xmm0 - psrlq xmm5, 7 - pxor xmm6, xmm0 - pxor xmm6, xmm3 - pxor xmm6, xmm2 - pxor xmm6, xmm5 - movdqa xmm3, xmm1 - psllq xmm3, 63 - movdqa xmm2, xmm1 - psllq xmm2, 58 - pxor xmm7, xmm3 - pxor xmm7, xmm2 + pxor xmm2, xmm5 // ZHi ^= Z1 >> 64 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 2 + pxor xmm0, xmm2 // ^= ZHi -> reflected state + movdqa [esp + 0], xmm0 // carry state to the next batch - // Pack and byte-reverse. - movq xmm0, xmm7 - pslldq xmm6, 8 - por xmm0, xmm6 - movdqu xmm7, [eax] - pshufb xmm0, xmm7 +{$IFDEF GCM_GHASH_FULL_BLOCKS_8} + add esi, 128 +{$ELSE} + add esi, 64 +{$ENDIF} + dec ecx + jnz @@GhashFullLoop + + movdqa xmm0, [esp + 0] + db $66, $0F, $38, $00, $44, $24, $10 // pshufb xmm0, [esp+16] (-> canonical) movdqu [ebx], xmm0 - pop edi - pop esi - pop ebx + mov esp, [esp + 48] // unwind the frame + pop edi + pop esi + pop ebx + diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc index 8a88bd6e..bbfeb6f1 100644 --- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc @@ -1,19 +1,20 @@ -// SSE2 + PCLMULQDQ fused N-way GHASH batch kernel (x86-64). +// SSE2 + PCLMULQDQ N-way GHASH batch kernel (x86-64). // ------------------------------------------------------------------- -// One monolithic body that performs a full GHASH batch fold: for each -// block loads the (byte-reversed) ciphertext, XORs the running state -// into block 0, multiplies by the matching H power via four PCLMULQDQ -// products (Karatsuba), accumulates across blocks into three 128-bit -// limbs Z0/Z1/Z2, then folds Z1 and reduces (Z0, Z2) modulo the GCM -// polynomial x^128 + x^7 + x^2 + x + 1 to a single field element, -// byte-reverses, and stores back to PFS. Replaces the previous chain -// of 11 Pascal -> ASM calls (one byte-reverse per block, one XOR fold, -// multiply-accumulate, 3-limb reduction, byte-reverse back) with a -// single kernel. +// Loops over ABatchCount batches inside the asm body. Per batch: for each +// block loads the (byte-reversed) data, XORs the running state into +// block 0, multiplies by the matching H power via four PCLMULQDQ products, +// accumulates across blocks into three 128-bit limbs Z0/Z1/Z2, then folds +// Z1 and reduces with two carry-less multiplies by the folded constant +// 0xC2000000_00000000 (PHPow holds the powers PRE-MULTIPLIED BY x in the +// reflected representation - see TGcmUtilities.InitEightWayHPowFromH - so +// no bit-shift correction is needed). The running state stays reflected +// across batches; the canonical byte order is only touched at entry/exit, +// and the nonvolatile save, mask load and reduction constant are paid +// once per call instead of once per batch. // // Variant selection (set EXACTLY ONE of the block counts): -// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1. -// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1. +// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batches, H^4..H^1. +// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batches, H^8..H^1. // Optional: // GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so // skip the per-block input byte-reverse. Used by the @@ -22,253 +23,223 @@ // still byte-reversed. Unset for GCM (canonical // input), leaving that path byte-identical. // -// On entry (after ClpSimdProc4Begin_x86_64.inc): +// On entry (after ClpSimdProc5Begin_x86_64.inc): // rcx = PFS (16-byte running GHASH state, canonical; in/out) -// rdx = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT) -// r8 = PHPow (H^N..H^1 as 16-byte limbs, field form) +// rdx = PC0 (data blocks; canonical, or field form with RAW_INPUT; advances) +// r8 = PHPow (H^N..H^1 as 16-byte limbs, x-pre-multiplied) // r9 = PMask (16-byte PSHUFB byte-reverse control) +// r10 = ABatchCount (> 0) // -// XMM (9/16 used): -// xmm0..xmm2 : Z0 / Z1 / Z2 accumulators (persistent across blocks) -// xmm3 : current byte-reversed data block -// xmm4 : current H-power limb -// xmm5..xmm7 : pclmul product destinations (per-block) -// xmm8 : byte-reverse mask (held for the whole kernel) +// XMM (10/16 used): xmm0..xmm2 = Z0/Z1/Z2, xmm3 = data block / reduction +// constant, xmm4..xmm7 = scratch, xmm8 = mask (whole call), xmm9 = +// reflected running state (whole call). [rsp] (16-aligned) holds the +// reduction constant. // -// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. +// PCLMULQDQ / PSHUFD and pxor forms touching xmm8/xmm9 are db-encoded +// for broad assembler compatibility. // {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - movdqu xmm8, [r9] - pxor xmm0, xmm0 - pxor xmm1, xmm1 - pxor xmm2, xmm2 + sub rsp, 24 // [rsp] 16-aligned + mov rax, $C200000000000000 + mov [rsp], rax // reduction constant (low qword) + xor eax, eax + mov [rsp + 8], rax + movdqu xmm8, [r9] // byte-reverse mask + movdqu xmm9, [rcx] + pshufb xmm9, xmm8 // reflected running state - // Block 0: byte-reverse + fold running state +@@GhashFullLoop: + pxor xmm0, xmm0 // Z0 = 0 + pxor xmm1, xmm1 // Z1 = 0 + pxor xmm2, xmm2 // Z2 = 0 + + // Block 0 (folds the carried state) movdqu xmm3, [rdx] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} - movdqu xmm5, [rcx] - pshufb xmm5, xmm8 - pxor xmm3, xmm5 + db $66, $41, $0F, $EF, $D9 // pxor xmm3, xmm9 (fold carried state) movdqu xmm4, [r8] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 1 + // Block 1 (folds the carried state) movdqu xmm3, [rdx + 16] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 16] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 2 + // Block 2 (folds the carried state) movdqu xmm3, [rdx + 32] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 32] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 3 + // Block 3 (folds the carried state) movdqu xmm3, [rdx + 48] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 48] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 - + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += {$IFDEF GCM_GHASH_FULL_BLOCKS_8} - // Block 4 + + // Block 4 (folds the carried state) movdqu xmm3, [rdx + 64] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 64] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 5 + // Block 5 (folds the carried state) movdqu xmm3, [rdx + 80] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 80] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 6 + // Block 6 (folds the carried state) movdqu xmm3, [rdx + 96] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 96] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += - // Block 7 + // Block 7 (folds the carried state) movdqu xmm3, [rdx + 112] {$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 {$ENDIF} movdqu xmm4, [r8 + 112] movdqa xmm5, xmm3 - db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 + db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00 movdqa xmm6, xmm3 - db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1 + db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16 + db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10 pxor xmm6, xmm7 movdqa xmm7, xmm3 - db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17 - pxor xmm0, xmm5 - pxor xmm1, xmm6 - pxor xmm2, xmm7 + db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11 + pxor xmm0, xmm5 // Z0 += + pxor xmm1, xmm6 // Z1 += + pxor xmm2, xmm7 // Z2 += {$ENDIF GCM_GHASH_FULL_BLOCKS_8} - // Inline 3-limb GHASH reduction (Z0=xmm0, Z1=xmm1, Z2=xmm2): fold Z1 into - // (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1. - // Step 1 -- fold Z1 into Z0/Z2. + // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into + // the 256-bit product, then two carry-less multiplies by the + // folded constant 0xC2000000_00000000 (the H powers are + // pre-multiplied by x, so no bit-shift correction is needed). --- + movdqa xmm3, [rsp] movdqa xmm4, xmm1 - movdqa xmm5, xmm1 pslldq xmm4, 8 + pxor xmm0, xmm4 // ZLo ^= Z1 << 64 + movdqa xmm5, xmm1 psrldq xmm5, 8 - pxor xmm0, xmm4 - pxor xmm2, xmm5 - // Extract T3, T1 as lower-64 halves; T2, T0 as upper-64 halves. - movq xmm3, xmm0 // xmm3 = T3 (low 64 of folded Z0) - movq xmm4, xmm2 // xmm4 = T1 (low 64 of folded Z2) - movdqa xmm5, xmm3 - psrlq xmm5, 1 - movdqa xmm6, xmm3 - psrlq xmm6, 2 - movdqa xmm7, xmm3 - psrlq xmm7, 7 - pxor xmm4, xmm3 - pxor xmm4, xmm5 - pxor xmm4, xmm6 - pxor xmm4, xmm7 // xmm4 = T1 ^ T3 ^ (T3>>1) ^ (T3>>2) ^ (T3>>7) - movdqa xmm1, xmm0 - psrldq xmm1, 8 // xmm1 = T2 (upper 64 of folded Z0, now in lower) - movdqa xmm5, xmm3 - psllq xmm5, 63 - movdqa xmm6, xmm3 - psllq xmm6, 62 - movdqa xmm7, xmm3 - psllq xmm7, 57 - pxor xmm1, xmm5 - pxor xmm1, xmm6 - pxor xmm1, xmm7 // xmm1 = T2 ^ (T3<<63) ^ (T3<<62) ^ (T3<<57) - movdqa xmm5, xmm2 - psrldq xmm5, 8 // xmm5 = T0 (upper 64 of folded Z2) - movdqa xmm6, xmm5 - psllq xmm6, 1 - movdqa xmm7, xmm4 - psrlq xmm7, 63 - por xmm6, xmm7 // xmm6 = LZ0 initial = (T0<<1) | (T1>>63) - movdqa xmm7, xmm4 - psllq xmm7, 1 - movdqa xmm3, xmm1 - psrlq xmm3, 63 - por xmm7, xmm3 // xmm7 = LZ1 initial = (T1<<1) | (T2>>63) - movdqa xmm0, xmm1 - psllq xmm0, 1 // xmm0 = LZ2 = T2<<1 - movdqa xmm3, xmm0 - psrlq xmm3, 1 - movdqa xmm2, xmm0 - psrlq xmm2, 2 - movdqa xmm5, xmm0 - psrlq xmm5, 7 - pxor xmm6, xmm0 - pxor xmm6, xmm3 - pxor xmm6, xmm2 - pxor xmm6, xmm5 // xmm6 (low 64) = final LZ0 - movdqa xmm3, xmm1 - psllq xmm3, 63 - movdqa xmm2, xmm1 - psllq xmm2, 58 - pxor xmm7, xmm3 - pxor xmm7, xmm2 // xmm7 (low 64) = final LZ1 + pxor xmm2, xmm5 // ZHi ^= Z1 >> 64 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00 + db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves) + pxor xmm0, xmm4 // fold 2 + pxor xmm0, xmm2 // ^= ZHi -> reflected state + movdqa xmm9, xmm0 // carry state to the next batch - // Pack 16-byte result: bytes 0..7 = xmm7.q0, bytes 8..15 = xmm6.q0. - movq xmm0, xmm7 - pslldq xmm6, 8 - por xmm0, xmm6 - // Byte-reverse to canonical form and store to PFS. - pshufb xmm0, xmm8 - movdqu [rcx], xmm0 +{$IFDEF GCM_GHASH_FULL_BLOCKS_8} + add rdx, 128 +{$ELSE} + add rdx, 64 +{$ENDIF} + dec r10 + jnz @@GhashFullLoop + + pshufb xmm9, xmm8 // reflected -> canonical + movdqu [rcx], xmm9 + add rsp, 24 {$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} +