diff --git a/CryptoLib.Tests/src/Crypto/CcmTests.pas b/CryptoLib.Tests/src/Crypto/CcmTests.pas
index 3bf91d66..c0c405aa 100644
--- a/CryptoLib.Tests/src/Crypto/CcmTests.pas
+++ b/CryptoLib.Tests/src/Crypto/CcmTests.pas
@@ -668,9 +668,11 @@ procedure TTestCcm.DoTestNoUnverifiedPlaintextOnFailure;
except
on E: EInvalidCipherTextCryptoLibException do
begin
+ // The tentative plaintext must be wiped before the exception leaves:
+ // every byte of the output region has to read back as zero.
for LI := 0 to High(LOutput) do
begin
- if LOutput[LI] <> $55 then
+ if LOutput[LI] <> $00 then
Fail('CCM left unverified plaintext in the output buffer on tag failure');
end;
end;
diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas
index 56773b1d..8d841fea 100644
--- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas
+++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas
@@ -22,6 +22,7 @@ interface
uses
SysUtils,
+ ClpCryptoLibTypes,
ClpIBlockCipher,
ClpIAesEngineX86,
ClpCipherKernelTypes,
@@ -55,6 +56,10 @@ TAesNiGcmKernel = class sealed(TInterfacedObject, IGcmKernel)
FRounds: Int32;
FHPow128: Pointer;
FMask: Pointer;
+ // Kernel-owned copy of the mode's H^8..H^1 table (already x-pre-multiplied
+ // by TGcmUtilities.InitEightWayHPowFromH, matching the carry-less-multiply
+ // folding reduction inside the fused kernel). FHPow128 points at it.
+ FHPowShifted: TCryptoLibByteArray;
public
constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer;
ARounds: Int32; AHPow128, AMask: Pointer);
@@ -82,11 +87,12 @@ implementation
// Natural pointer-sized alignment, no padding: the field offsets match the
// [rcx + N] / [ebx + N] accesses in AesGcmFusedCtrGhashEight_x86_64.inc and
// AesGcmFusedCtrGhashEight_i386.inc (they differ with pointer / NativeUInt
- // width: 8-byte fields on x86_64, 4-byte on i386). The kernel advances
- // PXorIn/POut/PPrevCipher and Counter32 in place across the batch run and
+ // width: 8-byte fields on x86_64, 4-byte on i386). The kernel holds the loop
+ // state in registers across the batch run, writing only Counter32 back, and
// builds each batch's counter blocks in-register from Counter32 + PJ0Template
// (both arches); the GHASH-from-output flag is 1 for encrypt (fold the output
- // ciphertext) and 0 for decrypt (fold the input).
+ // ciphertext) and 0 for decrypt (fold the input). PHPow128 points at the
+ // kernel-owned x-pre-multiplied H-power table (see TAesNiGcmKernel.Create).
TGcmFusedCtx = record
PXorIn: Pointer;
POut: Pointer;
@@ -157,8 +163,13 @@ constructor TAesNiGcmKernel.Create(const AEngine: IAesEngineX86;
FEngine := AEngine;
FKeys := AKeys;
FRounds := ARounds;
- FHPow128 := AHPow128;
FMask := AMask;
+ // The mode's table already holds the powers pre-multiplied by x (see
+ // TGcmUtilities.InitEightWayHPowFromH); keep a kernel-owned copy so the
+ // pointer stays valid for the kernel's lifetime.
+ System.SetLength(FHPowShifted, 128);
+ System.Move(AHPow128^, FHPowShifted[0], 128);
+ FHPow128 := @FHPowShifted[0];
end;
function TAesNiGcmKernel.MinimumBlockCount: Int32;
diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas
index 0aa6142f..5b300ffc 100644
--- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas
+++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas
@@ -49,9 +49,9 @@ TAesNiOcbKernel = class sealed(TInterfacedObject, IOcbKernel)
strict private
const
{$IFDEF CRYPTOLIB_X86_64_ASM}
- FUSED_OCB_MIN_BLOCKS = 8;
+ FUSED_OCB_MIN_BLOCKS = 16;
{$ELSE}
- FUSED_OCB_MIN_BLOCKS = 4;
+ FUSED_OCB_MIN_BLOCKS = 12;
{$ENDIF}
strict private
FEngine: IAesEngineX86;
diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas
index c99b1ce1..d9238560 100644
--- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas
+++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas
@@ -85,9 +85,11 @@ function TPclmulGcmSivKernel.MinimumBlockCount: Int32;
procedure TPclmulGcmSivKernel.ProcessPolyvalBatch(AInPtr, AAccumulator: Pointer;
ABlockCount: Int32);
begin
- if ABlockCount <> FUSED_POLYVAL_MIN_BLOCKS then
+ if (ABlockCount < FUSED_POLYVAL_MIN_BLOCKS) or
+ (ABlockCount mod FUSED_POLYVAL_MIN_BLOCKS <> 0) then
Exit;
- TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask);
+ TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask,
+ ABlockCount div FUSED_POLYVAL_MIN_BLOCKS);
end;
{ TPclmulGcmSivKernelFactory }
diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas
index d77e3a1e..e96a57f8 100644
--- a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas
+++ b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas
@@ -57,11 +57,14 @@ interface
TChaChaBaseEngine = class(TSalsa20Engine)
strict private
- // Pointer-form, unvalidated inner helpers (1 / 2 / 4 / 8 blocks).
+ // Pointer-form, unvalidated inner helpers (1 / 2 blocks).
procedure ProcessBlockFast(AIn, AOut: PByte);
procedure ProcessBlocks2Fast(AIn, AOut: PByte);
- procedure ProcessBlocks4Fast(AIn, AOut: PByte);
- procedure ProcessBlocks8Fast(AIn, AOut: PByte);
+ // One streaming SIMD tier (AGroupBlocks = 8 or 4): clamps to the wrap-safe
+ // span, runs the kernel once over every whole group, advances the cursors.
+ // False = no SIMD kernel for that width or no safe groups (degrade a tier).
+ function TryProcessWide(AGroupBlocks: Int32; var AIn, AOut: PByte;
+ var ABlockCount: Int32): Boolean;
strict protected
// 8 -> 4 -> 2 -> 1 bulk ladder (widest kernel first).
@@ -69,9 +72,10 @@ TChaChaBaseEngine = class(TSalsa20Engine)
// True for the DJB 64-bit counter (words 12-13); False (IETF).
function CounterIs64Bit: Boolean; virtual;
- // Wide kernels broadcast word 13; for the 64-bit counter that holds only while
- // word 12 does not wrap over the N-block span (else the caller degrades a tier).
- function WideBlocksSafe(ABlocks: Int32): Boolean; inline;
+ // Wide kernels broadcast word 13, valid only while word 12 does not wrap
+ // inside the processed span: clamps a span of AGroups x AGroupBlocks
+ // blocks to the groups fully before the wrap (identity for IETF).
+ function WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32;
procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override;
@@ -133,12 +137,20 @@ function TChaChaBaseEngine.CounterIs64Bit: Boolean;
Result := False;
end;
-function TChaChaBaseEngine.WideBlocksSafe(ABlocks: Int32): Boolean;
+function TChaChaBaseEngine.WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32;
+var
+ LSafeBlocks: UInt64;
begin
- Result := (not CounterIs64Bit) or
- (FEngineState[12] <= (UInt32($FFFFFFFF) - UInt32(ABlocks - 1)));
+ Result := AGroups;
+ if CounterIs64Bit then
+ begin
+ LSafeBlocks := UInt64($100000000) - FEngineState[12];
+ if (UInt64(Result) * UInt32(AGroupBlocks)) > LSafeBlocks then
+ Result := Int32(LSafeBlocks div UInt32(AGroupBlocks));
+ end;
end;
+
procedure TChaChaBaseEngine.GenerateKeyStream(const AOutput: TCryptoLibByteArray);
begin
ChaChaCore(FRounds, FEngineState, AOutput);
@@ -299,26 +311,29 @@ procedure TChaChaBaseEngine.ProcessBlocks2Fast(AIn, AOut: PByte);
ProcessBlockFast(AIn + 64, AOut + 64);
end;
-procedure TChaChaBaseEngine.ProcessBlocks4Fast(AIn, AOut: PByte);
-begin
- if WideBlocksSafe(4) then
- if TChaChaSimd.TryProcessBlocks4(FRounds, PByte(@FEngineState[0]),
- AIn, AOut, CounterIs64Bit) then
- Exit;
-
- ProcessBlocks2Fast(AIn, AOut);
- ProcessBlocks2Fast(AIn + 128, AOut + 128);
-end;
-
-procedure TChaChaBaseEngine.ProcessBlocks8Fast(AIn, AOut: PByte);
+function TChaChaBaseEngine.TryProcessWide(AGroupBlocks: Int32;
+ var AIn, AOut: PByte; var ABlockCount: Int32): Boolean;
+var
+ LGroups: Int32;
begin
- if WideBlocksSafe(8) then
- if TChaChaSimd.TryProcessBlocks8(FRounds, PByte(@FEngineState[0]),
- AIn, AOut, CounterIs64Bit) then
- Exit;
-
- ProcessBlocks4Fast(AIn, AOut);
- ProcessBlocks4Fast(AIn + 256, AOut + 256);
+ Result := False;
+ LGroups := WideGroupsSafe(ABlockCount div AGroupBlocks, AGroupBlocks);
+ if LGroups = 0 then
+ Exit;
+ case AGroupBlocks of
+ 8:
+ Result := TChaChaSimd.TryProcessBlocks8(FRounds,
+ PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit);
+ else
+ Result := TChaChaSimd.TryProcessBlocks4(FRounds,
+ PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit);
+ end;
+ if Result then
+ begin
+ AIn := AIn + LGroups * (AGroupBlocks * 64);
+ AOut := AOut + LGroups * (AGroupBlocks * 64);
+ System.Dec(ABlockCount, LGroups * AGroupBlocks);
+ end;
end;
function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte;
@@ -327,14 +342,24 @@ function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte;
Result := ABlockCount * 64;
while ABlockCount >= 8 do
begin
- ProcessBlocks8Fast(AIn, AOut);
+ // Widest streaming tier that lands, over all safe groups at once.
+ if TryProcessWide(8, AIn, AOut, ABlockCount) then
+ continue;
+ if TryProcessWide(4, AIn, AOut, ABlockCount) then
+ continue;
+ // No wide SIMD (or at the DJB wrap boundary): one 8-block group via 2/1.
+ ProcessBlocks2Fast(AIn, AOut);
+ ProcessBlocks2Fast(AIn + 128, AOut + 128);
+ ProcessBlocks2Fast(AIn + 256, AOut + 256);
+ ProcessBlocks2Fast(AIn + 384, AOut + 384);
AIn := AIn + 512;
AOut := AOut + 512;
System.Dec(ABlockCount, 8);
end;
- if ABlockCount >= 4 then
+ if (ABlockCount >= 4) and (not TryProcessWide(4, AIn, AOut, ABlockCount)) then
begin
- ProcessBlocks4Fast(AIn, AOut);
+ ProcessBlocks2Fast(AIn, AOut);
+ ProcessBlocks2Fast(AIn + 128, AOut + 128);
AIn := AIn + 256;
AOut := AOut + 256;
System.Dec(ABlockCount, 4);
diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas
index bf10519e..ac6ca3ee 100644
--- a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas
+++ b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas
@@ -42,10 +42,14 @@ TChaChaSimd = class sealed
/// Two-block ChaCha keystream (128 bytes). ACtr64 = DJB 64-bit
/// counter (no SIMD 2-block body; returns False).
class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
- /// Four-block ChaCha keystream (256 bytes). ACtr64 = DJB variant.
- class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
- /// Eight-block ChaCha keystream (512 bytes). ACtr64 = DJB variant.
- class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
+ /// Four-block ChaCha keystream, streaming AGroups consecutive
+ /// 256-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the
+ /// caller guarantees no counter low-word wrap inside the span.
+ class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
+ /// Eight-block ChaCha keystream, streaming AGroups consecutive
+ /// 512-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the
+ /// caller guarantees no counter low-word wrap inside the span.
+ class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
end;
implementation
@@ -70,19 +74,19 @@ class function TChaChaSimd.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut:
{$IFEND}
end;
-class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
begin
{$IF DEFINED(CRYPTOLIB_X86_SIMD)}
- Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, ACtr64);
+ Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, AGroups, ACtr64);
{$ELSE}
Result := False;
{$IFEND}
end;
-class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
begin
{$IF DEFINED(CRYPTOLIB_X86_SIMD)}
- Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, ACtr64);
+ Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, AGroups, ACtr64);
{$ELSE}
Result := False;
{$IFEND}
diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas
index 04f13d17..249b0c06 100644
--- a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas
+++ b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas
@@ -45,12 +45,16 @@ TChaChaX86Backend = class sealed
/// 64-bit-counter (DJB) variant; no SIMD 2-block kernel exists for it, so it
/// returns False and the caller runs its scalar path.
class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
- /// Four-block ChaCha keystream (256 bytes). ACtr64 selects the DJB
- /// 64-bit-counter kernel.
- class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
- /// AVX2 eight-block ChaCha keystream (512 bytes). ACtr64 selects the
- /// DJB 64-bit-counter kernel.
- class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
+ /// Four-block ChaCha keystream, streaming AGroups consecutive
+ /// 256-byte groups (AGroups >= 1) in one call. ACtr64 selects the DJB
+ /// 64-bit-counter kernel; the caller must guarantee the counter low word
+ /// does not wrap inside the 4*AGroups-block span.
+ class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
+ /// AVX2 eight-block ChaCha keystream, streaming AGroups consecutive
+ /// 512-byte groups (AGroups >= 1) in one call. ACtr64 selects the DJB
+ /// 64-bit-counter kernel; the caller must guarantee the counter low word
+ /// does not wrap inside the 8*AGroups-block span.
+ class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
end;
implementation
@@ -83,79 +87,89 @@ procedure ChaCha7539ProcessBlocks2Sse2(ARounds: Int32; AState, AIn, AOut: PByte)
{$ENDIF}
end;
-procedure ChaCha7539ProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte);
+procedure ChaCha7539ProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc}
{$ENDIF}
end;
-procedure ChaCha7539ProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte);
+procedure ChaCha7539ProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc}
{$ENDIF}
end;
-// AVX2 8-way (512B): x86_64 only (needs 16 ymm; i386 has 8, so it falls back to
-// two 256B ProcessBlocks4 in the engine).
+// AVX2 8-way: streams AGroups x 512B per call. x86_64 keeps the 16 state rows
+// in ymm registers; i386 (8 ymm) works them from a 64B-aligned stack frame.
+procedure ChaCha7539ProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-procedure ChaCha7539ProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte);
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc}
-end;
{$ENDIF}
+{$IFDEF CRYPTOLIB_I386_ASM}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
+{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_i386.inc}
+{$ENDIF}
+end;
// DJB ChaCha (64-bit counter, words 12-13): the vertical kernels are identical to
// the 7539 ones except the counter advance carries into word 13, which the shared
// includes emit under CHACHA_CTR64. Callers guard against a low-word wrap across the
-// lanes before invoking these (see the engine's WideBlocksSafe).
-procedure ChaChaProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte);
+// lanes before invoking these (see the engine's WideGroupsSafe).
+procedure ChaChaProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$DEFINE CHACHA_CTR64}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc}
{$UNDEF CHACHA_CTR64}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$DEFINE CHACHA_CTR64}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc}
{$UNDEF CHACHA_CTR64}
{$ENDIF}
end;
-procedure ChaChaProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte);
+procedure ChaChaProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$DEFINE CHACHA_CTR64}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc}
{$UNDEF CHACHA_CTR64}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$DEFINE CHACHA_CTR64}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc}
{$UNDEF CHACHA_CTR64}
{$ENDIF}
end;
+procedure ChaChaProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-procedure ChaChaProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte);
-{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$DEFINE CHACHA_CTR64}
{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc}
{$UNDEF CHACHA_CTR64}
-end;
{$ENDIF}
+{$IFDEF CRYPTOLIB_I386_ASM}
+{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
+{$DEFINE CHACHA_CTR64}
+{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_i386.inc}
+{$UNDEF CHACHA_CTR64}
+{$ENDIF}
+end;
{$ENDIF CRYPTOLIB_X86_SIMD}
{ TChaChaX86Backend }
@@ -193,7 +207,7 @@ class function TChaChaX86Backend.TryProcessBlocks2(ARounds: Int32; AState, AIn,
Result := False;
end;
-class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
begin
{$IFDEF CRYPTOLIB_X86_SIMD}
// 4-block (256B) tier: AVX2 gives no extra lanes at 4-way, and AVX2 CPUs run the
@@ -202,17 +216,17 @@ class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn,
TX86SimdLevel.SSSE3:
begin
if ACtr64 then
- ChaChaProcessBlocks4Ssse3(ARounds, AState, AIn, AOut)
+ ChaChaProcessBlocks4Ssse3(ARounds, AState, AIn, AOut, AGroups)
else
- ChaCha7539ProcessBlocks4Ssse3(ARounds, AState, AIn, AOut);
+ ChaCha7539ProcessBlocks4Ssse3(ARounds, AState, AIn, AOut, AGroups);
Exit(True);
end;
TX86SimdLevel.SSE2:
begin
if ACtr64 then
- ChaChaProcessBlocks4Sse2(ARounds, AState, AIn, AOut)
+ ChaChaProcessBlocks4Sse2(ARounds, AState, AIn, AOut, AGroups)
else
- ChaCha7539ProcessBlocks4Sse2(ARounds, AState, AIn, AOut);
+ ChaCha7539ProcessBlocks4Sse2(ARounds, AState, AIn, AOut, AGroups);
Exit(True);
end;
end;
@@ -220,16 +234,16 @@ class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn,
Result := False;
end;
-class function TChaChaX86Backend.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaX86Backend.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
begin
-{$IFDEF CRYPTOLIB_X86_64_ASM}
+{$IFDEF CRYPTOLIB_X86_SIMD}
case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2]) of
TX86SimdLevel.AVX2:
begin
if ACtr64 then
- ChaChaProcessBlocks8Avx2(ARounds, AState, AIn, AOut)
+ ChaChaProcessBlocks8Avx2(ARounds, AState, AIn, AOut, AGroups)
else
- ChaCha7539ProcessBlocks8Avx2(ARounds, AState, AIn, AOut);
+ ChaCha7539ProcessBlocks8Avx2(ARounds, AState, AIn, AOut, AGroups);
Exit(True);
end;
end;
diff --git a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas
index 3def6d53..5695dad0 100644
--- a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas
+++ b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas
@@ -102,6 +102,11 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher,
FMacSize: Int32;
FKeyParam: ICipherParameters;
FLastKey: TCryptoLibByteArray;
+ // CTR wrapper cached across packets (created once per keyed cipher); each
+ // packet re-inits it IV-only, so the AES schedule is never recomputed.
+ FCtrCipher: ISicBlockCipher;
+ FBulkCtr: IBulkBlockCipherMode;
+ FKernelForEnc: Boolean;
FAssociatedText: TMemoryStream;
// Accumulated body input (plaintext on encrypt, ciphertext+tag on decrypt).
// CCM needs the total length before processing (B_0 encodes it), so input is
@@ -115,7 +120,6 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher,
// the plaintext copied to the caller's output (which stays untouched on
// failure). Grown by doubling and reused across packets, and wiped after
// every packet (in a finally) so no recovered plaintext lingers.
- FPlainScratch: TCryptoLibByteArray;
// Cached once per Init; non-nil when the registry resolved a fused
// CCM kernel for the underlying cipher and current direction.
FCcmKernel: ICcmKernel;
@@ -189,7 +193,6 @@ constructor TCcmBlockCipher.Create(const ACipher: IBlockCipher);
FAssociatedText := TMemoryStream.Create;
FData := nil;
FDataLen := 0;
- FPlainScratch := nil;
if (ACipher.GetBlockSize() <> BlockSize) then
raise EArgumentCryptoLibException.CreateResFmt(@SCipherRequired, [BlockSize]);
@@ -233,6 +236,8 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean;
LChoice: TCipherAeadChoice;
LRequestedMacSizeBits: Int32;
LDirection: TCipherKernelDirection;
+ LPrevKey: TCryptoLibByteArray;
+ LSameKey: Boolean;
begin
FForEncryption := AForEncryption;
@@ -245,6 +250,7 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean;
raise EArgumentCryptoLibException.CreateResFmt(@SInvalidParameters, ['CCM']);
CheckNonceReuse(FForEncryption, LChoice.Nonce, LChoice.KeyParameter);
+ LPrevKey := FLastKey;
FNonce := LChoice.Nonce;
FInitialAssociatedText := LChoice.AssociatedText;
@@ -264,19 +270,35 @@ procedure TCcmBlockCipher.Init(AForEncryption: Boolean;
if (System.Length(FNonce) < 7) or (System.Length(FNonce) > 13) then
raise EArgumentCryptoLibException.CreateRes(@SNonceLengthRange);
- FCcmKernel := nil;
+ // Nonce-only rotation is the hot AEAD path: with the key and direction
+ // unchanged the AES schedule, the CTR wrapper and the fused-kernel binding
+ // all stay valid, so none of them are recomputed.
+ LSameKey := (LChoice.CipherKey = nil) or
+ ((LPrevKey <> nil) and (LChoice.KeyParameter <> nil) and
+ LChoice.KeyParameter.FixedTimeEquals(LPrevKey));
if FKeyParam <> nil then
begin
- // Key FCipher now so the factory can read the finalised AES
- // schedule. ProcessPacket re-keys through the SIC wrapper later; the
- // schedule contents are identical across those re-keys.
- FCipher.Init(True, FKeyParam);
- if AForEncryption then
- LDirection := TCipherKernelDirection.Encrypt
- else
- LDirection := TCipherKernelDirection.Decrypt;
- TCipherKernelRegistry.TryAcquireCcm(FCipher, LDirection, FCcmKernel);
- end;
+ if not LSameKey then
+ FCipher.Init(True, FKeyParam);
+ if (FCcmKernel = nil) or (not LSameKey) or
+ (FKernelForEnc <> AForEncryption) then
+ begin
+ FCcmKernel := nil;
+ if AForEncryption then
+ LDirection := TCipherKernelDirection.Encrypt
+ else
+ LDirection := TCipherKernelDirection.Decrypt;
+ TCipherKernelRegistry.TryAcquireCcm(FCipher, LDirection, FCcmKernel);
+ FKernelForEnc := AForEncryption;
+ end;
+ if FCtrCipher = nil then
+ begin
+ FCtrCipher := TSicBlockCipher.Create(FCipher);
+ TBlockCipherBulkUtilities.TryResolveBulkCipherMode(FCtrCipher, FBulkCtr);
+ end;
+ end
+ else
+ FCcmKernel := nil;
Reset();
end;
@@ -420,9 +442,11 @@ function TCcmBlockCipher.ProcessPacket(const AInput: TCryptoLibByteArray;
LIV[0] := Byte((LQ - 1) and $7);
System.Move(FNonce[0], LIV[1], LN);
- LCtrCipher := TSicBlockCipher.Create(FCipher);
- LCtrCipher.Init(FForEncryption, TParametersWithIV.Create(FKeyParam, LIV) as IParametersWithIV);
- TBlockCipherBulkUtilities.TryResolveBulkCipherMode(LCtrCipher, LBulkCtr);
+ // IV-only re-init of the cached CTR wrapper: the inner parameters are nil,
+ // so the (unchanged) AES schedule is not recomputed per packet.
+ LCtrCipher := FCtrCipher;
+ LCtrCipher.Init(FForEncryption, TParametersWithIV.Create(nil, LIV) as IParametersWithIV);
+ LBulkCtr := FBulkCtr;
if FForEncryption then
begin
@@ -856,21 +880,21 @@ function TCcmBlockCipher.RunDecrypt(AUseFused: Boolean;
var
LOk: Boolean;
begin
- TArrayUtilities.EnsureCapacity(FPlainScratch, ACtx.OutputLen);
- ACtx.Dest := FPlainScratch;
- ACtx.DestOff := 0;
- try
- if AUseFused then
- LOk := DecryptBodyFused(ACtx)
- else
- LOk := DecryptBodyScalar(ACtx);
- if not LOk then
- raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']);
- System.Move(FPlainScratch[0], AOutput[AOutOff], ACtx.OutputLen);
- Result := True;
- finally
- TArrayUtilities.Fill(FPlainScratch, 0, ACtx.OutputLen, Byte(0));
+ // Decrypt straight into the caller's buffer (no scratch bounce); on a MAC
+ // failure the tentative plaintext is wiped before the exception leaves, so
+ // no unverified plaintext survives the call.
+ ACtx.Dest := AOutput;
+ ACtx.DestOff := AOutOff;
+ if AUseFused then
+ LOk := DecryptBodyFused(ACtx)
+ else
+ LOk := DecryptBodyScalar(ACtx);
+ if not LOk then
+ begin
+ TArrayUtilities.Fill(AOutput, AOutOff, AOutOff + ACtx.OutputLen, Byte(0));
+ raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']);
end;
+ Result := True;
end;
end.
diff --git a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas
index 4193c979..1ebb2aa0 100644
--- a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas
+++ b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas
@@ -414,7 +414,7 @@ function TChaCha20Poly1305.ProcessByte(AInput: Byte;
function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray;
AInOff, ALen: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32;
var
- LResultLen, LAvailable, LInLimit1, LInLimit2, LRemBlocks: Int32;
+ LResultLen, LAvailable, LInLimit1, LRemBlocks: Int32;
begin
if (AInput = nil) then
raise EArgumentNilCryptoLibException.CreateRes(@SInBytesNil);
@@ -461,7 +461,6 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray;
end;
LInLimit1 := AInOff + ALen - System.Length(FBuf);
- LInLimit2 := LInLimit1 - BufSize;
LAvailable := BufSize - FBufPos;
System.Move(AInput[AInOff], FBuf[FBufPos], LAvailable);
@@ -472,15 +471,8 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray;
if (FBulkChaCha <> nil) then
begin
- // Bulk: MAC + decrypt in 512B (8-way) strides, then the remaining whole
- // blocks (< 8) in a single engine-laddered call.
- while (AInOff <= LInLimit2 - (BufSize * 6)) do
- begin
- FPoly1305.BlockUpdate(AInput, AInOff, BufSize * 8);
- ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen);
- AInOff := AInOff + BufSize * 8;
- LResultLen := LResultLen + BufSize * 8;
- end;
+ // Bulk: MAC then decrypt every whole block, one streaming call each
+ // (MAC first reads all ciphertext, so in-place decrypt stays safe).
if (AInOff <= LInLimit1) then
begin
LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1;
@@ -517,7 +509,6 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray;
end;
LInLimit1 := AInOff + ALen - BufSize;
- LInLimit2 := LInLimit1 - BufSize;
if (FBufPos > 0) then
begin
@@ -529,14 +520,8 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray;
if (FBulkChaCha <> nil) then
begin
- // Bulk: encrypt in 512B (8-way) strides, then the remaining whole blocks
- // (< 8) in a single engine-laddered call; MAC over the ciphertext below.
- while (AInOff <= LInLimit2 - (BufSize * 6)) do
- begin
- ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen);
- AInOff := AInOff + BufSize * 8;
- LResultLen := LResultLen + BufSize * 8;
- end;
+ // Bulk: encrypt every whole block in one streaming engine call (the
+ // ladder runs the widest kernels internally); MAC the ciphertext below.
if (AInOff <= LInLimit1) then
begin
LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1;
diff --git a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas
index bfacaa53..0784cfd5 100644
--- a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas
+++ b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas
@@ -522,6 +522,7 @@ procedure TGcmBlockCipher.Init(AForEncryption: Boolean;
LKeyParam: IKeyParameter;
LNewNonce: TCryptoLibByteArray;
LBufLength: Int32;
+ LSameKey: Boolean;
begin
FForEncryption := AForEncryption;
FMacBlock := nil;
@@ -551,8 +552,14 @@ procedure TGcmBlockCipher.Init(AForEncryption: Boolean;
if LKeyParam <> nil then
begin
+ // Same-key fast path: the key schedule, hash subkey, multiplier state,
+ // H-power table and fused kernel all depend only on the key, so a
+ // re-Init with the same key (fresh nonce per message) keeps them all.
+ LSameKey := (FH <> nil) and (FLastKey <> nil) and
+ LKeyParam.FixedTimeEquals(FLastKey);
FLastKey := LKeyParam.GetKey();
- InitCipherAndHashSubKey(LKeyParam);
+ if not LSameKey then
+ InitCipherAndHashSubKey(LKeyParam);
end
else if FH = nil then
begin
@@ -629,7 +636,7 @@ procedure TGcmBlockCipher.ProcessAadByte(AInput: Byte);
procedure TGcmBlockCipher.ProcessAadBytes(const AInput: TCryptoLibByteArray;
AInOff, ALen: Int32);
var
- LAvailable, LInLimit: Int32;
+ LAvailable, LInLimit, LBatches: Int32;
begin
CheckStatus();
@@ -650,6 +657,20 @@ procedure TGcmBlockCipher.ProcessAadBytes(const AInput: TCryptoLibByteArray;
ALen := ALen - LAvailable;
end;
+ // Bulk fast path: fold whole 128-byte spans through the fused 8-way GHASH
+ // kernel in one call; the per-block loop below handles the remainder.
+ if (FHPow <> nil) and (ALen >= 128) then
+ begin
+ LBatches := ALen div 128;
+ if TGhashSimd.TryFusedEightShuffledGhash(@FS_at[0], @AInput[AInOff],
+ @FHPow[0], LBatches) then
+ begin
+ AInOff := AInOff + LBatches * 128;
+ ALen := ALen - LBatches * 128;
+ FAtLength := FAtLength + UInt64(LBatches) * 128;
+ end;
+ end;
+
LInLimit := AInOff + ALen - BlockSize;
while AInOff <= LInLimit do
@@ -1097,19 +1118,14 @@ class procedure TGcmBlockCipher.GcmReverse16(const ASrc, ADst: PByte);
procedure TGcmBlockCipher.GhashFourShuffledBlocks(PC0, PC16, PC32, PC48: PByte);
var
- LB, LI: Int32;
- LDblk: array[0..15] of Byte;
- LBuf48: array[0..47] of Byte;
- LU0, LU1, LU2: array[0..15] of Byte;
- LSRev: array[0..15] of Byte;
+ LB: Int32;
LPCiph: PByte;
begin
- if TGhashSimd.TryFusedFourShuffledGhash(@FS[0], PC0, @FHPow[64]) then
+ if TGhashSimd.TryFusedFourShuffledGhash(@FS[0], PC0, @FHPow[64], 1) then
Exit;
- GcmReverse16(@FS[0], @LSRev[0]);
- FillChar(LU0, 16, 0);
- FillChar(LU1, 16, 0);
- FillChar(LU2, 16, 0);
+ // Unreachable in practice (the caller gates on the same predicate as the
+ // kernel); fold per block through the multiplier, which is independent of
+ // the x-pre-multiplied FHPow layout.
for LB := 0 to 3 do
begin
case LB of
@@ -1122,15 +1138,9 @@ procedure TGcmBlockCipher.GhashFourShuffledBlocks(PC0, PC16, PC32, PC48: PByte);
else
LPCiph := PC48;
end;
- GcmReverse16(LPCiph, @LDblk[0]);
- if LB = 0 then
- for LI := 0 to 15 do
- LDblk[LI] := LDblk[LI] xor LSRev[LI];
- TGcmUtilities.MultiplyExt(@LDblk[0], @FHPow[64 + (LB * 16)], @LBuf48[0]);
- TGcmUtilities.XorMultiplyExtLimbs48(@LU0[0], @LU1[0], @LU2[0], @LBuf48[0]);
+ TByteUtilities.&Xor(BlockSize, @FS[0], LPCiph, @FS[0]);
+ FMultiplier.MultiplyH(FS);
end;
- TGcmUtilities.Reduce3(@LU0[0], @LU1[0], @LU2[0], @LSRev[0]);
- GcmReverse16(@LSRev[0], @FS[0]);
end;
// =======================================================================
@@ -1168,31 +1178,18 @@ procedure TGcmBlockCipher.ProcessBlocks4Fused(const AInBuf: TCryptoLibByteArray;
procedure TGcmBlockCipher.GhashEightShuffledBlocks(PBase: PByte);
var
- LB, LI: Int32;
- LDblk: array [0 .. 15] of Byte;
- LBuf48: array [0 .. 47] of Byte;
- LU0, LU1, LU2: array [0 .. 15] of Byte;
- LSRev: array [0 .. 15] of Byte;
- LPCiph: PByte;
+ LB: Int32;
begin
- if TGhashSimd.TryFusedEightShuffledGhash(@FS[0], PBase, @FHPow[0]) then
+ if TGhashSimd.TryFusedEightShuffledGhash(@FS[0], PBase, @FHPow[0], 1) then
Exit;
- GcmReverse16(@FS[0], @LSRev[0]);
- FillChar(LU0, 16, 0);
- FillChar(LU1, 16, 0);
- FillChar(LU2, 16, 0);
+ // Unreachable in practice (the caller gates on the same predicate as the
+ // kernel); fold per block through the multiplier, which is independent of
+ // the x-pre-multiplied FHPow layout.
for LB := 0 to 7 do
begin
- LPCiph := PBase + (LB * 16);
- GcmReverse16(LPCiph, @LDblk[0]);
- if LB = 0 then
- for LI := 0 to 15 do
- LDblk[LI] := LDblk[LI] xor LSRev[LI];
- TGcmUtilities.MultiplyExt(@LDblk[0], @FHPow[LB * 16], @LBuf48[0]);
- TGcmUtilities.XorMultiplyExtLimbs48(@LU0[0], @LU1[0], @LU2[0], @LBuf48[0]);
+ TByteUtilities.&Xor(BlockSize, @FS[0], PBase + (LB * BlockSize), @FS[0]);
+ FMultiplier.MultiplyH(FS);
end;
- TGcmUtilities.Reduce3(@LU0[0], @LU1[0], @LU2[0], @LSRev[0]);
- GcmReverse16(@LSRev[0], @FS[0]);
end;
// Single-batch fused 8-way GCM step. See ProcessBlocks4Fused for direction semantics.
diff --git a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas
index 04240926..5def24aa 100644
--- a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas
+++ b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas
@@ -268,6 +268,8 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean;
LMacSizeBits, LBottom, LBits, LBytes, LI: Int32;
LB1, LB2: UInt32;
LFusedDirection: TCipherKernelDirection;
+ LPrevKey: TCryptoLibByteArray;
+ LSameKey, LSameKeyDir: Boolean;
begin
LOldForEncryption := FForEncryption;
FForEncryption := AForEncryption;
@@ -283,6 +285,7 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean;
LN := LChoice.Nonce;
CheckNonceReuse(FForEncryption, LN, LChoice.KeyParameter);
+ LPrevKey := FLastKey;
FLastN := System.Copy(LN);
if LChoice.KeyParameter <> nil then
FLastKey := LChoice.KeyParameter.GetKey();
@@ -290,6 +293,13 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean;
FInitialAssociatedText := LChoice.AssociatedText;
LKeyParameter := LChoice.KeyParameter;
+ // Nonce-only rotation is the hot AEAD path: when the key (and, for the main
+ // cipher / fused kernel, the direction) is unchanged, the key schedules, the
+ // L table and the kernel binding all stay valid and are not recomputed.
+ LSameKey := (LKeyParameter = nil) or
+ ((LPrevKey <> nil) and LKeyParameter.FixedTimeEquals(LPrevKey));
+ LSameKeyDir := LSameKey and (LOldForEncryption = AForEncryption);
+
if LChoice.IsAead then
begin
LMacSizeBits := LChoice.MacSizeBits;
@@ -316,11 +326,14 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean;
if (System.Length(LN) > 15) then
raise EArgumentCryptoLibException.CreateRes(@SIVTooLong);
- if (LKeyParameter <> nil) then
+ if (LKeyParameter <> nil) and (not LSameKeyDir) then
begin
- FHashCipher.Init(True, LKeyParameter);
+ if not LSameKey then
+ begin
+ FHashCipher.Init(True, LKeyParameter);
+ FKTopInput := nil;
+ end;
FMainCipher.Init(AForEncryption, LKeyParameter);
- FKTopInput := nil;
end
else if (LOldForEncryption <> AForEncryption) then
begin
@@ -332,41 +345,48 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean;
// same capability set. Kept as a plain interface QI (no algorithm-name
// assumption); any engine that advertises the contract in ClpIBulkBlockCipher
// is eligible for the 8-wide path in ProcessEightBlocksBulk.
- TBlockCipherBulkUtilities.TryResolveBulkCipher(FMainCipher, FMainBulk);
+ if not LSameKeyDir then
+ begin
+ TBlockCipherBulkUtilities.TryResolveBulkCipher(FMainCipher, FMainBulk);
- // Resolve a fused OCB kernel via the open factory registry; the
- // first factory whose TryCreate accepts the cipher / direction pair
- // wins, and the result is cached for the whole Init cycle.
- FOcbKernel := nil;
- FOcbKernelMinBlocks := 0;
- if FForEncryption then
- LFusedDirection := TCipherKernelDirection.Encrypt
- else
- LFusedDirection := TCipherKernelDirection.Decrypt;
- if TCipherKernelRegistry.TryAcquireOcb(FMainCipher, LFusedDirection,
- FOcbKernel) and (FOcbKernel <> nil) then
- begin
- FOcbKernelMinBlocks := FOcbKernel.MinimumBlockCount;
- // The fused batch buffer holds FUSED_BATCH_BLOCKS offsets; reject
- // kernels whose stride does not divide that capacity so the mode
- // can always present a full-stride batch.
- if (FOcbKernelMinBlocks <= 0) or
- (FUSED_BATCH_BLOCKS mod FOcbKernelMinBlocks <> 0) then
+ // Resolve a fused OCB kernel via the open factory registry; the
+ // first factory whose TryCreate accepts the cipher / direction pair
+ // wins, and the result stays bound until the key or direction changes
+ // (the engine schedule the kernel captured is untouched until then).
+ FOcbKernel := nil;
+ FOcbKernelMinBlocks := 0;
+ if FForEncryption then
+ LFusedDirection := TCipherKernelDirection.Encrypt
+ else
+ LFusedDirection := TCipherKernelDirection.Decrypt;
+ if TCipherKernelRegistry.TryAcquireOcb(FMainCipher, LFusedDirection,
+ FOcbKernel) and (FOcbKernel <> nil) then
begin
- FOcbKernel := nil;
- FOcbKernelMinBlocks := 0;
+ FOcbKernelMinBlocks := FOcbKernel.MinimumBlockCount;
+ // The fused batch buffer holds FUSED_BATCH_BLOCKS offsets; reject
+ // kernels whose stride does not divide that capacity so the mode
+ // can always present a full-stride batch.
+ if (FOcbKernelMinBlocks <= 0) or
+ (FUSED_BATCH_BLOCKS mod FOcbKernelMinBlocks <> 0) then
+ begin
+ FOcbKernel := nil;
+ FOcbKernelMinBlocks := 0;
+ end;
end;
end;
- System.SetLength(FL_Asterisk, 16);
- TArrayUtilities.Fill(FL_Asterisk, 0, 16, Byte(0));
- FHashCipher.ProcessBlock(FL_Asterisk, 0, FL_Asterisk, 0);
+ if not LSameKey then
+ begin
+ System.SetLength(FL_Asterisk, 16);
+ TArrayUtilities.Fill(FL_Asterisk, 0, 16, Byte(0));
+ FHashCipher.ProcessBlock(FL_Asterisk, 0, FL_Asterisk, 0);
- FL_Dollar := OCB_double(FL_Asterisk);
+ FL_Dollar := OCB_double(FL_Asterisk);
- FL.Clear;
- FL.Add(OCB_double(FL_Dollar));
- FLTableFlatCount := 0; // FL rebuilt for the new key; drop the flattened cache
+ FL.Clear;
+ FL.Add(OCB_double(FL_Dollar));
+ FLTableFlatCount := 0; // FL rebuilt for the new key; drop the flattened cache
+ end;
LBottom := ProcessNonce(LN);
diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas
index 0bc82c8c..d5e1ab38 100644
--- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas
+++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivSimd.pas
@@ -39,7 +39,8 @@ TGcmSivSimd = class sealed
/// True when a POLYVAL batch kernel is usable on this CPU.
class function IsSupported: Boolean; static;
/// Eight-block POLYVAL Horner batch. Precondition: IsSupported.
- class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); static;
+ class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer;
+ ABatchCount: NativeInt); static;
end;
implementation
@@ -55,10 +56,11 @@ class function TGcmSivSimd.IsSupported: Boolean;
{$IFEND}
end;
-class procedure TGcmSivSimd.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer);
+class procedure TGcmSivSimd.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer;
+ ABatchCount: NativeInt);
begin
{$IF DEFINED(CRYPTOLIB_X86_SIMD)}
- TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask);
+ TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask, ABatchCount);
{$IFEND}
end;
diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas
index 1b2f84e8..d9c584e2 100644
--- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas
+++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmSivX86Backend.pas
@@ -36,20 +36,22 @@ TGcmSivX86Backend = class sealed
public
/// True when the PCLMULQDQ POLYVAL kernel is usable on this CPU.
class function IsSupported: Boolean; static;
- /// Eight-block POLYVAL Horner batch. Precondition: IsSupported.
- class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer); static;
+ /// ABatchCount eight-block POLYVAL Horner batches. Precondition: IsSupported.
+ class procedure ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer;
+ ABatchCount: NativeInt); static;
end;
implementation
{$IFDEF CRYPTOLIB_X86_SIMD}
-procedure GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask: Pointer);
+procedure GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask: Pointer;
+ ABatchCount: NativeInt);
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\GcmSiv\PolyvalHornerEight_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$I ..\..\..\Include\Simd\GcmSiv\PolyvalHornerEight_i386.inc}
{$ENDIF}
end;
@@ -67,10 +69,11 @@ class function TGcmSivX86Backend.IsSupported: Boolean;
{$ENDIF}
end;
-class procedure TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer);
+class procedure TGcmSivX86Backend.ProcessPolyvalBatch(PFS, PC0, PHPow128, PMask: Pointer;
+ ABatchCount: NativeInt);
begin
{$IFDEF CRYPTOLIB_X86_SIMD}
- GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask);
+ GcmSivPolyvalHornerEight(PFS, PC0, PHPow128, PMask, ABatchCount);
{$ENDIF}
end;
diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas
index bc574749..a434ebff 100644
--- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas
+++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGcmUtilities.pas
@@ -69,7 +69,10 @@ TGcmUtilities = class sealed(TObject)
class procedure Reduce3(PZ0, PZ1, PZ2, PSVector16: PByte); static;
/// Xor three 16-byte limbs with three 16-byte slices from a 48-byte MultiplyExt output.
class procedure XorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte); static;
- /// HPow[0..7] = H^8..H^1 as 16-byte limbs at offsets 0,16,...,112 (index 0 = H^8). Four-way fused GHASH uses offsets 64..112 (H^4..H^1).
+ /// HPow[0..7] = H^8..H^1 as 16-byte limbs at offsets 0,16,...,112 (index 0 = H^8),
+ /// each PRE-MULTIPLIED BY x in the reflected representation (DivideP) to match the
+ /// carry-less-multiply folding reduction in the fused GHASH / POLYVAL kernels.
+ /// Four-way fused GHASH uses offsets 64..112 (H^4..H^1).
class procedure InitEightWayHPowFromH(const AH: TCryptoLibByteArray; const AHPow128: TCryptoLibByteArray); static;
class procedure &Xor(const AX, AY: TCryptoLibByteArray); overload; static;
@@ -387,6 +390,17 @@ class procedure TGcmUtilities.InitEightWayHPowFromH(const AH: TCryptoLibByteArra
LY := LF4;
Multiply(LAcc, LY);
LF8 := LAcc;
+ // Pre-multiply every power by x (DivideP in this field convention) so the
+ // fused kernels can reduce the batch product with two carry-less multiplies
+ // by 0xC2000000_00000000 instead of a bit-shift ladder.
+ LY := LF1; DivideP(LY, LF1);
+ LY := LF2; DivideP(LY, LF2);
+ LY := LF3; DivideP(LY, LF3);
+ LY := LF4; DivideP(LY, LF4);
+ LY := LF5; DivideP(LY, LF5);
+ LY := LF6; DivideP(LY, LF6);
+ LY := LF7; DivideP(LY, LF7);
+ LY := LF8; DivideP(LY, LF8);
TPack.UInt64_To_LE(LF8.N1, AHPow128, 0);
TPack.UInt64_To_LE(LF8.N0, AHPow128, 8);
TPack.UInt64_To_LE(LF7.N1, AHPow128, 16);
diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas
index 6adee681..ffc204e1 100644
--- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas
+++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas
@@ -46,10 +46,12 @@ TGhashSimd = class sealed
class function TryReduce3(PZ0, PZ1, PZ2, PSVector16: PByte): Boolean; static;
/// Xor three 16-byte limbs with three slices of a 48-byte MultiplyExt output.
class function TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte): Boolean; static;
- /// Fused 4-way GHASH over 64 contiguous ciphertext bytes.
- class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; static;
- /// Fused 8-way GHASH over 128 contiguous ciphertext bytes.
- class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; static;
+ /// Fused 4-way GHASH over ABatchCount contiguous 64-byte batches.
+ class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte;
+ ABatchCount: NativeInt): Boolean; static;
+ /// Fused 8-way GHASH over ABatchCount contiguous 128-byte batches.
+ class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte;
+ ABatchCount: NativeInt): Boolean; static;
/// True when the shuffled/fused GHASH path (4-/8-way) is usable on this build/CPU. Gates the batch dispatch and the H-power precompute.
class function IsShuffledGhashSupported: Boolean; static;
@@ -99,19 +101,21 @@ class function TGhashSimd.TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte)
{$IFEND}
end;
-class function TGhashSimd.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean;
+class function TGhashSimd.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte;
+ ABatchCount: NativeInt): Boolean;
begin
{$IF DEFINED(CRYPTOLIB_X86_SIMD)}
- Result := TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64);
+ Result := TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64, ABatchCount);
{$ELSE}
Result := False;
{$IFEND}
end;
-class function TGhashSimd.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean;
+class function TGhashSimd.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte;
+ ABatchCount: NativeInt): Boolean;
begin
{$IF DEFINED(CRYPTOLIB_X86_SIMD)}
- Result := TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128);
+ Result := TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128, ABatchCount);
{$ELSE}
Result := False;
{$IFEND}
diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas
index 60b67233..c1fa43e4 100644
--- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas
+++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas
@@ -45,10 +45,12 @@ TGhashX86Backend = class sealed
class function TryReduce3(PZ0, PZ1, PZ2, PSVector16: PByte): Boolean; static;
/// SIMD xor of three 16-byte limbs with three slices of a 48-byte MultiplyExt output.
class function TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48: PByte): Boolean; static;
- /// PCLMULQDQ fused 4-way GHASH (requires packed vector layout). Uses the backend's own byte-reverse mask.
- class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean; static;
- /// PCLMULQDQ fused 8-way GHASH (requires packed vector layout). Uses the backend's own byte-reverse mask.
- class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean; static;
+ /// PCLMULQDQ fused 4-way GHASH over ABatchCount 64-byte batches (requires packed vector layout). Uses the backend's own byte-reverse mask.
+ class function TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte;
+ ABatchCount: NativeInt): Boolean; static;
+ /// PCLMULQDQ fused 8-way GHASH over ABatchCount 128-byte batches (requires packed vector layout). Uses the backend's own byte-reverse mask.
+ class function TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte;
+ ABatchCount: NativeInt): Boolean; static;
/// True when the fused shuffled-GHASH path is usable on this CPU (needs packed vector layout). Gates the 4-/8-way batch dispatch and the H-power precompute.
class function IsShuffledGhashSupported: Boolean; static;
@@ -119,27 +121,27 @@ procedure GcmXorMultiplyExtLimbs48Sse2(PA0, PA1, PA2, PSrc48: Pointer);
{$ENDIF}
end;
-procedure GcmGhashFourFull(PFS, PC0, PHPow64, PMask: Pointer);
+procedure GcmGhashFourFull(PFS, PC0, PHPow64, PMask: Pointer; ABatchCount: NativeInt);
{$DEFINE GCM_GHASH_FULL_BLOCKS_4}
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_i386.inc}
{$ENDIF}
{$UNDEF GCM_GHASH_FULL_BLOCKS_4}
end;
-procedure GcmGhashEightFull(PFS, PC0, PHPow128, PMask: Pointer);
+procedure GcmGhashEightFull(PFS, PC0, PHPow128, PMask: Pointer; ABatchCount: NativeInt);
{$DEFINE GCM_GHASH_FULL_BLOCKS_8}
{$IFDEF CRYPTOLIB_X86_64_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
-{$I ..\..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc}
+{$I ..\..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc}
{$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_i386.inc}
{$ENDIF}
{$UNDEF GCM_GHASH_FULL_BLOCKS_8}
@@ -242,28 +244,32 @@ class function TGhashX86Backend.TryXorMultiplyExtLimbs48(PA0, PA1, PA2, PSrc48:
Result := False;
end;
-class function TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte): Boolean;
+class function TGhashX86Backend.TryFusedFourShuffledGhash(PFS, PC0, PHPow64: PByte;
+ ABatchCount: NativeInt): Boolean;
begin
{$IFDEF CRYPTOLIB_X86_SIMD}
if TCpuFeatures.X86.HasSSSE3 and TCpuFeatures.X86.HasPCLMULQDQ and TIntrinsicsVector.IsPacked then
begin
- // Monolithic kernel: byte-reverse + state fold + 4-way multiply-accumulate +
- // Reduce3 + byte-reverse back, all in a single assembly body (one call boundary).
- GcmGhashFourFull(PFS, PC0, PHPow64, @ReverseBytesMask[0]);
+ // Monolithic kernel: the whole ABatchCount-batch run - byte-reverse, state
+ // fold, 4-way multiply-accumulate and folding reduction per batch - in a
+ // single assembly body (one call boundary).
+ GcmGhashFourFull(PFS, PC0, PHPow64, @ReverseBytesMask[0], ABatchCount);
Exit(True);
end;
{$ENDIF}
Result := False;
end;
-class function TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte): Boolean;
+class function TGhashX86Backend.TryFusedEightShuffledGhash(PFS, PC0, PHPow128: PByte;
+ ABatchCount: NativeInt): Boolean;
begin
{$IFDEF CRYPTOLIB_X86_SIMD}
if TCpuFeatures.X86.HasSSSE3 and TCpuFeatures.X86.HasPCLMULQDQ and TIntrinsicsVector.IsPacked then
begin
- // Monolithic kernel: byte-reverse + state fold + 8-way multiply-accumulate +
- // Reduce3 + byte-reverse back, all in a single assembly body (one call boundary).
- GcmGhashEightFull(PFS, PC0, PHPow128, @ReverseBytesMask[0]);
+ // Monolithic kernel: the whole ABatchCount-batch run - byte-reverse, state
+ // fold, 8-way multiply-accumulate and folding reduction per batch - in a
+ // single assembly body (one call boundary).
+ GcmGhashEightFull(PFS, PC0, PHPow128, @ReverseBytesMask[0], ABatchCount);
Exit(True);
end;
{$ENDIF}
diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc
index fade6bc6..c22d356a 100644
--- a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc
@@ -22,13 +22,15 @@
// direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must
// NOT be defined when including this file.
//
-// Constraints: i386 has only xmm0..7 (all volatile) and, after unpacking the
-// context, only edx as a free scratch GP register. The 128-bit counter lives in
-// memory at [eax] and is bumped in place with a byte-wise big-endian carry (edx =
-// byte index). Each staged block copies the current [eax] to the scratch (four
-// dword moves through edx, since no xmm is free mid-round) then advances. The
-// pipeline builds one batch ahead, so [eax] over-runs by exactly one batch (8) -
-// undone before returning. The keystream^plaintext XOR has no free xmm for the
+// Constraints: i386 has only xmm0..7 (all volatile). BlockCount parks in a frame
+// slot so both edx and ecx serve the staging: edx holds the counter's low dword
+// native-LE across the whole span (bswap/store/bswap/add per staged block - no
+// read-modify-write of counter memory, so no store-forward stalls), while ecx
+// copies the constant 12-byte prefix from [eax+0..11] (written only by the rare
+// carry-out path, which bumps those bytes big-endian). The pipeline builds one
+// batch ahead, so edx over-runs by exactly one batch (8) - undone (with a
+// byte-wise borrow when it crosses back over a 2^32 boundary) before the low
+// dword is stored back to [eax+12]. The keystream^plaintext XOR has no free xmm for the
// plaintext (nor can it use a memory-operand pxor, which would require the input
// to be 16-byte aligned), so it spills lane 7 to a scratch slot above the counter
// area and reuses xmm7 for the movdqu plaintext load. AES-NI and register pxor
@@ -42,81 +44,156 @@
mov ecx, [ebx + 16] // BlockCount
mov ebx, [ebx + 0] // InPtr (overwrites PCtx)
- sub esp, 144 // next-batch counters [esp+0..127]; ks7 spill [esp+128..143]
+ sub esp, 152 // next-batch counters [esp+0..127]; ks7 spill [esp+128..143]; countdown [esp+144]
+ mov [esp + 144], ecx // park BlockCount; ecx = staging scratch
+ mov edx, [eax + 12] // counter low dword, kept native-LE in edx
+ bswap edx
// ---- stage batch 0 into the scratch (xmm0 is free here), then load it ----
- movdqu xmm0, [eax]
- movdqu [esp + 0], xmm0
- mov edx, 15
-@@PreA0:
- inc byte ptr [eax + edx]
- jnz @@PreD0
- dec edx
- jns @@PreA0
-@@PreD0:
- movdqu xmm0, [eax]
- movdqu [esp + 16], xmm0
- mov edx, 15
-@@PreA1:
- inc byte ptr [eax + edx]
- jnz @@PreD1
- dec edx
- jns @@PreA1
-@@PreD1:
- movdqu xmm0, [eax]
- movdqu [esp + 32], xmm0
- mov edx, 15
-@@PreA2:
- inc byte ptr [eax + edx]
- jnz @@PreD2
- dec edx
- jns @@PreA2
-@@PreD2:
- movdqu xmm0, [eax]
- movdqu [esp + 48], xmm0
- mov edx, 15
-@@PreA3:
- inc byte ptr [eax + edx]
- jnz @@PreD3
- dec edx
- jns @@PreA3
-@@PreD3:
- movdqu xmm0, [eax]
- movdqu [esp + 64], xmm0
- mov edx, 15
-@@PreA4:
- inc byte ptr [eax + edx]
- jnz @@PreD4
- dec edx
- jns @@PreA4
-@@PreD4:
- movdqu xmm0, [eax]
- movdqu [esp + 80], xmm0
- mov edx, 15
-@@PreA5:
- inc byte ptr [eax + edx]
- jnz @@PreD5
- dec edx
- jns @@PreA5
-@@PreD5:
- movdqu xmm0, [eax]
- movdqu [esp + 96], xmm0
- mov edx, 15
-@@PreA6:
- inc byte ptr [eax + edx]
- jnz @@PreD6
- dec edx
- jns @@PreA6
-@@PreD6:
- movdqu xmm0, [eax]
- movdqu [esp + 112], xmm0
- mov edx, 15
-@@PreA7:
- inc byte ptr [eax + edx]
- jnz @@PreD7
- dec edx
- jns @@PreA7
-@@PreD7:
+ mov ecx, [eax]
+ mov [esp + 0], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 4], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 8], ecx
+ bswap edx
+ mov [esp + 12], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP0
+ mov ecx, 11
+@@StgCyP0:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP0
+ dec ecx
+ jns @@StgCyP0
+@@StgOkP0:
+ mov ecx, [eax]
+ mov [esp + 16], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 20], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 24], ecx
+ bswap edx
+ mov [esp + 28], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP1
+ mov ecx, 11
+@@StgCyP1:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP1
+ dec ecx
+ jns @@StgCyP1
+@@StgOkP1:
+ mov ecx, [eax]
+ mov [esp + 32], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 36], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 40], ecx
+ bswap edx
+ mov [esp + 44], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP2
+ mov ecx, 11
+@@StgCyP2:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP2
+ dec ecx
+ jns @@StgCyP2
+@@StgOkP2:
+ mov ecx, [eax]
+ mov [esp + 48], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 52], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 56], ecx
+ bswap edx
+ mov [esp + 60], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP3
+ mov ecx, 11
+@@StgCyP3:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP3
+ dec ecx
+ jns @@StgCyP3
+@@StgOkP3:
+ mov ecx, [eax]
+ mov [esp + 64], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 68], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 72], ecx
+ bswap edx
+ mov [esp + 76], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP4
+ mov ecx, 11
+@@StgCyP4:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP4
+ dec ecx
+ jns @@StgCyP4
+@@StgOkP4:
+ mov ecx, [eax]
+ mov [esp + 80], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 84], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 88], ecx
+ bswap edx
+ mov [esp + 92], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP5
+ mov ecx, 11
+@@StgCyP5:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP5
+ dec ecx
+ jns @@StgCyP5
+@@StgOkP5:
+ mov ecx, [eax]
+ mov [esp + 96], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 100], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 104], ecx
+ bswap edx
+ mov [esp + 108], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP6
+ mov ecx, 11
+@@StgCyP6:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP6
+ dec ecx
+ jns @@StgCyP6
+@@StgOkP6:
+ mov ecx, [eax]
+ mov [esp + 112], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 116], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 120], ecx
+ bswap edx
+ mov [esp + 124], edx
+ bswap edx
+ add edx, 1
+ jnc @@StgOkP7
+ mov ecx, 11
+@@StgCyP7:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkP7
+ dec ecx
+ jns @@StgCyP7
+@@StgOkP7:
movdqu xmm0, [esp + 0]
movdqu xmm1, [esp + 16]
@@ -147,21 +224,24 @@
db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16]
db $66, $0F, $38, $DC, $77, $10 // aesenc xmm6, [edi+16]
db $66, $0F, $38, $DC, $7F, $10 // aesenc xmm7, [edi+16]
- mov edx, [eax]
- mov [esp + 0], edx
- mov edx, [eax + 4]
- mov [esp + 4], edx
- mov edx, [eax + 8]
- mov [esp + 8], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 0], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 4], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 8], ecx
+ bswap edx
mov [esp + 12], edx
- mov edx, 15
-@@LpA0:
- inc byte ptr [eax + edx]
- jnz @@LpD0
- dec edx
- jns @@LpA0
-@@LpD0:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL0
+ mov ecx, 11
+@@StgCyL0:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL0
+ dec ecx
+ jns @@StgCyL0
+@@StgOkL0:
db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32]
db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32]
@@ -171,21 +251,24 @@
db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32]
db $66, $0F, $38, $DC, $77, $20 // aesenc xmm6, [edi+32]
db $66, $0F, $38, $DC, $7F, $20 // aesenc xmm7, [edi+32]
- mov edx, [eax]
- mov [esp + 16], edx
- mov edx, [eax + 4]
- mov [esp + 20], edx
- mov edx, [eax + 8]
- mov [esp + 24], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 16], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 20], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 24], ecx
+ bswap edx
mov [esp + 28], edx
- mov edx, 15
-@@LpA1:
- inc byte ptr [eax + edx]
- jnz @@LpD1
- dec edx
- jns @@LpA1
-@@LpD1:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL1
+ mov ecx, 11
+@@StgCyL1:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL1
+ dec ecx
+ jns @@StgCyL1
+@@StgOkL1:
db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48]
db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48]
@@ -195,21 +278,24 @@
db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48]
db $66, $0F, $38, $DC, $77, $30 // aesenc xmm6, [edi+48]
db $66, $0F, $38, $DC, $7F, $30 // aesenc xmm7, [edi+48]
- mov edx, [eax]
- mov [esp + 32], edx
- mov edx, [eax + 4]
- mov [esp + 36], edx
- mov edx, [eax + 8]
- mov [esp + 40], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 32], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 36], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 40], ecx
+ bswap edx
mov [esp + 44], edx
- mov edx, 15
-@@LpA2:
- inc byte ptr [eax + edx]
- jnz @@LpD2
- dec edx
- jns @@LpA2
-@@LpD2:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL2
+ mov ecx, 11
+@@StgCyL2:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL2
+ dec ecx
+ jns @@StgCyL2
+@@StgOkL2:
db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64]
db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64]
@@ -219,21 +305,24 @@
db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64]
db $66, $0F, $38, $DC, $77, $40 // aesenc xmm6, [edi+64]
db $66, $0F, $38, $DC, $7F, $40 // aesenc xmm7, [edi+64]
- mov edx, [eax]
- mov [esp + 48], edx
- mov edx, [eax + 4]
- mov [esp + 52], edx
- mov edx, [eax + 8]
- mov [esp + 56], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 48], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 52], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 56], ecx
+ bswap edx
mov [esp + 60], edx
- mov edx, 15
-@@LpA3:
- inc byte ptr [eax + edx]
- jnz @@LpD3
- dec edx
- jns @@LpA3
-@@LpD3:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL3
+ mov ecx, 11
+@@StgCyL3:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL3
+ dec ecx
+ jns @@StgCyL3
+@@StgOkL3:
db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80]
db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80]
@@ -243,21 +332,24 @@
db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80]
db $66, $0F, $38, $DC, $77, $50 // aesenc xmm6, [edi+80]
db $66, $0F, $38, $DC, $7F, $50 // aesenc xmm7, [edi+80]
- mov edx, [eax]
- mov [esp + 64], edx
- mov edx, [eax + 4]
- mov [esp + 68], edx
- mov edx, [eax + 8]
- mov [esp + 72], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 64], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 68], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 72], ecx
+ bswap edx
mov [esp + 76], edx
- mov edx, 15
-@@LpA4:
- inc byte ptr [eax + edx]
- jnz @@LpD4
- dec edx
- jns @@LpA4
-@@LpD4:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL4
+ mov ecx, 11
+@@StgCyL4:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL4
+ dec ecx
+ jns @@StgCyL4
+@@StgOkL4:
db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96]
db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96]
@@ -267,21 +359,24 @@
db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96]
db $66, $0F, $38, $DC, $77, $60 // aesenc xmm6, [edi+96]
db $66, $0F, $38, $DC, $7F, $60 // aesenc xmm7, [edi+96]
- mov edx, [eax]
- mov [esp + 80], edx
- mov edx, [eax + 4]
- mov [esp + 84], edx
- mov edx, [eax + 8]
- mov [esp + 88], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 80], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 84], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 88], ecx
+ bswap edx
mov [esp + 92], edx
- mov edx, 15
-@@LpA5:
- inc byte ptr [eax + edx]
- jnz @@LpD5
- dec edx
- jns @@LpA5
-@@LpD5:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL5
+ mov ecx, 11
+@@StgCyL5:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL5
+ dec ecx
+ jns @@StgCyL5
+@@StgOkL5:
db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112]
db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112]
@@ -291,21 +386,24 @@
db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112]
db $66, $0F, $38, $DC, $77, $70 // aesenc xmm6, [edi+112]
db $66, $0F, $38, $DC, $7F, $70 // aesenc xmm7, [edi+112]
- mov edx, [eax]
- mov [esp + 96], edx
- mov edx, [eax + 4]
- mov [esp + 100], edx
- mov edx, [eax + 8]
- mov [esp + 104], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 96], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 100], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 104], ecx
+ bswap edx
mov [esp + 108], edx
- mov edx, 15
-@@LpA6:
- inc byte ptr [eax + edx]
- jnz @@LpD6
- dec edx
- jns @@LpA6
-@@LpD6:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL6
+ mov ecx, 11
+@@StgCyL6:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL6
+ dec ecx
+ jns @@StgCyL6
+@@StgOkL6:
db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128]
db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128]
@@ -315,21 +413,24 @@
db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128]
db $66, $0F, $38, $DC, $B7, $80, $00, $00, $00 // aesenc xmm6, [edi+128]
db $66, $0F, $38, $DC, $BF, $80, $00, $00, $00 // aesenc xmm7, [edi+128]
- mov edx, [eax]
- mov [esp + 112], edx
- mov edx, [eax + 4]
- mov [esp + 116], edx
- mov edx, [eax + 8]
- mov [esp + 120], edx
- mov edx, [eax + 12]
+ mov ecx, [eax]
+ mov [esp + 112], ecx
+ mov ecx, [eax + 4]
+ mov [esp + 116], ecx
+ mov ecx, [eax + 8]
+ mov [esp + 120], ecx
+ bswap edx
mov [esp + 124], edx
- mov edx, 15
-@@LpA7:
- inc byte ptr [eax + edx]
- jnz @@LpD7
- dec edx
- jns @@LpA7
-@@LpD7:
+ bswap edx
+ add edx, 1
+ jnc @@StgOkL7
+ mov ecx, 11
+@@StgCyL7:
+ inc byte ptr [eax + ecx]
+ jnz @@StgOkL7
+ dec ecx
+ jns @@StgCyL7
+@@StgOkL7:
// ---- round 9 (all key sizes): aesenc, no more staging ----
db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144]
@@ -453,21 +554,24 @@
add ebx, 128
add esi, 128
- sub ecx, 8
+ sub dword ptr [esp + 144], 8
jnz @@CtrEightLoopI386
- // ---- undo the one-batch build-ahead: counter (at [eax]) -= 8, big-endian ----
- sub byte ptr [eax + 15], 8
- jnc @@CtrWbDone
- mov edx, 14
+ // ---- undo the one-batch build-ahead (edx is 8 blocks ahead); a borrow
+ // unwinds the prefix bytes, then the low dword stores big-endian ----
+ sub edx, 8
+ jnc @@CtrWbStore
+ mov ecx, 11
@@CtrWbBorrow:
- sub byte ptr [eax + edx], 1
- jnc @@CtrWbDone
- dec edx
+ sub byte ptr [eax + ecx], 1
+ jnc @@CtrWbStore
+ dec ecx
jns @@CtrWbBorrow
-@@CtrWbDone:
+@@CtrWbStore:
+ bswap edx
+ mov [eax + 12], edx
- add esp, 144
+ add esp, 152
pop edi
pop esi
pop ebx
diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc
index ad14023f..aecf45f7 100644
--- a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc
@@ -24,11 +24,11 @@
// direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must
// NOT be defined when including this file.
//
-// The 128-bit counter is held natively as r11 (high 64 bits) : rbx (low 64 bits)
-// across the whole span; the low half is bumped with add/adc so carry propagates
-// through the full 128 bits, matching SIC's big-endian whole-counter increment.
-// Each staged block writes bswap(r11) || bswap(rbx) (big-endian) to the scratch,
-// then advances. The pipeline builds one batch ahead, so on exit the counter has
+// The 128-bit counter is held as r11 (high 64 bits, KEPT big-endian since the
+// fast path only stores it) : rbx (low 64 bits, native-LE). Each staged block
+// writes r11 || bswap(rbx) to the scratch and bumps rbx; a low-half carry-out
+// (once per 2^64 blocks) increments r11 through a bswap round-trip, matching
+// SIC's big-endian whole-counter increment. The pipeline builds one batch ahead, so on exit the counter has
// over-run by exactly one batch (8) - undone before write-back.
//
// Register/stack notes: rbx is callee-saved (push/pop). xmm6-15 are saved once
@@ -47,76 +47,91 @@
mov rdx, [rcx + 8] // OutPtr
mov rcx, [rcx + 0] // InPtr (overwrites PCtx)
- mov r11, [r9] // counter high 64 bits (big-endian)
- bswap r11
- mov rbx, [r9 + 8] // counter low 64 bits (big-endian)
+ mov r11, [r9] // counter high 64 bits, KEPT big-endian (store-only)
+ mov rbx, [r9 + 8] // counter low 64 bits, native-LE in rbx
bswap rbx
// ---- stage batch 0 into the scratch, then load it into xmm0..7 ----
- mov rax, r11
- bswap rax
- mov [rsp + 0], rax
+ mov [rsp + 0], r11
mov rax, rbx
bswap rax
mov [rsp + 8], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 16], rax
+ jnc @@StgOk0
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk0:
+ mov [rsp + 16], r11
mov rax, rbx
bswap rax
mov [rsp + 24], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 32], rax
+ jnc @@StgOk1
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk1:
+ mov [rsp + 32], r11
mov rax, rbx
bswap rax
mov [rsp + 40], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 48], rax
+ jnc @@StgOk2
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk2:
+ mov [rsp + 48], r11
mov rax, rbx
bswap rax
mov [rsp + 56], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 64], rax
+ jnc @@StgOk3
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk3:
+ mov [rsp + 64], r11
mov rax, rbx
bswap rax
mov [rsp + 72], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 80], rax
+ jnc @@StgOk4
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk4:
+ mov [rsp + 80], r11
mov rax, rbx
bswap rax
mov [rsp + 88], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 96], rax
+ jnc @@StgOk5
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk5:
+ mov [rsp + 96], r11
mov rax, rbx
bswap rax
mov [rsp + 104], rax
add rbx, 1
- adc r11, 0
- mov rax, r11
- bswap rax
- mov [rsp + 112], rax
+ jnc @@StgOk6
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk6:
+ mov [rsp + 112], r11
mov rax, rbx
bswap rax
mov [rsp + 120], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk7
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk7:
movdqu xmm0, [rsp]
movdqu xmm1, [rsp + 16]
@@ -149,14 +164,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 0], rax
+ mov [rsp + 0], r11
mov rax, rbx
bswap rax
mov [rsp + 8], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk8
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk8:
movdqa xmm8, [r8 + 32]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -167,14 +184,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 16], rax
+ mov [rsp + 16], r11
mov rax, rbx
bswap rax
mov [rsp + 24], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk9
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk9:
movdqa xmm8, [r8 + 48]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -185,14 +204,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 32], rax
+ mov [rsp + 32], r11
mov rax, rbx
bswap rax
mov [rsp + 40], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk10
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk10:
movdqa xmm8, [r8 + 64]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -203,14 +224,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 48], rax
+ mov [rsp + 48], r11
mov rax, rbx
bswap rax
mov [rsp + 56], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk11
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk11:
movdqa xmm8, [r8 + 80]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -221,14 +244,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 64], rax
+ mov [rsp + 64], r11
mov rax, rbx
bswap rax
mov [rsp + 72], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk12
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk12:
movdqa xmm8, [r8 + 96]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -239,14 +264,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 80], rax
+ mov [rsp + 80], r11
mov rax, rbx
bswap rax
mov [rsp + 88], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk13
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk13:
movdqa xmm8, [r8 + 112]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -257,14 +284,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 96], rax
+ mov [rsp + 96], r11
mov rax, rbx
bswap rax
mov [rsp + 104], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk14
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk14:
movdqa xmm8, [r8 + 128]
db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
@@ -275,14 +304,16 @@
db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- mov rax, r11
- bswap rax
- mov [rsp + 112], rax
+ mov [rsp + 112], r11
mov rax, rbx
bswap rax
mov [rsp + 120], rax
add rbx, 1
- adc r11, 0
+ jnc @@StgOk15
+ bswap r11
+ add r11, 1
+ bswap r11
+@@StgOk15:
// ---- round 9 (all key sizes): aesenc, no more staging ----
movdqa xmm8, [r8 + 144]
@@ -399,13 +430,14 @@
// ---- undo the one-batch build-ahead, write the counter back (big-endian) ----
sub rbx, 8
- sbb r11, 0
- mov rax, r11
- bswap rax
- mov [r9], rax
- mov rax, rbx
- bswap rax
- mov [r9 + 8], rax
+ jnc @@CtrWbStore
+ bswap r11
+ sub r11, 1
+ bswap r11
+@@CtrWbStore:
+ mov [r9], r11
+ bswap rbx
+ mov [r9 + 8], rbx
add rsp, 128
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc
index 7b401a64..19d08026 100644
--- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc
@@ -1,324 +1,280 @@
// Fused AES-NI CTR keystream + 8-way GHASH kernel (i386).
// -------------------------------------------------------------------
// Technique: Shay Gueron, "Intel AES New Instructions" Rev 3.01 - AES
-// and PCLMULQDQ instructions are interleaved so the AES port (p0 on
-// Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding
-// GHASH latency inside the AES critical path.
+// and PCLMULQDQ instructions are interleaved so the AES port and the
+// carry-less-multiply port operate in parallel, hiding GHASH latency
+// inside the AES critical path.
//
// Loops over BatchCount pipelined batches. Each batch AES-encrypts eight
-// counter blocks to produce a keystream, XORs it with PXorIn to produce POut,
-// and concurrently GHASHes the PREVIOUS batch's 8 ciphertext blocks into the
-// running authentication state (the pipeline lags by one batch). AES inside GCM
-// is always CTR keystream construction; GhashFromOutput picks which buffer feeds
-// GHASH (output on encrypt, input on decrypt). The 8-xmm budget does not fit a
-// single-pass 8-wide layout, so the body processes the batch as two back-to-back
-// 4-block halves sharing the same running Z0/Z1/Z2 accumulators; each half packs
-// 4 GHASH iters into AES rounds 1..4 to keep both ports busy, rounds 5..N-1 are
-// AES-only, and the 3-limb reduction runs once after half 2. Only 4 counter
-// blocks are live per half, so each half builds its four in-register (eax/edx
-// are free there) - no counter scratch buffer is needed. The mode primes batch 0
-// and drains the last.
+// counter blocks (generated in-register from Counter32 + PJ0Template) to
+// produce a keystream, XORs it with PXorIn to produce POut, and concurrently
+// GHASHes the PREVIOUS batch's 8 ciphertext blocks into the running
+// authentication state (the pipeline lags by one batch). AES inside GCM is
+// always CTR keystream construction; GhashFromOutput picks which buffer feeds
+// GHASH (output on encrypt, input on decrypt). The mode primes batch 0 and
+// drains the last.
+//
+// PHPow128 holds the kernel-owned H^8..H^1 table PRE-MULTIPLIED BY x in the
+// reflected representation, so the batch product reduces with two
+// carry-less multiplies by 0xC2000000_00000000 instead of a shift/XOR
+// ladder. The running GHASH state stays reflected between batches and is
+// only byte-reversed at kernel entry/exit.
//
// Variant selection (set EXACTLY ONE):
// GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes).
// GCM_FUSED_AES_ROUNDS_12 -> AES-192 (schedule = 13 x 16 bytes).
// GCM_FUSED_AES_ROUNDS_14 -> AES-256 (schedule = 15 x 16 bytes).
//
+// The 8-xmm budget does not fit a single-pass 8-wide layout, so the body
+// processes the batch as two back-to-back 4-block halves sharing the same
+// Z0/Z1/Z2 accumulator slots; each half packs 4 GHASH iters into AES
+// rounds 1..4, rounds 5..N-1 are AES-only, and the reduction runs once
+// after half 2. Only 4 counter blocks are live per half, so each half
+// builds its four in-register right before that half runs.
+//
// On entry (after ClpSimdProc1Begin_i386.inc): ebx = PCtx.
+// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 4-byte fields):
+// [ebx + 0] PXorIn [ebx + 4] POut [ebx + 8] PPrevCipher
+// [ebx + 12] PRoundKeys (16-byte aligned) [ebx + 16] PHPow128
+// [ebx + 20] PFS [ebx + 24] PMask [ebx + 28] Counter32
+// [ebx + 32] PJ0Template [ebx + 36] BatchCount
+// [ebx + 40] GhashFromOutput (1 = encrypt)
//
-// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 4-byte fields).
-// [ebx + 0] PXorIn : 128-byte input to XOR with keystream (advances)
-// [ebx + 4] POut : 128-byte output (advances; may alias PXorIn)
-// [ebx + 8] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch)
-// [ebx + 12] PRoundKeys : AES encrypt round-key schedule (16-byte aligned)
-// [ebx + 16] PHPow128 : H^8..H^1, eight 16-byte limbs (field form)
-// [ebx + 20] PFS : 16-byte running GHASH state (in/out, canonical)
-// [ebx + 24] PMask : 16-byte PSHUFB byte-reverse mask
-// [ebx + 28] Counter32 : 32-bit GCM counter, last-used value (advances by 8)
-// [ebx + 32] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block)
-// [ebx + 36] BatchCount : number of 8-block batches to process
-// [ebx + 40] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt)
+// GPR: ebx=PCtx, edi=Keys (pinned), ebp=HPow (pinned), esi=PPrevCipher
+// (rotates per batch), ecx=counter32 (in-register across the call),
+// eax/edx=scratch.
//
-// XMM (8/8 used; i386 has only xmm0..xmm7):
-// xmm0..xmm3 : 4 AES keystream lanes (per half; reloaded between halves)
-// xmm4 : GHASH block cache (byte-reversed prev-cipher block, per iter)
-// xmm5 : current H-power limb (per iter)
-// xmm6 : pclmul product destination (re-inited from xmm4 per imm)
-// xmm7 : transient scratch (byte-reverse mask, then Z-reload fold)
+// XMM: xmm0..xmm3 = 4 AES keystream lanes (per half); xmm4 = GHASH block;
+// xmm5 = H-power limb; xmm6/xmm7 = pclmul product scratch.
//
-// Stack scratch (64 bytes, base unaligned - movdqu only):
-// [esp + 0..15] Z0 accumulator (updated 8x per batch)
-// [esp + 16..31] Z1 accumulator (updated 16x per batch; imm=1 + imm=16)
-// [esp + 32..47] Z2 accumulator (updated 8x per batch)
-// [esp + 48..63] byte-reversed PFS (computed once, folded into iter 0)
+// Stack scratch (base 16-aligned; caller esp parked inside):
+// [esp + 0] Z0 [esp + 16] Z1 [esp + 32] Z2 (movdqa)
+// [esp + 48] reflected running state (carried across batches)
+// [esp + 64] byte-reverse mask copy (PSHUFB memory operand)
+// [esp + 80] reduction constant [esp + 96] parked caller esp
//
-// AES-NI (aesenc / aesenclast), PCLMULQDQ, and PINSRD (counter build)
-// instructions are db-encoded for broad assembler compatibility.
+// AES-NI (aesenc / aesenclast), PCLMULQDQ, PINSRD, PSHUFD and the
+// PSHUFB memory-operand form are db-encoded for broad assembler
+// compatibility.
// -------------------------------------------------------------------
- // --- Callee-save prologue. ClpSimdProc1Begin_i386.inc already pushed ebx
- // and moved PCtx into ebx, so this block only pushes the other
- // three callee-saved GPRs and reserves the Z / br(PFS) scratch. ---
+ // --- Callee-save prologue (ClpSimdProc1Begin_i386.inc already pushed
+ // ebx); park the caller esp inside the 16-aligned frame. ---
push edi
push esi
push ebp
- sub esp, 64
-
- // --- Batch loop: process BatchCount batches, generating GCM counters and
- // rotating the GHASH pipeline in-asm. i386 has no spare GP registers, so
- // the loop state lives in the context: the 32-bit counter [ebx+28], the
- // J0 template [ebx+32], the batch count [ebx+36], and the
- // "GHASH-from-output" flag [ebx+40] (1=encrypt). Each batch's 8 counter
- // blocks are built in-register, 4 per half, right before that half runs
- // (see the per-half counter builds below). ---
-@@GcmFusedLoop:
+ mov eax, esp
+ sub esp, 120
+ and esp, -16 // align the frame
+ mov [esp + 96], eax // park the caller esp
+
+ // --- Frame constants: reduction constant, mask copy, reflected state. ---
+ mov dword ptr [esp + 80], 0
+ mov dword ptr [esp + 84], $C2000000
+ mov dword ptr [esp + 88], 0
+ mov dword ptr [esp + 92], 0
+ mov eax, [ebx + 24] // PMask
+ movdqu xmm0, [eax]
+ movdqa [esp + 64], xmm0
+ mov eax, [ebx + 20] // PFS
+ movdqu xmm1, [eax]
+ pshufb xmm1, xmm0 // canonical -> reflected
+ movdqa [esp + 48], xmm1
- // --- Load context pointers. ---
- mov edi, [ebx + 12] // edi = PRoundKeys (pinned)
- mov esi, [ebx + 8] // esi = PPrevCipher (pinned base)
- mov ebp, [ebx + 16] // ebp = PHPow128 (pinned base)
- mov ecx, [ebx + 24] // ecx = PMask (pinned)
+ // --- Pinned loop registers. ---
+ mov edi, [ebx + 12] // edi = PRoundKeys (aligned)
+ mov ebp, [ebx + 16] // ebp = PHPow128 (x-shifted)
+ mov esi, [ebx + 8] // esi = PPrevCipher
+ mov ecx, [ebx + 28] // ecx = counter32
- // --- Zero Z0 / Z1 / Z2 accumulator slots. ---
+@@GcmFusedLoop:
+ // --- Zero the Z accumulator slots. ---
pxor xmm0, xmm0
- movdqu [esp + 0], xmm0 // Z0 = 0
- movdqu [esp + 16], xmm0 // Z1 = 0
- movdqu [esp + 32], xmm0 // Z2 = 0
-
- // --- Compute byte-reversed running state once, stash for iter-0 fold. ---
- mov eax, [ebx + 20] // eax = PFS
- movdqu xmm0, [eax] // load canonical PFS
- movdqu xmm7, [ecx] // mask
- pshufb xmm0, xmm7 // byte-reverse
- movdqu [esp + 48], xmm0 // stash byte-reversed PFS
+ movdqa [esp + 0], xmm0 // Z0 = 0
+ movdqa [esp + 16], xmm0 // Z1 = 0
+ movdqa [esp + 32], xmm0 // Z2 = 0
// --- Build 4 counter blocks for HALF 1 (blocks 0..3): J0[0..11] ||
- // BE(counter32+1..+4). GCM's counter is the last-used value, so the batch
- // based at counter32 spans counter32+1..counter32+8 (matching the scalar
- // GetNextCtrBlocks8). Load J0 into each lane, then PINSRD (db-encoded) the
- // big-endian counter word into dword 3; eax/edx are free here. ---
- mov eax, [ebx + 32] // PJ0Template
+ // BE(counter32+1..+4); the counter holds the last-used value. ---
+ mov eax, [ebx + 32] // PJ0Template
movdqu xmm0, [eax]
movdqu xmm1, [eax]
movdqu xmm2, [eax]
movdqu xmm3, [eax]
- mov eax, [ebx + 28] // counter32
- lea edx, [eax + 1]
+ lea edx, [ecx + 1]
bswap edx
- db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3
- lea edx, [eax + 2]
+ db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3
+ lea edx, [ecx + 2]
bswap edx
- db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3
- lea edx, [eax + 3]
+ db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3
+ lea edx, [ecx + 3]
bswap edx
- db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3
- lea edx, [eax + 4]
+ db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3
+ lea edx, [ecx + 4]
bswap edx
- db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3
+ db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3
// ======================================================================
- // HALF 1: blocks 0..3, GHASH iters 0..3 (prev_cipher[0..3], H^8..H^5)
- // iter 0 additionally folds byte-reversed PFS into the block.
+ // HALF 1: blocks 0..3, GHASH iters 0..3 (prev_cipher[0..3], H^8..H^5);
+ // iter 0 additionally folds the carried reflected state.
// ======================================================================
- // --- Round 0: AddRoundKey (pxor with rk[0]). Round keys are aligned. ---
+ // --- Round 0: AddRoundKey (aligned key memory operands). ---
pxor xmm0, [edi + 0]
pxor xmm1, [edi + 0]
pxor xmm2, [edi + 0]
pxor xmm3, [edi + 0]
- // --- Round 1 + GHASH iter 0 (with state fold). ---
- db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10]
- db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10]
- db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10]
- db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10]
-
- // GHASH iter 0: block 0 byte-reverse + state fold
- movdqu xmm4, [esi + 0] // prev cipher block 0
- movdqu xmm7, [ecx] // mask
- pshufb xmm4, xmm7 // byte-reverse
- movdqu xmm7, [esp + 48] // byte-reversed PFS
- pxor xmm4, xmm7 // fold running state
- movdqu xmm5, [ebp + 0] // H^8
-
- // 4 pclmul -> Z0 (imm=0), Z1 (imm=1), Z1 (imm=16), Z2 (imm=17).
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7 // Z0 +=
+ // --- Round 1 + GHASH iter 0 (with carried-state fold). ---
+ db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10]
+ db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10]
+ db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10]
+ db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10]
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7 // Z1 +=
+ movdqu xmm4, [esi + 0] // prev cipher block 0
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ pxor xmm4, [esp + 48] // fold carried state
+ movdqu xmm5, [ebp + 0] // H^8 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7 // Z1 +=
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7 // Z2 +=
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
// --- Round 2 + GHASH iter 1. ---
- db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20]
- db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20]
- db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20]
- db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20]
+ db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20]
+ db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20]
+ db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20]
+ db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20]
movdqu xmm4, [esi + 16] // prev cipher block 1
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 16] // H^7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 16] // H^7 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
// --- Round 3 + GHASH iter 2. ---
- db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30]
- db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30]
- db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30]
- db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30]
+ db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30]
+ db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30]
+ db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30]
+ db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30]
movdqu xmm4, [esi + 32] // prev cipher block 2
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 32] // H^6
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 32] // H^6 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
- // --- Round 4 + GHASH iter 3 (last GHASH iter of half 1). ---
- db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40]
- db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40]
- db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40]
- db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40]
+ // --- Round 4 + GHASH iter 3. ---
+ db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40]
+ db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40]
+ db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40]
+ db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40]
movdqu xmm4, [esi + 48] // prev cipher block 3
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 48] // H^5
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 48] // H^5 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
// --- Rounds 5..Nr-1 (variant-specific): AES-only. ---
-
- // Round 5 (disp8 = 80)
- db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50]
- db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50]
- db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50]
- db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50]
- // Round 6 (disp8 = 96)
- db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60]
- db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60]
- db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60]
- db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60]
- // Round 7 (disp8 = 112)
- db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70]
- db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70]
- db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70]
- db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70]
- // Round 8 (disp32 = 128)
+ // Round 5
+ db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50]
+ db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50]
+ db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50]
+ db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50]
+ // Round 6
+ db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60]
+ db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60]
+ db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60]
+ db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60]
+ // Round 7
+ db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70]
+ db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70]
+ db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70]
+ db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70]
+ // Round 8
db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+0x80]
db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+0x80]
db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+0x80]
db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+0x80]
- // Round 9 (disp32 = 144)
+ // Round 9
db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+0x90]
db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+0x90]
db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+0x90]
db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+0x90]
-
{$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)}
- // Round 10 (disp32 = 160)
+ // Round 10
db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+0xA0]
db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+0xA0]
db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+0xA0]
db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+0xA0]
- // Round 11 (disp32 = 176)
+ // Round 11
db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+0xB0]
db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+0xB0]
db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+0xB0]
db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+0xB0]
{$IFEND}
-
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- // Round 12 (disp32 = 192)
+ // Round 12
db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+0xC0]
db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+0xC0]
db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+0xC0]
db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+0xC0]
- // Round 13 (disp32 = 208)
+ // Round 13
db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+0xD0]
db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+0xD0]
db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+0xD0]
@@ -327,69 +283,63 @@
// --- Final AES round (aesenclast on rk[Nr]). ---
{$IFDEF GCM_FUSED_AES_ROUNDS_10}
- // AES-128: aesenclast at disp32 = 160
db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+0xA0]
db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+0xA0]
db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+0xA0]
db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+0xA0]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_12}
- // AES-192: aesenclast at disp32 = 192
db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+0xC0]
db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+0xC0]
db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+0xC0]
db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+0xC0]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- // AES-256: aesenclast at disp32 = 224
db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+0xE0]
db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+0xE0]
db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+0xE0]
db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+0xE0]
{$ENDIF}
- // --- Post-AES (half 1): XOR with PXorIn[0..48] and store to POut[0..48]. ---
- mov eax, [ebx + 0] // eax = PXorIn
- mov edx, [ebx + 4] // edx = POut
-
- movdqu xmm7, [eax + 0]
- pxor xmm0, xmm7
+ // --- Post-AES (half 1): XOR with PXorIn[0..48], store POut. ---
+ mov eax, [ebx + 0] // eax = PXorIn
+ mov edx, [ebx + 4] // edx = POut
+ movdqu xmm4, [eax + 0]
+ pxor xmm0, xmm4
movdqu [edx + 0], xmm0
- movdqu xmm7, [eax + 16]
- pxor xmm1, xmm7
+ movdqu xmm4, [eax + 16]
+ pxor xmm1, xmm4
movdqu [edx + 16], xmm1
- movdqu xmm7, [eax + 32]
- pxor xmm2, xmm7
+ movdqu xmm4, [eax + 32]
+ pxor xmm2, xmm4
movdqu [edx + 32], xmm2
- movdqu xmm7, [eax + 48]
- pxor xmm3, xmm7
+ movdqu xmm4, [eax + 48]
+ pxor xmm3, xmm4
movdqu [edx + 48], xmm3
- // ======================================================================
- // HALF 2: blocks 4..7, GHASH iters 4..7 (prev_cipher[4..7], H^4..H^1)
- // no state fold (already consumed by iter 0 of half 1).
- // ======================================================================
-
- // --- Build 4 counter blocks for half 2 (blocks 4..7): counter32+5..+8. The
- // pinned regs (edi/esi/ebp/ecx) are untouched; eax/edx are free here. ---
- mov eax, [ebx + 32] // PJ0Template
+ // --- Build 4 counter blocks for HALF 2 (blocks 4..7): counter32+5..+8. ---
+ mov eax, [ebx + 32] // PJ0Template
movdqu xmm0, [eax]
movdqu xmm1, [eax]
movdqu xmm2, [eax]
movdqu xmm3, [eax]
- mov eax, [ebx + 28] // counter32
- lea edx, [eax + 5]
+ lea edx, [ecx + 5]
bswap edx
- db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3
- lea edx, [eax + 6]
+ db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3
+ lea edx, [ecx + 6]
bswap edx
- db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3
- lea edx, [eax + 7]
+ db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3
+ lea edx, [ecx + 7]
bswap edx
- db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3
- lea edx, [eax + 8]
+ db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3
+ lea edx, [ecx + 8]
bswap edx
- db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3
+ db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3
+
+ // ======================================================================
+ // HALF 2: blocks 4..7, GHASH iters 4..7 (prev_cipher[4..7], H^4..H^1);
+ // no state fold (already consumed by iter 0 of half 1).
+ // ======================================================================
// --- Round 0: AddRoundKey. ---
pxor xmm0, [edi + 0]
@@ -398,337 +348,245 @@
pxor xmm3, [edi + 0]
// --- Round 1 + GHASH iter 4. ---
- db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10]
- db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10]
- db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10]
- db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10]
+ db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+0x10]
+ db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+0x10]
+ db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+0x10]
+ db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+0x10]
movdqu xmm4, [esi + 64] // prev cipher block 4
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 64] // H^4
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 64] // H^4 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
// --- Round 2 + GHASH iter 5. ---
- db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20]
- db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20]
- db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20]
- db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20]
+ db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+0x20]
+ db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+0x20]
+ db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+0x20]
+ db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+0x20]
movdqu xmm4, [esi + 80] // prev cipher block 5
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 80] // H^3
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 80] // H^3 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
// --- Round 3 + GHASH iter 6. ---
- db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30]
- db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30]
- db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30]
- db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30]
+ db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+0x30]
+ db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+0x30]
+ db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+0x30]
+ db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+0x30]
movdqu xmm4, [esi + 96] // prev cipher block 6
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 96] // H^2
-
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 96] // H^2 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
-
- // --- Round 4 + GHASH iter 7 (last GHASH iter of batch). ---
- db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40]
- db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40]
- db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40]
- db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40]
-
- movdqu xmm4, [esi + 112] // prev cipher block 7
- movdqu xmm7, [ecx]
- pshufb xmm4, xmm7
- movdqu xmm5, [ebp + 112] // H^1
+ // --- Round 4 + GHASH iter 7. ---
+ db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+0x40]
+ db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+0x40]
+ db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+0x40]
+ db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+0x40]
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
- movdqu xmm7, [esp + 0]
- pxor xmm7, xmm6
- movdqu [esp + 0], xmm7
+ movdqu xmm4, [esi + 112] // prev cipher block 7
+ db $66, $0F, $38, $00, $64, $24, $40 // pshufb xmm4, [esp+64] (byte-reverse)
+ movdqu xmm5, [ebp + 112] // H^1 (x-shifted)
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
-
+ db $66, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm6, xmm5, 0x00
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $11 // pclmulqdq xmm7, xmm5, 0x11
+ pxor xmm6, [esp + 0]
+ movdqa [esp + 0], xmm6 // Z0 +=
+ pxor xmm7, [esp + 32]
+ movdqa [esp + 32], xmm7 // Z2 +=
movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm6, xmm5, 0x10
- movdqu xmm7, [esp + 16]
- pxor xmm7, xmm6
- movdqu [esp + 16], xmm7
+ db $66, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm6, xmm5, 0x01
+ movdqa xmm7, xmm4
+ db $66, $0F, $3A, $44, $FD, $10 // pclmulqdq xmm7, xmm5, 0x10
+ pxor xmm6, xmm7 // both cross products
+ pxor xmm6, [esp + 16]
+ movdqa [esp + 16], xmm6 // Z1 +=
- movdqa xmm6, xmm4
- db $66, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm6, xmm5, 0x11
- movdqu xmm7, [esp + 32]
- pxor xmm7, xmm6
- movdqu [esp + 32], xmm7
-
- // --- Rounds 5..Nr-1 AES-only (variant-specific). ---
- // Round 5 (disp8 = 80)
- db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50]
- db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50]
- db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50]
- db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50]
- // Round 6 (disp8 = 96)
- db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60]
- db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60]
- db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60]
- db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60]
- // Round 7 (disp8 = 112)
- db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70]
- db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70]
- db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70]
- db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70]
- // Round 8 (disp32 = 128)
+ // --- Rounds 5..Nr-1 (variant-specific): AES-only. ---
+ // Round 5
+ db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+0x50]
+ db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+0x50]
+ db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+0x50]
+ db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+0x50]
+ // Round 6
+ db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+0x60]
+ db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+0x60]
+ db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+0x60]
+ db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+0x60]
+ // Round 7
+ db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+0x70]
+ db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+0x70]
+ db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+0x70]
+ db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+0x70]
+ // Round 8
db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+0x80]
db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+0x80]
db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+0x80]
db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+0x80]
- // Round 9 (disp32 = 144)
+ // Round 9
db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+0x90]
db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+0x90]
db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+0x90]
db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+0x90]
-
{$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)}
- // Round 10 (disp32 = 160)
+ // Round 10
db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+0xA0]
db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+0xA0]
db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+0xA0]
db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+0xA0]
- // Round 11 (disp32 = 176)
+ // Round 11
db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+0xB0]
db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+0xB0]
db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+0xB0]
db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+0xB0]
{$IFEND}
-
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- // Round 12 (disp32 = 192)
+ // Round 12
db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+0xC0]
db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+0xC0]
db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+0xC0]
db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+0xC0]
- // Round 13 (disp32 = 208)
+ // Round 13
db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+0xD0]
db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+0xD0]
db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+0xD0]
db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+0xD0]
{$ENDIF}
- // --- Final AES round (aesenclast) for half 2. ---
+ // --- Final AES round (aesenclast on rk[Nr]). ---
{$IFDEF GCM_FUSED_AES_ROUNDS_10}
- // AES-128: aesenclast at disp32 = 160
db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+0xA0]
db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+0xA0]
db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+0xA0]
db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+0xA0]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_12}
- // AES-192: aesenclast at disp32 = 192
db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+0xC0]
db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+0xC0]
db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+0xC0]
db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+0xC0]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- // AES-256: aesenclast at disp32 = 224
db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+0xE0]
db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+0xE0]
db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+0xE0]
db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+0xE0]
{$ENDIF}
- // --- Post-AES (half 2): XOR with PXorIn[64..112] and store POut[64..112]. ---
- mov eax, [ebx + 0] // eax = PXorIn
- mov edx, [ebx + 4] // edx = POut
-
- movdqu xmm7, [eax + 64]
- pxor xmm0, xmm7
+ // --- Post-AES (half 2): XOR with PXorIn[64..112], store POut. ---
+ mov eax, [ebx + 0] // eax = PXorIn
+ mov edx, [ebx + 4] // edx = POut
+ movdqu xmm4, [eax + 64]
+ pxor xmm0, xmm4
movdqu [edx + 64], xmm0
- movdqu xmm7, [eax + 80]
- pxor xmm1, xmm7
+ movdqu xmm4, [eax + 80]
+ pxor xmm1, xmm4
movdqu [edx + 80], xmm1
- movdqu xmm7, [eax + 96]
- pxor xmm2, xmm7
+ movdqu xmm4, [eax + 96]
+ pxor xmm2, xmm4
movdqu [edx + 96], xmm2
- movdqu xmm7, [eax + 112]
- pxor xmm3, xmm7
+ movdqu xmm4, [eax + 112]
+ pxor xmm3, xmm4
movdqu [edx + 112], xmm3
- // ======================================================================
- // Inline 3-limb GHASH reduction on Z0 / Z1 / Z2 (copied from
- // GcmGhashFull_i386.inc). Load the stack-resident accumulators into
- // xmm0..xmm2 (keystream lanes are no longer needed after the stores
- // above), then fold Z1 into (Z0, Z2) and reduce modulo
- // x^128 + x^7 + x^2 + x + 1.
- // ======================================================================
- movdqu xmm0, [esp + 0] // Z0
- movdqu xmm1, [esp + 16] // Z1
- movdqu xmm2, [esp + 32] // Z2
-
+ // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into
+ // the 256-bit product, then two carry-less multiplies by the
+ // folded constant 0xC2000000_00000000 (the H powers are
+ // pre-multiplied by x, so no bit-shift correction is needed). ---
+ movdqa xmm0, [esp + 0] // Z0
+ movdqa xmm1, [esp + 16] // Z1
+ movdqa xmm2, [esp + 32] // Z2
+ movdqa xmm3, [esp + 80]
movdqa xmm4, xmm1
- movdqa xmm5, xmm1
pslldq xmm4, 8
+ pxor xmm0, xmm4 // ZLo ^= Z1 << 64
+ movdqa xmm5, xmm1
psrldq xmm5, 8
- pxor xmm0, xmm4
- pxor xmm2, xmm5
- movq xmm3, xmm0
- movq xmm4, xmm2
- movdqa xmm5, xmm3
- psrlq xmm5, 1
- movdqa xmm6, xmm3
- psrlq xmm6, 2
- movdqa xmm7, xmm3
- psrlq xmm7, 7
- pxor xmm4, xmm3
- pxor xmm4, xmm5
- pxor xmm4, xmm6
- pxor xmm4, xmm7
- movdqa xmm1, xmm0
- psrldq xmm1, 8
- movdqa xmm5, xmm3
- psllq xmm5, 63
- movdqa xmm6, xmm3
- psllq xmm6, 62
- movdqa xmm7, xmm3
- psllq xmm7, 57
- pxor xmm1, xmm5
- pxor xmm1, xmm6
- pxor xmm1, xmm7
- movdqa xmm5, xmm2
- psrldq xmm5, 8
- movdqa xmm6, xmm5
- psllq xmm6, 1
- movdqa xmm7, xmm4
- psrlq xmm7, 63
- por xmm6, xmm7
- movdqa xmm7, xmm4
- psllq xmm7, 1
- movdqa xmm3, xmm1
- psrlq xmm3, 63
- por xmm7, xmm3
- movdqa xmm0, xmm1
- psllq xmm0, 1
- movdqa xmm3, xmm0
- psrlq xmm3, 1
- movdqa xmm2, xmm0
- psrlq xmm2, 2
- movdqa xmm5, xmm0
- psrlq xmm5, 7
- pxor xmm6, xmm0
- pxor xmm6, xmm3
- pxor xmm6, xmm2
- pxor xmm6, xmm5
- movdqa xmm3, xmm1
- psllq xmm3, 63
- movdqa xmm2, xmm1
- psllq xmm2, 58
- pxor xmm7, xmm3
- pxor xmm7, xmm2
-
- // Pack and byte-reverse final GHASH state; store to PFS.
- movq xmm0, xmm7
- pslldq xmm6, 8
- por xmm0, xmm6
- movdqu xmm7, [ecx] // mask reload
- pshufb xmm0, xmm7 // byte-reverse -> canonical
- mov eax, [ebx + 20] // PFS
- movdqu [eax], xmm0
-
- // --- Advance the context for the next batch, then loop. PPrevCipher becomes
- // this batch's ciphertext (output on encrypt, input on decrypt); the
- // counter advances by 8 in place so the caller reads the final value. ---
- mov eax, [ebx + 4] // POut (encrypt: prev = output)
- cmp dword ptr [ebx + 40], 0 // GHASH-from-output flag (1=encrypt)
+ pxor xmm2, xmm5 // ZHi ^= Z1 >> 64
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 1
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 2
+ pxor xmm0, xmm2 // ^= ZHi -> reflected state
+ movdqa [esp + 48], xmm0 // carry state to the next batch
+
+ // --- Advance for the next batch: PPrevCipher becomes this batch's
+ // ciphertext (output on encrypt, input on decrypt). ---
+ mov esi, [ebx + 4] // POut (encrypt: prev = output)
+ cmp dword ptr [ebx + 40], 0 // GHASH-from-output flag
jne @@GcmPrevSet
- mov eax, [ebx + 0] // PXorIn (decrypt: prev = input)
+ mov esi, [ebx + 0] // PXorIn (decrypt: prev = input)
@@GcmPrevSet:
- mov [ebx + 8], eax // PPrevCipher = this batch's ciphertext
- add dword ptr [ebx + 0], 128 // PXorIn += 128
- add dword ptr [ebx + 4], 128 // POut += 128
- add dword ptr [ebx + 28], 8 // counter32 += 8
- dec dword ptr [ebx + 36] // batch count
+ add dword ptr [ebx + 0], 128 // PXorIn += 128
+ add dword ptr [ebx + 4], 128 // POut += 128
+ add ecx, 8 // counter32 += 8
+ dec dword ptr [ebx + 36] // batch count
jnz @@GcmFusedLoop
- // --- Epilogue: unwind stack scratch and pop callee-saved GPRs. ebx
- // is popped last because ClpSimdProc1Begin_i386.inc pushed it first. ---
- add esp, 64
+ // --- Epilogue: store counter, canonicalize + store the state, unwind.
+ // ebx is popped last (ClpSimdProc1Begin_i386.inc pushed it first). ---
+ mov [ebx + 28], ecx // store advanced counter32
+ movdqa xmm0, [esp + 48]
+ db $66, $0F, $38, $00, $44, $24, $40 // pshufb xmm0, [esp+64] (-> canonical)
+ mov eax, [ebx + 20] // PFS
+ movdqu [eax], xmm0
+
+ mov esp, [esp + 96]
pop ebp
pop esi
pop edi
pop ebx
+
diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc
index 7fa864b7..6abc36a2 100644
--- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc
@@ -1,9 +1,9 @@
// Fused AES-NI CTR keystream + 8-way GHASH kernel (x86-64).
// -------------------------------------------------------------------
// Technique: Shay Gueron, "Intel AES New Instructions" Rev 3.01 - AES
-// and PCLMULQDQ instructions are interleaved so the AES port (p0 on
-// Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding
-// GHASH latency inside the AES critical path.
+// and PCLMULQDQ instructions are interleaved so the AES port and the
+// carry-less-multiply port operate in parallel, hiding GHASH latency
+// inside the AES critical path.
//
// Loops over BatchCount pipelined batches. Each batch AES-encrypts eight
// counter blocks (generated in-register from Counter32 + PJ0Template) to
@@ -11,9 +11,14 @@
// GHASHes the PREVIOUS batch's 8 ciphertext blocks into the running
// authentication state (the pipeline lags by one batch). AES inside GCM is
// always CTR keystream construction; GhashFromOutput picks which buffer feeds
-// GHASH (output on encrypt, input on decrypt). Rounds 0..8 (with all 8 GHASH
-// iters) are shared across key sizes; only rounds 9..N-1 (extra aesenc) and
-// round N (aesenclast) depend on N. The mode primes batch 0 and drains the last.
+// GHASH (output on encrypt, input on decrypt). The mode primes batch 0 and
+// drains the last.
+//
+// PHPow128 holds the kernel-owned H^8..H^1 table PRE-MULTIPLIED BY x in the
+// reflected representation, so the batch product reduces with two
+// carry-less multiplies by 0xC2000000_00000000 instead of a shift/XOR
+// ladder. The running GHASH state stays reflected between batches and is
+// only byte-reversed at kernel entry/exit.
//
// Variant selection (set EXACTLY ONE):
// GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes).
@@ -21,571 +26,528 @@
// GCM_FUSED_AES_ROUNDS_14 -> AES-256 (schedule = 15 x 16 bytes).
//
// On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx.
+// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 8-byte fields):
+// [rcx + 0] PXorIn [rcx + 8] POut [rcx + 16] PPrevCipher
+// [rcx + 24] PRoundKeys (16-byte aligned) [rcx + 32] PHPow128
+// [rcx + 40] PFS [rcx + 48] PMask [rcx + 56] Counter32
+// [rcx + 64] PJ0Template [rcx + 72] BatchCount
+// [rcx + 80] GhashFromOutput (1 = encrypt)
//
-// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 8-byte fields).
-// [rcx + 0] PXorIn : 128-byte input to XOR with keystream (advances)
-// [rcx + 8] POut : 128-byte output (advances; may alias PXorIn)
-// [rcx + 16] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch)
-// [rcx + 24] PRoundKeys : AES encrypt round-key schedule (16-byte aligned)
-// [rcx + 32] PHPow128 : H^8..H^1, eight 16-byte limbs (field form)
-// [rcx + 40] PFS : 16-byte running GHASH state (in/out, canonical)
-// [rcx + 48] PMask : 16-byte PSHUFB byte-reverse mask
-// [rcx + 56] Counter32 : 32-bit GCM counter, last-used value (advances by 8)
-// [rcx + 64] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block)
-// [rcx + 72] BatchCount : number of 8-block batches to process
-// [rcx + 80] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt)
+// All loop-carried state lives in registers - no context memory is
+// touched inside the batch loop:
+// rsi/rdi = PXorIn/POut (advance) r10 = PPrevCipher (rotates)
+// rdx = PRoundKeys r11 = PHPow128 r8d = counter32
+// rbx = batch count ebp = GHASH-from-output flag rax = scratch
//
// XMM (16/16 used):
-// xmm0..xmm7 : 8 AES keystream lanes (persistent across rounds)
-// xmm8 : current AES round key (reloaded per round)
-// xmm9..xmm11 : Z0 / Z1 / Z2 GHASH accumulators (across all 8 iters)
+// xmm0..xmm7 : 8 AES keystream lanes (round keys via memory operands)
+// xmm8 : keystream-XOR scratch (free during the rounds)
+// xmm9..xmm11 : Z0 / Z1 / Z2 GHASH accumulators; xmm9 doubles as the
+// reflected running state carried across batches
// xmm12 : GHASH input block (byte-reversed, per iter)
// xmm13 : current H-power limb (per iter)
// xmm14 : pclmul scratch destination
// xmm15 : byte-reverse mask (held for the whole kernel)
//
+// Stack (below the nonvolatile-save area; [rsp] is 16-aligned):
+// [rsp + 0] reduction constant (low qword 0xC2000000_00000000)
+// [rsp + 16..143] 8 aligned counter-block slots. The fixed 12-byte J0
+// prefix is staged once in the prologue, and each batch's AES-only
+// tail rounds stage the NEXT batch's counter words (GPR stores, no
+// shuffle-port inserts), so the loop-top movdqa loads never read a
+// freshly written slot (no store-forward stalls). The prologue
+// pre-stages batch 0; the final batch over-stages into scratch.
+//
// AES-NI and PCLMULQDQ instructions, plus selected forms touching
-// xmm8..xmm15 (pxor, movdqa), are db-encoded for broad assembler
+// xmm8..xmm15 (pxor) and PSHUFD, are db-encoded for broad assembler
// compatibility.
// -------------------------------------------------------------------
+ push rbx
+ push rsi
+ push rdi
+ push rbp
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- // --- Batch loop. The context (rcx) carries advancing PXorIn/POut/PPrevCipher,
- // the 32-bit GCM counter [rcx+64], the J0 template [rcx+72], the batch
- // count [rcx+80], and the "GHASH-from-output" flag [rcx+88] (1=encrypt).
- // r8d = counter32, r9 = J0 template (persist across the loop; both are
- // volatile and untouched by the shared body). ---
- mov r8d, [rcx + 56] // counter32
- mov r9, [rcx + 64] // PJ0Template
-@@GcmFusedLoop:
- // Build 8 counter blocks into xmm0..7: block j = J0[0..11] || BE(counter32+j+1).
- // GCM's 32-bit counter holds the last-used value, so the batch based at
- // counter32 spans counter32+1..counter32+8 (matching FillCtr8BlocksRaw); only
- // bytes 12-15 vary. PINSRD is db-encoded.
- movdqu xmm0, [r9]
+ sub rsp, 152 // [rsp] 16-aligned
+ mov rax, $C200000000000000
+ mov [rsp], rax // reduction constant (low qword)
+ xor eax, eax
+ mov [rsp + 8], rax
+
+ // --- Stage the fixed J0 prefix into the 8 counter slots once; only
+ // bytes 12..15 (the counter word) change per batch. ---
+ mov rax, [rcx + 64] // PJ0Template
+ mov rdx, [rax] // J0 bytes 0..7
+ mov eax, [rax + 8] // J0 bytes 8..11
+ mov [rsp + 16], rdx
+ mov [rsp + 24], eax
+ mov [rsp + 32], rdx
+ mov [rsp + 40], eax
+ mov [rsp + 48], rdx
+ mov [rsp + 56], eax
+ mov [rsp + 64], rdx
+ mov [rsp + 72], eax
+ mov [rsp + 80], rdx
+ mov [rsp + 88], eax
+ mov [rsp + 96], rdx
+ mov [rsp + 104], eax
+ mov [rsp + 112], rdx
+ mov [rsp + 120], eax
+ mov [rsp + 128], rdx
+ mov [rsp + 136], eax
+
+ // --- Hoist the whole loop state into registers. ---
+ mov rdx, [rcx + 24] // rdx = PRoundKeys (aligned)
+ mov r11, [rcx + 32] // r11 = PHPow128 (x-shifted)
+ mov rax, [rcx + 48]
+ movdqu xmm15, [rax] // xmm15 = byte-reverse mask
+ mov rsi, [rcx + 0] // rsi = PXorIn
+ mov rdi, [rcx + 8] // rdi = POut
+ mov r10, [rcx + 16] // r10 = PPrevCipher
+ mov r8d, [rcx + 56] // r8d = counter32
+ mov rbx, [rcx + 72] // rbx = batch count
+ mov ebp, [rcx + 80] // ebp = GHASH-from-output flag
+ mov rax, [rcx + 40] // rax = PFS
+ movdqu xmm9, [rax]
+ pshufb xmm9, xmm15 // xmm9 = reflected running state
+
+ // --- Pre-stage batch 0's counter words (counter32+1..+8, big-endian). ---
lea eax, [r8 + 1]
bswap eax
- db $66, $0F, $3A, $22, $C0, $03 // pinsrd xmm0, eax, 3
- movdqu xmm1, [r9]
+ mov [rsp + 28], eax
lea eax, [r8 + 2]
bswap eax
- db $66, $0F, $3A, $22, $C8, $03 // pinsrd xmm1, eax, 3
- movdqu xmm2, [r9]
+ mov [rsp + 44], eax
lea eax, [r8 + 3]
bswap eax
- db $66, $0F, $3A, $22, $D0, $03 // pinsrd xmm2, eax, 3
- movdqu xmm3, [r9]
+ mov [rsp + 60], eax
lea eax, [r8 + 4]
bswap eax
- db $66, $0F, $3A, $22, $D8, $03 // pinsrd xmm3, eax, 3
- movdqu xmm4, [r9]
+ mov [rsp + 76], eax
lea eax, [r8 + 5]
bswap eax
- db $66, $0F, $3A, $22, $E0, $03 // pinsrd xmm4, eax, 3
- movdqu xmm5, [r9]
+ mov [rsp + 92], eax
lea eax, [r8 + 6]
bswap eax
- db $66, $0F, $3A, $22, $E8, $03 // pinsrd xmm5, eax, 3
- movdqu xmm6, [r9]
+ mov [rsp + 108], eax
lea eax, [r8 + 7]
bswap eax
- db $66, $0F, $3A, $22, $F0, $03 // pinsrd xmm6, eax, 3
- movdqu xmm7, [r9]
+ mov [rsp + 124], eax
lea eax, [r8 + 8]
bswap eax
- db $66, $0F, $3A, $22, $F8, $03 // pinsrd xmm7, eax, 3
-
- mov rax, [rcx + 48] // rax = PMask
- movdqu xmm15, [rax] // xmm15 = byte-reverse mask
-
- mov rdx, [rcx + 24] // rdx = PRoundKeys
- mov r10, [rcx + 16] // r10 = PPrevCipher
- mov r11, [rcx + 32] // r11 = PHPow128
-
- // --- Round 0: AddRoundKey + zero Z0/Z1/Z2 ----------------------------
- movdqa xmm8, [rdx] // rk[0]
- db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8
- db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8
- db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8
- db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $45, $0F, $EF, $C9 // pxor xmm9, xmm9 (Z0=0)
- db $66, $45, $0F, $EF, $D2 // pxor xmm10, xmm10 (Z1=0)
- db $66, $45, $0F, $EF, $DB // pxor xmm11, xmm11 (Z2=0)
+ mov [rsp + 140], eax
- // --- Round 1 + GHASH iter 0 (with state fold) ------------------------
- movdqa xmm8, [rdx + 16] // rk[1]
-
- // GHASH iter 0 setup: load block 0, byte-reverse, fold running state
- movdqu xmm12, [r10] // prev cipher block 0
+@@GcmFusedLoop:
+ // Load the 8 pre-staged counter blocks: block j = J0[0..11] ||
+ // BE(counter32+j+1) (GCM's counter holds the last-used value, matching
+ // FillCtr8BlocksRaw). The words were staged a full round-chain ago.
+ movdqa xmm0, [rsp + 16]
+ movdqa xmm1, [rsp + 32]
+ movdqa xmm2, [rsp + 48]
+ movdqa xmm3, [rsp + 64]
+ movdqa xmm4, [rsp + 80]
+ movdqa xmm5, [rsp + 96]
+ movdqa xmm6, [rsp + 112]
+ movdqa xmm7, [rsp + 128]
+
+ // --- Round 0: AddRoundKey (aligned key memory operands) + zero Z1/Z2.
+ // Z0 (xmm9) still holds the carried state; iter 0 folds and clears it. ---
+ pxor xmm0, [rdx]
+ pxor xmm1, [rdx]
+ pxor xmm2, [rdx]
+ pxor xmm3, [rdx]
+ pxor xmm4, [rdx]
+ pxor xmm5, [rdx]
+ pxor xmm6, [rdx]
+ pxor xmm7, [rdx]
+ db $66, $45, $0F, $EF, $D2 // pxor xmm10, xmm10 (Z1=0)
+ db $66, $45, $0F, $EF, $DB // pxor xmm11, xmm11 (Z2=0)
+
+ // --- Round 1 + GHASH iter 0 (with carried-state fold). ---
+ movdqu xmm12, [r10 + 0] // prev cipher block 0
pshufb xmm12, xmm15 // byte-reverse
- mov rax, [rcx + 40] // rax = PFS
- movdqu xmm14, [rax] // load running state
- pshufb xmm14, xmm15 // byte-reverse state
- db $66, $45, $0F, $EF, $E6 // pxor xmm12, xmm14 (fold)
- movdqu xmm13, [r11] // H^8
+ db $66, $45, $0F, $EF, $E1 // pxor xmm12, xmm9 (fold carried state)
+ db $66, $45, $0F, $EF, $C9 // pxor xmm9, xmm9 (Z0=0)
+ movdqu xmm13, [r11 + 0] // H^8 (x-shifted)
- // Interleave 8 aesenc with 4 pclmul + 4 xor-accumulate
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0+=)
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1+=)
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1+=)
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2+=)
+ db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 2 + GHASH iter 1 ------------------------------------------
- movdqa xmm8, [rdx + 32] // rk[2]
- movdqu xmm12, [r10 + 16] // prev cipher block 1
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 16] // H^7
+ // --- Round 2 + GHASH iter 1. ---
+ movdqu xmm12, [r10 + 16] // prev cipher block 1
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 16] // H^7 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 3 + GHASH iter 2 ------------------------------------------
- movdqa xmm8, [rdx + 48] // rk[3]
- movdqu xmm12, [r10 + 32] // prev cipher block 2
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 32] // H^6
+ // --- Round 3 + GHASH iter 2. ---
+ movdqu xmm12, [r10 + 32] // prev cipher block 2
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 32] // H^6 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 4 + GHASH iter 3 ------------------------------------------
- movdqa xmm8, [rdx + 64] // rk[4]
- movdqu xmm12, [r10 + 48] // prev cipher block 3
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 48] // H^5
+ // --- Round 4 + GHASH iter 3. ---
+ movdqu xmm12, [r10 + 48] // prev cipher block 3
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 48] // H^5 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 5 + GHASH iter 4 ------------------------------------------
- movdqa xmm8, [rdx + 80] // rk[5]
- movdqu xmm12, [r10 + 64] // prev cipher block 4
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 64] // H^4
+ // --- Round 5 + GHASH iter 4. ---
+ movdqu xmm12, [r10 + 64] // prev cipher block 4
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 64] // H^4 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 6 + GHASH iter 5 ------------------------------------------
- movdqa xmm8, [rdx + 96] // rk[6]
- movdqu xmm12, [r10 + 80] // prev cipher block 5
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 80] // H^3
+ // --- Round 6 + GHASH iter 5. ---
+ movdqu xmm12, [r10 + 80] // prev cipher block 5
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 80] // H^3 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 7 + GHASH iter 6 ------------------------------------------
- movdqa xmm8, [rdx + 112] // rk[7]
- movdqu xmm12, [r10 + 96] // prev cipher block 6
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 96] // H^2
+ // --- Round 7 + GHASH iter 6. ---
+ movdqu xmm12, [r10 + 96] // prev cipher block 6
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 96] // H^2 (x-shifted)
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
+ db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
- // --- Round 8 + GHASH iter 7 (last GHASH) -----------------------------
- movdqa xmm8, [rdx + 128] // rk[8]
+ // --- Round 8 + GHASH iter 7. ---
movdqu xmm12, [r10 + 112] // prev cipher block 7
- pshufb xmm12, xmm15
- movdqu xmm13, [r11 + 112] // H^1
-
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14,xmm13,0
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14,xmm13,1
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14,xmm13,16
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- movdqa xmm14, xmm12
- db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14,xmm13,17
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
- db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14
-
- // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast ---
- // AES-128 (Nr=10): aesenc rk[9]; aesenclast on rk[10] (offset 160).
- // AES-192 (Nr=12): aesenc rk[9..11]; aesenclast on rk[12] (offset 192).
- // AES-256 (Nr=14): aesenc rk[9..13]; aesenclast on rk[14] (offset 224).
- // All three variants consume rk[9] (offset 144) as their first tail key.
-
-{$IFDEF GCM_FUSED_AES_ROUNDS_10}
- movdqa xmm8, [rdx + 144] // rk[9]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-{$ENDIF}
-
-{$IFDEF GCM_FUSED_AES_ROUNDS_12}
- movdqa xmm8, [rdx + 144] // rk[9]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 160] // rk[10]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 176] // rk[11]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-{$ENDIF}
-
+ pshufb xmm12, xmm15 // byte-reverse
+ movdqu xmm13, [r11 + 112] // H^1 (x-shifted)
+
+ db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $00 // pclmulqdq xmm14, xmm13, 0x00
+ db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128]
+ db $66, $45, $0F, $EF, $CE // pxor xmm9, xmm14 (Z0 +=)
+ db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $01 // pclmulqdq xmm14, xmm13, 0x01
+ db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $10 // pclmulqdq xmm14, xmm13, 0x10
+ db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128]
+ db $66, $45, $0F, $EF, $D6 // pxor xmm10, xmm14 (Z1 +=)
+ db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128]
+ movdqa xmm14, xmm12
+ db $66, $45, $0F, $3A, $44, $F5, $11 // pclmulqdq xmm14, xmm13, 0x11
+ db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128]
+ db $66, $45, $0F, $EF, $DE // pxor xmm11, xmm14 (Z2 +=)
+
+ // --- Variant-specific tail: aesenc rounds 9..Nr-1, then aesenclast.
+ // Round 9 interleaves the staging of the NEXT batch's counter
+ // words (counter32+9..+16) into the slots. ---
+ db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144]
+ lea eax, [r8 + 9]
+ bswap eax
+ mov [rsp + 28], eax
+ db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144]
+ lea eax, [r8 + 10]
+ bswap eax
+ mov [rsp + 44], eax
+ db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144]
+ lea eax, [r8 + 11]
+ bswap eax
+ mov [rsp + 60], eax
+ db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144]
+ lea eax, [r8 + 12]
+ bswap eax
+ mov [rsp + 76], eax
+ db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144]
+ lea eax, [r8 + 13]
+ bswap eax
+ mov [rsp + 92], eax
+ db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144]
+ lea eax, [r8 + 14]
+ bswap eax
+ mov [rsp + 108], eax
+ db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144]
+ lea eax, [r8 + 15]
+ bswap eax
+ mov [rsp + 124], eax
+ db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144]
+ lea eax, [r8 + 16]
+ bswap eax
+ mov [rsp + 140], eax
+{$IF DEFINED(GCM_FUSED_AES_ROUNDS_12) OR DEFINED(GCM_FUSED_AES_ROUNDS_14)}
+ db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160]
+ db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160]
+ db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160]
+ db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160]
+ db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160]
+ db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160]
+ db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160]
+ db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160]
+ db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176]
+ db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176]
+ db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176]
+ db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176]
+ db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176]
+ db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176]
+ db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176]
+ db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176]
+{$IFEND}
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- movdqa xmm8, [rdx + 144] // rk[9]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 160] // rk[10]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 176] // rk[11]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 192] // rk[12]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
-
- movdqa xmm8, [rdx + 208] // rk[13]
- db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8
- db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8
- db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8
- db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8
- db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8
- db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8
- db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8
- db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8
+ db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192]
+ db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192]
+ db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192]
+ db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192]
+ db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192]
+ db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192]
+ db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192]
+ db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192]
+ db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208]
+ db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208]
+ db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208]
+ db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208]
+ db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208]
+ db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208]
+ db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208]
+ db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208]
{$ENDIF}
-
- // --- Final AES round: aesenclast on rk[Nr] ---------------------------
{$IFDEF GCM_FUSED_AES_ROUNDS_10}
- movdqa xmm8, [rdx + 160] // rk[10] (last for AES-128)
+ db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160]
+ db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160]
+ db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160]
+ db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160]
+ db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160]
+ db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160]
+ db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160]
+ db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_12}
- movdqa xmm8, [rdx + 192] // rk[12] (last for AES-192)
+ db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192]
+ db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192]
+ db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192]
+ db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192]
+ db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192]
+ db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192]
+ db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192]
+ db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192]
{$ENDIF}
{$IFDEF GCM_FUSED_AES_ROUNDS_14}
- movdqa xmm8, [rdx + 224] // rk[14] (last for AES-256)
+ db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224]
+ db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224]
+ db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224]
+ db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224]
+ db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224]
+ db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224]
+ db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224]
+ db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224]
{$ENDIF}
- db $66, $41, $0F, $38, $DD, $C0 // aesenclast xmm0, xmm8
- db $66, $41, $0F, $38, $DD, $C8 // aesenclast xmm1, xmm8
- db $66, $41, $0F, $38, $DD, $D0 // aesenclast xmm2, xmm8
- db $66, $41, $0F, $38, $DD, $D8 // aesenclast xmm3, xmm8
- db $66, $41, $0F, $38, $DD, $E0 // aesenclast xmm4, xmm8
- db $66, $41, $0F, $38, $DD, $E8 // aesenclast xmm5, xmm8
- db $66, $41, $0F, $38, $DD, $F0 // aesenclast xmm6, xmm8
- db $66, $41, $0F, $38, $DD, $F8 // aesenclast xmm7, xmm8
-
- // --- Post-AES: XOR keystream with PXorIn, store to POut --------------
- mov rax, [rcx + 0] // rax = PXorIn
- mov rdx, [rcx + 8] // rdx = POut (was PRoundKeys; no longer needed)
- movdqu xmm8, [rax]
- db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8
- movdqu [rdx], xmm0
- movdqu xmm8, [rax + 16]
- db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8
- movdqu [rdx + 16], xmm1
- movdqu xmm8, [rax + 32]
- db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8
- movdqu [rdx + 32], xmm2
- movdqu xmm8, [rax + 48]
- db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8
- movdqu [rdx + 48], xmm3
- movdqu xmm8, [rax + 64]
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- movdqu [rdx + 64], xmm4
- movdqu xmm8, [rax + 80]
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- movdqu [rdx + 80], xmm5
- movdqu xmm8, [rax + 96]
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- movdqu [rdx + 96], xmm6
- movdqu xmm8, [rax + 112]
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- movdqu [rdx + 112], xmm7
-
- // --- Inline 3-limb GHASH reduction on Z0 (xmm9), Z1 (xmm10), Z2 (xmm11):
- // fold Z1 into (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1. ---
- // Migrate to xmm0/xmm1/xmm2 to mirror GcmReduce3FoldSse2 layout so the
- // scalar shift/XOR sequence below reuses xmm0..xmm7 as scratch freely.
- movdqa xmm0, xmm9 // xmm0 = Z0
- movdqa xmm1, xmm10 // xmm1 = Z1
- movdqa xmm2, xmm11 // xmm2 = Z2
-
- // Fold Z1 into Z0/Z2
+ // --- Post-AES: XOR keystream with PXorIn, store to POut. ---
+ movdqu xmm8, [rsi]
+ db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8
+ movdqu [rdi], xmm0
+ movdqu xmm8, [rsi + 16]
+ db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8
+ movdqu [rdi + 16], xmm1
+ movdqu xmm8, [rsi + 32]
+ db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8
+ movdqu [rdi + 32], xmm2
+ movdqu xmm8, [rsi + 48]
+ db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8
+ movdqu [rdi + 48], xmm3
+ movdqu xmm8, [rsi + 64]
+ db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
+ movdqu [rdi + 64], xmm4
+ movdqu xmm8, [rsi + 80]
+ db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
+ movdqu [rdi + 80], xmm5
+ movdqu xmm8, [rsi + 96]
+ db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
+ movdqu [rdi + 96], xmm6
+ movdqu xmm8, [rsi + 112]
+ db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
+ movdqu [rdi + 112], xmm7
+
+ // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into
+ // the 256-bit product, then two carry-less multiplies by the
+ // folded constant 0xC2000000_00000000 (the H powers are
+ // pre-multiplied by x, so no bit-shift correction is needed). ---
+ movdqa xmm0, xmm9 // Z0
+ movdqa xmm1, xmm10 // Z1
+ movdqa xmm2, xmm11 // Z2
+ movdqa xmm3, [rsp]
movdqa xmm4, xmm1
- movdqa xmm5, xmm1
pslldq xmm4, 8
+ pxor xmm0, xmm4 // ZLo ^= Z1 << 64
+ movdqa xmm5, xmm1
psrldq xmm5, 8
- pxor xmm0, xmm4 // Z0 ^= (Z1 << 64)
- pxor xmm2, xmm5 // Z2 ^= (Z1 >> 64)
-
- // Extract T3, T1 (low 64s); T2, T0 (upper 64s).
- movq xmm3, xmm0 // T3 = low64(Z0 folded)
- movq xmm4, xmm2 // T1 = low64(Z2 folded)
- movdqa xmm5, xmm3
- psrlq xmm5, 1
- movdqa xmm6, xmm3
- psrlq xmm6, 2
- movdqa xmm7, xmm3
- psrlq xmm7, 7
- pxor xmm4, xmm3
- pxor xmm4, xmm5
- pxor xmm4, xmm6
- pxor xmm4, xmm7 // T1 ^ T3 ^ (T3>>1)^(>>2)^(>>7)
-
- movdqa xmm1, xmm0
- psrldq xmm1, 8 // T2 = upper64(Z0 folded)
- movdqa xmm5, xmm3
- psllq xmm5, 63
- movdqa xmm6, xmm3
- psllq xmm6, 62
- movdqa xmm7, xmm3
- psllq xmm7, 57
- pxor xmm1, xmm5
- pxor xmm1, xmm6
- pxor xmm1, xmm7 // T2 ^ (T3<<63)^(<<62)^(<<57)
-
- movdqa xmm5, xmm2
- psrldq xmm5, 8 // T0 = upper64(Z2 folded)
- movdqa xmm6, xmm5
- psllq xmm6, 1
- movdqa xmm7, xmm4
- psrlq xmm7, 63
- por xmm6, xmm7 // LZ0 init = (T0<<1)|(T1>>63)
-
- movdqa xmm7, xmm4
- psllq xmm7, 1
- movdqa xmm3, xmm1
- psrlq xmm3, 63
- por xmm7, xmm3 // LZ1 init = (T1<<1)|(T2>>63)
-
- movdqa xmm0, xmm1
- psllq xmm0, 1 // LZ2 = T2<<1
-
- movdqa xmm3, xmm0
- psrlq xmm3, 1
- movdqa xmm2, xmm0
- psrlq xmm2, 2
- movdqa xmm5, xmm0
- psrlq xmm5, 7
- pxor xmm6, xmm0
- pxor xmm6, xmm3
- pxor xmm6, xmm2
- pxor xmm6, xmm5 // xmm6.low = final LZ0
-
- movdqa xmm3, xmm1
- psllq xmm3, 63
- movdqa xmm2, xmm1
- psllq xmm2, 58
- pxor xmm7, xmm3
- pxor xmm7, xmm2 // xmm7.low = final LZ1
-
- // Pack and byte-reverse final GHASH state; store to PFS.
- movq xmm0, xmm7
- pslldq xmm6, 8
- por xmm0, xmm6 // xmm0 = [LZ0 | LZ1]
- pshufb xmm0, xmm15 // byte-reverse -> canonical
- mov rax, [rcx + 40] // PFS
- movdqu [rax], xmm0
-
- // --- Advance the context for the next batch, then loop. PPrevCipher becomes
- // this batch's ciphertext (output on encrypt, input on decrypt). ---
- mov rax, [rcx + 8] // POut (out[i]); encrypt: prev = output
- cmp qword ptr [rcx + 80], 0 // GHASH-from-output flag (1 = encrypt)
- jne @@GcmPrevSet
- mov rax, [rcx + 0] // PXorIn (in[i]); decrypt: prev = input
+ pxor xmm2, xmm5 // ZHi ^= Z1 >> 64
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 1
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 2
+ pxor xmm0, xmm2 // ^= ZHi -> reflected state
+ movdqa xmm9, xmm0 // carry state to the next batch
+
+ // --- Advance for the next batch: PPrevCipher becomes this batch's
+ // ciphertext (output on encrypt, input on decrypt). ---
+ mov r10, rdi
+ test ebp, ebp // GHASH-from-output flag
+ jnz @@GcmPrevSet
+ mov r10, rsi // decrypt: prev = input
@@GcmPrevSet:
- mov [rcx + 16], rax // PPrevCipher = this batch's ciphertext
- add qword ptr [rcx + 0], 128 // PXorIn += 128
- add qword ptr [rcx + 8], 128 // POut += 128
- add r8d, 8 // counter32 += 8
- dec qword ptr [rcx + 72] // batch count
+ add rsi, 128
+ add rdi, 128
+ add r8d, 8
+ dec rbx
jnz @@GcmFusedLoop
- mov [rcx + 56], r8d // store advanced counter32
+
+ // --- Epilogue: canonicalize the state, store counter + PFS, unwind. ---
+ mov [rcx + 56], r8d // store advanced counter32
+ pshufb xmm9, xmm15 // reflected -> canonical
+ mov rax, [rcx + 40] // PFS
+ movdqu [rax], xmm9
+ add rsp, 152
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
+ pop rbp
+ pop rdi
+ pop rsi
+ pop rbx
+
diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc
index 01d10874..2f470206 100644
--- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc
@@ -1,446 +1,1135 @@
-// OCB fused 4-wide AES-NI kernel (i386) with in-kernel offset ladder,
-// L-table lookup and checksum fold.
+// OCB fused 6-wide AES-NI kernel (i386) with a round-interleaved offset
+// ladder, L-table lookup and checksum fold.
// -------------------------------------------------------------------
-// Processes ABlockCount consecutive 16-byte OCB body blocks in chunks
-// of 4 blocks (64 bytes) per iteration. 4 is the widest fused lane
-// count that fits the 8-xmm budget once xmm_offset + xmm_checksum +
-// scratch are reserved; memory-form aesenc / aesdec is therefore
-// used throughout against the 16-byte-aligned round-key schedule.
-// The kernel owns all per-block OCB bookkeeping across the batch:
-// on entry it seeds xmm_offset from OffsetPtr, xmm_checksum from
-// ChecksumPtr and a running block counter from StartBlockCount; the
-// loop derives ntz per block itself (bsf on the incremented counter),
-// XORs the L-table entry into xmm_offset to derive each Delta, runs
-// P_i XOR delta XOR K_0 through the AES rounds, XORs delta into
-// the post-aesenclast state to produce C_i, and folds plaintext
-// into xmm_checksum; on exit it writes xmm_offset and xmm_checksum
-// back. The mode layer never touches per-block state.
+// Processes ABlockCount consecutive 16-byte OCB body blocks, two 6-block
+// iterations (192 bytes) per loop pass. All 8 xmm registers are
+// productive: xmm0..5 data lanes, xmm6 offset accumulator, xmm7
+// checksum accumulator outside the rounds / L-table scratch inside them
+// (the checksum spills to an aligned frame slot across the round chain,
+// where it is never read). The offset ladder for the NEXT 6-block
+// iteration is staged between the AES round groups of the current one
+// into the opposite of two ping-pong delta banks, so the ladder's
+// serial bsf/load/xor chain rides the AES latency instead of running
+// as a prefix; the prologue pre-stages iteration 0, and the final
+// pass over-stages one iteration, undone afterwards by re-XORing the
+// same six L entries (XOR is self-inverse; the counter is not
+// persisted, so only the offset accumulator needs the rollback).
+// Whitening / unwhitening take the aligned stashes and the aligned key
+// schedule as pxor memory operands. The kernel owns all per-block OCB
+// bookkeeping across the batch: it seeds xmm_offset / xmm_checksum and
+// the 64-bit block counter on entry, derives ntz per block (bsf; the
+// counter-high dword is only touched on a low-dword wrap), and writes
+// both accumulators back on exit.
//
// Variant selection:
// CRYPTOLIB_AESNI_DECRYPT -> aesdec / aesdeclast; else aesenc / aesenclast.
// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 selects N_R.
//
// On entry (after ClpSimdProc1Begin_i386.inc): ebx = PCtx.
-//
-// Context layout (7 pointers + NativeUInt + UInt64 = 40 bytes, 4-byte aligned).
-// See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas.
-// [ebx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned)
-// [ebx + 4] InPtr : base of the main 16-byte-block stream
-// [ebx + 8] OutPtr : base of BlockCount output blocks (may alias InPtr)
-// [ebx + 12] OffsetPtr : 16-byte live FOffsetMAIN (r/w)
-// [ebx + 16] ChecksumPtr : 16-byte live FChecksum (r/w)
-// [ebx + 20] LTablePtr : flat L[0..LMax] * 16 bytes (read-only; cached on stack)
-// [ebx + 24] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr)
-// [ebx + 28] BlockCount : NativeUInt; multiple of 4
-// [ebx + 32] StartBlockCount : UInt64; block count before this span (ntz counter seed).
-// lo at +32, hi at +36.
-//
-// XMM (7/8 used; i386 has only xmm0..xmm7):
-// xmm0..xmm3 : 4 data lanes
-// xmm4 : xmm_offset accumulator (seeded/stored once per batch)
-// xmm5 : xmm_checksum accumulator (seeded/stored once per batch)
-// xmm6 : scratch (L-table loads; stack-offset reloads during whitening)
-// xmm7 : free
+// Context layout: see TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas.
+// BlockCount must be a positive multiple of 12.
//
// GPR: ebx=PCtx, edi=Keys, edx=InPtr, esi=OutPtr, ecx=BlockCount,
-// ebp=block counter low dword (bsf source), eax=scratch. The counter
-// high dword lives on the stack (only touched on a 2^32 low-dword wrap),
-// so the full 64-bit ntz is exact yet the hot path stays register-only.
+// ebp=block counter low dword (bsf source), eax=scratch.
//
-// Stack scratch (72 bytes, base unaligned - movdqu only):
-// [esp + 0..63] 4 x 16-byte per-block offset stashes
-// [esp + 64..67] LTablePtr cache (one dword; GPR budget cannot pin it)
-// [esp + 68..71] block counter high dword
+// Stack scratch (16-aligned; caller esp parked inside):
+// [esp + 0..95] delta bank 0 (6 x 16) [esp + 96..191] delta bank 1
+// [esp + 192] LTablePtr cache [esp + 196] counter high dword
+// [esp + 200] parked caller esp [esp + 204] block-0 source ptr
+// [esp + 208..223] checksum spill during the round chain
//
// AES-NI instructions are db-encoded for broad assembler compatibility.
// -------------------------------------------------------------------
- // --- Callee-save prologue. ClpSimdProc1Begin_i386.inc already pushed ebx
- // and moved PCtx into ebx, so this block only pushes the other
- // three callee-saved GPRs and reserves stack scratch. ---
push edi
push esi
push ebp
- sub esp, 72
+ mov eax, esp
+ sub esp, 240
+ and esp, -16 // align the stash frame
+ mov [esp + 200], eax // park the caller esp
- // --- Load context pointers and seed accumulators. ---
+ // --- Load context, seed accumulators and the block counter. ---
mov edi, [ebx + 0] // edi = Keys (pinned)
mov edx, [ebx + 4] // edx = InPtr (advancing)
mov esi, [ebx + 8] // esi = OutPtr (advancing)
- mov ecx, [ebx + 28] // ecx = BlockCount (>= 4, multiple of 4)
-
- // Seed the running block counter (64-bit) from StartBlockCount: low
- // dword in ebp (bsf source), high dword in the [esp + 68] stack slot.
+ mov ecx, [ebx + 28] // ecx = BlockCount (>= 12, multiple of 12)
mov ebp, [ebx + 32] // ebp = StartBlockCount low
- mov eax, [ebx + 36] // eax = StartBlockCount high
- mov [esp + 68], eax
-
- // Stash LTablePtr into the stack slot at [esp + 64]; the inner loop
- // re-adds it to ntz*16 to form each L-table entry address.
- mov eax, [ebx + 20] // eax = LTablePtr
- mov [esp + 64], eax
-
- // Seed xmm_offset from [OffsetPtr] and xmm_checksum from [ChecksumPtr].
- mov eax, [ebx + 12] // eax = OffsetPtr
- movdqu xmm4, [eax] // xmm4 = FOffsetMAIN
- mov eax, [ebx + 16] // eax = ChecksumPtr
- movdqu xmm5, [eax] // xmm5 = FChecksum
+ mov eax, [ebx + 36]
+ mov [esp + 196], eax // counter high dword
+ mov eax, [ebx + 20]
+ mov [esp + 192], eax // LTablePtr cache
+ mov eax, [ebx + 24]
+ mov [esp + 204], eax // iteration-0 block-0 source
+ mov eax, [ebx + 12]
+ movdqu xmm6, [eax] // xmm6 = FOffsetMAIN
+ mov eax, [ebx + 16]
+ movdqu xmm7, [eax] // xmm7 = FChecksum
- // Seed the block-0 pointer. eax is consumed by Phase 2 at the top
- // of each iteration (before Phase 1 clobbers it), and refreshed to
- // edx at the bottom of every iteration so iter >= 1 transparently
- // reads block 0 from the advancing main stream at offset 0.
- mov eax, [ebx + 24] // eax = Block0Ptr
+ // --- Pre-stage iteration 0's Deltas into bank 0 (lanes dead: xmm0
+ // is the table scratch here). ---
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p0
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 0], xmm6
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p1
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p1:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 16], xmm6
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p2
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p2:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 32], xmm6
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p3
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p3:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 48], xmm6
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p4
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p4:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 64], xmm6
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_p5
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_p5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ movdqa [esp + 80], xmm6
@@ocb_i386_loop:
- // --- Phase 2: load 4 input blocks. Must run BEFORE the ladder
- // because Phase 1 below clobbers eax (ntz scratch / L-table
- // index). Block 0 comes from eax (iter 0: Block0Ptr; iter >= 1:
- // main stream via the mov eax, edx at the bottom of iter k-1).
- // Blocks 1..3 always come from the advancing main stream edx
- // at [edx + 16..48]. ---
+ // --- Iteration A: load 6 blocks (block 0 via the staged source). ---
+ mov eax, [esp + 204]
movdqu xmm0, [eax]
movdqu xmm1, [edx + 16]
movdqu xmm2, [edx + 32]
movdqu xmm3, [edx + 48]
+ movdqu xmm4, [edx + 64]
+ movdqu xmm5, [edx + 80]
+
+{$IFNDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Fold plaintext into the checksum (still live in xmm7).
+ pxor xmm7, xmm0
+ pxor xmm7, xmm1
+ pxor xmm7, xmm2
+ pxor xmm7, xmm3
+ pxor xmm7, xmm4
+ pxor xmm7, xmm5
+{$ENDIF}
+ movdqa [esp + 208], xmm7 // spill checksum; xmm7 = ladder scratch
+
+ // Pre-whiten with K_0 and this iteration's stashed Deltas.
+ pxor xmm0, [edi + 0]
+ pxor xmm1, [edi + 0]
+ pxor xmm2, [edi + 0]
+ pxor xmm3, [edi + 0]
+ pxor xmm4, [edi + 0]
+ pxor xmm5, [edi + 0]
+ pxor xmm0, [esp + 0]
+ pxor xmm1, [esp + 16]
+ pxor xmm2, [esp + 32]
+ pxor xmm3, [esp + 48]
+ pxor xmm4, [esp + 64]
+ pxor xmm5, [esp + 80]
- // ------------------------------------------------------------------
- // Phase 1: offset ladder. For each block i in 0..3:
- // advance the 64-bit block counter (ebp low + [esp+68] high) by one
- // eax = ntz(counter) (bsf ebp; on a low-dword wrap, 32 + bsf hi)
- // eax = eax * 16 (shl eax, 4)
- // eax = LTablePtr + eax (add from stashed slot)
- // xmm6 = L[ntz] (movdqu from [eax])
- // xmm4 ^= xmm6 (pxor)
- // [esp + 16*i] <- xmm4 (movdqu stash of offset_i)
- // After this phase xmm4 holds offset_3 which is the Delta carried
- // into the next iteration, and [esp + 0..63] holds offsets_0..3.
- // `add ebp, 1` (not inc) sets CF so the rare high-dword carry
- // propagates; the bsf-high branch is essentially never taken.
- // ------------------------------------------------------------------
- // Block 0
+{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Round 1
+ db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+16]
+ db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16]
+ db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16]
+ db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16]
+ db $66, $0F, $38, $DE, $67, $10 // aesdec xmm4, [edi+16]
+ db $66, $0F, $38, $DE, $6F, $10 // aesdec xmm5, [edi+16]
+ // stage next-iteration Delta 0 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA0
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 96], xmm6
+ // Round 2
+ db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32]
+ db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32]
+ db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32]
+ db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32]
+ db $66, $0F, $38, $DE, $67, $20 // aesdec xmm4, [edi+32]
+ db $66, $0F, $38, $DE, $6F, $20 // aesdec xmm5, [edi+32]
+ // stage next-iteration Delta 1 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA1
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA1:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 112], xmm6
+ // Round 3
+ db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48]
+ db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48]
+ db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48]
+ db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48]
+ db $66, $0F, $38, $DE, $67, $30 // aesdec xmm4, [edi+48]
+ db $66, $0F, $38, $DE, $6F, $30 // aesdec xmm5, [edi+48]
+ // stage next-iteration Delta 2 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA2
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA2:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 128], xmm6
+ // Round 4
+ db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64]
+ db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64]
+ db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64]
+ db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64]
+ db $66, $0F, $38, $DE, $67, $40 // aesdec xmm4, [edi+64]
+ db $66, $0F, $38, $DE, $6F, $40 // aesdec xmm5, [edi+64]
+ // stage next-iteration Delta 3 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA3
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA3:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 144], xmm6
+ // Round 5
+ db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80]
+ db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80]
+ db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80]
+ db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80]
+ db $66, $0F, $38, $DE, $67, $50 // aesdec xmm4, [edi+80]
+ db $66, $0F, $38, $DE, $6F, $50 // aesdec xmm5, [edi+80]
+ // stage next-iteration Delta 4 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA4
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA4:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 160], xmm6
+ // Round 6
+ db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96]
+ db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96]
+ db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96]
+ db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96]
+ db $66, $0F, $38, $DE, $67, $60 // aesdec xmm4, [edi+96]
+ db $66, $0F, $38, $DE, $6F, $60 // aesdec xmm5, [edi+96]
+ // stage next-iteration Delta 5 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA5
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 176], xmm6
+ // Round 7
+ db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112]
+ db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112]
+ db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112]
+ db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112]
+ db $66, $0F, $38, $DE, $67, $70 // aesdec xmm4, [edi+112]
+ db $66, $0F, $38, $DE, $6F, $70 // aesdec xmm5, [edi+112]
+ // Round 8
+ db $66, $0F, $38, $DE, $87, $80, $00, $00, $00 // aesdec xmm0, [edi+128]
+ db $66, $0F, $38, $DE, $8F, $80, $00, $00, $00 // aesdec xmm1, [edi+128]
+ db $66, $0F, $38, $DE, $97, $80, $00, $00, $00 // aesdec xmm2, [edi+128]
+ db $66, $0F, $38, $DE, $9F, $80, $00, $00, $00 // aesdec xmm3, [edi+128]
+ db $66, $0F, $38, $DE, $A7, $80, $00, $00, $00 // aesdec xmm4, [edi+128]
+ db $66, $0F, $38, $DE, $AF, $80, $00, $00, $00 // aesdec xmm5, [edi+128]
+ // Round 9
+ db $66, $0F, $38, $DE, $87, $90, $00, $00, $00 // aesdec xmm0, [edi+144]
+ db $66, $0F, $38, $DE, $8F, $90, $00, $00, $00 // aesdec xmm1, [edi+144]
+ db $66, $0F, $38, $DE, $97, $90, $00, $00, $00 // aesdec xmm2, [edi+144]
+ db $66, $0F, $38, $DE, $9F, $90, $00, $00, $00 // aesdec xmm3, [edi+144]
+ db $66, $0F, $38, $DE, $A7, $90, $00, $00, $00 // aesdec xmm4, [edi+144]
+ db $66, $0F, $38, $DE, $AF, $90, $00, $00, $00 // aesdec xmm5, [edi+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DE, $87, $A0, $00, $00, $00 // aesdec xmm0, [edi+160]
+ db $66, $0F, $38, $DE, $8F, $A0, $00, $00, $00 // aesdec xmm1, [edi+160]
+ db $66, $0F, $38, $DE, $97, $A0, $00, $00, $00 // aesdec xmm2, [edi+160]
+ db $66, $0F, $38, $DE, $9F, $A0, $00, $00, $00 // aesdec xmm3, [edi+160]
+ db $66, $0F, $38, $DE, $A7, $A0, $00, $00, $00 // aesdec xmm4, [edi+160]
+ db $66, $0F, $38, $DE, $AF, $A0, $00, $00, $00 // aesdec xmm5, [edi+160]
+ db $66, $0F, $38, $DE, $87, $B0, $00, $00, $00 // aesdec xmm0, [edi+176]
+ db $66, $0F, $38, $DE, $8F, $B0, $00, $00, $00 // aesdec xmm1, [edi+176]
+ db $66, $0F, $38, $DE, $97, $B0, $00, $00, $00 // aesdec xmm2, [edi+176]
+ db $66, $0F, $38, $DE, $9F, $B0, $00, $00, $00 // aesdec xmm3, [edi+176]
+ db $66, $0F, $38, $DE, $A7, $B0, $00, $00, $00 // aesdec xmm4, [edi+176]
+ db $66, $0F, $38, $DE, $AF, $B0, $00, $00, $00 // aesdec xmm5, [edi+176]
+{$IFEND}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DE, $87, $C0, $00, $00, $00 // aesdec xmm0, [edi+192]
+ db $66, $0F, $38, $DE, $8F, $C0, $00, $00, $00 // aesdec xmm1, [edi+192]
+ db $66, $0F, $38, $DE, $97, $C0, $00, $00, $00 // aesdec xmm2, [edi+192]
+ db $66, $0F, $38, $DE, $9F, $C0, $00, $00, $00 // aesdec xmm3, [edi+192]
+ db $66, $0F, $38, $DE, $A7, $C0, $00, $00, $00 // aesdec xmm4, [edi+192]
+ db $66, $0F, $38, $DE, $AF, $C0, $00, $00, $00 // aesdec xmm5, [edi+192]
+ db $66, $0F, $38, $DE, $87, $D0, $00, $00, $00 // aesdec xmm0, [edi+208]
+ db $66, $0F, $38, $DE, $8F, $D0, $00, $00, $00 // aesdec xmm1, [edi+208]
+ db $66, $0F, $38, $DE, $97, $D0, $00, $00, $00 // aesdec xmm2, [edi+208]
+ db $66, $0F, $38, $DE, $9F, $D0, $00, $00, $00 // aesdec xmm3, [edi+208]
+ db $66, $0F, $38, $DE, $A7, $D0, $00, $00, $00 // aesdec xmm4, [edi+208]
+ db $66, $0F, $38, $DE, $AF, $D0, $00, $00, $00 // aesdec xmm5, [edi+208]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY128}
+ db $66, $0F, $38, $DF, $87, $A0, $00, $00, $00 // aesdeclast xmm0, [edi+160]
+ db $66, $0F, $38, $DF, $8F, $A0, $00, $00, $00 // aesdeclast xmm1, [edi+160]
+ db $66, $0F, $38, $DF, $97, $A0, $00, $00, $00 // aesdeclast xmm2, [edi+160]
+ db $66, $0F, $38, $DF, $9F, $A0, $00, $00, $00 // aesdeclast xmm3, [edi+160]
+ db $66, $0F, $38, $DF, $A7, $A0, $00, $00, $00 // aesdeclast xmm4, [edi+160]
+ db $66, $0F, $38, $DF, $AF, $A0, $00, $00, $00 // aesdeclast xmm5, [edi+160]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY192}
+ db $66, $0F, $38, $DF, $87, $C0, $00, $00, $00 // aesdeclast xmm0, [edi+192]
+ db $66, $0F, $38, $DF, $8F, $C0, $00, $00, $00 // aesdeclast xmm1, [edi+192]
+ db $66, $0F, $38, $DF, $97, $C0, $00, $00, $00 // aesdeclast xmm2, [edi+192]
+ db $66, $0F, $38, $DF, $9F, $C0, $00, $00, $00 // aesdeclast xmm3, [edi+192]
+ db $66, $0F, $38, $DF, $A7, $C0, $00, $00, $00 // aesdeclast xmm4, [edi+192]
+ db $66, $0F, $38, $DF, $AF, $C0, $00, $00, $00 // aesdeclast xmm5, [edi+192]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DF, $87, $E0, $00, $00, $00 // aesdeclast xmm0, [edi+224]
+ db $66, $0F, $38, $DF, $8F, $E0, $00, $00, $00 // aesdeclast xmm1, [edi+224]
+ db $66, $0F, $38, $DF, $97, $E0, $00, $00, $00 // aesdeclast xmm2, [edi+224]
+ db $66, $0F, $38, $DF, $9F, $E0, $00, $00, $00 // aesdeclast xmm3, [edi+224]
+ db $66, $0F, $38, $DF, $A7, $E0, $00, $00, $00 // aesdeclast xmm4, [edi+224]
+ db $66, $0F, $38, $DF, $AF, $E0, $00, $00, $00 // aesdeclast xmm5, [edi+224]
+{$ENDIF}
+{$ELSE}
+ // Round 1
+ db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16]
+ db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16]
+ db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16]
+ db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16]
+ db $66, $0F, $38, $DC, $67, $10 // aesenc xmm4, [edi+16]
+ db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16]
+ // stage next-iteration Delta 0 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA0
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 96], xmm6
+ // Round 2
+ db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32]
+ db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32]
+ db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32]
+ db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32]
+ db $66, $0F, $38, $DC, $67, $20 // aesenc xmm4, [edi+32]
+ db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32]
+ // stage next-iteration Delta 1 -> bank 1
add ebp, 1
- adc dword [esp + 68], 0
bsf eax, ebp
- jnz @@ocb_i386_ntz0
- bsf eax, [esp + 68]
+ jnz @@ocb_i386_sA1
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
add eax, 32
-@@ocb_i386_ntz0:
+@@ocb_i386_sA1:
shl eax, 4
- add eax, [esp + 64]
- movdqu xmm6, [eax]
- pxor xmm4, xmm6
- movdqu [esp + 0], xmm4
- // Block 1
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 112], xmm6
+ // Round 3
+ db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48]
+ db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48]
+ db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48]
+ db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48]
+ db $66, $0F, $38, $DC, $67, $30 // aesenc xmm4, [edi+48]
+ db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48]
+ // stage next-iteration Delta 2 -> bank 1
add ebp, 1
- adc dword [esp + 68], 0
bsf eax, ebp
- jnz @@ocb_i386_ntz1
- bsf eax, [esp + 68]
+ jnz @@ocb_i386_sA2
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
add eax, 32
-@@ocb_i386_ntz1:
+@@ocb_i386_sA2:
shl eax, 4
- add eax, [esp + 64]
- movdqu xmm6, [eax]
- pxor xmm4, xmm6
- movdqu [esp + 16], xmm4
- // Block 2
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 128], xmm6
+ // Round 4
+ db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64]
+ db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64]
+ db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64]
+ db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64]
+ db $66, $0F, $38, $DC, $67, $40 // aesenc xmm4, [edi+64]
+ db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64]
+ // stage next-iteration Delta 3 -> bank 1
add ebp, 1
- adc dword [esp + 68], 0
bsf eax, ebp
- jnz @@ocb_i386_ntz2
- bsf eax, [esp + 68]
+ jnz @@ocb_i386_sA3
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
add eax, 32
-@@ocb_i386_ntz2:
+@@ocb_i386_sA3:
shl eax, 4
- add eax, [esp + 64]
- movdqu xmm6, [eax]
- pxor xmm4, xmm6
- movdqu [esp + 32], xmm4
- // Block 3
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 144], xmm6
+ // Round 5
+ db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80]
+ db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80]
+ db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80]
+ db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80]
+ db $66, $0F, $38, $DC, $67, $50 // aesenc xmm4, [edi+80]
+ db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80]
+ // stage next-iteration Delta 4 -> bank 1
add ebp, 1
- adc dword [esp + 68], 0
bsf eax, ebp
- jnz @@ocb_i386_ntz3
- bsf eax, [esp + 68]
+ jnz @@ocb_i386_sA4
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
add eax, 32
-@@ocb_i386_ntz3:
+@@ocb_i386_sA4:
shl eax, 4
- add eax, [esp + 64]
- movdqu xmm6, [eax]
- pxor xmm4, xmm6
- movdqu [esp + 48], xmm4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 160], xmm6
+ // Round 6
+ db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96]
+ db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96]
+ db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96]
+ db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96]
+ db $66, $0F, $38, $DC, $67, $60 // aesenc xmm4, [edi+96]
+ db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96]
+ // stage next-iteration Delta 5 -> bank 1
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sA5
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sA5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 176], xmm6
+ // Round 7
+ db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112]
+ db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112]
+ db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112]
+ db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112]
+ db $66, $0F, $38, $DC, $67, $70 // aesenc xmm4, [edi+112]
+ db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112]
+ // Round 8
+ db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128]
+ db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128]
+ db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+128]
+ db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+128]
+ db $66, $0F, $38, $DC, $A7, $80, $00, $00, $00 // aesenc xmm4, [edi+128]
+ db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128]
+ // Round 9
+ db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144]
+ db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+144]
+ db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+144]
+ db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+144]
+ db $66, $0F, $38, $DC, $A7, $90, $00, $00, $00 // aesenc xmm4, [edi+144]
+ db $66, $0F, $38, $DC, $AF, $90, $00, $00, $00 // aesenc xmm5, [edi+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+160]
+ db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+160]
+ db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+160]
+ db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+160]
+ db $66, $0F, $38, $DC, $A7, $A0, $00, $00, $00 // aesenc xmm4, [edi+160]
+ db $66, $0F, $38, $DC, $AF, $A0, $00, $00, $00 // aesenc xmm5, [edi+160]
+ db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+176]
+ db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+176]
+ db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+176]
+ db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+176]
+ db $66, $0F, $38, $DC, $A7, $B0, $00, $00, $00 // aesenc xmm4, [edi+176]
+ db $66, $0F, $38, $DC, $AF, $B0, $00, $00, $00 // aesenc xmm5, [edi+176]
+{$IFEND}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+192]
+ db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+192]
+ db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+192]
+ db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+192]
+ db $66, $0F, $38, $DC, $A7, $C0, $00, $00, $00 // aesenc xmm4, [edi+192]
+ db $66, $0F, $38, $DC, $AF, $C0, $00, $00, $00 // aesenc xmm5, [edi+192]
+ db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+208]
+ db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+208]
+ db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+208]
+ db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+208]
+ db $66, $0F, $38, $DC, $A7, $D0, $00, $00, $00 // aesenc xmm4, [edi+208]
+ db $66, $0F, $38, $DC, $AF, $D0, $00, $00, $00 // aesenc xmm5, [edi+208]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY128}
+ db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+160]
+ db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+160]
+ db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+160]
+ db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+160]
+ db $66, $0F, $38, $DD, $A7, $A0, $00, $00, $00 // aesenclast xmm4, [edi+160]
+ db $66, $0F, $38, $DD, $AF, $A0, $00, $00, $00 // aesenclast xmm5, [edi+160]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY192}
+ db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+192]
+ db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+192]
+ db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+192]
+ db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+192]
+ db $66, $0F, $38, $DD, $A7, $C0, $00, $00, $00 // aesenclast xmm4, [edi+192]
+ db $66, $0F, $38, $DD, $AF, $C0, $00, $00, $00 // aesenclast xmm5, [edi+192]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+224]
+ db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+224]
+ db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+224]
+ db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+224]
+ db $66, $0F, $38, $DD, $A7, $E0, $00, $00, $00 // aesenclast xmm4, [edi+224]
+ db $66, $0F, $38, $DD, $AF, $E0, $00, $00, $00 // aesenclast xmm5, [edi+224]
+{$ENDIF}
+{$ENDIF}
+
+ movdqa xmm7, [esp + 208] // reload checksum
+ // Post-whiten with this iteration's Deltas.
+ pxor xmm0, [esp + 0]
+ pxor xmm1, [esp + 16]
+ pxor xmm2, [esp + 32]
+ pxor xmm3, [esp + 48]
+ pxor xmm4, [esp + 64]
+ pxor xmm5, [esp + 80]
+
+{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Fold recovered plaintext into the checksum.
+ pxor xmm7, xmm0
+ pxor xmm7, xmm1
+ pxor xmm7, xmm2
+ pxor xmm7, xmm3
+ pxor xmm7, xmm4
+ pxor xmm7, xmm5
+{$ENDIF}
+
+ movdqu [esi + 0], xmm0
+ movdqu [esi + 16], xmm1
+ movdqu [esi + 32], xmm2
+ movdqu [esi + 48], xmm3
+ movdqu [esi + 64], xmm4
+ movdqu [esi + 80], xmm5
+ add edx, 96
+ add esi, 96
+ mov [esp + 204], edx // next iteration's block-0 source
+
+ // --- Iteration B: load 6 blocks (block 0 via the staged source). ---
+ mov eax, [esp + 204]
+ movdqu xmm0, [eax]
+ movdqu xmm1, [edx + 16]
+ movdqu xmm2, [edx + 32]
+ movdqu xmm3, [edx + 48]
+ movdqu xmm4, [edx + 64]
+ movdqu xmm5, [edx + 80]
{$IFNDEF CRYPTOLIB_AESNI_DECRYPT}
- // --- Phase 3 (encrypt): fold plaintext into xmm_checksum. ---
- pxor xmm5, xmm0
- pxor xmm5, xmm1
- pxor xmm5, xmm2
- pxor xmm5, xmm3
+ // Fold plaintext into the checksum (still live in xmm7).
+ pxor xmm7, xmm0
+ pxor xmm7, xmm1
+ pxor xmm7, xmm2
+ pxor xmm7, xmm3
+ pxor xmm7, xmm4
+ pxor xmm7, xmm5
{$ENDIF}
+ movdqa [esp + 208], xmm7 // spill checksum; xmm7 = ladder scratch
- // --- Phase 4: pre-whiten with K_0 (memory-form, aligned schedule). ---
+ // Pre-whiten with K_0 and this iteration's stashed Deltas.
pxor xmm0, [edi + 0]
pxor xmm1, [edi + 0]
pxor xmm2, [edi + 0]
pxor xmm3, [edi + 0]
+ pxor xmm4, [edi + 0]
+ pxor xmm5, [edi + 0]
+ pxor xmm0, [esp + 96]
+ pxor xmm1, [esp + 112]
+ pxor xmm2, [esp + 128]
+ pxor xmm3, [esp + 144]
+ pxor xmm4, [esp + 160]
+ pxor xmm5, [esp + 176]
- // --- Phase 5: pre-whiten with stashed per-block offsets (unaligned
- // stack -> movdqu + reg-reg pxor). ---
- movdqu xmm6, [esp + 0]
- pxor xmm0, xmm6
- movdqu xmm6, [esp + 16]
- pxor xmm1, xmm6
- movdqu xmm6, [esp + 32]
- pxor xmm2, xmm6
- movdqu xmm6, [esp + 48]
- pxor xmm3, xmm6
-
- // ------------------------------------------------------------------
- // Phase 6: AES middle rounds 1..N-1 via memory-form aesenc / aesdec.
- // Key size drives the number of rounds; shared through the same
- // pattern used by the previous 6-wide kernel.
- // ------------------------------------------------------------------
{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
-
- // Round 1 (disp8 = 16)
- db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+0x10]
- db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16]
- db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16]
- db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16]
- // Round 2 (disp8 = 32)
- db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32]
- db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32]
- db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32]
- db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32]
- // Round 3 (disp8 = 48)
- db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48]
- db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48]
- db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48]
- db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48]
- // Round 4 (disp8 = 64)
- db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64]
- db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64]
- db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64]
- db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64]
- // Round 5 (disp8 = 80)
- db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80]
- db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80]
- db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80]
- db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80]
- // Round 6 (disp8 = 96)
- db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96]
- db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96]
- db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96]
- db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96]
- // Round 7 (disp8 = 112)
- db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112]
- db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112]
- db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112]
- db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112]
- // Round 8 (disp32 = 128)
+ // Round 1
+ db $66, $0F, $38, $DE, $47, $10 // aesdec xmm0, [edi+16]
+ db $66, $0F, $38, $DE, $4F, $10 // aesdec xmm1, [edi+16]
+ db $66, $0F, $38, $DE, $57, $10 // aesdec xmm2, [edi+16]
+ db $66, $0F, $38, $DE, $5F, $10 // aesdec xmm3, [edi+16]
+ db $66, $0F, $38, $DE, $67, $10 // aesdec xmm4, [edi+16]
+ db $66, $0F, $38, $DE, $6F, $10 // aesdec xmm5, [edi+16]
+ // stage next-iteration Delta 0 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB0
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 0], xmm6
+ // Round 2
+ db $66, $0F, $38, $DE, $47, $20 // aesdec xmm0, [edi+32]
+ db $66, $0F, $38, $DE, $4F, $20 // aesdec xmm1, [edi+32]
+ db $66, $0F, $38, $DE, $57, $20 // aesdec xmm2, [edi+32]
+ db $66, $0F, $38, $DE, $5F, $20 // aesdec xmm3, [edi+32]
+ db $66, $0F, $38, $DE, $67, $20 // aesdec xmm4, [edi+32]
+ db $66, $0F, $38, $DE, $6F, $20 // aesdec xmm5, [edi+32]
+ // stage next-iteration Delta 1 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB1
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB1:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 16], xmm6
+ // Round 3
+ db $66, $0F, $38, $DE, $47, $30 // aesdec xmm0, [edi+48]
+ db $66, $0F, $38, $DE, $4F, $30 // aesdec xmm1, [edi+48]
+ db $66, $0F, $38, $DE, $57, $30 // aesdec xmm2, [edi+48]
+ db $66, $0F, $38, $DE, $5F, $30 // aesdec xmm3, [edi+48]
+ db $66, $0F, $38, $DE, $67, $30 // aesdec xmm4, [edi+48]
+ db $66, $0F, $38, $DE, $6F, $30 // aesdec xmm5, [edi+48]
+ // stage next-iteration Delta 2 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB2
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB2:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 32], xmm6
+ // Round 4
+ db $66, $0F, $38, $DE, $47, $40 // aesdec xmm0, [edi+64]
+ db $66, $0F, $38, $DE, $4F, $40 // aesdec xmm1, [edi+64]
+ db $66, $0F, $38, $DE, $57, $40 // aesdec xmm2, [edi+64]
+ db $66, $0F, $38, $DE, $5F, $40 // aesdec xmm3, [edi+64]
+ db $66, $0F, $38, $DE, $67, $40 // aesdec xmm4, [edi+64]
+ db $66, $0F, $38, $DE, $6F, $40 // aesdec xmm5, [edi+64]
+ // stage next-iteration Delta 3 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB3
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB3:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 48], xmm6
+ // Round 5
+ db $66, $0F, $38, $DE, $47, $50 // aesdec xmm0, [edi+80]
+ db $66, $0F, $38, $DE, $4F, $50 // aesdec xmm1, [edi+80]
+ db $66, $0F, $38, $DE, $57, $50 // aesdec xmm2, [edi+80]
+ db $66, $0F, $38, $DE, $5F, $50 // aesdec xmm3, [edi+80]
+ db $66, $0F, $38, $DE, $67, $50 // aesdec xmm4, [edi+80]
+ db $66, $0F, $38, $DE, $6F, $50 // aesdec xmm5, [edi+80]
+ // stage next-iteration Delta 4 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB4
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB4:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 64], xmm6
+ // Round 6
+ db $66, $0F, $38, $DE, $47, $60 // aesdec xmm0, [edi+96]
+ db $66, $0F, $38, $DE, $4F, $60 // aesdec xmm1, [edi+96]
+ db $66, $0F, $38, $DE, $57, $60 // aesdec xmm2, [edi+96]
+ db $66, $0F, $38, $DE, $5F, $60 // aesdec xmm3, [edi+96]
+ db $66, $0F, $38, $DE, $67, $60 // aesdec xmm4, [edi+96]
+ db $66, $0F, $38, $DE, $6F, $60 // aesdec xmm5, [edi+96]
+ // stage next-iteration Delta 5 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB5
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 80], xmm6
+ // Round 7
+ db $66, $0F, $38, $DE, $47, $70 // aesdec xmm0, [edi+112]
+ db $66, $0F, $38, $DE, $4F, $70 // aesdec xmm1, [edi+112]
+ db $66, $0F, $38, $DE, $57, $70 // aesdec xmm2, [edi+112]
+ db $66, $0F, $38, $DE, $5F, $70 // aesdec xmm3, [edi+112]
+ db $66, $0F, $38, $DE, $67, $70 // aesdec xmm4, [edi+112]
+ db $66, $0F, $38, $DE, $6F, $70 // aesdec xmm5, [edi+112]
+ // Round 8
db $66, $0F, $38, $DE, $87, $80, $00, $00, $00 // aesdec xmm0, [edi+128]
db $66, $0F, $38, $DE, $8F, $80, $00, $00, $00 // aesdec xmm1, [edi+128]
db $66, $0F, $38, $DE, $97, $80, $00, $00, $00 // aesdec xmm2, [edi+128]
db $66, $0F, $38, $DE, $9F, $80, $00, $00, $00 // aesdec xmm3, [edi+128]
- // Round 9 (disp32 = 144)
+ db $66, $0F, $38, $DE, $A7, $80, $00, $00, $00 // aesdec xmm4, [edi+128]
+ db $66, $0F, $38, $DE, $AF, $80, $00, $00, $00 // aesdec xmm5, [edi+128]
+ // Round 9
db $66, $0F, $38, $DE, $87, $90, $00, $00, $00 // aesdec xmm0, [edi+144]
db $66, $0F, $38, $DE, $8F, $90, $00, $00, $00 // aesdec xmm1, [edi+144]
db $66, $0F, $38, $DE, $97, $90, $00, $00, $00 // aesdec xmm2, [edi+144]
db $66, $0F, $38, $DE, $9F, $90, $00, $00, $00 // aesdec xmm3, [edi+144]
-
+ db $66, $0F, $38, $DE, $A7, $90, $00, $00, $00 // aesdec xmm4, [edi+144]
+ db $66, $0F, $38, $DE, $AF, $90, $00, $00, $00 // aesdec xmm5, [edi+144]
{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
- // Round 10 (disp32 = 160)
db $66, $0F, $38, $DE, $87, $A0, $00, $00, $00 // aesdec xmm0, [edi+160]
db $66, $0F, $38, $DE, $8F, $A0, $00, $00, $00 // aesdec xmm1, [edi+160]
db $66, $0F, $38, $DE, $97, $A0, $00, $00, $00 // aesdec xmm2, [edi+160]
db $66, $0F, $38, $DE, $9F, $A0, $00, $00, $00 // aesdec xmm3, [edi+160]
- // Round 11 (disp32 = 176)
+ db $66, $0F, $38, $DE, $A7, $A0, $00, $00, $00 // aesdec xmm4, [edi+160]
+ db $66, $0F, $38, $DE, $AF, $A0, $00, $00, $00 // aesdec xmm5, [edi+160]
db $66, $0F, $38, $DE, $87, $B0, $00, $00, $00 // aesdec xmm0, [edi+176]
db $66, $0F, $38, $DE, $8F, $B0, $00, $00, $00 // aesdec xmm1, [edi+176]
db $66, $0F, $38, $DE, $97, $B0, $00, $00, $00 // aesdec xmm2, [edi+176]
db $66, $0F, $38, $DE, $9F, $B0, $00, $00, $00 // aesdec xmm3, [edi+176]
+ db $66, $0F, $38, $DE, $A7, $B0, $00, $00, $00 // aesdec xmm4, [edi+176]
+ db $66, $0F, $38, $DE, $AF, $B0, $00, $00, $00 // aesdec xmm5, [edi+176]
{$IFEND}
-
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // Round 12 (disp32 = 192)
db $66, $0F, $38, $DE, $87, $C0, $00, $00, $00 // aesdec xmm0, [edi+192]
db $66, $0F, $38, $DE, $8F, $C0, $00, $00, $00 // aesdec xmm1, [edi+192]
db $66, $0F, $38, $DE, $97, $C0, $00, $00, $00 // aesdec xmm2, [edi+192]
db $66, $0F, $38, $DE, $9F, $C0, $00, $00, $00 // aesdec xmm3, [edi+192]
- // Round 13 (disp32 = 208)
+ db $66, $0F, $38, $DE, $A7, $C0, $00, $00, $00 // aesdec xmm4, [edi+192]
+ db $66, $0F, $38, $DE, $AF, $C0, $00, $00, $00 // aesdec xmm5, [edi+192]
db $66, $0F, $38, $DE, $87, $D0, $00, $00, $00 // aesdec xmm0, [edi+208]
db $66, $0F, $38, $DE, $8F, $D0, $00, $00, $00 // aesdec xmm1, [edi+208]
db $66, $0F, $38, $DE, $97, $D0, $00, $00, $00 // aesdec xmm2, [edi+208]
db $66, $0F, $38, $DE, $9F, $D0, $00, $00, $00 // aesdec xmm3, [edi+208]
+ db $66, $0F, $38, $DE, $A7, $D0, $00, $00, $00 // aesdec xmm4, [edi+208]
+ db $66, $0F, $38, $DE, $AF, $D0, $00, $00, $00 // aesdec xmm5, [edi+208]
{$ENDIF}
-
- // --- Phase 7: final round (aesdeclast). ---
{$IFDEF CRYPTOLIB_AESNI_KEY128}
- // AES-128 final at disp32 = 160
db $66, $0F, $38, $DF, $87, $A0, $00, $00, $00 // aesdeclast xmm0, [edi+160]
db $66, $0F, $38, $DF, $8F, $A0, $00, $00, $00 // aesdeclast xmm1, [edi+160]
db $66, $0F, $38, $DF, $97, $A0, $00, $00, $00 // aesdeclast xmm2, [edi+160]
db $66, $0F, $38, $DF, $9F, $A0, $00, $00, $00 // aesdeclast xmm3, [edi+160]
+ db $66, $0F, $38, $DF, $A7, $A0, $00, $00, $00 // aesdeclast xmm4, [edi+160]
+ db $66, $0F, $38, $DF, $AF, $A0, $00, $00, $00 // aesdeclast xmm5, [edi+160]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY192}
- // AES-192 final at disp32 = 192
db $66, $0F, $38, $DF, $87, $C0, $00, $00, $00 // aesdeclast xmm0, [edi+192]
db $66, $0F, $38, $DF, $8F, $C0, $00, $00, $00 // aesdeclast xmm1, [edi+192]
db $66, $0F, $38, $DF, $97, $C0, $00, $00, $00 // aesdeclast xmm2, [edi+192]
db $66, $0F, $38, $DF, $9F, $C0, $00, $00, $00 // aesdeclast xmm3, [edi+192]
+ db $66, $0F, $38, $DF, $A7, $C0, $00, $00, $00 // aesdeclast xmm4, [edi+192]
+ db $66, $0F, $38, $DF, $AF, $C0, $00, $00, $00 // aesdeclast xmm5, [edi+192]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // AES-256 final at disp32 = 224
db $66, $0F, $38, $DF, $87, $E0, $00, $00, $00 // aesdeclast xmm0, [edi+224]
db $66, $0F, $38, $DF, $8F, $E0, $00, $00, $00 // aesdeclast xmm1, [edi+224]
db $66, $0F, $38, $DF, $97, $E0, $00, $00, $00 // aesdeclast xmm2, [edi+224]
db $66, $0F, $38, $DF, $9F, $E0, $00, $00, $00 // aesdeclast xmm3, [edi+224]
+ db $66, $0F, $38, $DF, $A7, $E0, $00, $00, $00 // aesdeclast xmm4, [edi+224]
+ db $66, $0F, $38, $DF, $AF, $E0, $00, $00, $00 // aesdeclast xmm5, [edi+224]
{$ENDIF}
-
-{$ELSE} // ====================== ENCRYPT variant ======================
-
- // Rounds 1..9 : aesenc xmm_i, [edi + 16*r] (opcode $DC)
- // Round 1 (disp8 = 16)
- db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16]
- db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16]
- db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16]
- db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16]
- // Round 2 (disp8 = 32)
- db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32]
- db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32]
- db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32]
- db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32]
- // Round 3 (disp8 = 48)
- db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48]
- db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48]
- db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48]
- db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48]
- // Round 4 (disp8 = 64)
- db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64]
- db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64]
- db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64]
- db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64]
- // Round 5 (disp8 = 80)
- db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80]
- db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80]
- db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80]
- db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80]
- // Round 6 (disp8 = 96)
- db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96]
- db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96]
- db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96]
- db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96]
- // Round 7 (disp8 = 112)
- db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112]
- db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112]
- db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112]
- db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112]
- // Round 8 (disp32 = 128)
+{$ELSE}
+ // Round 1
+ db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16]
+ db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16]
+ db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16]
+ db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16]
+ db $66, $0F, $38, $DC, $67, $10 // aesenc xmm4, [edi+16]
+ db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16]
+ // stage next-iteration Delta 0 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB0
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 0], xmm6
+ // Round 2
+ db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32]
+ db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32]
+ db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32]
+ db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32]
+ db $66, $0F, $38, $DC, $67, $20 // aesenc xmm4, [edi+32]
+ db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32]
+ // stage next-iteration Delta 1 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB1
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB1:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 16], xmm6
+ // Round 3
+ db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48]
+ db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48]
+ db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48]
+ db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48]
+ db $66, $0F, $38, $DC, $67, $30 // aesenc xmm4, [edi+48]
+ db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48]
+ // stage next-iteration Delta 2 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB2
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB2:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 32], xmm6
+ // Round 4
+ db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64]
+ db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64]
+ db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64]
+ db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64]
+ db $66, $0F, $38, $DC, $67, $40 // aesenc xmm4, [edi+64]
+ db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64]
+ // stage next-iteration Delta 3 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB3
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB3:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 48], xmm6
+ // Round 5
+ db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80]
+ db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80]
+ db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80]
+ db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80]
+ db $66, $0F, $38, $DC, $67, $50 // aesenc xmm4, [edi+80]
+ db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80]
+ // stage next-iteration Delta 4 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB4
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB4:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 64], xmm6
+ // Round 6
+ db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96]
+ db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96]
+ db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96]
+ db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96]
+ db $66, $0F, $38, $DC, $67, $60 // aesenc xmm4, [edi+96]
+ db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96]
+ // stage next-iteration Delta 5 -> bank 0
+ add ebp, 1
+ bsf eax, ebp
+ jnz @@ocb_i386_sB5
+ inc dword [esp + 196]
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_sB5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm7, [eax]
+ pxor xmm6, xmm7
+ movdqa [esp + 80], xmm6
+ // Round 7
+ db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112]
+ db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112]
+ db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112]
+ db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112]
+ db $66, $0F, $38, $DC, $67, $70 // aesenc xmm4, [edi+112]
+ db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112]
+ // Round 8
db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128]
db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128]
db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+128]
db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+128]
- // Round 9 (disp32 = 144)
+ db $66, $0F, $38, $DC, $A7, $80, $00, $00, $00 // aesenc xmm4, [edi+128]
+ db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128]
+ // Round 9
db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144]
db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+144]
db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+144]
db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+144]
-
+ db $66, $0F, $38, $DC, $A7, $90, $00, $00, $00 // aesenc xmm4, [edi+144]
+ db $66, $0F, $38, $DC, $AF, $90, $00, $00, $00 // aesenc xmm5, [edi+144]
{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
- // Round 10 (disp32 = 160)
db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+160]
db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+160]
db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+160]
db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+160]
- // Round 11 (disp32 = 176)
+ db $66, $0F, $38, $DC, $A7, $A0, $00, $00, $00 // aesenc xmm4, [edi+160]
+ db $66, $0F, $38, $DC, $AF, $A0, $00, $00, $00 // aesenc xmm5, [edi+160]
db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+176]
db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+176]
db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+176]
db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+176]
+ db $66, $0F, $38, $DC, $A7, $B0, $00, $00, $00 // aesenc xmm4, [edi+176]
+ db $66, $0F, $38, $DC, $AF, $B0, $00, $00, $00 // aesenc xmm5, [edi+176]
{$IFEND}
-
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // Round 12 (disp32 = 192)
db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+192]
db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+192]
db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+192]
db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+192]
- // Round 13 (disp32 = 208)
+ db $66, $0F, $38, $DC, $A7, $C0, $00, $00, $00 // aesenc xmm4, [edi+192]
+ db $66, $0F, $38, $DC, $AF, $C0, $00, $00, $00 // aesenc xmm5, [edi+192]
db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+208]
db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+208]
db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+208]
db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+208]
+ db $66, $0F, $38, $DC, $A7, $D0, $00, $00, $00 // aesenc xmm4, [edi+208]
+ db $66, $0F, $38, $DC, $AF, $D0, $00, $00, $00 // aesenc xmm5, [edi+208]
{$ENDIF}
-
- // --- Phase 7: final round (aesenclast). ---
{$IFDEF CRYPTOLIB_AESNI_KEY128}
- // AES-128 final at disp32 = 160
db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+160]
db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+160]
db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+160]
db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+160]
+ db $66, $0F, $38, $DD, $A7, $A0, $00, $00, $00 // aesenclast xmm4, [edi+160]
+ db $66, $0F, $38, $DD, $AF, $A0, $00, $00, $00 // aesenclast xmm5, [edi+160]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY192}
- // AES-192 final at disp32 = 192
db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+192]
db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+192]
db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+192]
db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+192]
+ db $66, $0F, $38, $DD, $A7, $C0, $00, $00, $00 // aesenclast xmm4, [edi+192]
+ db $66, $0F, $38, $DD, $AF, $C0, $00, $00, $00 // aesenclast xmm5, [edi+192]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // AES-256 final at disp32 = 224
db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+224]
db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+224]
db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+224]
db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+224]
+ db $66, $0F, $38, $DD, $A7, $E0, $00, $00, $00 // aesenclast xmm4, [edi+224]
+ db $66, $0F, $38, $DD, $AF, $E0, $00, $00, $00 // aesenclast xmm5, [edi+224]
+{$ENDIF}
{$ENDIF}
-{$ENDIF} // CRYPTOLIB_AESNI_DECRYPT
-
- // --- Phase 8: post-whiten with stashed per-block offsets. ---
- movdqu xmm6, [esp + 0]
- pxor xmm0, xmm6
- movdqu xmm6, [esp + 16]
- pxor xmm1, xmm6
- movdqu xmm6, [esp + 32]
- pxor xmm2, xmm6
- movdqu xmm6, [esp + 48]
- pxor xmm3, xmm6
+ movdqa xmm7, [esp + 208] // reload checksum
+ // Post-whiten with this iteration's Deltas.
+ pxor xmm0, [esp + 96]
+ pxor xmm1, [esp + 112]
+ pxor xmm2, [esp + 128]
+ pxor xmm3, [esp + 144]
+ pxor xmm4, [esp + 160]
+ pxor xmm5, [esp + 176]
{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
- // --- Phase 9 (decrypt): fold recovered plaintext into checksum. ---
- pxor xmm5, xmm0
- pxor xmm5, xmm1
- pxor xmm5, xmm2
- pxor xmm5, xmm3
+ // Fold recovered plaintext into the checksum.
+ pxor xmm7, xmm0
+ pxor xmm7, xmm1
+ pxor xmm7, xmm2
+ pxor xmm7, xmm3
+ pxor xmm7, xmm4
+ pxor xmm7, xmm5
{$ENDIF}
- // --- Phase 10: store 4 output blocks. ---
movdqu [esi + 0], xmm0
movdqu [esi + 16], xmm1
movdqu [esi + 32], xmm2
movdqu [esi + 48], xmm3
+ movdqu [esi + 64], xmm4
+ movdqu [esi + 80], xmm5
+ add edx, 96
+ add esi, 96
+ mov [esp + 204], edx // next iteration's block-0 source
- // --- Advance pointers and decrement loop counter. The block counter
- // (ebp + [esp+68]) is advanced per block inside the ladder above. ---
- add edx, 64
- add esi, 64
- mov eax, edx // next iter's block-0 source
- // tracks the main stream
- sub ecx, 4
+ sub ecx, 12
jnz @@ocb_i386_loop
- // --- Epilogue: flush accumulators back to memory, then unwind the
- // stack scratch and pop the callee-saved GPRs. ebx is popped
- // last because ClpSimdProc1Begin_i386.inc pushed it first. ---
- mov eax, [ebx + 12] // eax = OffsetPtr
- movdqu [eax], xmm4
- mov eax, [ebx + 16] // eax = ChecksumPtr
- movdqu [eax], xmm5
+ // --- Undo the one-iteration ladder over-stage: re-XOR the same six
+ // L entries while walking the counter back (XOR is self-inverse;
+ // only xmm_offset needs this - the counter is not persisted). ---
+ bsf eax, ebp
+ jnz @@ocb_i386_u0
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u0:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+ bsf eax, ebp
+ jnz @@ocb_i386_u1
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u1:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+ bsf eax, ebp
+ jnz @@ocb_i386_u2
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u2:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+ bsf eax, ebp
+ jnz @@ocb_i386_u3
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u3:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+ bsf eax, ebp
+ jnz @@ocb_i386_u4
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u4:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+ bsf eax, ebp
+ jnz @@ocb_i386_u5
+ bsf eax, [esp + 196]
+ add eax, 32
+@@ocb_i386_u5:
+ shl eax, 4
+ add eax, [esp + 192]
+ movdqu xmm0, [eax]
+ pxor xmm6, xmm0
+ sub ebp, 1
+ sbb dword [esp + 196], 0
+
+ // --- Epilogue: flush accumulators, unwind, pop callee-saves
+ // (ebx last: ClpSimdProc1Begin_i386.inc pushed it first). ---
+ mov eax, [ebx + 12]
+ movdqu [eax], xmm6
+ mov eax, [ebx + 16]
+ movdqu [eax], xmm7
- add esp, 72
+ mov esp, [esp + 200]
pop ebp
pop esi
pop edi
diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc
index c414e16a..638ad1bc 100644
--- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc
@@ -1,207 +1,149 @@
-// OCB fused 8-wide AES-NI kernel (x86-64) with in-kernel offset ladder,
-// L-table lookup and checksum fold.
+// OCB fused 8-wide AES-NI kernel (x86-64) with a round-interleaved offset
+// ladder, L-table lookup and checksum fold.
// -------------------------------------------------------------------
-// Processes ABlockCount consecutive 16-byte OCB body blocks in chunks
-// of 8 blocks (128 bytes) per iteration. The kernel owns all per-block
-// OCB bookkeeping across the batch: on entry it seeds xmm_offset from
-// OffsetPtr and xmm_checksum from ChecksumPtr and a running block counter
-// from StartBlockCount; the loop derives ntz per block in-register (bsf on
-// the incremented counter), XORs the L-table entry into xmm_offset
-// to derive each Delta, runs P_i XOR delta XOR K_0 through the AES
-// rounds, XORs delta into the post-aesenclast state to produce C_i,
-// and folds plaintext into xmm_checksum; on exit it writes xmm_offset
-// and xmm_checksum back. The mode layer never touches per-block state.
+// Processes ABlockCount consecutive 16-byte OCB body blocks, two 8-block
+// iterations (256 bytes) per loop pass. xmm0..7 data lanes, xmm8 offset
+// accumulator, xmm9 checksum accumulator, xmm11 L-table scratch. Round
+// keys are aesenc/aesdec memory operands against the 16-aligned
+// schedule. The offset ladder for the NEXT 8-block iteration is staged
+// between the AES round groups of the current one into the opposite of
+// two ping-pong delta banks, so the ladder's serial bsf/load/xor chain
+// rides the AES latency instead of running as a prefix; the prologue
+// pre-stages iteration 0, and the final pass over-stages one iteration,
+// undone afterwards by re-XORing the same eight L entries (XOR is
+// self-inverse; the counter is not persisted, so only the offset
+// accumulator needs the rollback). The 64-bit block counter lives whole
+// in r11, so the ladder is branch-free (bsf sees zero only at the 2^64
+// wrap, unreachable under OCB limits).
//
// Variant selection:
// CRYPTOLIB_AESNI_DECRYPT -> aesdec / aesdeclast; else aesenc / aesenclast.
// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 selects N_R.
//
-// On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx.
+// On entry (after ClpSimdProc1Begin_x86_64.inc): rcx = PCtx (pinned).
+// Context layout: see TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas.
+// BlockCount must be a positive multiple of 16.
//
-// Context layout (7 pointers + NativeUInt + UInt64 = 72 bytes, 8-byte aligned).
-// See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas.
-// [rcx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned)
-// [rcx + 8] InPtr : base of the main 16-byte-block stream
-// [rcx + 16] OutPtr : base of BlockCount output blocks (may alias InPtr)
-// [rcx + 24] OffsetPtr : 16-byte live FOffsetMAIN (r/w)
-// [rcx + 32] ChecksumPtr : 16-byte live FChecksum (r/w)
-// [rcx + 40] LTablePtr : flat L[0..LMax] * 16 bytes (read-only)
-// [rcx + 48] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr)
-// [rcx + 56] BlockCount : NativeUInt; multiple of 8
-// [rcx + 64] StartBlockCount : UInt64; block count before this span (ntz counter seed)
+// GPR: rcx=PCtx, rdx=Keys, rbx=InPtr, r8=OutPtr, r9=LTablePtr,
+// r10=pass countdown, r11=block counter, rax=scratch/block-0 source.
//
-// XMM (12/16 used):
-// xmm0..xmm7 : 8 data lanes
-// xmm8 : xmm_offset accumulator (seeded/stored once per batch)
-// xmm9 : xmm_checksum accumulator (seeded/stored once per batch)
-// xmm10 : AES round-key scratch (movdqa reload per round)
-// xmm11 : L-table load scratch
-// xmm12..xmm15 : free
+// Stack scratch (16-aligned): [rsp+0..127] delta bank 0, [rsp+128..255]
+// delta bank 1.
//
-// Stack scratch (128 bytes, 16-aligned):
-// [rsp + 0 .. 127] 8 x 16-byte per-block offset stashes (movdqa-safe)
-//
-// AES-NI instructions and selected forms touching xmm8..xmm15 (pxor,
-// movdqa) are db-encoded for broad assembler compatibility.
+// AES-NI instructions and forms touching xmm8..xmm15 are db-encoded for
+// broad assembler compatibility.
// -------------------------------------------------------------------
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- // --- Prologue: preserve rbx, reserve 128B aligned stack scratch. ---
+ // --- Prologue: preserve rbx, reserve the aligned delta banks. ---
push rbx // rsp: 8 -> 0 mod 16
- sub rsp, 128 // rsp stays 0 mod 16
+ sub rsp, 256
- // --- Load context pointers. rcx (PCtx) stays live for the
- // epilogue store-backs; everything else derives from rcx now. ---
mov rdx, [rcx + 0] // rdx = Keys
mov rbx, [rcx + 8] // rbx = InPtr
mov r8, [rcx + 16] // r8 = OutPtr
mov r9, [rcx + 40] // r9 = LTablePtr
mov r11, [rcx + 64] // r11 = running block counter
- // (seeded from StartBlockCount);
- // ntz derived per block via bsf
mov r10, [rcx + 56] // r10 = BlockCount
- shr r10, 3 // r10 /= 8 (>= 1)
+ shr r10, 4 // r10 = pass count (BlockCount/16)
- // --- Seed the persistent accumulators from memory (one-shot).
- // rax is used as a scratch for the two-hop ptr-of-ptr loads
- // and is then overwritten with Block0Ptr for the loop. ---
- mov rax, [rcx + 24] // rax = OffsetPtr
- movdqu xmm8, [rax] // xmm8 = FOffsetMAIN
- mov rax, [rcx + 32] // rax = ChecksumPtr
- movdqu xmm9, [rax] // xmm9 = FChecksum
+ mov rax, [rcx + 24]
+ db $F3, $44, $0F, $6F, $00 // movdqu xmm8, [rax] (FOffsetMAIN)
+ mov rax, [rcx + 32]
+ db $F3, $44, $0F, $6F, $08 // movdqu xmm9, [rax] (FChecksum)
- // --- Seed the block-0 pointer. rax persists across loop
- // iterations: iter 0 consumes it via Phase 2 below, and
- // every iteration's bottom refreshes it to rbx (the main
- // stream advance) so iter >= 1 loads block 0 from the
- // normal +0 offset transparently. ---
- mov rax, [rcx + 48] // rax = Block0Ptr
+ // --- Pre-stage iteration 0's Deltas into bank 0. ---
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8
+ mov rax, [rcx + 48] // iteration-0 block-0 source
@@ocb_x64_loop:
- // ------------------------------------------------------------------
- // Phase 2: load 8 input blocks into xmm0..xmm7. Must run BEFORE the
- // ladder because Phase 1 below clobbers rax (ntz scratch and
- // L-table index). On iter 0 rax points at Block0Ptr (which may be
- // unrelated to rbx, e.g. FMainBlock for full-MAC decrypt); on every
- // subsequent iter the end-of-iter epilogue sets rax := rbx (the
- // freshly advanced main stream), so `movdqu xmm0, [rax]` becomes
- // effectively identical to `movdqu xmm0, [rbx + 0]`. Blocks 1..7
- // always come from the advancing main stream at rbx + 16..112.
- // ------------------------------------------------------------------
+ // --- Iteration A: load 8 blocks (block 0 via the staged source). ---
movdqu xmm0, [rax]
- movdqu xmm1, [rbx + 16]
- movdqu xmm2, [rbx + 32]
- movdqu xmm3, [rbx + 48]
- movdqu xmm4, [rbx + 64]
- movdqu xmm5, [rbx + 80]
- movdqu xmm6, [rbx + 96]
+ movdqu xmm1, [rbx + 16]
+ movdqu xmm2, [rbx + 32]
+ movdqu xmm3, [rbx + 48]
+ movdqu xmm4, [rbx + 64]
+ movdqu xmm5, [rbx + 80]
+ movdqu xmm6, [rbx + 96]
movdqu xmm7, [rbx + 112]
- // ------------------------------------------------------------------
- // Phase 1: offset ladder. For each block i in 0..7 read ntz[i],
- // scale to a 16-byte L-table offset, XOR the entry into xmm_offset
- // (xmm8) and stash the post-XOR value into [rsp + 16*i]. After this
- // phase [rsp + 0..127] holds eight consecutive 16-byte Delta values
- // and xmm8 holds the Delta after the final block of this chunk
- // (which is also the starting Delta for the next chunk).
- //
- // Each step emits:
- // movzx eax, byte [r11 + i] ; eax = ntz[i]
- // shl rax, 4 ; rax = ntz[i] * 16
- // movdqu xmm11, [r9 + rax] ; xmm11 = L[ntz[i]]
- // pxor xmm8, xmm11 ; xmm8 ^= L[ntz[i]] (db)
- // movdqa [rsp + 16*i], xmm8 ; stash offset_i (db)
- // ------------------------------------------------------------------
- // Block 0
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $00 // movdqa [rsp+0], xmm8
- // Block 1
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8
- // Block 2
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8
- // Block 3
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8
- // Block 4
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8
- // Block 5
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8
- // Block 6
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8
- // Block 7
- inc r11
- bsf rax, r11 // rax = ntz(block counter)
- shl rax, 4
- movdqu xmm11, [r9 + rax]
- db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
- db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8
-
{$IFNDEF CRYPTOLIB_AESNI_DECRYPT}
- // --- Phase 3 (encrypt): fold plaintext into xmm_checksum BEFORE
- // any data XOR. ---
- // pxor xmm9, xmmX : REX.R only ($44), ModR/M = 11 001 rrr (reg=xmm9,
- // r/m=xmmX). bytes: $C8 $C9 $CA $CB $CC $CD $CE $CF for xmm0..xmm7.
- db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
- db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
- db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
- db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
- db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
- db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
- db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
- db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
+ // Fold plaintext into the checksum.
+ db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
+ db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
+ db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
+ db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
+ db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
+ db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
{$ENDIF}
- // --- Phase 4: pre-whiten with K_0 (aligned reg reload into xmm10
- // broadcast across all 8 data lanes via db-encoded pxor). ---
- movdqa xmm10, [rdx] // xmm10 = K_0
- // pxor xmmX, xmm10 : REX.B ($41), ModR/M = 11 rrr 010 (r/m=xmm10,
- // low-3 bits = 010). bytes for reg=xmm0..xmm7:
- // $C2 $CA $D2 $DA $E2 $EA $F2 $FA
- db $66, $41, $0F, $EF, $C2 // pxor xmm0, xmm10
- db $66, $41, $0F, $EF, $CA // pxor xmm1, xmm10
- db $66, $41, $0F, $EF, $D2 // pxor xmm2, xmm10
- db $66, $41, $0F, $EF, $DA // pxor xmm3, xmm10
- db $66, $41, $0F, $EF, $E2 // pxor xmm4, xmm10
- db $66, $41, $0F, $EF, $EA // pxor xmm5, xmm10
- db $66, $41, $0F, $EF, $F2 // pxor xmm6, xmm10
- db $66, $41, $0F, $EF, $FA // pxor xmm7, xmm10
-
- // --- Phase 5: pre-whiten with stashed per-block offsets. Stack
- // scratch is 16-aligned so memory-form pxor is safe; no REX. ---
+ // Pre-whiten with K_0 and this iteration's stashed Deltas.
+ pxor xmm0, [rdx + 0]
+ pxor xmm1, [rdx + 0]
+ pxor xmm2, [rdx + 0]
+ pxor xmm3, [rdx + 0]
+ pxor xmm4, [rdx + 0]
+ pxor xmm5, [rdx + 0]
+ pxor xmm6, [rdx + 0]
+ pxor xmm7, [rdx + 0]
pxor xmm0, [rsp + 0]
pxor xmm1, [rsp + 16]
pxor xmm2, [rsp + 32]
@@ -211,373 +153,433 @@
pxor xmm6, [rsp + 96]
pxor xmm7, [rsp + 112]
- // ------------------------------------------------------------------
- // Phase 6: AES middle rounds 1..N-1. xmm10 is reloaded per round
- // with the round key and broadcast-XORed/AES'd across all 8 data
- // lanes. Opcode byte selects direction:
- // $DC = aesenc $DE = aesdec
- // ModR/M bytes with r/m=xmm10 follow the same $C2..$FA sequence
- // used in Phase 4.
- // ------------------------------------------------------------------
{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
-
// Round 1
- movdqa xmm10, [rdx + 16]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $10 // aesdec xmm0, [rdx+16]
+ db $66, $0F, $38, $DE, $4A, $10 // aesdec xmm1, [rdx+16]
+ db $66, $0F, $38, $DE, $52, $10 // aesdec xmm2, [rdx+16]
+ db $66, $0F, $38, $DE, $5A, $10 // aesdec xmm3, [rdx+16]
+ db $66, $0F, $38, $DE, $62, $10 // aesdec xmm4, [rdx+16]
+ db $66, $0F, $38, $DE, $6A, $10 // aesdec xmm5, [rdx+16]
+ db $66, $0F, $38, $DE, $72, $10 // aesdec xmm6, [rdx+16]
+ db $66, $0F, $38, $DE, $7A, $10 // aesdec xmm7, [rdx+16]
+ // stage next-iteration Delta 0 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqa [rsp+128], xmm8
// Round 2
- movdqa xmm10, [rdx + 32]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $20 // aesdec xmm0, [rdx+32]
+ db $66, $0F, $38, $DE, $4A, $20 // aesdec xmm1, [rdx+32]
+ db $66, $0F, $38, $DE, $52, $20 // aesdec xmm2, [rdx+32]
+ db $66, $0F, $38, $DE, $5A, $20 // aesdec xmm3, [rdx+32]
+ db $66, $0F, $38, $DE, $62, $20 // aesdec xmm4, [rdx+32]
+ db $66, $0F, $38, $DE, $6A, $20 // aesdec xmm5, [rdx+32]
+ db $66, $0F, $38, $DE, $72, $20 // aesdec xmm6, [rdx+32]
+ db $66, $0F, $38, $DE, $7A, $20 // aesdec xmm7, [rdx+32]
+ // stage next-iteration Delta 1 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqa [rsp+144], xmm8
// Round 3
- movdqa xmm10, [rdx + 48]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $30 // aesdec xmm0, [rdx+48]
+ db $66, $0F, $38, $DE, $4A, $30 // aesdec xmm1, [rdx+48]
+ db $66, $0F, $38, $DE, $52, $30 // aesdec xmm2, [rdx+48]
+ db $66, $0F, $38, $DE, $5A, $30 // aesdec xmm3, [rdx+48]
+ db $66, $0F, $38, $DE, $62, $30 // aesdec xmm4, [rdx+48]
+ db $66, $0F, $38, $DE, $6A, $30 // aesdec xmm5, [rdx+48]
+ db $66, $0F, $38, $DE, $72, $30 // aesdec xmm6, [rdx+48]
+ db $66, $0F, $38, $DE, $7A, $30 // aesdec xmm7, [rdx+48]
+ // stage next-iteration Delta 2 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $A0, $00, $00, $00 // movdqa [rsp+160], xmm8
// Round 4
- movdqa xmm10, [rdx + 64]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $40 // aesdec xmm0, [rdx+64]
+ db $66, $0F, $38, $DE, $4A, $40 // aesdec xmm1, [rdx+64]
+ db $66, $0F, $38, $DE, $52, $40 // aesdec xmm2, [rdx+64]
+ db $66, $0F, $38, $DE, $5A, $40 // aesdec xmm3, [rdx+64]
+ db $66, $0F, $38, $DE, $62, $40 // aesdec xmm4, [rdx+64]
+ db $66, $0F, $38, $DE, $6A, $40 // aesdec xmm5, [rdx+64]
+ db $66, $0F, $38, $DE, $72, $40 // aesdec xmm6, [rdx+64]
+ db $66, $0F, $38, $DE, $7A, $40 // aesdec xmm7, [rdx+64]
+ // stage next-iteration Delta 3 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $B0, $00, $00, $00 // movdqa [rsp+176], xmm8
// Round 5
- movdqa xmm10, [rdx + 80]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $50 // aesdec xmm0, [rdx+80]
+ db $66, $0F, $38, $DE, $4A, $50 // aesdec xmm1, [rdx+80]
+ db $66, $0F, $38, $DE, $52, $50 // aesdec xmm2, [rdx+80]
+ db $66, $0F, $38, $DE, $5A, $50 // aesdec xmm3, [rdx+80]
+ db $66, $0F, $38, $DE, $62, $50 // aesdec xmm4, [rdx+80]
+ db $66, $0F, $38, $DE, $6A, $50 // aesdec xmm5, [rdx+80]
+ db $66, $0F, $38, $DE, $72, $50 // aesdec xmm6, [rdx+80]
+ db $66, $0F, $38, $DE, $7A, $50 // aesdec xmm7, [rdx+80]
+ // stage next-iteration Delta 4 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $C0, $00, $00, $00 // movdqa [rsp+192], xmm8
// Round 6
- movdqa xmm10, [rdx + 96]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $60 // aesdec xmm0, [rdx+96]
+ db $66, $0F, $38, $DE, $4A, $60 // aesdec xmm1, [rdx+96]
+ db $66, $0F, $38, $DE, $52, $60 // aesdec xmm2, [rdx+96]
+ db $66, $0F, $38, $DE, $5A, $60 // aesdec xmm3, [rdx+96]
+ db $66, $0F, $38, $DE, $62, $60 // aesdec xmm4, [rdx+96]
+ db $66, $0F, $38, $DE, $6A, $60 // aesdec xmm5, [rdx+96]
+ db $66, $0F, $38, $DE, $72, $60 // aesdec xmm6, [rdx+96]
+ db $66, $0F, $38, $DE, $7A, $60 // aesdec xmm7, [rdx+96]
+ // stage next-iteration Delta 5 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $D0, $00, $00, $00 // movdqa [rsp+208], xmm8
// Round 7
- movdqa xmm10, [rdx + 112]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $42, $70 // aesdec xmm0, [rdx+112]
+ db $66, $0F, $38, $DE, $4A, $70 // aesdec xmm1, [rdx+112]
+ db $66, $0F, $38, $DE, $52, $70 // aesdec xmm2, [rdx+112]
+ db $66, $0F, $38, $DE, $5A, $70 // aesdec xmm3, [rdx+112]
+ db $66, $0F, $38, $DE, $62, $70 // aesdec xmm4, [rdx+112]
+ db $66, $0F, $38, $DE, $6A, $70 // aesdec xmm5, [rdx+112]
+ db $66, $0F, $38, $DE, $72, $70 // aesdec xmm6, [rdx+112]
+ db $66, $0F, $38, $DE, $7A, $70 // aesdec xmm7, [rdx+112]
+ // stage next-iteration Delta 6 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $E0, $00, $00, $00 // movdqa [rsp+224], xmm8
// Round 8
- movdqa xmm10, [rdx + 128]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $82, $80, $00, $00, $00 // aesdec xmm0, [rdx+128]
+ db $66, $0F, $38, $DE, $8A, $80, $00, $00, $00 // aesdec xmm1, [rdx+128]
+ db $66, $0F, $38, $DE, $92, $80, $00, $00, $00 // aesdec xmm2, [rdx+128]
+ db $66, $0F, $38, $DE, $9A, $80, $00, $00, $00 // aesdec xmm3, [rdx+128]
+ db $66, $0F, $38, $DE, $A2, $80, $00, $00, $00 // aesdec xmm4, [rdx+128]
+ db $66, $0F, $38, $DE, $AA, $80, $00, $00, $00 // aesdec xmm5, [rdx+128]
+ db $66, $0F, $38, $DE, $B2, $80, $00, $00, $00 // aesdec xmm6, [rdx+128]
+ db $66, $0F, $38, $DE, $BA, $80, $00, $00, $00 // aesdec xmm7, [rdx+128]
+ // stage next-iteration Delta 7 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $F0, $00, $00, $00 // movdqa [rsp+240], xmm8
// Round 9
- movdqa xmm10, [rdx + 144]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
-
-{$IFDEF CRYPTOLIB_AESNI_KEY192}
- // Round 10
- movdqa xmm10, [rdx + 160]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
- // Round 11
- movdqa xmm10, [rdx + 176]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
-{$ENDIF}
-
+ db $66, $0F, $38, $DE, $82, $90, $00, $00, $00 // aesdec xmm0, [rdx+144]
+ db $66, $0F, $38, $DE, $8A, $90, $00, $00, $00 // aesdec xmm1, [rdx+144]
+ db $66, $0F, $38, $DE, $92, $90, $00, $00, $00 // aesdec xmm2, [rdx+144]
+ db $66, $0F, $38, $DE, $9A, $90, $00, $00, $00 // aesdec xmm3, [rdx+144]
+ db $66, $0F, $38, $DE, $A2, $90, $00, $00, $00 // aesdec xmm4, [rdx+144]
+ db $66, $0F, $38, $DE, $AA, $90, $00, $00, $00 // aesdec xmm5, [rdx+144]
+ db $66, $0F, $38, $DE, $B2, $90, $00, $00, $00 // aesdec xmm6, [rdx+144]
+ db $66, $0F, $38, $DE, $BA, $90, $00, $00, $00 // aesdec xmm7, [rdx+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DE, $82, $A0, $00, $00, $00 // aesdec xmm0, [rdx+160]
+ db $66, $0F, $38, $DE, $8A, $A0, $00, $00, $00 // aesdec xmm1, [rdx+160]
+ db $66, $0F, $38, $DE, $92, $A0, $00, $00, $00 // aesdec xmm2, [rdx+160]
+ db $66, $0F, $38, $DE, $9A, $A0, $00, $00, $00 // aesdec xmm3, [rdx+160]
+ db $66, $0F, $38, $DE, $A2, $A0, $00, $00, $00 // aesdec xmm4, [rdx+160]
+ db $66, $0F, $38, $DE, $AA, $A0, $00, $00, $00 // aesdec xmm5, [rdx+160]
+ db $66, $0F, $38, $DE, $B2, $A0, $00, $00, $00 // aesdec xmm6, [rdx+160]
+ db $66, $0F, $38, $DE, $BA, $A0, $00, $00, $00 // aesdec xmm7, [rdx+160]
+ db $66, $0F, $38, $DE, $82, $B0, $00, $00, $00 // aesdec xmm0, [rdx+176]
+ db $66, $0F, $38, $DE, $8A, $B0, $00, $00, $00 // aesdec xmm1, [rdx+176]
+ db $66, $0F, $38, $DE, $92, $B0, $00, $00, $00 // aesdec xmm2, [rdx+176]
+ db $66, $0F, $38, $DE, $9A, $B0, $00, $00, $00 // aesdec xmm3, [rdx+176]
+ db $66, $0F, $38, $DE, $A2, $B0, $00, $00, $00 // aesdec xmm4, [rdx+176]
+ db $66, $0F, $38, $DE, $AA, $B0, $00, $00, $00 // aesdec xmm5, [rdx+176]
+ db $66, $0F, $38, $DE, $B2, $B0, $00, $00, $00 // aesdec xmm6, [rdx+176]
+ db $66, $0F, $38, $DE, $BA, $B0, $00, $00, $00 // aesdec xmm7, [rdx+176]
+{$IFEND}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // Round 10
- movdqa xmm10, [rdx + 160]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
- // Round 11
- movdqa xmm10, [rdx + 176]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
- // Round 12
- movdqa xmm10, [rdx + 192]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
- // Round 13
- movdqa xmm10, [rdx + 208]
- db $66, $41, $0F, $38, $DE, $C2 // aesdec xmm0, xmm10
- db $66, $41, $0F, $38, $DE, $CA // aesdec xmm1, xmm10
- db $66, $41, $0F, $38, $DE, $D2 // aesdec xmm2, xmm10
- db $66, $41, $0F, $38, $DE, $DA // aesdec xmm3, xmm10
- db $66, $41, $0F, $38, $DE, $E2 // aesdec xmm4, xmm10
- db $66, $41, $0F, $38, $DE, $EA // aesdec xmm5, xmm10
- db $66, $41, $0F, $38, $DE, $F2 // aesdec xmm6, xmm10
- db $66, $41, $0F, $38, $DE, $FA // aesdec xmm7, xmm10
+ db $66, $0F, $38, $DE, $82, $C0, $00, $00, $00 // aesdec xmm0, [rdx+192]
+ db $66, $0F, $38, $DE, $8A, $C0, $00, $00, $00 // aesdec xmm1, [rdx+192]
+ db $66, $0F, $38, $DE, $92, $C0, $00, $00, $00 // aesdec xmm2, [rdx+192]
+ db $66, $0F, $38, $DE, $9A, $C0, $00, $00, $00 // aesdec xmm3, [rdx+192]
+ db $66, $0F, $38, $DE, $A2, $C0, $00, $00, $00 // aesdec xmm4, [rdx+192]
+ db $66, $0F, $38, $DE, $AA, $C0, $00, $00, $00 // aesdec xmm5, [rdx+192]
+ db $66, $0F, $38, $DE, $B2, $C0, $00, $00, $00 // aesdec xmm6, [rdx+192]
+ db $66, $0F, $38, $DE, $BA, $C0, $00, $00, $00 // aesdec xmm7, [rdx+192]
+ db $66, $0F, $38, $DE, $82, $D0, $00, $00, $00 // aesdec xmm0, [rdx+208]
+ db $66, $0F, $38, $DE, $8A, $D0, $00, $00, $00 // aesdec xmm1, [rdx+208]
+ db $66, $0F, $38, $DE, $92, $D0, $00, $00, $00 // aesdec xmm2, [rdx+208]
+ db $66, $0F, $38, $DE, $9A, $D0, $00, $00, $00 // aesdec xmm3, [rdx+208]
+ db $66, $0F, $38, $DE, $A2, $D0, $00, $00, $00 // aesdec xmm4, [rdx+208]
+ db $66, $0F, $38, $DE, $AA, $D0, $00, $00, $00 // aesdec xmm5, [rdx+208]
+ db $66, $0F, $38, $DE, $B2, $D0, $00, $00, $00 // aesdec xmm6, [rdx+208]
+ db $66, $0F, $38, $DE, $BA, $D0, $00, $00, $00 // aesdec xmm7, [rdx+208]
{$ENDIF}
-
- // --- Phase 7: final round. Opcode $DF = aesdeclast. ---
{$IFDEF CRYPTOLIB_AESNI_KEY128}
- movdqa xmm10, [rdx + 160]
+ db $66, $0F, $38, $DF, $82, $A0, $00, $00, $00 // aesdeclast xmm0, [rdx+160]
+ db $66, $0F, $38, $DF, $8A, $A0, $00, $00, $00 // aesdeclast xmm1, [rdx+160]
+ db $66, $0F, $38, $DF, $92, $A0, $00, $00, $00 // aesdeclast xmm2, [rdx+160]
+ db $66, $0F, $38, $DF, $9A, $A0, $00, $00, $00 // aesdeclast xmm3, [rdx+160]
+ db $66, $0F, $38, $DF, $A2, $A0, $00, $00, $00 // aesdeclast xmm4, [rdx+160]
+ db $66, $0F, $38, $DF, $AA, $A0, $00, $00, $00 // aesdeclast xmm5, [rdx+160]
+ db $66, $0F, $38, $DF, $B2, $A0, $00, $00, $00 // aesdeclast xmm6, [rdx+160]
+ db $66, $0F, $38, $DF, $BA, $A0, $00, $00, $00 // aesdeclast xmm7, [rdx+160]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY192}
- movdqa xmm10, [rdx + 192]
+ db $66, $0F, $38, $DF, $82, $C0, $00, $00, $00 // aesdeclast xmm0, [rdx+192]
+ db $66, $0F, $38, $DF, $8A, $C0, $00, $00, $00 // aesdeclast xmm1, [rdx+192]
+ db $66, $0F, $38, $DF, $92, $C0, $00, $00, $00 // aesdeclast xmm2, [rdx+192]
+ db $66, $0F, $38, $DF, $9A, $C0, $00, $00, $00 // aesdeclast xmm3, [rdx+192]
+ db $66, $0F, $38, $DF, $A2, $C0, $00, $00, $00 // aesdeclast xmm4, [rdx+192]
+ db $66, $0F, $38, $DF, $AA, $C0, $00, $00, $00 // aesdeclast xmm5, [rdx+192]
+ db $66, $0F, $38, $DF, $B2, $C0, $00, $00, $00 // aesdeclast xmm6, [rdx+192]
+ db $66, $0F, $38, $DF, $BA, $C0, $00, $00, $00 // aesdeclast xmm7, [rdx+192]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- movdqa xmm10, [rdx + 224]
+ db $66, $0F, $38, $DF, $82, $E0, $00, $00, $00 // aesdeclast xmm0, [rdx+224]
+ db $66, $0F, $38, $DF, $8A, $E0, $00, $00, $00 // aesdeclast xmm1, [rdx+224]
+ db $66, $0F, $38, $DF, $92, $E0, $00, $00, $00 // aesdeclast xmm2, [rdx+224]
+ db $66, $0F, $38, $DF, $9A, $E0, $00, $00, $00 // aesdeclast xmm3, [rdx+224]
+ db $66, $0F, $38, $DF, $A2, $E0, $00, $00, $00 // aesdeclast xmm4, [rdx+224]
+ db $66, $0F, $38, $DF, $AA, $E0, $00, $00, $00 // aesdeclast xmm5, [rdx+224]
+ db $66, $0F, $38, $DF, $B2, $E0, $00, $00, $00 // aesdeclast xmm6, [rdx+224]
+ db $66, $0F, $38, $DF, $BA, $E0, $00, $00, $00 // aesdeclast xmm7, [rdx+224]
{$ENDIF}
- db $66, $41, $0F, $38, $DF, $C2 // aesdeclast xmm0, xmm10
- db $66, $41, $0F, $38, $DF, $CA // aesdeclast xmm1, xmm10
- db $66, $41, $0F, $38, $DF, $D2 // aesdeclast xmm2, xmm10
- db $66, $41, $0F, $38, $DF, $DA // aesdeclast xmm3, xmm10
- db $66, $41, $0F, $38, $DF, $E2 // aesdeclast xmm4, xmm10
- db $66, $41, $0F, $38, $DF, $EA // aesdeclast xmm5, xmm10
- db $66, $41, $0F, $38, $DF, $F2 // aesdeclast xmm6, xmm10
- db $66, $41, $0F, $38, $DF, $FA // aesdeclast xmm7, xmm10
-
-{$ELSE} // ===================== ENCRYPT variant =====================
-
+{$ELSE}
// Round 1
- movdqa xmm10, [rdx + 16]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16]
+ db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16]
+ db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16]
+ db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16]
+ db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16]
+ db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16]
+ db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16]
+ db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16]
+ // stage next-iteration Delta 0 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqa [rsp+128], xmm8
// Round 2
- movdqa xmm10, [rdx + 32]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32]
+ db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32]
+ db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32]
+ db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32]
+ db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32]
+ db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32]
+ db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32]
+ db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32]
+ // stage next-iteration Delta 1 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqa [rsp+144], xmm8
// Round 3
- movdqa xmm10, [rdx + 48]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48]
+ db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48]
+ db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48]
+ db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48]
+ db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48]
+ db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48]
+ db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48]
+ db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48]
+ // stage next-iteration Delta 2 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $A0, $00, $00, $00 // movdqa [rsp+160], xmm8
// Round 4
- movdqa xmm10, [rdx + 64]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64]
+ db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64]
+ db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64]
+ db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64]
+ db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64]
+ db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64]
+ db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64]
+ db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64]
+ // stage next-iteration Delta 3 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $B0, $00, $00, $00 // movdqa [rsp+176], xmm8
// Round 5
- movdqa xmm10, [rdx + 80]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80]
+ db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80]
+ db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80]
+ db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80]
+ db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80]
+ db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80]
+ db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80]
+ db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80]
+ // stage next-iteration Delta 4 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $C0, $00, $00, $00 // movdqa [rsp+192], xmm8
// Round 6
- movdqa xmm10, [rdx + 96]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96]
+ db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96]
+ db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96]
+ db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96]
+ db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96]
+ db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96]
+ db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96]
+ db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96]
+ // stage next-iteration Delta 5 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $D0, $00, $00, $00 // movdqa [rsp+208], xmm8
// Round 7
- movdqa xmm10, [rdx + 112]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112]
+ db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112]
+ db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112]
+ db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112]
+ db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112]
+ db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112]
+ db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112]
+ db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112]
+ // stage next-iteration Delta 6 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $E0, $00, $00, $00 // movdqa [rsp+224], xmm8
// Round 8
- movdqa xmm10, [rdx + 128]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128]
+ db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128]
+ db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128]
+ db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128]
+ db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128]
+ db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128]
+ db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128]
+ db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128]
+ // stage next-iteration Delta 7 -> bank 1
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $84, $24, $F0, $00, $00, $00 // movdqa [rsp+240], xmm8
// Round 9
- movdqa xmm10, [rdx + 144]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
-
-{$IFDEF CRYPTOLIB_AESNI_KEY192}
- // Round 10
- movdqa xmm10, [rdx + 160]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
- // Round 11
- movdqa xmm10, [rdx + 176]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
-{$ENDIF}
-
+ db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144]
+ db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144]
+ db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144]
+ db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144]
+ db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144]
+ db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144]
+ db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144]
+ db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160]
+ db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160]
+ db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160]
+ db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160]
+ db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160]
+ db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160]
+ db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160]
+ db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160]
+ db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176]
+ db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176]
+ db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176]
+ db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176]
+ db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176]
+ db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176]
+ db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176]
+ db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176]
+{$IFEND}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- // Round 10
- movdqa xmm10, [rdx + 160]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
- // Round 11
- movdqa xmm10, [rdx + 176]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
- // Round 12
- movdqa xmm10, [rdx + 192]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
- // Round 13
- movdqa xmm10, [rdx + 208]
- db $66, $41, $0F, $38, $DC, $C2 // aesenc xmm0, xmm10
- db $66, $41, $0F, $38, $DC, $CA // aesenc xmm1, xmm10
- db $66, $41, $0F, $38, $DC, $D2 // aesenc xmm2, xmm10
- db $66, $41, $0F, $38, $DC, $DA // aesenc xmm3, xmm10
- db $66, $41, $0F, $38, $DC, $E2 // aesenc xmm4, xmm10
- db $66, $41, $0F, $38, $DC, $EA // aesenc xmm5, xmm10
- db $66, $41, $0F, $38, $DC, $F2 // aesenc xmm6, xmm10
- db $66, $41, $0F, $38, $DC, $FA // aesenc xmm7, xmm10
+ db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192]
+ db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192]
+ db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192]
+ db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192]
+ db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192]
+ db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192]
+ db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192]
+ db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192]
+ db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208]
+ db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208]
+ db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208]
+ db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208]
+ db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208]
+ db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208]
+ db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208]
+ db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208]
{$ENDIF}
-
- // --- Phase 7: final round. Opcode $DD = aesenclast. ---
{$IFDEF CRYPTOLIB_AESNI_KEY128}
- movdqa xmm10, [rdx + 160]
+ db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160]
+ db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160]
+ db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160]
+ db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160]
+ db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160]
+ db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160]
+ db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160]
+ db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY192}
- movdqa xmm10, [rdx + 192]
+ db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192]
+ db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192]
+ db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192]
+ db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192]
+ db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192]
+ db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192]
+ db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192]
+ db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192]
{$ENDIF}
{$IFDEF CRYPTOLIB_AESNI_KEY256}
- movdqa xmm10, [rdx + 224]
+ db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224]
+ db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224]
+ db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224]
+ db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224]
+ db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224]
+ db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224]
+ db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224]
+ db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224]
{$ENDIF}
- db $66, $41, $0F, $38, $DD, $C2 // aesenclast xmm0, xmm10
- db $66, $41, $0F, $38, $DD, $CA // aesenclast xmm1, xmm10
- db $66, $41, $0F, $38, $DD, $D2 // aesenclast xmm2, xmm10
- db $66, $41, $0F, $38, $DD, $DA // aesenclast xmm3, xmm10
- db $66, $41, $0F, $38, $DD, $E2 // aesenclast xmm4, xmm10
- db $66, $41, $0F, $38, $DD, $EA // aesenclast xmm5, xmm10
- db $66, $41, $0F, $38, $DD, $F2 // aesenclast xmm6, xmm10
- db $66, $41, $0F, $38, $DD, $FA // aesenclast xmm7, xmm10
-
{$ENDIF}
- // --- Phase 8: post-whiten with stashed per-block offsets. ---
+ // Post-whiten with this iteration's Deltas.
pxor xmm0, [rsp + 0]
pxor xmm1, [rsp + 16]
pxor xmm2, [rsp + 32]
@@ -588,19 +590,17 @@
pxor xmm7, [rsp + 112]
{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
- // --- Phase 9 (decrypt): fold recovered plaintext into xmm_checksum
- // AFTER post-whiten. ---
- db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
- db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
- db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
- db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
- db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
- db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
- db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
- db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
+ // Fold recovered plaintext into the checksum.
+ db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
+ db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
+ db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
+ db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
+ db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
+ db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
{$ENDIF}
- // --- Phase 10: store 8 output blocks. ---
movdqu [r8 + 0], xmm0
movdqu [r8 + 16], xmm1
movdqu [r8 + 32], xmm2
@@ -609,23 +609,570 @@
movdqu [r8 + 80], xmm5
movdqu [r8 + 96], xmm6
movdqu [r8 + 112], xmm7
+ add rbx, 128
+ add r8, 128
+ mov rax, rbx // next iteration's block-0 source
+
+ // --- Iteration B: load 8 blocks (block 0 via the staged source). ---
+ movdqu xmm0, [rax]
+ movdqu xmm1, [rbx + 16]
+ movdqu xmm2, [rbx + 32]
+ movdqu xmm3, [rbx + 48]
+ movdqu xmm4, [rbx + 64]
+ movdqu xmm5, [rbx + 80]
+ movdqu xmm6, [rbx + 96]
+ movdqu xmm7, [rbx + 112]
+
+{$IFNDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Fold plaintext into the checksum.
+ db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
+ db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
+ db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
+ db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
+ db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
+ db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
+{$ENDIF}
+
+ // Pre-whiten with K_0 and this iteration's stashed Deltas.
+ pxor xmm0, [rdx + 0]
+ pxor xmm1, [rdx + 0]
+ pxor xmm2, [rdx + 0]
+ pxor xmm3, [rdx + 0]
+ pxor xmm4, [rdx + 0]
+ pxor xmm5, [rdx + 0]
+ pxor xmm6, [rdx + 0]
+ pxor xmm7, [rdx + 0]
+ pxor xmm0, [rsp + 128]
+ pxor xmm1, [rsp + 144]
+ pxor xmm2, [rsp + 160]
+ pxor xmm3, [rsp + 176]
+ pxor xmm4, [rsp + 192]
+ pxor xmm5, [rsp + 208]
+ pxor xmm6, [rsp + 224]
+ pxor xmm7, [rsp + 240]
+
+{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Round 1
+ db $66, $0F, $38, $DE, $42, $10 // aesdec xmm0, [rdx+16]
+ db $66, $0F, $38, $DE, $4A, $10 // aesdec xmm1, [rdx+16]
+ db $66, $0F, $38, $DE, $52, $10 // aesdec xmm2, [rdx+16]
+ db $66, $0F, $38, $DE, $5A, $10 // aesdec xmm3, [rdx+16]
+ db $66, $0F, $38, $DE, $62, $10 // aesdec xmm4, [rdx+16]
+ db $66, $0F, $38, $DE, $6A, $10 // aesdec xmm5, [rdx+16]
+ db $66, $0F, $38, $DE, $72, $10 // aesdec xmm6, [rdx+16]
+ db $66, $0F, $38, $DE, $7A, $10 // aesdec xmm7, [rdx+16]
+ // stage next-iteration Delta 0 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8
+ // Round 2
+ db $66, $0F, $38, $DE, $42, $20 // aesdec xmm0, [rdx+32]
+ db $66, $0F, $38, $DE, $4A, $20 // aesdec xmm1, [rdx+32]
+ db $66, $0F, $38, $DE, $52, $20 // aesdec xmm2, [rdx+32]
+ db $66, $0F, $38, $DE, $5A, $20 // aesdec xmm3, [rdx+32]
+ db $66, $0F, $38, $DE, $62, $20 // aesdec xmm4, [rdx+32]
+ db $66, $0F, $38, $DE, $6A, $20 // aesdec xmm5, [rdx+32]
+ db $66, $0F, $38, $DE, $72, $20 // aesdec xmm6, [rdx+32]
+ db $66, $0F, $38, $DE, $7A, $20 // aesdec xmm7, [rdx+32]
+ // stage next-iteration Delta 1 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8
+ // Round 3
+ db $66, $0F, $38, $DE, $42, $30 // aesdec xmm0, [rdx+48]
+ db $66, $0F, $38, $DE, $4A, $30 // aesdec xmm1, [rdx+48]
+ db $66, $0F, $38, $DE, $52, $30 // aesdec xmm2, [rdx+48]
+ db $66, $0F, $38, $DE, $5A, $30 // aesdec xmm3, [rdx+48]
+ db $66, $0F, $38, $DE, $62, $30 // aesdec xmm4, [rdx+48]
+ db $66, $0F, $38, $DE, $6A, $30 // aesdec xmm5, [rdx+48]
+ db $66, $0F, $38, $DE, $72, $30 // aesdec xmm6, [rdx+48]
+ db $66, $0F, $38, $DE, $7A, $30 // aesdec xmm7, [rdx+48]
+ // stage next-iteration Delta 2 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8
+ // Round 4
+ db $66, $0F, $38, $DE, $42, $40 // aesdec xmm0, [rdx+64]
+ db $66, $0F, $38, $DE, $4A, $40 // aesdec xmm1, [rdx+64]
+ db $66, $0F, $38, $DE, $52, $40 // aesdec xmm2, [rdx+64]
+ db $66, $0F, $38, $DE, $5A, $40 // aesdec xmm3, [rdx+64]
+ db $66, $0F, $38, $DE, $62, $40 // aesdec xmm4, [rdx+64]
+ db $66, $0F, $38, $DE, $6A, $40 // aesdec xmm5, [rdx+64]
+ db $66, $0F, $38, $DE, $72, $40 // aesdec xmm6, [rdx+64]
+ db $66, $0F, $38, $DE, $7A, $40 // aesdec xmm7, [rdx+64]
+ // stage next-iteration Delta 3 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8
+ // Round 5
+ db $66, $0F, $38, $DE, $42, $50 // aesdec xmm0, [rdx+80]
+ db $66, $0F, $38, $DE, $4A, $50 // aesdec xmm1, [rdx+80]
+ db $66, $0F, $38, $DE, $52, $50 // aesdec xmm2, [rdx+80]
+ db $66, $0F, $38, $DE, $5A, $50 // aesdec xmm3, [rdx+80]
+ db $66, $0F, $38, $DE, $62, $50 // aesdec xmm4, [rdx+80]
+ db $66, $0F, $38, $DE, $6A, $50 // aesdec xmm5, [rdx+80]
+ db $66, $0F, $38, $DE, $72, $50 // aesdec xmm6, [rdx+80]
+ db $66, $0F, $38, $DE, $7A, $50 // aesdec xmm7, [rdx+80]
+ // stage next-iteration Delta 4 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8
+ // Round 6
+ db $66, $0F, $38, $DE, $42, $60 // aesdec xmm0, [rdx+96]
+ db $66, $0F, $38, $DE, $4A, $60 // aesdec xmm1, [rdx+96]
+ db $66, $0F, $38, $DE, $52, $60 // aesdec xmm2, [rdx+96]
+ db $66, $0F, $38, $DE, $5A, $60 // aesdec xmm3, [rdx+96]
+ db $66, $0F, $38, $DE, $62, $60 // aesdec xmm4, [rdx+96]
+ db $66, $0F, $38, $DE, $6A, $60 // aesdec xmm5, [rdx+96]
+ db $66, $0F, $38, $DE, $72, $60 // aesdec xmm6, [rdx+96]
+ db $66, $0F, $38, $DE, $7A, $60 // aesdec xmm7, [rdx+96]
+ // stage next-iteration Delta 5 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8
+ // Round 7
+ db $66, $0F, $38, $DE, $42, $70 // aesdec xmm0, [rdx+112]
+ db $66, $0F, $38, $DE, $4A, $70 // aesdec xmm1, [rdx+112]
+ db $66, $0F, $38, $DE, $52, $70 // aesdec xmm2, [rdx+112]
+ db $66, $0F, $38, $DE, $5A, $70 // aesdec xmm3, [rdx+112]
+ db $66, $0F, $38, $DE, $62, $70 // aesdec xmm4, [rdx+112]
+ db $66, $0F, $38, $DE, $6A, $70 // aesdec xmm5, [rdx+112]
+ db $66, $0F, $38, $DE, $72, $70 // aesdec xmm6, [rdx+112]
+ db $66, $0F, $38, $DE, $7A, $70 // aesdec xmm7, [rdx+112]
+ // stage next-iteration Delta 6 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8
+ // Round 8
+ db $66, $0F, $38, $DE, $82, $80, $00, $00, $00 // aesdec xmm0, [rdx+128]
+ db $66, $0F, $38, $DE, $8A, $80, $00, $00, $00 // aesdec xmm1, [rdx+128]
+ db $66, $0F, $38, $DE, $92, $80, $00, $00, $00 // aesdec xmm2, [rdx+128]
+ db $66, $0F, $38, $DE, $9A, $80, $00, $00, $00 // aesdec xmm3, [rdx+128]
+ db $66, $0F, $38, $DE, $A2, $80, $00, $00, $00 // aesdec xmm4, [rdx+128]
+ db $66, $0F, $38, $DE, $AA, $80, $00, $00, $00 // aesdec xmm5, [rdx+128]
+ db $66, $0F, $38, $DE, $B2, $80, $00, $00, $00 // aesdec xmm6, [rdx+128]
+ db $66, $0F, $38, $DE, $BA, $80, $00, $00, $00 // aesdec xmm7, [rdx+128]
+ // stage next-iteration Delta 7 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8
+ // Round 9
+ db $66, $0F, $38, $DE, $82, $90, $00, $00, $00 // aesdec xmm0, [rdx+144]
+ db $66, $0F, $38, $DE, $8A, $90, $00, $00, $00 // aesdec xmm1, [rdx+144]
+ db $66, $0F, $38, $DE, $92, $90, $00, $00, $00 // aesdec xmm2, [rdx+144]
+ db $66, $0F, $38, $DE, $9A, $90, $00, $00, $00 // aesdec xmm3, [rdx+144]
+ db $66, $0F, $38, $DE, $A2, $90, $00, $00, $00 // aesdec xmm4, [rdx+144]
+ db $66, $0F, $38, $DE, $AA, $90, $00, $00, $00 // aesdec xmm5, [rdx+144]
+ db $66, $0F, $38, $DE, $B2, $90, $00, $00, $00 // aesdec xmm6, [rdx+144]
+ db $66, $0F, $38, $DE, $BA, $90, $00, $00, $00 // aesdec xmm7, [rdx+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DE, $82, $A0, $00, $00, $00 // aesdec xmm0, [rdx+160]
+ db $66, $0F, $38, $DE, $8A, $A0, $00, $00, $00 // aesdec xmm1, [rdx+160]
+ db $66, $0F, $38, $DE, $92, $A0, $00, $00, $00 // aesdec xmm2, [rdx+160]
+ db $66, $0F, $38, $DE, $9A, $A0, $00, $00, $00 // aesdec xmm3, [rdx+160]
+ db $66, $0F, $38, $DE, $A2, $A0, $00, $00, $00 // aesdec xmm4, [rdx+160]
+ db $66, $0F, $38, $DE, $AA, $A0, $00, $00, $00 // aesdec xmm5, [rdx+160]
+ db $66, $0F, $38, $DE, $B2, $A0, $00, $00, $00 // aesdec xmm6, [rdx+160]
+ db $66, $0F, $38, $DE, $BA, $A0, $00, $00, $00 // aesdec xmm7, [rdx+160]
+ db $66, $0F, $38, $DE, $82, $B0, $00, $00, $00 // aesdec xmm0, [rdx+176]
+ db $66, $0F, $38, $DE, $8A, $B0, $00, $00, $00 // aesdec xmm1, [rdx+176]
+ db $66, $0F, $38, $DE, $92, $B0, $00, $00, $00 // aesdec xmm2, [rdx+176]
+ db $66, $0F, $38, $DE, $9A, $B0, $00, $00, $00 // aesdec xmm3, [rdx+176]
+ db $66, $0F, $38, $DE, $A2, $B0, $00, $00, $00 // aesdec xmm4, [rdx+176]
+ db $66, $0F, $38, $DE, $AA, $B0, $00, $00, $00 // aesdec xmm5, [rdx+176]
+ db $66, $0F, $38, $DE, $B2, $B0, $00, $00, $00 // aesdec xmm6, [rdx+176]
+ db $66, $0F, $38, $DE, $BA, $B0, $00, $00, $00 // aesdec xmm7, [rdx+176]
+{$IFEND}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DE, $82, $C0, $00, $00, $00 // aesdec xmm0, [rdx+192]
+ db $66, $0F, $38, $DE, $8A, $C0, $00, $00, $00 // aesdec xmm1, [rdx+192]
+ db $66, $0F, $38, $DE, $92, $C0, $00, $00, $00 // aesdec xmm2, [rdx+192]
+ db $66, $0F, $38, $DE, $9A, $C0, $00, $00, $00 // aesdec xmm3, [rdx+192]
+ db $66, $0F, $38, $DE, $A2, $C0, $00, $00, $00 // aesdec xmm4, [rdx+192]
+ db $66, $0F, $38, $DE, $AA, $C0, $00, $00, $00 // aesdec xmm5, [rdx+192]
+ db $66, $0F, $38, $DE, $B2, $C0, $00, $00, $00 // aesdec xmm6, [rdx+192]
+ db $66, $0F, $38, $DE, $BA, $C0, $00, $00, $00 // aesdec xmm7, [rdx+192]
+ db $66, $0F, $38, $DE, $82, $D0, $00, $00, $00 // aesdec xmm0, [rdx+208]
+ db $66, $0F, $38, $DE, $8A, $D0, $00, $00, $00 // aesdec xmm1, [rdx+208]
+ db $66, $0F, $38, $DE, $92, $D0, $00, $00, $00 // aesdec xmm2, [rdx+208]
+ db $66, $0F, $38, $DE, $9A, $D0, $00, $00, $00 // aesdec xmm3, [rdx+208]
+ db $66, $0F, $38, $DE, $A2, $D0, $00, $00, $00 // aesdec xmm4, [rdx+208]
+ db $66, $0F, $38, $DE, $AA, $D0, $00, $00, $00 // aesdec xmm5, [rdx+208]
+ db $66, $0F, $38, $DE, $B2, $D0, $00, $00, $00 // aesdec xmm6, [rdx+208]
+ db $66, $0F, $38, $DE, $BA, $D0, $00, $00, $00 // aesdec xmm7, [rdx+208]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY128}
+ db $66, $0F, $38, $DF, $82, $A0, $00, $00, $00 // aesdeclast xmm0, [rdx+160]
+ db $66, $0F, $38, $DF, $8A, $A0, $00, $00, $00 // aesdeclast xmm1, [rdx+160]
+ db $66, $0F, $38, $DF, $92, $A0, $00, $00, $00 // aesdeclast xmm2, [rdx+160]
+ db $66, $0F, $38, $DF, $9A, $A0, $00, $00, $00 // aesdeclast xmm3, [rdx+160]
+ db $66, $0F, $38, $DF, $A2, $A0, $00, $00, $00 // aesdeclast xmm4, [rdx+160]
+ db $66, $0F, $38, $DF, $AA, $A0, $00, $00, $00 // aesdeclast xmm5, [rdx+160]
+ db $66, $0F, $38, $DF, $B2, $A0, $00, $00, $00 // aesdeclast xmm6, [rdx+160]
+ db $66, $0F, $38, $DF, $BA, $A0, $00, $00, $00 // aesdeclast xmm7, [rdx+160]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY192}
+ db $66, $0F, $38, $DF, $82, $C0, $00, $00, $00 // aesdeclast xmm0, [rdx+192]
+ db $66, $0F, $38, $DF, $8A, $C0, $00, $00, $00 // aesdeclast xmm1, [rdx+192]
+ db $66, $0F, $38, $DF, $92, $C0, $00, $00, $00 // aesdeclast xmm2, [rdx+192]
+ db $66, $0F, $38, $DF, $9A, $C0, $00, $00, $00 // aesdeclast xmm3, [rdx+192]
+ db $66, $0F, $38, $DF, $A2, $C0, $00, $00, $00 // aesdeclast xmm4, [rdx+192]
+ db $66, $0F, $38, $DF, $AA, $C0, $00, $00, $00 // aesdeclast xmm5, [rdx+192]
+ db $66, $0F, $38, $DF, $B2, $C0, $00, $00, $00 // aesdeclast xmm6, [rdx+192]
+ db $66, $0F, $38, $DF, $BA, $C0, $00, $00, $00 // aesdeclast xmm7, [rdx+192]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DF, $82, $E0, $00, $00, $00 // aesdeclast xmm0, [rdx+224]
+ db $66, $0F, $38, $DF, $8A, $E0, $00, $00, $00 // aesdeclast xmm1, [rdx+224]
+ db $66, $0F, $38, $DF, $92, $E0, $00, $00, $00 // aesdeclast xmm2, [rdx+224]
+ db $66, $0F, $38, $DF, $9A, $E0, $00, $00, $00 // aesdeclast xmm3, [rdx+224]
+ db $66, $0F, $38, $DF, $A2, $E0, $00, $00, $00 // aesdeclast xmm4, [rdx+224]
+ db $66, $0F, $38, $DF, $AA, $E0, $00, $00, $00 // aesdeclast xmm5, [rdx+224]
+ db $66, $0F, $38, $DF, $B2, $E0, $00, $00, $00 // aesdeclast xmm6, [rdx+224]
+ db $66, $0F, $38, $DF, $BA, $E0, $00, $00, $00 // aesdeclast xmm7, [rdx+224]
+{$ENDIF}
+{$ELSE}
+ // Round 1
+ db $66, $0F, $38, $DC, $42, $10 // aesenc xmm0, [rdx+16]
+ db $66, $0F, $38, $DC, $4A, $10 // aesenc xmm1, [rdx+16]
+ db $66, $0F, $38, $DC, $52, $10 // aesenc xmm2, [rdx+16]
+ db $66, $0F, $38, $DC, $5A, $10 // aesenc xmm3, [rdx+16]
+ db $66, $0F, $38, $DC, $62, $10 // aesenc xmm4, [rdx+16]
+ db $66, $0F, $38, $DC, $6A, $10 // aesenc xmm5, [rdx+16]
+ db $66, $0F, $38, $DC, $72, $10 // aesenc xmm6, [rdx+16]
+ db $66, $0F, $38, $DC, $7A, $10 // aesenc xmm7, [rdx+16]
+ // stage next-iteration Delta 0 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $04, $24 // movdqa [rsp+0], xmm8
+ // Round 2
+ db $66, $0F, $38, $DC, $42, $20 // aesenc xmm0, [rdx+32]
+ db $66, $0F, $38, $DC, $4A, $20 // aesenc xmm1, [rdx+32]
+ db $66, $0F, $38, $DC, $52, $20 // aesenc xmm2, [rdx+32]
+ db $66, $0F, $38, $DC, $5A, $20 // aesenc xmm3, [rdx+32]
+ db $66, $0F, $38, $DC, $62, $20 // aesenc xmm4, [rdx+32]
+ db $66, $0F, $38, $DC, $6A, $20 // aesenc xmm5, [rdx+32]
+ db $66, $0F, $38, $DC, $72, $20 // aesenc xmm6, [rdx+32]
+ db $66, $0F, $38, $DC, $7A, $20 // aesenc xmm7, [rdx+32]
+ // stage next-iteration Delta 1 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8
+ // Round 3
+ db $66, $0F, $38, $DC, $42, $30 // aesenc xmm0, [rdx+48]
+ db $66, $0F, $38, $DC, $4A, $30 // aesenc xmm1, [rdx+48]
+ db $66, $0F, $38, $DC, $52, $30 // aesenc xmm2, [rdx+48]
+ db $66, $0F, $38, $DC, $5A, $30 // aesenc xmm3, [rdx+48]
+ db $66, $0F, $38, $DC, $62, $30 // aesenc xmm4, [rdx+48]
+ db $66, $0F, $38, $DC, $6A, $30 // aesenc xmm5, [rdx+48]
+ db $66, $0F, $38, $DC, $72, $30 // aesenc xmm6, [rdx+48]
+ db $66, $0F, $38, $DC, $7A, $30 // aesenc xmm7, [rdx+48]
+ // stage next-iteration Delta 2 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8
+ // Round 4
+ db $66, $0F, $38, $DC, $42, $40 // aesenc xmm0, [rdx+64]
+ db $66, $0F, $38, $DC, $4A, $40 // aesenc xmm1, [rdx+64]
+ db $66, $0F, $38, $DC, $52, $40 // aesenc xmm2, [rdx+64]
+ db $66, $0F, $38, $DC, $5A, $40 // aesenc xmm3, [rdx+64]
+ db $66, $0F, $38, $DC, $62, $40 // aesenc xmm4, [rdx+64]
+ db $66, $0F, $38, $DC, $6A, $40 // aesenc xmm5, [rdx+64]
+ db $66, $0F, $38, $DC, $72, $40 // aesenc xmm6, [rdx+64]
+ db $66, $0F, $38, $DC, $7A, $40 // aesenc xmm7, [rdx+64]
+ // stage next-iteration Delta 3 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8
+ // Round 5
+ db $66, $0F, $38, $DC, $42, $50 // aesenc xmm0, [rdx+80]
+ db $66, $0F, $38, $DC, $4A, $50 // aesenc xmm1, [rdx+80]
+ db $66, $0F, $38, $DC, $52, $50 // aesenc xmm2, [rdx+80]
+ db $66, $0F, $38, $DC, $5A, $50 // aesenc xmm3, [rdx+80]
+ db $66, $0F, $38, $DC, $62, $50 // aesenc xmm4, [rdx+80]
+ db $66, $0F, $38, $DC, $6A, $50 // aesenc xmm5, [rdx+80]
+ db $66, $0F, $38, $DC, $72, $50 // aesenc xmm6, [rdx+80]
+ db $66, $0F, $38, $DC, $7A, $50 // aesenc xmm7, [rdx+80]
+ // stage next-iteration Delta 4 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8
+ // Round 6
+ db $66, $0F, $38, $DC, $42, $60 // aesenc xmm0, [rdx+96]
+ db $66, $0F, $38, $DC, $4A, $60 // aesenc xmm1, [rdx+96]
+ db $66, $0F, $38, $DC, $52, $60 // aesenc xmm2, [rdx+96]
+ db $66, $0F, $38, $DC, $5A, $60 // aesenc xmm3, [rdx+96]
+ db $66, $0F, $38, $DC, $62, $60 // aesenc xmm4, [rdx+96]
+ db $66, $0F, $38, $DC, $6A, $60 // aesenc xmm5, [rdx+96]
+ db $66, $0F, $38, $DC, $72, $60 // aesenc xmm6, [rdx+96]
+ db $66, $0F, $38, $DC, $7A, $60 // aesenc xmm7, [rdx+96]
+ // stage next-iteration Delta 5 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8
+ // Round 7
+ db $66, $0F, $38, $DC, $42, $70 // aesenc xmm0, [rdx+112]
+ db $66, $0F, $38, $DC, $4A, $70 // aesenc xmm1, [rdx+112]
+ db $66, $0F, $38, $DC, $52, $70 // aesenc xmm2, [rdx+112]
+ db $66, $0F, $38, $DC, $5A, $70 // aesenc xmm3, [rdx+112]
+ db $66, $0F, $38, $DC, $62, $70 // aesenc xmm4, [rdx+112]
+ db $66, $0F, $38, $DC, $6A, $70 // aesenc xmm5, [rdx+112]
+ db $66, $0F, $38, $DC, $72, $70 // aesenc xmm6, [rdx+112]
+ db $66, $0F, $38, $DC, $7A, $70 // aesenc xmm7, [rdx+112]
+ // stage next-iteration Delta 6 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8
+ // Round 8
+ db $66, $0F, $38, $DC, $82, $80, $00, $00, $00 // aesenc xmm0, [rdx+128]
+ db $66, $0F, $38, $DC, $8A, $80, $00, $00, $00 // aesenc xmm1, [rdx+128]
+ db $66, $0F, $38, $DC, $92, $80, $00, $00, $00 // aesenc xmm2, [rdx+128]
+ db $66, $0F, $38, $DC, $9A, $80, $00, $00, $00 // aesenc xmm3, [rdx+128]
+ db $66, $0F, $38, $DC, $A2, $80, $00, $00, $00 // aesenc xmm4, [rdx+128]
+ db $66, $0F, $38, $DC, $AA, $80, $00, $00, $00 // aesenc xmm5, [rdx+128]
+ db $66, $0F, $38, $DC, $B2, $80, $00, $00, $00 // aesenc xmm6, [rdx+128]
+ db $66, $0F, $38, $DC, $BA, $80, $00, $00, $00 // aesenc xmm7, [rdx+128]
+ // stage next-iteration Delta 7 -> bank 0
+ add r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ db $66, $44, $0F, $7F, $44, $24, $70 // movdqa [rsp+112], xmm8
+ // Round 9
+ db $66, $0F, $38, $DC, $82, $90, $00, $00, $00 // aesenc xmm0, [rdx+144]
+ db $66, $0F, $38, $DC, $8A, $90, $00, $00, $00 // aesenc xmm1, [rdx+144]
+ db $66, $0F, $38, $DC, $92, $90, $00, $00, $00 // aesenc xmm2, [rdx+144]
+ db $66, $0F, $38, $DC, $9A, $90, $00, $00, $00 // aesenc xmm3, [rdx+144]
+ db $66, $0F, $38, $DC, $A2, $90, $00, $00, $00 // aesenc xmm4, [rdx+144]
+ db $66, $0F, $38, $DC, $AA, $90, $00, $00, $00 // aesenc xmm5, [rdx+144]
+ db $66, $0F, $38, $DC, $B2, $90, $00, $00, $00 // aesenc xmm6, [rdx+144]
+ db $66, $0F, $38, $DC, $BA, $90, $00, $00, $00 // aesenc xmm7, [rdx+144]
+{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)}
+ db $66, $0F, $38, $DC, $82, $A0, $00, $00, $00 // aesenc xmm0, [rdx+160]
+ db $66, $0F, $38, $DC, $8A, $A0, $00, $00, $00 // aesenc xmm1, [rdx+160]
+ db $66, $0F, $38, $DC, $92, $A0, $00, $00, $00 // aesenc xmm2, [rdx+160]
+ db $66, $0F, $38, $DC, $9A, $A0, $00, $00, $00 // aesenc xmm3, [rdx+160]
+ db $66, $0F, $38, $DC, $A2, $A0, $00, $00, $00 // aesenc xmm4, [rdx+160]
+ db $66, $0F, $38, $DC, $AA, $A0, $00, $00, $00 // aesenc xmm5, [rdx+160]
+ db $66, $0F, $38, $DC, $B2, $A0, $00, $00, $00 // aesenc xmm6, [rdx+160]
+ db $66, $0F, $38, $DC, $BA, $A0, $00, $00, $00 // aesenc xmm7, [rdx+160]
+ db $66, $0F, $38, $DC, $82, $B0, $00, $00, $00 // aesenc xmm0, [rdx+176]
+ db $66, $0F, $38, $DC, $8A, $B0, $00, $00, $00 // aesenc xmm1, [rdx+176]
+ db $66, $0F, $38, $DC, $92, $B0, $00, $00, $00 // aesenc xmm2, [rdx+176]
+ db $66, $0F, $38, $DC, $9A, $B0, $00, $00, $00 // aesenc xmm3, [rdx+176]
+ db $66, $0F, $38, $DC, $A2, $B0, $00, $00, $00 // aesenc xmm4, [rdx+176]
+ db $66, $0F, $38, $DC, $AA, $B0, $00, $00, $00 // aesenc xmm5, [rdx+176]
+ db $66, $0F, $38, $DC, $B2, $B0, $00, $00, $00 // aesenc xmm6, [rdx+176]
+ db $66, $0F, $38, $DC, $BA, $B0, $00, $00, $00 // aesenc xmm7, [rdx+176]
+{$IFEND}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DC, $82, $C0, $00, $00, $00 // aesenc xmm0, [rdx+192]
+ db $66, $0F, $38, $DC, $8A, $C0, $00, $00, $00 // aesenc xmm1, [rdx+192]
+ db $66, $0F, $38, $DC, $92, $C0, $00, $00, $00 // aesenc xmm2, [rdx+192]
+ db $66, $0F, $38, $DC, $9A, $C0, $00, $00, $00 // aesenc xmm3, [rdx+192]
+ db $66, $0F, $38, $DC, $A2, $C0, $00, $00, $00 // aesenc xmm4, [rdx+192]
+ db $66, $0F, $38, $DC, $AA, $C0, $00, $00, $00 // aesenc xmm5, [rdx+192]
+ db $66, $0F, $38, $DC, $B2, $C0, $00, $00, $00 // aesenc xmm6, [rdx+192]
+ db $66, $0F, $38, $DC, $BA, $C0, $00, $00, $00 // aesenc xmm7, [rdx+192]
+ db $66, $0F, $38, $DC, $82, $D0, $00, $00, $00 // aesenc xmm0, [rdx+208]
+ db $66, $0F, $38, $DC, $8A, $D0, $00, $00, $00 // aesenc xmm1, [rdx+208]
+ db $66, $0F, $38, $DC, $92, $D0, $00, $00, $00 // aesenc xmm2, [rdx+208]
+ db $66, $0F, $38, $DC, $9A, $D0, $00, $00, $00 // aesenc xmm3, [rdx+208]
+ db $66, $0F, $38, $DC, $A2, $D0, $00, $00, $00 // aesenc xmm4, [rdx+208]
+ db $66, $0F, $38, $DC, $AA, $D0, $00, $00, $00 // aesenc xmm5, [rdx+208]
+ db $66, $0F, $38, $DC, $B2, $D0, $00, $00, $00 // aesenc xmm6, [rdx+208]
+ db $66, $0F, $38, $DC, $BA, $D0, $00, $00, $00 // aesenc xmm7, [rdx+208]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY128}
+ db $66, $0F, $38, $DD, $82, $A0, $00, $00, $00 // aesenclast xmm0, [rdx+160]
+ db $66, $0F, $38, $DD, $8A, $A0, $00, $00, $00 // aesenclast xmm1, [rdx+160]
+ db $66, $0F, $38, $DD, $92, $A0, $00, $00, $00 // aesenclast xmm2, [rdx+160]
+ db $66, $0F, $38, $DD, $9A, $A0, $00, $00, $00 // aesenclast xmm3, [rdx+160]
+ db $66, $0F, $38, $DD, $A2, $A0, $00, $00, $00 // aesenclast xmm4, [rdx+160]
+ db $66, $0F, $38, $DD, $AA, $A0, $00, $00, $00 // aesenclast xmm5, [rdx+160]
+ db $66, $0F, $38, $DD, $B2, $A0, $00, $00, $00 // aesenclast xmm6, [rdx+160]
+ db $66, $0F, $38, $DD, $BA, $A0, $00, $00, $00 // aesenclast xmm7, [rdx+160]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY192}
+ db $66, $0F, $38, $DD, $82, $C0, $00, $00, $00 // aesenclast xmm0, [rdx+192]
+ db $66, $0F, $38, $DD, $8A, $C0, $00, $00, $00 // aesenclast xmm1, [rdx+192]
+ db $66, $0F, $38, $DD, $92, $C0, $00, $00, $00 // aesenclast xmm2, [rdx+192]
+ db $66, $0F, $38, $DD, $9A, $C0, $00, $00, $00 // aesenclast xmm3, [rdx+192]
+ db $66, $0F, $38, $DD, $A2, $C0, $00, $00, $00 // aesenclast xmm4, [rdx+192]
+ db $66, $0F, $38, $DD, $AA, $C0, $00, $00, $00 // aesenclast xmm5, [rdx+192]
+ db $66, $0F, $38, $DD, $B2, $C0, $00, $00, $00 // aesenclast xmm6, [rdx+192]
+ db $66, $0F, $38, $DD, $BA, $C0, $00, $00, $00 // aesenclast xmm7, [rdx+192]
+{$ENDIF}
+{$IFDEF CRYPTOLIB_AESNI_KEY256}
+ db $66, $0F, $38, $DD, $82, $E0, $00, $00, $00 // aesenclast xmm0, [rdx+224]
+ db $66, $0F, $38, $DD, $8A, $E0, $00, $00, $00 // aesenclast xmm1, [rdx+224]
+ db $66, $0F, $38, $DD, $92, $E0, $00, $00, $00 // aesenclast xmm2, [rdx+224]
+ db $66, $0F, $38, $DD, $9A, $E0, $00, $00, $00 // aesenclast xmm3, [rdx+224]
+ db $66, $0F, $38, $DD, $A2, $E0, $00, $00, $00 // aesenclast xmm4, [rdx+224]
+ db $66, $0F, $38, $DD, $AA, $E0, $00, $00, $00 // aesenclast xmm5, [rdx+224]
+ db $66, $0F, $38, $DD, $B2, $E0, $00, $00, $00 // aesenclast xmm6, [rdx+224]
+ db $66, $0F, $38, $DD, $BA, $E0, $00, $00, $00 // aesenclast xmm7, [rdx+224]
+{$ENDIF}
+{$ENDIF}
+
+ // Post-whiten with this iteration's Deltas.
+ pxor xmm0, [rsp + 128]
+ pxor xmm1, [rsp + 144]
+ pxor xmm2, [rsp + 160]
+ pxor xmm3, [rsp + 176]
+ pxor xmm4, [rsp + 192]
+ pxor xmm5, [rsp + 208]
+ pxor xmm6, [rsp + 224]
+ pxor xmm7, [rsp + 240]
+
+{$IFDEF CRYPTOLIB_AESNI_DECRYPT}
+ // Fold recovered plaintext into the checksum.
+ db $66, $44, $0F, $EF, $C8 // pxor xmm9, xmm0
+ db $66, $44, $0F, $EF, $C9 // pxor xmm9, xmm1
+ db $66, $44, $0F, $EF, $CA // pxor xmm9, xmm2
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ db $66, $44, $0F, $EF, $CC // pxor xmm9, xmm4
+ db $66, $44, $0F, $EF, $CD // pxor xmm9, xmm5
+ db $66, $44, $0F, $EF, $CE // pxor xmm9, xmm6
+ db $66, $44, $0F, $EF, $CF // pxor xmm9, xmm7
+{$ENDIF}
- // --- Advance pointers and decrement loop counter. ---
+ movdqu [r8 + 0], xmm0
+ movdqu [r8 + 16], xmm1
+ movdqu [r8 + 32], xmm2
+ movdqu [r8 + 48], xmm3
+ movdqu [r8 + 64], xmm4
+ movdqu [r8 + 80], xmm5
+ movdqu [r8 + 96], xmm6
+ movdqu [r8 + 112], xmm7
add rbx, 128
- add r8, 128
- mov rax, rbx // next iter's block-0 source
- // tracks the main stream
- dec r10
+ add r8, 128
+ mov rax, rbx // next iteration's block-0 source
+
+ sub r10, 1
jnz @@ocb_x64_loop
- // --- Epilogue: flush xmm_offset and xmm_checksum back to memory,
- // release stack scratch, restore rbx. ---
- mov rax, [rcx + 24] // rax = OffsetPtr
- movdqu [rax], xmm8
- mov rax, [rcx + 32] // rax = ChecksumPtr
- movdqu [rax], xmm9
+ // --- Undo the one-iteration ladder over-stage (XOR self-inverse). ---
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+ bsf rax, r11
+ shl rax, 4
+ add rax, r9
+ db $F3, $44, $0F, $6F, $18 // movdqu xmm11, [rax]
+ db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11
+ sub r11, 1
+
+ // --- Epilogue: flush accumulators, unwind. ---
+ mov rax, [rcx + 24]
+ db $F3, $44, $0F, $7F, $00 // movdqu [rax], xmm8
+ mov rax, [rcx + 32]
+ db $F3, $44, $0F, $7F, $08 // movdqu [rax], xmm9
- add rsp, 128
+ add rsp, 256
pop rbx
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc
index a5031baf..2ae80d0e 100644
--- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc
@@ -1,95 +1,144 @@
-// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (i386).
+// ChaCha7539 (IETF) 4-way SSE2 keystream kernel, vertical layout (i386).
// -------------------------------------------------------------------
//
-// Four 64-byte ChaCha20 blocks, state words held vertically (one block per
-// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame;
-// each quarter-round loads its 4 words into xmm0-3 (temp xmm4).
+// Four 64-byte ChaCha20 blocks per group, state words held vertically (one
+// block per xmm lane). i386 has only 8 xmm, so the 16 state words live in a
+// 64B-aligned stack frame (original esp parked inside it); each quarter-round
+// loads its 4 words into xmm0-3 (temp xmm4).
// rot16/rot8 use shift+or; rot12/rot7 use shift+or.
//
-// On entry (after ClpSimdProc4Begin_i386.inc):
-// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut.
+// Streams AGroups x 256B in one call: prologue once, per-group ORIG->STATE
+// reload (counter lanes +4), fused accumulate/transpose/xor tail to AOut.
+//
+// On entry (after ClpSimdProc5Begin_i386.inc):
+// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut,
+// ecx = AGroups (4-block groups, >= 1).
//
// On exit:
-// AState word-12 counter (byte 48) advanced by 4; the keystream is xored
-// with AIn into AOut (256 bytes).
+// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is
+// xored with AIn into AOut (256*AGroups bytes).
//
// punpck* / pshufb are db-encoded for broad assembler compatibility.
- sub esp, 528
+ mov edx, esp
+ sub esp, 560
+ and esp, -64
+ mov dword ptr [esp + 544], edx
mov dword ptr [esp + 512], 0
mov dword ptr [esp + 516], 1
mov dword ptr [esp + 520], 2
mov dword ptr [esp + 524], 3
+ mov dword ptr [esp + 528], 4
+ mov dword ptr [esp + 532], 4
+ mov dword ptr [esp + 536], 4
+ mov dword ptr [esp + 540], 4
movd xmm0, [esi]
pshufd xmm0, xmm0, $00
- movdqu [esp], xmm0
- movdqu [esp + 256], xmm0
+ movdqa [esp], xmm0
+ movdqa [esp + 256], xmm0
movd xmm0, [esi + 4]
pshufd xmm0, xmm0, $00
- movdqu [esp + 16], xmm0
- movdqu [esp + 272], xmm0
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 272], xmm0
movd xmm0, [esi + 8]
pshufd xmm0, xmm0, $00
- movdqu [esp + 32], xmm0
- movdqu [esp + 288], xmm0
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 288], xmm0
movd xmm0, [esi + 12]
pshufd xmm0, xmm0, $00
- movdqu [esp + 48], xmm0
- movdqu [esp + 304], xmm0
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 304], xmm0
movd xmm0, [esi + 16]
pshufd xmm0, xmm0, $00
- movdqu [esp + 64], xmm0
- movdqu [esp + 320], xmm0
+ movdqa [esp + 64], xmm0
+ movdqa [esp + 320], xmm0
movd xmm0, [esi + 20]
pshufd xmm0, xmm0, $00
- movdqu [esp + 80], xmm0
- movdqu [esp + 336], xmm0
+ movdqa [esp + 80], xmm0
+ movdqa [esp + 336], xmm0
movd xmm0, [esi + 24]
pshufd xmm0, xmm0, $00
- movdqu [esp + 96], xmm0
- movdqu [esp + 352], xmm0
+ movdqa [esp + 96], xmm0
+ movdqa [esp + 352], xmm0
movd xmm0, [esi + 28]
pshufd xmm0, xmm0, $00
- movdqu [esp + 112], xmm0
- movdqu [esp + 368], xmm0
+ movdqa [esp + 112], xmm0
+ movdqa [esp + 368], xmm0
movd xmm0, [esi + 32]
pshufd xmm0, xmm0, $00
- movdqu [esp + 128], xmm0
- movdqu [esp + 384], xmm0
+ movdqa [esp + 128], xmm0
+ movdqa [esp + 384], xmm0
movd xmm0, [esi + 36]
pshufd xmm0, xmm0, $00
- movdqu [esp + 144], xmm0
- movdqu [esp + 400], xmm0
+ movdqa [esp + 144], xmm0
+ movdqa [esp + 400], xmm0
movd xmm0, [esi + 40]
pshufd xmm0, xmm0, $00
- movdqu [esp + 160], xmm0
- movdqu [esp + 416], xmm0
+ movdqa [esp + 160], xmm0
+ movdqa [esp + 416], xmm0
movd xmm0, [esi + 44]
pshufd xmm0, xmm0, $00
- movdqu [esp + 176], xmm0
- movdqu [esp + 432], xmm0
+ movdqa [esp + 176], xmm0
+ movdqa [esp + 432], xmm0
movd xmm0, [esi + 48]
pshufd xmm0, xmm0, $00
- movdqu xmm1, [esp + 512]
- paddd xmm0, xmm1
- movdqu [esp + 192], xmm0
- movdqu [esp + 448], xmm0
+ db $66, $0F, $FE, $84, $24, $00, $02, $00, $00 // paddd xmm0, [esp + 512]
+ movdqa [esp + 192], xmm0
+ movdqa [esp + 448], xmm0
movd xmm0, [esi + 52]
pshufd xmm0, xmm0, $00
- movdqu [esp + 208], xmm0
- movdqu [esp + 464], xmm0
+ movdqa [esp + 208], xmm0
+ movdqa [esp + 464], xmm0
movd xmm0, [esi + 56]
pshufd xmm0, xmm0, $00
- movdqu [esp + 224], xmm0
- movdqu [esp + 480], xmm0
+ movdqa [esp + 224], xmm0
+ movdqa [esp + 480], xmm0
movd xmm0, [esi + 60]
pshufd xmm0, xmm0, $00
- movdqu [esp + 240], xmm0
- movdqu [esp + 496], xmm0
+ movdqa [esp + 240], xmm0
+ movdqa [esp + 496], xmm0
+ jmp @@c7539_sse2_enter
+@@c7539_sse2_outer:
+ movdqa xmm0, [esp + 256]
+ movdqa [esp], xmm0
+ movdqa xmm0, [esp + 272]
+ movdqa [esp + 16], xmm0
+ movdqa xmm0, [esp + 288]
+ movdqa [esp + 32], xmm0
+ movdqa xmm0, [esp + 304]
+ movdqa [esp + 48], xmm0
+ movdqa xmm0, [esp + 320]
+ movdqa [esp + 64], xmm0
+ movdqa xmm0, [esp + 336]
+ movdqa [esp + 80], xmm0
+ movdqa xmm0, [esp + 352]
+ movdqa [esp + 96], xmm0
+ movdqa xmm0, [esp + 368]
+ movdqa [esp + 112], xmm0
+ movdqa xmm0, [esp + 384]
+ movdqa [esp + 128], xmm0
+ movdqa xmm0, [esp + 400]
+ movdqa [esp + 144], xmm0
+ movdqa xmm0, [esp + 416]
+ movdqa [esp + 160], xmm0
+ movdqa xmm0, [esp + 432]
+ movdqa [esp + 176], xmm0
+ movdqa xmm0, [esp + 448]
+ db $66, $0F, $FE, $84, $24, $10, $02, $00, $00 // paddd xmm0, [esp + 528]
+ movdqa [esp + 448], xmm0
+ movdqa [esp + 192], xmm0
+ movdqa xmm0, [esp + 464]
+ movdqa [esp + 208], xmm0
+ movdqa xmm0, [esp + 480]
+ movdqa [esp + 224], xmm0
+ movdqa xmm0, [esp + 496]
+ movdqa [esp + 240], xmm0
+@@c7539_sse2_enter:
+ mov edx, ebx
@@c7539_sse2_round:
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 64]
- movdqu xmm2, [esp + 128]
- movdqu xmm3, [esp + 192]
+ movdqa xmm0, [esp]
+ movdqa xmm1, [esp + 64]
+ movdqa xmm2, [esp + 128]
+ movdqa xmm3, [esp + 192]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -114,14 +163,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp], xmm0
- movdqu [esp + 64], xmm1
- movdqu [esp + 128], xmm2
- movdqu [esp + 192], xmm3
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 144]
- movdqu xmm3, [esp + 208]
+ movdqa [esp], xmm0
+ movdqa [esp + 64], xmm1
+ movdqa [esp + 128], xmm2
+ movdqa [esp + 192], xmm3
+ movdqa xmm0, [esp + 16]
+ movdqa xmm1, [esp + 80]
+ movdqa xmm2, [esp + 144]
+ movdqa xmm3, [esp + 208]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -146,14 +195,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 16], xmm0
- movdqu [esp + 80], xmm1
- movdqu [esp + 144], xmm2
- movdqu [esp + 208], xmm3
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 96]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 224]
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 80], xmm1
+ movdqa [esp + 144], xmm2
+ movdqa [esp + 208], xmm3
+ movdqa xmm0, [esp + 32]
+ movdqa xmm1, [esp + 96]
+ movdqa xmm2, [esp + 160]
+ movdqa xmm3, [esp + 224]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -178,14 +227,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 32], xmm0
- movdqu [esp + 96], xmm1
- movdqu [esp + 160], xmm2
- movdqu [esp + 224], xmm3
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 112]
- movdqu xmm2, [esp + 176]
- movdqu xmm3, [esp + 240]
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 96], xmm1
+ movdqa [esp + 160], xmm2
+ movdqa [esp + 224], xmm3
+ movdqa xmm0, [esp + 48]
+ movdqa xmm1, [esp + 112]
+ movdqa xmm2, [esp + 176]
+ movdqa xmm3, [esp + 240]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -210,14 +259,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 48], xmm0
- movdqu [esp + 112], xmm1
- movdqu [esp + 176], xmm2
- movdqu [esp + 240], xmm3
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 240]
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 112], xmm1
+ movdqa [esp + 176], xmm2
+ movdqa [esp + 240], xmm3
+ movdqa xmm0, [esp]
+ movdqa xmm1, [esp + 80]
+ movdqa xmm2, [esp + 160]
+ movdqa xmm3, [esp + 240]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -242,14 +291,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp], xmm0
- movdqu [esp + 80], xmm1
- movdqu [esp + 160], xmm2
- movdqu [esp + 240], xmm3
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 96]
- movdqu xmm2, [esp + 176]
- movdqu xmm3, [esp + 192]
+ movdqa [esp], xmm0
+ movdqa [esp + 80], xmm1
+ movdqa [esp + 160], xmm2
+ movdqa [esp + 240], xmm3
+ movdqa xmm0, [esp + 16]
+ movdqa xmm1, [esp + 96]
+ movdqa xmm2, [esp + 176]
+ movdqa xmm3, [esp + 192]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -274,14 +323,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 16], xmm0
- movdqu [esp + 96], xmm1
- movdqu [esp + 176], xmm2
- movdqu [esp + 192], xmm3
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 112]
- movdqu xmm2, [esp + 128]
- movdqu xmm3, [esp + 208]
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 96], xmm1
+ movdqa [esp + 176], xmm2
+ movdqa [esp + 192], xmm3
+ movdqa xmm0, [esp + 32]
+ movdqa xmm1, [esp + 112]
+ movdqa xmm2, [esp + 128]
+ movdqa xmm3, [esp + 208]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -306,14 +355,14 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 32], xmm0
- movdqu [esp + 112], xmm1
- movdqu [esp + 128], xmm2
- movdqu [esp + 208], xmm3
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 64]
- movdqu xmm2, [esp + 144]
- movdqu xmm3, [esp + 224]
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 112], xmm1
+ movdqa [esp + 128], xmm2
+ movdqa [esp + 208], xmm3
+ movdqa xmm0, [esp + 48]
+ movdqa xmm1, [esp + 64]
+ movdqa xmm2, [esp + 144]
+ movdqa xmm3, [esp + 224]
paddd xmm0, xmm1
pxor xmm3, xmm0
movdqa xmm4, xmm3
@@ -338,96 +387,36 @@
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 48], xmm0
- movdqu [esp + 64], xmm1
- movdqu [esp + 144], xmm2
- movdqu [esp + 224], xmm3
- sub ebx, 2
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 64], xmm1
+ movdqa [esp + 144], xmm2
+ movdqa [esp + 224], xmm3
+ sub edx, 2
jg @@c7539_sse2_round
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 256]
- paddd xmm0, xmm1
- movdqu [esp], xmm0
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 272]
- paddd xmm0, xmm1
- movdqu [esp + 16], xmm0
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 288]
- paddd xmm0, xmm1
- movdqu [esp + 32], xmm0
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 304]
- paddd xmm0, xmm1
- movdqu [esp + 48], xmm0
- movdqu xmm0, [esp + 64]
- movdqu xmm1, [esp + 320]
- paddd xmm0, xmm1
- movdqu [esp + 64], xmm0
- movdqu xmm0, [esp + 80]
- movdqu xmm1, [esp + 336]
- paddd xmm0, xmm1
- movdqu [esp + 80], xmm0
- movdqu xmm0, [esp + 96]
- movdqu xmm1, [esp + 352]
- paddd xmm0, xmm1
- movdqu [esp + 96], xmm0
- movdqu xmm0, [esp + 112]
- movdqu xmm1, [esp + 368]
- paddd xmm0, xmm1
- movdqu [esp + 112], xmm0
- movdqu xmm0, [esp + 128]
- movdqu xmm1, [esp + 384]
- paddd xmm0, xmm1
- movdqu [esp + 128], xmm0
- movdqu xmm0, [esp + 144]
- movdqu xmm1, [esp + 400]
- paddd xmm0, xmm1
- movdqu [esp + 144], xmm0
- movdqu xmm0, [esp + 160]
- movdqu xmm1, [esp + 416]
- paddd xmm0, xmm1
- movdqu [esp + 160], xmm0
- movdqu xmm0, [esp + 176]
- movdqu xmm1, [esp + 432]
- paddd xmm0, xmm1
- movdqu [esp + 176], xmm0
- movdqu xmm0, [esp + 192]
- movdqu xmm1, [esp + 448]
- paddd xmm0, xmm1
- movdqu [esp + 192], xmm0
- movdqu xmm0, [esp + 208]
- movdqu xmm1, [esp + 464]
- paddd xmm0, xmm1
- movdqu [esp + 208], xmm0
- movdqu xmm0, [esp + 224]
- movdqu xmm1, [esp + 480]
- paddd xmm0, xmm1
- movdqu [esp + 224], xmm0
- movdqu xmm0, [esp + 240]
- movdqu xmm1, [esp + 496]
- paddd xmm0, xmm1
- movdqu [esp + 240], xmm0
add dword ptr [esi + 48], 4
{$IFDEF CHACHA_CTR64}
adc dword ptr [esi + 52], 0
{$ENDIF}
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 16]
- movdqu xmm2, [esp + 32]
- movdqu xmm3, [esp + 48]
+ movdqa xmm0, [esp]
+ db $66, $0F, $FE, $84, $24, $00, $01, $00, $00 // paddd xmm0, [esp + 256]
+ movdqa xmm1, [esp + 16]
+ db $66, $0F, $FE, $8C, $24, $10, $01, $00, $00 // paddd xmm1, [esp + 272]
+ movdqa xmm2, [esp + 32]
+ db $66, $0F, $FE, $94, $24, $20, $01, $00, $00 // paddd xmm2, [esp + 288]
+ movdqa xmm3, [esp + 48]
+ db $66, $0F, $FE, $9C, $24, $30, $01, $00, $00 // paddd xmm3, [esp + 304]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi]
pxor xmm0, xmm5
movdqu [eax], xmm0
@@ -440,22 +429,26 @@
movdqu xmm5, [edi + 192]
pxor xmm3, xmm5
movdqu [eax + 192], xmm3
- movdqu xmm0, [esp + 64]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 96]
- movdqu xmm3, [esp + 112]
+ movdqa xmm0, [esp + 64]
+ db $66, $0F, $FE, $84, $24, $40, $01, $00, $00 // paddd xmm0, [esp + 320]
+ movdqa xmm1, [esp + 80]
+ db $66, $0F, $FE, $8C, $24, $50, $01, $00, $00 // paddd xmm1, [esp + 336]
+ movdqa xmm2, [esp + 96]
+ db $66, $0F, $FE, $94, $24, $60, $01, $00, $00 // paddd xmm2, [esp + 352]
+ movdqa xmm3, [esp + 112]
+ db $66, $0F, $FE, $9C, $24, $70, $01, $00, $00 // paddd xmm3, [esp + 368]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 16]
pxor xmm0, xmm5
movdqu [eax + 16], xmm0
@@ -468,22 +461,26 @@
movdqu xmm5, [edi + 208]
pxor xmm3, xmm5
movdqu [eax + 208], xmm3
- movdqu xmm0, [esp + 128]
- movdqu xmm1, [esp + 144]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 176]
+ movdqa xmm0, [esp + 128]
+ db $66, $0F, $FE, $84, $24, $80, $01, $00, $00 // paddd xmm0, [esp + 384]
+ movdqa xmm1, [esp + 144]
+ db $66, $0F, $FE, $8C, $24, $90, $01, $00, $00 // paddd xmm1, [esp + 400]
+ movdqa xmm2, [esp + 160]
+ db $66, $0F, $FE, $94, $24, $A0, $01, $00, $00 // paddd xmm2, [esp + 416]
+ movdqa xmm3, [esp + 176]
+ db $66, $0F, $FE, $9C, $24, $B0, $01, $00, $00 // paddd xmm3, [esp + 432]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 32]
pxor xmm0, xmm5
movdqu [eax + 32], xmm0
@@ -496,22 +493,26 @@
movdqu xmm5, [edi + 224]
pxor xmm3, xmm5
movdqu [eax + 224], xmm3
- movdqu xmm0, [esp + 192]
- movdqu xmm1, [esp + 208]
- movdqu xmm2, [esp + 224]
- movdqu xmm3, [esp + 240]
+ movdqa xmm0, [esp + 192]
+ db $66, $0F, $FE, $84, $24, $C0, $01, $00, $00 // paddd xmm0, [esp + 448]
+ movdqa xmm1, [esp + 208]
+ db $66, $0F, $FE, $8C, $24, $D0, $01, $00, $00 // paddd xmm1, [esp + 464]
+ movdqa xmm2, [esp + 224]
+ db $66, $0F, $FE, $94, $24, $E0, $01, $00, $00 // paddd xmm2, [esp + 480]
+ movdqa xmm3, [esp + 240]
+ db $66, $0F, $FE, $9C, $24, $F0, $01, $00, $00 // paddd xmm3, [esp + 496]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 48]
pxor xmm0, xmm5
movdqu [eax + 48], xmm0
@@ -524,7 +525,11 @@
movdqu xmm5, [edi + 240]
pxor xmm3, xmm5
movdqu [eax + 240], xmm3
- add esp, 528
+ add edi, 256
+ add eax, 256
+ dec ecx
+ jnz @@c7539_sse2_outer
+ mov esp, dword ptr [esp + 544]
pop edi
pop esi
pop ebx
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc
index 080a9a32..0028fc49 100644
--- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc
@@ -1,364 +1,354 @@
-// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (x86-64).
+// ChaCha7539 (IETF) 4-way SSE2 keystream kernel, vertical layout (x86-64).
// -------------------------------------------------------------------
//
-// Four 64-byte ChaCha20 blocks, state words held vertically (one block per
-// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words
-// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch.
-// rot16/rot8 use shift+or; rot12/rot7 use shift+or.
+// Four 64-byte ChaCha20 blocks per group, state words held vertically (one
+// block per xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7),
+// xmm12-15 = words d (12-15). Words c (8-11): two are resident in xmm8/xmm9,
+// the other two wait in stack slots; quarter-rounds run as two resident pairs
+// with one pair swap per round (2 stores + 2 loads). xmm10/xmm11 are the
+// rotate temps. rot16/rot8 use shift+or;
+// rot12/rot7 use shift+or. The frame is 64B-aligned (r11 = frame register).
//
-// On entry (after ClpSimdProc4Begin_x86_64.inc):
-// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut.
+// Streams AGroups x 256B in one call: prologue once, light per-group reload
+// (counter lanes +4), fused accumulate/transpose/xor tail straight to AOut.
+//
+// On entry (after ClpSimdProc5Begin_x86_64.inc):
+// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut,
+// r10 = AGroups (4-block groups, >= 1).
//
// On exit:
-// AState word-12 counter (byte 48) advanced by 4; the keystream is xored
-// with AIn into AOut (256 bytes).
+// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is
+// xored with AIn into AOut (256*AGroups bytes).
//
// Ops touching the extended xmm8-15 registers are db-encoded for broad
// assembler compatibility.
- mov r10, rcx
+ mov r10d, r10d
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- sub rsp, 336
+ mov r11, rsp
+ sub rsp, 352
+ and rsp, -64
mov dword ptr [rsp + 320], 0
mov dword ptr [rsp + 324], 1
mov dword ptr [rsp + 328], 2
mov dword ptr [rsp + 332], 3
+ mov dword ptr [rsp + 336], 4
+ mov dword ptr [rsp + 340], 4
+ mov dword ptr [rsp + 344], 4
+ mov dword ptr [rsp + 348], 4
movd xmm0, [rdx]
pshufd xmm0, xmm0, $00
- movdqu [rsp + 64], xmm0
+ movdqa [rsp + 64], xmm0
movd xmm1, [rdx + 4]
pshufd xmm1, xmm1, $00
- movdqu [rsp + 80], xmm1
+ movdqa [rsp + 80], xmm1
movd xmm2, [rdx + 8]
pshufd xmm2, xmm2, $00
- movdqu [rsp + 96], xmm2
+ movdqa [rsp + 96], xmm2
movd xmm3, [rdx + 12]
pshufd xmm3, xmm3, $00
- movdqu [rsp + 112], xmm3
+ movdqa [rsp + 112], xmm3
movd xmm4, [rdx + 16]
pshufd xmm4, xmm4, $00
- movdqu [rsp + 128], xmm4
+ movdqa [rsp + 128], xmm4
movd xmm5, [rdx + 20]
pshufd xmm5, xmm5, $00
- movdqu [rsp + 144], xmm5
+ movdqa [rsp + 144], xmm5
movd xmm6, [rdx + 24]
pshufd xmm6, xmm6, $00
- movdqu [rsp + 160], xmm6
+ movdqa [rsp + 160], xmm6
movd xmm7, [rdx + 28]
pshufd xmm7, xmm7, $00
- movdqu [rsp + 176], xmm7
+ movdqa [rsp + 176], xmm7
movd xmm8, [rdx + 32]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp], xmm8
- movdqu [rsp + 192], xmm8
- movd xmm8, [rdx + 36]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 16], xmm8
- movdqu [rsp + 208], xmm8
- movd xmm8, [rdx + 40]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 32], xmm8
- movdqu [rsp + 224], xmm8
- movd xmm8, [rdx + 44]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 48], xmm8
- movdqu [rsp + 240], xmm8
+ db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
+ movdqa [rsp + 192], xmm8
+ movd xmm9, [rdx + 36]
+ db $66, $45, $0F, $70, $C9, $00 // pshufd xmm9, xmm9, $00
+ movdqa [rsp + 208], xmm9
+ movd xmm10, [rdx + 40]
+ db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00
+ movdqa [rsp + 224], xmm10
+ movdqa [rsp + 32], xmm10
+ movd xmm10, [rdx + 44]
+ db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00
+ movdqa [rsp + 240], xmm10
+ movdqa [rsp + 48], xmm10
movd xmm12, [rdx + 48]
- db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00
- movdqu xmm8, [rsp + 320]
- db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8
- movdqu [rsp + 256], xmm12
+ db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00
+ db $66, $44, $0F, $FE, $A4, $24, $40, $01, $00, $00 // paddd xmm12, [rsp + 320]
+ movdqa [rsp + 256], xmm12
movd xmm13, [rdx + 52]
- db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00
- movdqu [rsp + 272], xmm13
+ db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00
+ movdqa [rsp + 272], xmm13
movd xmm14, [rdx + 56]
- db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00
- movdqu [rsp + 288], xmm14
+ db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00
+ movdqa [rsp + 288], xmm14
movd xmm15, [rdx + 60]
- db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00
- movdqu [rsp + 304], xmm15
+ db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00
+ movdqa [rsp + 304], xmm15
+ jmp @@c7539_sse2_enter
+@@c7539_sse2_outer:
+ movdqa xmm0, [rsp + 64]
+ movdqa xmm1, [rsp + 80]
+ movdqa xmm2, [rsp + 96]
+ movdqa xmm3, [rsp + 112]
+ movdqa xmm4, [rsp + 128]
+ movdqa xmm5, [rsp + 144]
+ movdqa xmm6, [rsp + 160]
+ movdqa xmm7, [rsp + 176]
+ movdqa xmm8, [rsp + 192]
+ movdqa xmm9, [rsp + 208]
+ movdqa xmm10, [rsp + 224]
+ movdqa [rsp + 32], xmm10
+ movdqa xmm10, [rsp + 240]
+ movdqa [rsp + 48], xmm10
+ movdqa xmm12, [rsp + 256]
+ db $66, $44, $0F, $FE, $A4, $24, $50, $01, $00, $00 // paddd xmm12, [rsp + 336]
+ movdqa [rsp + 256], xmm12
+ movdqa xmm13, [rsp + 272]
+ movdqa xmm14, [rsp + 288]
+ movdqa xmm15, [rsp + 304]
+@@c7539_sse2_enter:
+ mov eax, ecx
@@c7539_sse2_round:
- movdqu xmm8, [rsp]
- paddd xmm0, xmm4
- db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
- db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12
- db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
- pslld xmm4, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
paddd xmm0, xmm4
- db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
- db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12
- db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
- pslld xmm4, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
- movdqu [rsp], xmm8
- movdqu xmm8, [rsp + 16]
+ db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
+ db $66, $45, $0F, $6F, $D4 // movdqa xmm10, xmm12
+ db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16
+ db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16
+ db $66, $45, $0F, $EB, $E2 // por xmm12, xmm10
paddd xmm1, xmm5
- db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
- db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13
- db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
+ db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
+ db $66, $45, $0F, $6F, $DD // movdqa xmm11, xmm13
+ db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16
+ db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16
+ db $66, $45, $0F, $EB, $EB // por xmm13, xmm11
+ db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
+ db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
+ db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4
+ pslld xmm4, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10
+ db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13
+ db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9
+ db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5
pslld xmm5, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $EB // por xmm5, xmm11
+ paddd xmm0, xmm4
+ db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
+ db $66, $45, $0F, $6F, $D4 // movdqa xmm10, xmm12
+ db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8
+ db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24
+ db $66, $45, $0F, $EB, $E2 // por xmm12, xmm10
paddd xmm1, xmm5
- db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
- db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13
- db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
+ db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
+ db $66, $45, $0F, $6F, $DD // movdqa xmm11, xmm13
+ db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8
+ db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24
+ db $66, $45, $0F, $EB, $EB // por xmm13, xmm11
+ db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
+ db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
+ db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4
+ pslld xmm4, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11
+ db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13
+ db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9
+ db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5
pslld xmm5, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- movdqu [rsp + 16], xmm8
- movdqu xmm8, [rsp + 32]
- paddd xmm2, xmm6
- db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
- db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14
- db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
- pslld xmm6, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $EA // por xmm5, xmm10
+ movdqa [rsp], xmm8
+ movdqa [rsp + 16], xmm9
+ movdqa xmm8, [rsp + 32]
+ movdqa xmm9, [rsp + 48]
paddd xmm2, xmm6
- db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
- db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14
- db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
- pslld xmm6, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
- movdqu [rsp + 32], xmm8
- movdqu xmm8, [rsp + 48]
+ db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
+ db $66, $45, $0F, $6F, $D6 // movdqa xmm10, xmm14
+ db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16
+ db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16
+ db $66, $45, $0F, $EB, $F2 // por xmm14, xmm10
paddd xmm3, xmm7
- db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
- db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15
- db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ db $66, $45, $0F, $6F, $DF // movdqa xmm11, xmm15
+ db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16
+ db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16
+ db $66, $45, $0F, $EB, $FB // por xmm15, xmm11
+ db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
+ db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
+ db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6
+ pslld xmm6, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10
+ db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15
+ db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9
+ db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7
pslld xmm7, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $FB // por xmm7, xmm11
+ paddd xmm2, xmm6
+ db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
+ db $66, $45, $0F, $6F, $D6 // movdqa xmm10, xmm14
+ db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8
+ db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24
+ db $66, $45, $0F, $EB, $F2 // por xmm14, xmm10
paddd xmm3, xmm7
- db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
- db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15
- db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ db $66, $45, $0F, $6F, $DF // movdqa xmm11, xmm15
+ db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8
+ db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24
+ db $66, $45, $0F, $EB, $FB // por xmm15, xmm11
+ db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
+ db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
+ pslld xmm6, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11
+ db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15
+ db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9
+ db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7
pslld xmm7, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
- movdqu [rsp + 48], xmm8
- movdqu xmm8, [rsp + 32]
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $FA // por xmm7, xmm10
paddd xmm0, xmm5
- db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
- db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15
- db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
- pslld xmm5, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- paddd xmm0, xmm5
- db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
- db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15
- db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
- pslld xmm5, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- movdqu [rsp + 32], xmm8
- movdqu xmm8, [rsp + 48]
+ db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
+ db $66, $45, $0F, $6F, $D7 // movdqa xmm10, xmm15
+ db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16
+ db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16
+ db $66, $45, $0F, $EB, $FA // por xmm15, xmm10
paddd xmm1, xmm6
- db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
- db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12
- db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
+ db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
+ db $66, $45, $0F, $6F, $DC // movdqa xmm11, xmm12
+ db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16
+ db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16
+ db $66, $45, $0F, $EB, $E3 // por xmm12, xmm11
+ db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
+ db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
+ db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5
+ pslld xmm5, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $EA // por xmm5, xmm10
+ db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12
+ db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
pslld xmm6, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11
+ paddd xmm0, xmm5
+ db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
+ db $66, $45, $0F, $6F, $D7 // movdqa xmm10, xmm15
+ db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8
+ db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24
+ db $66, $45, $0F, $EB, $FA // por xmm15, xmm10
paddd xmm1, xmm6
- db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
- db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12
- db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
+ db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
+ db $66, $45, $0F, $6F, $DC // movdqa xmm11, xmm12
+ db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8
+ db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24
+ db $66, $45, $0F, $EB, $E3 // por xmm12, xmm11
+ db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
+ db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
+ db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5
+ pslld xmm5, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $EB // por xmm5, xmm11
+ db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12
+ db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9
+ db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6
pslld xmm6, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
- movdqu [rsp + 48], xmm8
- movdqu xmm8, [rsp]
- paddd xmm2, xmm7
- db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
- db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13
- db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
- pslld xmm7, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10
+ movdqa [rsp + 32], xmm8
+ movdqa [rsp + 48], xmm9
+ movdqa xmm8, [rsp]
+ movdqa xmm9, [rsp + 16]
paddd xmm2, xmm7
- db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
- db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13
- db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
- pslld xmm7, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
- movdqu [rsp], xmm8
- movdqu xmm8, [rsp + 16]
+ db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
+ db $66, $45, $0F, $6F, $D5 // movdqa xmm10, xmm13
+ db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16
+ db $66, $41, $0F, $72, $D2, $10 // psrld xmm10, 16
+ db $66, $45, $0F, $EB, $EA // por xmm13, xmm10
paddd xmm3, xmm4
- db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
- db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14
- db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16
- db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16
- db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
+ db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
+ db $66, $45, $0F, $6F, $DE // movdqa xmm11, xmm14
+ db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16
+ db $66, $41, $0F, $72, $D3, $10 // psrld xmm11, 16
+ db $66, $45, $0F, $EB, $F3 // por xmm14, xmm11
+ db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
+ db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
+ db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7
+ pslld xmm7, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $FA // por xmm7, xmm10
+ db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14
+ db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9
+ db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4
pslld xmm4, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11
+ paddd xmm2, xmm7
+ db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
+ db $66, $45, $0F, $6F, $D5 // movdqa xmm10, xmm13
+ db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8
+ db $66, $41, $0F, $72, $D2, $18 // psrld xmm10, 24
+ db $66, $45, $0F, $EB, $EA // por xmm13, xmm10
paddd xmm3, xmm4
- db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
- db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14
- db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8
- db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24
- db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
+ db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
+ db $66, $45, $0F, $6F, $DE // movdqa xmm11, xmm14
+ db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8
+ db $66, $41, $0F, $72, $D3, $18 // psrld xmm11, 24
+ db $66, $45, $0F, $EB, $F3 // por xmm14, xmm11
+ db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
+ db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
+ db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7
+ pslld xmm7, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $FB // por xmm7, xmm11
+ db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14
+ db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9
+ db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4
pslld xmm4, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
- movdqu [rsp + 16], xmm8
- sub r10, 2
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10
+ sub eax, 2
jg @@c7539_sse2_round
- movdqu xmm8, [rsp + 64]
- db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8
- movdqu [rsp + 64], xmm0
- movdqu xmm8, [rsp + 80]
- db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8
- movdqu [rsp + 80], xmm1
- movdqu xmm8, [rsp + 96]
- db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8
- movdqu [rsp + 96], xmm2
- movdqu xmm8, [rsp + 112]
- db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8
- movdqu [rsp + 112], xmm3
- movdqu xmm8, [rsp + 128]
- db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8
- movdqu [rsp + 128], xmm4
- movdqu xmm8, [rsp + 144]
- db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8
- movdqu [rsp + 144], xmm5
- movdqu xmm8, [rsp + 160]
- db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8
- movdqu [rsp + 160], xmm6
- movdqu xmm8, [rsp + 176]
- db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8
- movdqu [rsp + 176], xmm7
- movdqu xmm8, [rsp + 256]
- db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8
- movdqu [rsp + 256], xmm12
- movdqu xmm8, [rsp + 272]
- db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8
- movdqu [rsp + 272], xmm13
- movdqu xmm8, [rsp + 288]
- db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8
- movdqu [rsp + 288], xmm14
- movdqu xmm8, [rsp + 304]
- db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8
- movdqu [rsp + 304], xmm15
- movdqu xmm8, [rsp]
- movdqu xmm9, [rsp + 192]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 192], xmm8
- movdqu xmm8, [rsp + 16]
- movdqu xmm9, [rsp + 208]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 208], xmm8
- movdqu xmm8, [rsp + 32]
- movdqu xmm9, [rsp + 224]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 224], xmm8
- movdqu xmm8, [rsp + 48]
- movdqu xmm9, [rsp + 240]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 240], xmm8
add dword ptr [rdx + 48], 4
{$IFDEF CHACHA_CTR64}
adc dword ptr [rdx + 52], 0
{$ENDIF}
- movdqu xmm0, [rsp + 64]
- movdqu xmm1, [rsp + 80]
- movdqu xmm2, [rsp + 96]
- movdqu xmm3, [rsp + 112]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $FE, $44, $24, $40 // paddd xmm0, [rsp + 64]
+ db $66, $0F, $FE, $4C, $24, $50 // paddd xmm1, [rsp + 80]
+ db $66, $0F, $FE, $54, $24, $60 // paddd xmm2, [rsp + 96]
+ db $66, $0F, $FE, $5C, $24, $70 // paddd xmm3, [rsp + 112]
+ db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $44, $0F, $6A, $D1 // punpckhdq xmm10, xmm1
+ db $66, $44, $0F, $6F, $DA // movdqa xmm11, xmm2
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $44, $0F, $6A, $DB // punpckhdq xmm11, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
+ db $66, $45, $0F, $6C, $D3 // punpcklqdq xmm10, xmm11
+ db $66, $41, $0F, $6D, $DB // punpckhqdq xmm3, xmm11
+ db $66, $0F, $FE, $A4, $24, $80, $00, $00, $00 // paddd xmm4, [rsp + 128]
+ db $66, $0F, $FE, $AC, $24, $90, $00, $00, $00 // paddd xmm5, [rsp + 144]
+ db $66, $0F, $FE, $B4, $24, $A0, $00, $00, $00 // paddd xmm6, [rsp + 160]
+ db $66, $0F, $FE, $BC, $24, $B0, $00, $00, $00 // paddd xmm7, [rsp + 176]
+ movdqa xmm2, xmm4
+ db $66, $0F, $62, $E5 // punpckldq xmm4, xmm5
+ db $66, $0F, $6A, $D5 // punpckhdq xmm2, xmm5
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
+ db $66, $0F, $62, $F7 // punpckldq xmm6, xmm7
+ db $66, $44, $0F, $6A, $DF // punpckhdq xmm11, xmm7
+ movdqa xmm5, xmm4
+ db $66, $0F, $6C, $E6 // punpcklqdq xmm4, xmm6
+ db $66, $0F, $6D, $EE // punpckhqdq xmm5, xmm6
+ movdqa xmm7, xmm2
+ db $66, $41, $0F, $6C, $D3 // punpcklqdq xmm2, xmm11
+ db $66, $41, $0F, $6D, $FB // punpckhqdq xmm7, xmm11
movdqu xmm6, [r8]
pxor xmm0, xmm6
movdqu [r9], xmm0
@@ -366,95 +356,85 @@
pxor xmm1, xmm6
movdqu [r9 + 64], xmm1
movdqu xmm6, [r8 + 128]
- pxor xmm4, xmm6
- movdqu [r9 + 128], xmm4
+ db $66, $44, $0F, $EF, $D6 // pxor xmm10, xmm6
+ movdqu [r9 + 128], xmm10
movdqu xmm6, [r8 + 192]
pxor xmm3, xmm6
movdqu [r9 + 192], xmm3
- movdqu xmm0, [rsp + 128]
- movdqu xmm1, [rsp + 144]
- movdqu xmm2, [rsp + 160]
- movdqu xmm3, [rsp + 176]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm6, [r8 + 16]
- pxor xmm0, xmm6
- movdqu [r9 + 16], xmm0
+ pxor xmm4, xmm6
+ movdqu [r9 + 16], xmm4
movdqu xmm6, [r8 + 80]
- pxor xmm1, xmm6
- movdqu [r9 + 80], xmm1
+ pxor xmm5, xmm6
+ movdqu [r9 + 80], xmm5
movdqu xmm6, [r8 + 144]
- pxor xmm4, xmm6
- movdqu [r9 + 144], xmm4
+ pxor xmm2, xmm6
+ movdqu [r9 + 144], xmm2
movdqu xmm6, [r8 + 208]
- pxor xmm3, xmm6
- movdqu [r9 + 208], xmm3
- movdqu xmm0, [rsp + 192]
- movdqu xmm1, [rsp + 208]
- movdqu xmm2, [rsp + 224]
- movdqu xmm3, [rsp + 240]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
- movdqu xmm6, [r8 + 32]
- pxor xmm0, xmm6
- movdqu [r9 + 32], xmm0
- movdqu xmm6, [r8 + 96]
- pxor xmm1, xmm6
- movdqu [r9 + 96], xmm1
- movdqu xmm6, [r8 + 160]
- pxor xmm4, xmm6
- movdqu [r9 + 160], xmm4
- movdqu xmm6, [r8 + 224]
- pxor xmm3, xmm6
- movdqu [r9 + 224], xmm3
- movdqu xmm0, [rsp + 256]
- movdqu xmm1, [rsp + 272]
- movdqu xmm2, [rsp + 288]
- movdqu xmm3, [rsp + 304]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
- movdqu xmm6, [r8 + 48]
- pxor xmm0, xmm6
- movdqu [r9 + 48], xmm0
- movdqu xmm6, [r8 + 112]
- pxor xmm1, xmm6
- movdqu [r9 + 112], xmm1
- movdqu xmm6, [r8 + 176]
- pxor xmm4, xmm6
- movdqu [r9 + 176], xmm4
- movdqu xmm6, [r8 + 240]
- pxor xmm3, xmm6
- movdqu [r9 + 240], xmm3
- add rsp, 336
+ pxor xmm7, xmm6
+ movdqu [r9 + 208], xmm7
+ movdqa xmm0, [rsp + 32]
+ movdqa xmm1, [rsp + 48]
+ db $66, $44, $0F, $FE, $84, $24, $C0, $00, $00, $00 // paddd xmm8, [rsp + 192]
+ db $66, $44, $0F, $FE, $8C, $24, $D0, $00, $00, $00 // paddd xmm9, [rsp + 208]
+ db $66, $0F, $FE, $84, $24, $E0, $00, $00, $00 // paddd xmm0, [rsp + 224]
+ db $66, $0F, $FE, $8C, $24, $F0, $00, $00, $00 // paddd xmm1, [rsp + 240]
+ db $66, $41, $0F, $6F, $D0 // movdqa xmm2, xmm8
+ db $66, $45, $0F, $62, $C1 // punpckldq xmm8, xmm9
+ db $66, $41, $0F, $6A, $D1 // punpckhdq xmm2, xmm9
+ movdqa xmm3, xmm0
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $D9 // punpckhdq xmm3, xmm1
+ db $66, $45, $0F, $6F, $C8 // movdqa xmm9, xmm8
+ db $66, $44, $0F, $6C, $C0 // punpcklqdq xmm8, xmm0
+ db $66, $44, $0F, $6D, $C8 // punpckhqdq xmm9, xmm0
+ movdqa xmm1, xmm2
+ db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3
+ db $66, $0F, $6D, $CB // punpckhqdq xmm1, xmm3
+ db $66, $44, $0F, $FE, $A4, $24, $00, $01, $00, $00 // paddd xmm12, [rsp + 256]
+ db $66, $44, $0F, $FE, $AC, $24, $10, $01, $00, $00 // paddd xmm13, [rsp + 272]
+ db $66, $44, $0F, $FE, $B4, $24, $20, $01, $00, $00 // paddd xmm14, [rsp + 288]
+ db $66, $44, $0F, $FE, $BC, $24, $30, $01, $00, $00 // paddd xmm15, [rsp + 304]
+ db $66, $41, $0F, $6F, $C4 // movdqa xmm0, xmm12
+ db $66, $45, $0F, $62, $E5 // punpckldq xmm12, xmm13
+ db $66, $41, $0F, $6A, $C5 // punpckhdq xmm0, xmm13
+ db $66, $41, $0F, $6F, $DE // movdqa xmm3, xmm14
+ db $66, $45, $0F, $62, $F7 // punpckldq xmm14, xmm15
+ db $66, $41, $0F, $6A, $DF // punpckhdq xmm3, xmm15
+ db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
+ db $66, $45, $0F, $6C, $E6 // punpcklqdq xmm12, xmm14
+ db $66, $45, $0F, $6D, $EE // punpckhqdq xmm13, xmm14
+ db $66, $44, $0F, $6F, $F8 // movdqa xmm15, xmm0
+ db $66, $0F, $6C, $C3 // punpcklqdq xmm0, xmm3
+ db $66, $44, $0F, $6D, $FB // punpckhqdq xmm15, xmm3
+ movdqu xmm3, [r8 + 32]
+ db $66, $44, $0F, $EF, $C3 // pxor xmm8, xmm3
+ movdqu [r9 + 32], xmm8
+ movdqu xmm3, [r8 + 96]
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ movdqu [r9 + 96], xmm9
+ movdqu xmm3, [r8 + 160]
+ pxor xmm2, xmm3
+ movdqu [r9 + 160], xmm2
+ movdqu xmm3, [r8 + 224]
+ pxor xmm1, xmm3
+ movdqu [r9 + 224], xmm1
+ movdqu xmm3, [r8 + 48]
+ db $66, $44, $0F, $EF, $E3 // pxor xmm12, xmm3
+ movdqu [r9 + 48], xmm12
+ movdqu xmm3, [r8 + 112]
+ db $66, $44, $0F, $EF, $EB // pxor xmm13, xmm3
+ movdqu [r9 + 112], xmm13
+ movdqu xmm3, [r8 + 176]
+ pxor xmm0, xmm3
+ movdqu [r9 + 176], xmm0
+ movdqu xmm3, [r8 + 240]
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ movdqu [r9 + 240], xmm15
+ add r8, 256
+ add r9, 256
+ dec r10
+ jnz @@c7539_sse2_outer
+ mov rsp, r11
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc
index 7af122fb..c02a316a 100644
--- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc
@@ -1,108 +1,157 @@
-// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (i386).
+// ChaCha7539 (IETF) 4-way SSSE3 keystream kernel, vertical layout (i386).
// -------------------------------------------------------------------
//
-// Four 64-byte ChaCha20 blocks, state words held vertically (one block per
-// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame;
-// each quarter-round loads its 4 words into xmm0-3 (temp xmm4) and the rot16/rot8 masks in xmm6-7.
+// Four 64-byte ChaCha20 blocks per group, state words held vertically (one
+// block per xmm lane). i386 has only 8 xmm, so the 16 state words live in a
+// 64B-aligned stack frame (original esp parked inside it); each quarter-round
+// loads its 4 words into xmm0-3 (temp xmm4) and the rot16/rot8 masks in xmm6-7.
// rot16/rot8 use pshufb; rot12/rot7 use shift+or.
//
-// On entry (after ClpSimdProc4Begin_i386.inc):
-// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut.
+// Streams AGroups x 256B in one call: prologue once, per-group ORIG->STATE
+// reload (counter lanes +4), fused accumulate/transpose/xor tail to AOut.
+//
+// On entry (after ClpSimdProc5Begin_i386.inc):
+// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut,
+// ecx = AGroups (4-block groups, >= 1).
//
// On exit:
-// AState word-12 counter (byte 48) advanced by 4; the keystream is xored
-// with AIn into AOut (256 bytes).
+// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is
+// xored with AIn into AOut (256*AGroups bytes).
//
// punpck* / pshufb are db-encoded for broad assembler compatibility.
- sub esp, 560
- mov dword ptr [esp + 528], $01000302
- mov dword ptr [esp + 532], $05040706
- mov dword ptr [esp + 536], $09080B0A
- mov dword ptr [esp + 540], $0D0C0F0E
- mov dword ptr [esp + 544], $02010003
- mov dword ptr [esp + 548], $06050407
- mov dword ptr [esp + 552], $0A09080B
- mov dword ptr [esp + 556], $0E0D0C0F
- movdqu xmm6, [esp + 528]
- movdqu xmm7, [esp + 544]
+ mov edx, esp
+ sub esp, 592
+ and esp, -64
+ mov dword ptr [esp + 576], edx
+ mov dword ptr [esp + 544], $01000302
+ mov dword ptr [esp + 548], $05040706
+ mov dword ptr [esp + 552], $09080B0A
+ mov dword ptr [esp + 556], $0D0C0F0E
+ mov dword ptr [esp + 560], $02010003
+ mov dword ptr [esp + 564], $06050407
+ mov dword ptr [esp + 568], $0A09080B
+ mov dword ptr [esp + 572], $0E0D0C0F
+ movdqa xmm6, [esp + 544]
+ movdqa xmm7, [esp + 560]
mov dword ptr [esp + 512], 0
mov dword ptr [esp + 516], 1
mov dword ptr [esp + 520], 2
mov dword ptr [esp + 524], 3
+ mov dword ptr [esp + 528], 4
+ mov dword ptr [esp + 532], 4
+ mov dword ptr [esp + 536], 4
+ mov dword ptr [esp + 540], 4
movd xmm0, [esi]
pshufd xmm0, xmm0, $00
- movdqu [esp], xmm0
- movdqu [esp + 256], xmm0
+ movdqa [esp], xmm0
+ movdqa [esp + 256], xmm0
movd xmm0, [esi + 4]
pshufd xmm0, xmm0, $00
- movdqu [esp + 16], xmm0
- movdqu [esp + 272], xmm0
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 272], xmm0
movd xmm0, [esi + 8]
pshufd xmm0, xmm0, $00
- movdqu [esp + 32], xmm0
- movdqu [esp + 288], xmm0
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 288], xmm0
movd xmm0, [esi + 12]
pshufd xmm0, xmm0, $00
- movdqu [esp + 48], xmm0
- movdqu [esp + 304], xmm0
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 304], xmm0
movd xmm0, [esi + 16]
pshufd xmm0, xmm0, $00
- movdqu [esp + 64], xmm0
- movdqu [esp + 320], xmm0
+ movdqa [esp + 64], xmm0
+ movdqa [esp + 320], xmm0
movd xmm0, [esi + 20]
pshufd xmm0, xmm0, $00
- movdqu [esp + 80], xmm0
- movdqu [esp + 336], xmm0
+ movdqa [esp + 80], xmm0
+ movdqa [esp + 336], xmm0
movd xmm0, [esi + 24]
pshufd xmm0, xmm0, $00
- movdqu [esp + 96], xmm0
- movdqu [esp + 352], xmm0
+ movdqa [esp + 96], xmm0
+ movdqa [esp + 352], xmm0
movd xmm0, [esi + 28]
pshufd xmm0, xmm0, $00
- movdqu [esp + 112], xmm0
- movdqu [esp + 368], xmm0
+ movdqa [esp + 112], xmm0
+ movdqa [esp + 368], xmm0
movd xmm0, [esi + 32]
pshufd xmm0, xmm0, $00
- movdqu [esp + 128], xmm0
- movdqu [esp + 384], xmm0
+ movdqa [esp + 128], xmm0
+ movdqa [esp + 384], xmm0
movd xmm0, [esi + 36]
pshufd xmm0, xmm0, $00
- movdqu [esp + 144], xmm0
- movdqu [esp + 400], xmm0
+ movdqa [esp + 144], xmm0
+ movdqa [esp + 400], xmm0
movd xmm0, [esi + 40]
pshufd xmm0, xmm0, $00
- movdqu [esp + 160], xmm0
- movdqu [esp + 416], xmm0
+ movdqa [esp + 160], xmm0
+ movdqa [esp + 416], xmm0
movd xmm0, [esi + 44]
pshufd xmm0, xmm0, $00
- movdqu [esp + 176], xmm0
- movdqu [esp + 432], xmm0
+ movdqa [esp + 176], xmm0
+ movdqa [esp + 432], xmm0
movd xmm0, [esi + 48]
pshufd xmm0, xmm0, $00
- movdqu xmm1, [esp + 512]
- paddd xmm0, xmm1
- movdqu [esp + 192], xmm0
- movdqu [esp + 448], xmm0
+ db $66, $0F, $FE, $84, $24, $00, $02, $00, $00 // paddd xmm0, [esp + 512]
+ movdqa [esp + 192], xmm0
+ movdqa [esp + 448], xmm0
movd xmm0, [esi + 52]
pshufd xmm0, xmm0, $00
- movdqu [esp + 208], xmm0
- movdqu [esp + 464], xmm0
+ movdqa [esp + 208], xmm0
+ movdqa [esp + 464], xmm0
movd xmm0, [esi + 56]
pshufd xmm0, xmm0, $00
- movdqu [esp + 224], xmm0
- movdqu [esp + 480], xmm0
+ movdqa [esp + 224], xmm0
+ movdqa [esp + 480], xmm0
movd xmm0, [esi + 60]
pshufd xmm0, xmm0, $00
- movdqu [esp + 240], xmm0
- movdqu [esp + 496], xmm0
+ movdqa [esp + 240], xmm0
+ movdqa [esp + 496], xmm0
+ jmp @@c7539_ssse3_enter
+@@c7539_ssse3_outer:
+ movdqa xmm0, [esp + 256]
+ movdqa [esp], xmm0
+ movdqa xmm0, [esp + 272]
+ movdqa [esp + 16], xmm0
+ movdqa xmm0, [esp + 288]
+ movdqa [esp + 32], xmm0
+ movdqa xmm0, [esp + 304]
+ movdqa [esp + 48], xmm0
+ movdqa xmm0, [esp + 320]
+ movdqa [esp + 64], xmm0
+ movdqa xmm0, [esp + 336]
+ movdqa [esp + 80], xmm0
+ movdqa xmm0, [esp + 352]
+ movdqa [esp + 96], xmm0
+ movdqa xmm0, [esp + 368]
+ movdqa [esp + 112], xmm0
+ movdqa xmm0, [esp + 384]
+ movdqa [esp + 128], xmm0
+ movdqa xmm0, [esp + 400]
+ movdqa [esp + 144], xmm0
+ movdqa xmm0, [esp + 416]
+ movdqa [esp + 160], xmm0
+ movdqa xmm0, [esp + 432]
+ movdqa [esp + 176], xmm0
+ movdqa xmm0, [esp + 448]
+ db $66, $0F, $FE, $84, $24, $10, $02, $00, $00 // paddd xmm0, [esp + 528]
+ movdqa [esp + 448], xmm0
+ movdqa [esp + 192], xmm0
+ movdqa xmm0, [esp + 464]
+ movdqa [esp + 208], xmm0
+ movdqa xmm0, [esp + 480]
+ movdqa [esp + 224], xmm0
+ movdqa xmm0, [esp + 496]
+ movdqa [esp + 240], xmm0
+@@c7539_ssse3_enter:
+ mov edx, ebx
@@c7539_ssse3_round:
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 64]
- movdqu xmm2, [esp + 128]
- movdqu xmm3, [esp + 192]
+ movdqa xmm0, [esp]
+ movdqa xmm1, [esp + 64]
+ movdqa xmm2, [esp + 128]
+ movdqa xmm3, [esp + 192]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -111,24 +160,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp], xmm0
- movdqu [esp + 64], xmm1
- movdqu [esp + 128], xmm2
- movdqu [esp + 192], xmm3
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 144]
- movdqu xmm3, [esp + 208]
+ movdqa [esp], xmm0
+ movdqa [esp + 64], xmm1
+ movdqa [esp + 128], xmm2
+ movdqa [esp + 192], xmm3
+ movdqa xmm0, [esp + 16]
+ movdqa xmm1, [esp + 80]
+ movdqa xmm2, [esp + 144]
+ movdqa xmm3, [esp + 208]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -137,24 +186,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 16], xmm0
- movdqu [esp + 80], xmm1
- movdqu [esp + 144], xmm2
- movdqu [esp + 208], xmm3
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 96]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 224]
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 80], xmm1
+ movdqa [esp + 144], xmm2
+ movdqa [esp + 208], xmm3
+ movdqa xmm0, [esp + 32]
+ movdqa xmm1, [esp + 96]
+ movdqa xmm2, [esp + 160]
+ movdqa xmm3, [esp + 224]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -163,24 +212,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 32], xmm0
- movdqu [esp + 96], xmm1
- movdqu [esp + 160], xmm2
- movdqu [esp + 224], xmm3
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 112]
- movdqu xmm2, [esp + 176]
- movdqu xmm3, [esp + 240]
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 96], xmm1
+ movdqa [esp + 160], xmm2
+ movdqa [esp + 224], xmm3
+ movdqa xmm0, [esp + 48]
+ movdqa xmm1, [esp + 112]
+ movdqa xmm2, [esp + 176]
+ movdqa xmm3, [esp + 240]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -189,24 +238,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 48], xmm0
- movdqu [esp + 112], xmm1
- movdqu [esp + 176], xmm2
- movdqu [esp + 240], xmm3
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 240]
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 112], xmm1
+ movdqa [esp + 176], xmm2
+ movdqa [esp + 240], xmm3
+ movdqa xmm0, [esp]
+ movdqa xmm1, [esp + 80]
+ movdqa xmm2, [esp + 160]
+ movdqa xmm3, [esp + 240]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -215,24 +264,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp], xmm0
- movdqu [esp + 80], xmm1
- movdqu [esp + 160], xmm2
- movdqu [esp + 240], xmm3
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 96]
- movdqu xmm2, [esp + 176]
- movdqu xmm3, [esp + 192]
+ movdqa [esp], xmm0
+ movdqa [esp + 80], xmm1
+ movdqa [esp + 160], xmm2
+ movdqa [esp + 240], xmm3
+ movdqa xmm0, [esp + 16]
+ movdqa xmm1, [esp + 96]
+ movdqa xmm2, [esp + 176]
+ movdqa xmm3, [esp + 192]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -241,24 +290,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 16], xmm0
- movdqu [esp + 96], xmm1
- movdqu [esp + 176], xmm2
- movdqu [esp + 192], xmm3
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 112]
- movdqu xmm2, [esp + 128]
- movdqu xmm3, [esp + 208]
+ movdqa [esp + 16], xmm0
+ movdqa [esp + 96], xmm1
+ movdqa [esp + 176], xmm2
+ movdqa [esp + 192], xmm3
+ movdqa xmm0, [esp + 32]
+ movdqa xmm1, [esp + 112]
+ movdqa xmm2, [esp + 128]
+ movdqa xmm3, [esp + 208]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -267,24 +316,24 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 32], xmm0
- movdqu [esp + 112], xmm1
- movdqu [esp + 128], xmm2
- movdqu [esp + 208], xmm3
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 64]
- movdqu xmm2, [esp + 144]
- movdqu xmm3, [esp + 224]
+ movdqa [esp + 32], xmm0
+ movdqa [esp + 112], xmm1
+ movdqa [esp + 128], xmm2
+ movdqa [esp + 208], xmm3
+ movdqa xmm0, [esp + 48]
+ movdqa xmm1, [esp + 64]
+ movdqa xmm2, [esp + 144]
+ movdqa xmm3, [esp + 224]
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
+ db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
@@ -293,103 +342,43 @@
por xmm1, xmm4
paddd xmm0, xmm1
pxor xmm3, xmm0
- db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7
paddd xmm2, xmm3
pxor xmm1, xmm2
movdqa xmm4, xmm1
pslld xmm1, 7
psrld xmm4, 25
por xmm1, xmm4
- movdqu [esp + 48], xmm0
- movdqu [esp + 64], xmm1
- movdqu [esp + 144], xmm2
- movdqu [esp + 224], xmm3
- sub ebx, 2
+ movdqa [esp + 48], xmm0
+ movdqa [esp + 64], xmm1
+ movdqa [esp + 144], xmm2
+ movdqa [esp + 224], xmm3
+ sub edx, 2
jg @@c7539_ssse3_round
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 256]
- paddd xmm0, xmm1
- movdqu [esp], xmm0
- movdqu xmm0, [esp + 16]
- movdqu xmm1, [esp + 272]
- paddd xmm0, xmm1
- movdqu [esp + 16], xmm0
- movdqu xmm0, [esp + 32]
- movdqu xmm1, [esp + 288]
- paddd xmm0, xmm1
- movdqu [esp + 32], xmm0
- movdqu xmm0, [esp + 48]
- movdqu xmm1, [esp + 304]
- paddd xmm0, xmm1
- movdqu [esp + 48], xmm0
- movdqu xmm0, [esp + 64]
- movdqu xmm1, [esp + 320]
- paddd xmm0, xmm1
- movdqu [esp + 64], xmm0
- movdqu xmm0, [esp + 80]
- movdqu xmm1, [esp + 336]
- paddd xmm0, xmm1
- movdqu [esp + 80], xmm0
- movdqu xmm0, [esp + 96]
- movdqu xmm1, [esp + 352]
- paddd xmm0, xmm1
- movdqu [esp + 96], xmm0
- movdqu xmm0, [esp + 112]
- movdqu xmm1, [esp + 368]
- paddd xmm0, xmm1
- movdqu [esp + 112], xmm0
- movdqu xmm0, [esp + 128]
- movdqu xmm1, [esp + 384]
- paddd xmm0, xmm1
- movdqu [esp + 128], xmm0
- movdqu xmm0, [esp + 144]
- movdqu xmm1, [esp + 400]
- paddd xmm0, xmm1
- movdqu [esp + 144], xmm0
- movdqu xmm0, [esp + 160]
- movdqu xmm1, [esp + 416]
- paddd xmm0, xmm1
- movdqu [esp + 160], xmm0
- movdqu xmm0, [esp + 176]
- movdqu xmm1, [esp + 432]
- paddd xmm0, xmm1
- movdqu [esp + 176], xmm0
- movdqu xmm0, [esp + 192]
- movdqu xmm1, [esp + 448]
- paddd xmm0, xmm1
- movdqu [esp + 192], xmm0
- movdqu xmm0, [esp + 208]
- movdqu xmm1, [esp + 464]
- paddd xmm0, xmm1
- movdqu [esp + 208], xmm0
- movdqu xmm0, [esp + 224]
- movdqu xmm1, [esp + 480]
- paddd xmm0, xmm1
- movdqu [esp + 224], xmm0
- movdqu xmm0, [esp + 240]
- movdqu xmm1, [esp + 496]
- paddd xmm0, xmm1
- movdqu [esp + 240], xmm0
add dword ptr [esi + 48], 4
{$IFDEF CHACHA_CTR64}
adc dword ptr [esi + 52], 0
{$ENDIF}
- movdqu xmm0, [esp]
- movdqu xmm1, [esp + 16]
- movdqu xmm2, [esp + 32]
- movdqu xmm3, [esp + 48]
+ movdqa xmm0, [esp]
+ db $66, $0F, $FE, $84, $24, $00, $01, $00, $00 // paddd xmm0, [esp + 256]
+ movdqa xmm1, [esp + 16]
+ db $66, $0F, $FE, $8C, $24, $10, $01, $00, $00 // paddd xmm1, [esp + 272]
+ movdqa xmm2, [esp + 32]
+ db $66, $0F, $FE, $94, $24, $20, $01, $00, $00 // paddd xmm2, [esp + 288]
+ movdqa xmm3, [esp + 48]
+ db $66, $0F, $FE, $9C, $24, $30, $01, $00, $00 // paddd xmm3, [esp + 304]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi]
pxor xmm0, xmm5
movdqu [eax], xmm0
@@ -402,22 +391,26 @@
movdqu xmm5, [edi + 192]
pxor xmm3, xmm5
movdqu [eax + 192], xmm3
- movdqu xmm0, [esp + 64]
- movdqu xmm1, [esp + 80]
- movdqu xmm2, [esp + 96]
- movdqu xmm3, [esp + 112]
+ movdqa xmm0, [esp + 64]
+ db $66, $0F, $FE, $84, $24, $40, $01, $00, $00 // paddd xmm0, [esp + 320]
+ movdqa xmm1, [esp + 80]
+ db $66, $0F, $FE, $8C, $24, $50, $01, $00, $00 // paddd xmm1, [esp + 336]
+ movdqa xmm2, [esp + 96]
+ db $66, $0F, $FE, $94, $24, $60, $01, $00, $00 // paddd xmm2, [esp + 352]
+ movdqa xmm3, [esp + 112]
+ db $66, $0F, $FE, $9C, $24, $70, $01, $00, $00 // paddd xmm3, [esp + 368]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 16]
pxor xmm0, xmm5
movdqu [eax + 16], xmm0
@@ -430,22 +423,26 @@
movdqu xmm5, [edi + 208]
pxor xmm3, xmm5
movdqu [eax + 208], xmm3
- movdqu xmm0, [esp + 128]
- movdqu xmm1, [esp + 144]
- movdqu xmm2, [esp + 160]
- movdqu xmm3, [esp + 176]
+ movdqa xmm0, [esp + 128]
+ db $66, $0F, $FE, $84, $24, $80, $01, $00, $00 // paddd xmm0, [esp + 384]
+ movdqa xmm1, [esp + 144]
+ db $66, $0F, $FE, $8C, $24, $90, $01, $00, $00 // paddd xmm1, [esp + 400]
+ movdqa xmm2, [esp + 160]
+ db $66, $0F, $FE, $94, $24, $A0, $01, $00, $00 // paddd xmm2, [esp + 416]
+ movdqa xmm3, [esp + 176]
+ db $66, $0F, $FE, $9C, $24, $B0, $01, $00, $00 // paddd xmm3, [esp + 432]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 32]
pxor xmm0, xmm5
movdqu [eax + 32], xmm0
@@ -458,22 +455,26 @@
movdqu xmm5, [edi + 224]
pxor xmm3, xmm5
movdqu [eax + 224], xmm3
- movdqu xmm0, [esp + 192]
- movdqu xmm1, [esp + 208]
- movdqu xmm2, [esp + 224]
- movdqu xmm3, [esp + 240]
+ movdqa xmm0, [esp + 192]
+ db $66, $0F, $FE, $84, $24, $C0, $01, $00, $00 // paddd xmm0, [esp + 448]
+ movdqa xmm1, [esp + 208]
+ db $66, $0F, $FE, $8C, $24, $D0, $01, $00, $00 // paddd xmm1, [esp + 464]
+ movdqa xmm2, [esp + 224]
+ db $66, $0F, $FE, $94, $24, $E0, $01, $00, $00 // paddd xmm2, [esp + 480]
+ movdqa xmm3, [esp + 240]
+ db $66, $0F, $FE, $9C, $24, $F0, $01, $00, $00 // paddd xmm3, [esp + 496]
movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
+ db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm5, [edi + 48]
pxor xmm0, xmm5
movdqu [eax + 48], xmm0
@@ -486,7 +487,11 @@
movdqu xmm5, [edi + 240]
pxor xmm3, xmm5
movdqu [eax + 240], xmm3
- add esp, 560
+ add edi, 256
+ add eax, 256
+ dec ecx
+ jnz @@c7539_ssse3_outer
+ mov esp, dword ptr [esp + 576]
pop edi
pop esi
pop ebx
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc
index ac08b118..0a64aaf2 100644
--- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc
@@ -1,326 +1,324 @@
-// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (x86-64).
+// ChaCha7539 (IETF) 4-way SSSE3 keystream kernel, vertical layout (x86-64).
// -------------------------------------------------------------------
//
-// Four 64-byte ChaCha20 blocks, state words held vertically (one block per
-// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words
-// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch, xmm10-11 the rot16/rot8 pshufb masks.
-// rot16/rot8 use pshufb; rot12/rot7 use shift+or.
+// Four 64-byte ChaCha20 blocks per group, state words held vertically (one
+// block per xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7),
+// xmm12-15 = words d (12-15). Words c (8-11): two are resident in xmm8/xmm9,
+// the other two wait in stack slots; quarter-rounds run as two resident pairs
+// with one pair swap per round (2 stores + 2 loads). xmm10/xmm11 are the
+// rotate temps. rot16/rot8 use pshufb (mask reloaded into the just-freed temp);
+// rot12/rot7 use shift+or. The frame is 64B-aligned (r11 = frame register).
//
-// On entry (after ClpSimdProc4Begin_x86_64.inc):
-// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut.
+// Streams AGroups x 256B in one call: prologue once, light per-group reload
+// (counter lanes +4), fused accumulate/transpose/xor tail straight to AOut.
+//
+// On entry (after ClpSimdProc5Begin_x86_64.inc):
+// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut,
+// r10 = AGroups (4-block groups, >= 1).
//
// On exit:
-// AState word-12 counter (byte 48) advanced by 4; the keystream is xored
-// with AIn into AOut (256 bytes).
+// AState word-12 counter (byte 48) advanced by 4*AGroups; the keystream is
+// xored with AIn into AOut (256*AGroups bytes).
//
// Ops touching the extended xmm8-15 registers are db-encoded for broad
// assembler compatibility.
- mov r10, rcx
+ mov r10d, r10d
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- sub rsp, 368
- mov dword ptr [rsp + 336], $01000302
- mov dword ptr [rsp + 340], $05040706
- mov dword ptr [rsp + 344], $09080B0A
- mov dword ptr [rsp + 348], $0D0C0F0E
- mov dword ptr [rsp + 352], $02010003
- mov dword ptr [rsp + 356], $06050407
- mov dword ptr [rsp + 360], $0A09080B
- mov dword ptr [rsp + 364], $0E0D0C0F
- movdqu xmm10, [rsp + 336]
- movdqu xmm11, [rsp + 352]
+ mov r11, rsp
+ sub rsp, 384
+ and rsp, -64
mov dword ptr [rsp + 320], 0
mov dword ptr [rsp + 324], 1
mov dword ptr [rsp + 328], 2
mov dword ptr [rsp + 332], 3
+ mov dword ptr [rsp + 336], 4
+ mov dword ptr [rsp + 340], 4
+ mov dword ptr [rsp + 344], 4
+ mov dword ptr [rsp + 348], 4
+ mov dword ptr [rsp + 352], $01000302
+ mov dword ptr [rsp + 356], $05040706
+ mov dword ptr [rsp + 360], $09080B0A
+ mov dword ptr [rsp + 364], $0D0C0F0E
+ mov dword ptr [rsp + 368], $02010003
+ mov dword ptr [rsp + 372], $06050407
+ mov dword ptr [rsp + 376], $0A09080B
+ mov dword ptr [rsp + 380], $0E0D0C0F
movd xmm0, [rdx]
pshufd xmm0, xmm0, $00
- movdqu [rsp + 64], xmm0
+ movdqa [rsp + 64], xmm0
movd xmm1, [rdx + 4]
pshufd xmm1, xmm1, $00
- movdqu [rsp + 80], xmm1
+ movdqa [rsp + 80], xmm1
movd xmm2, [rdx + 8]
pshufd xmm2, xmm2, $00
- movdqu [rsp + 96], xmm2
+ movdqa [rsp + 96], xmm2
movd xmm3, [rdx + 12]
pshufd xmm3, xmm3, $00
- movdqu [rsp + 112], xmm3
+ movdqa [rsp + 112], xmm3
movd xmm4, [rdx + 16]
pshufd xmm4, xmm4, $00
- movdqu [rsp + 128], xmm4
+ movdqa [rsp + 128], xmm4
movd xmm5, [rdx + 20]
pshufd xmm5, xmm5, $00
- movdqu [rsp + 144], xmm5
+ movdqa [rsp + 144], xmm5
movd xmm6, [rdx + 24]
pshufd xmm6, xmm6, $00
- movdqu [rsp + 160], xmm6
+ movdqa [rsp + 160], xmm6
movd xmm7, [rdx + 28]
pshufd xmm7, xmm7, $00
- movdqu [rsp + 176], xmm7
+ movdqa [rsp + 176], xmm7
movd xmm8, [rdx + 32]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp], xmm8
- movdqu [rsp + 192], xmm8
- movd xmm8, [rdx + 36]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 16], xmm8
- movdqu [rsp + 208], xmm8
- movd xmm8, [rdx + 40]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 32], xmm8
- movdqu [rsp + 224], xmm8
- movd xmm8, [rdx + 44]
- db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
- movdqu [rsp + 48], xmm8
- movdqu [rsp + 240], xmm8
+ db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00
+ movdqa [rsp + 192], xmm8
+ movd xmm9, [rdx + 36]
+ db $66, $45, $0F, $70, $C9, $00 // pshufd xmm9, xmm9, $00
+ movdqa [rsp + 208], xmm9
+ movd xmm10, [rdx + 40]
+ db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00
+ movdqa [rsp + 224], xmm10
+ movdqa [rsp + 32], xmm10
+ movd xmm10, [rdx + 44]
+ db $66, $45, $0F, $70, $D2, $00 // pshufd xmm10, xmm10, $00
+ movdqa [rsp + 240], xmm10
+ movdqa [rsp + 48], xmm10
movd xmm12, [rdx + 48]
- db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00
- movdqu xmm8, [rsp + 320]
- db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8
- movdqu [rsp + 256], xmm12
+ db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00
+ db $66, $44, $0F, $FE, $A4, $24, $40, $01, $00, $00 // paddd xmm12, [rsp + 320]
+ movdqa [rsp + 256], xmm12
movd xmm13, [rdx + 52]
- db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00
- movdqu [rsp + 272], xmm13
+ db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00
+ movdqa [rsp + 272], xmm13
movd xmm14, [rdx + 56]
- db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00
- movdqu [rsp + 288], xmm14
+ db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00
+ movdqa [rsp + 288], xmm14
movd xmm15, [rdx + 60]
- db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00
- movdqu [rsp + 304], xmm15
+ db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00
+ movdqa [rsp + 304], xmm15
+ movdqa xmm11, [rsp + 352]
+ jmp @@c7539_ssse3_enter
+@@c7539_ssse3_outer:
+ movdqa xmm0, [rsp + 64]
+ movdqa xmm1, [rsp + 80]
+ movdqa xmm2, [rsp + 96]
+ movdqa xmm3, [rsp + 112]
+ movdqa xmm4, [rsp + 128]
+ movdqa xmm5, [rsp + 144]
+ movdqa xmm6, [rsp + 160]
+ movdqa xmm7, [rsp + 176]
+ movdqa xmm8, [rsp + 192]
+ movdqa xmm9, [rsp + 208]
+ movdqa xmm10, [rsp + 224]
+ movdqa [rsp + 32], xmm10
+ movdqa xmm10, [rsp + 240]
+ movdqa [rsp + 48], xmm10
+ movdqa xmm12, [rsp + 256]
+ db $66, $44, $0F, $FE, $A4, $24, $50, $01, $00, $00 // paddd xmm12, [rsp + 336]
+ movdqa [rsp + 256], xmm12
+ movdqa xmm13, [rsp + 272]
+ movdqa xmm14, [rsp + 288]
+ movdqa xmm15, [rsp + 304]
+ movdqa xmm11, [rsp + 352]
+@@c7539_ssse3_enter:
+ mov eax, ecx
@@c7539_ssse3_round:
- movdqu xmm8, [rsp]
- paddd xmm0, xmm4
- db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
- db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
- pslld xmm4, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
paddd xmm0, xmm4
- db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
- db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
- pslld xmm4, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
- movdqu [rsp], xmm8
- movdqu xmm8, [rsp + 16]
+ db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
+ db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11
paddd xmm1, xmm5
- db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
- db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
+ db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
+ db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11
+ db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
+ db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
+ db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4
+ pslld xmm4, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10
+ movdqa xmm10, [rsp + 368]
+ db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13
+ db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9
+ db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5
pslld xmm5, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $EB // por xmm5, xmm11
+ paddd xmm0, xmm4
+ db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0
+ db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10
paddd xmm1, xmm5
- db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
- db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
+ db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1
+ db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10
+ db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
+ db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
+ db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4
+ pslld xmm4, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11
+ movdqa xmm11, [rsp + 352]
+ db $66, $45, $0F, $FE, $CD // paddd xmm9, xmm13
+ db $66, $41, $0F, $EF, $E9 // pxor xmm5, xmm9
+ db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5
pslld xmm5, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- movdqu [rsp + 16], xmm8
- movdqu xmm8, [rsp + 32]
- paddd xmm2, xmm6
- db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
- db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
- pslld xmm6, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $EA // por xmm5, xmm10
+ movdqa [rsp], xmm8
+ movdqa [rsp + 16], xmm9
+ movdqa xmm8, [rsp + 32]
+ movdqa xmm9, [rsp + 48]
paddd xmm2, xmm6
- db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
- db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
- pslld xmm6, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
- movdqu [rsp + 32], xmm8
- movdqu xmm8, [rsp + 48]
+ db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
+ db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11
paddd xmm3, xmm7
- db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
- db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11
+ db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
+ db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
+ db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6
+ pslld xmm6, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10
+ movdqa xmm10, [rsp + 368]
+ db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15
+ db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9
+ db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7
pslld xmm7, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $FB // por xmm7, xmm11
+ paddd xmm2, xmm6
+ db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2
+ db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10
paddd xmm3, xmm7
- db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
- db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10
+ db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
+ db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
+ pslld xmm6, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11
+ movdqa xmm11, [rsp + 352]
+ db $66, $45, $0F, $FE, $CF // paddd xmm9, xmm15
+ db $66, $41, $0F, $EF, $F9 // pxor xmm7, xmm9
+ db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7
pslld xmm7, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
- movdqu [rsp + 48], xmm8
- movdqu xmm8, [rsp + 32]
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $FA // por xmm7, xmm10
paddd xmm0, xmm5
- db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
- db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
- pslld xmm5, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- paddd xmm0, xmm5
- db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
- db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11
- db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
- db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
- db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5
- pslld xmm5, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9
- movdqu [rsp + 32], xmm8
- movdqu xmm8, [rsp + 48]
+ db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
+ db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11
paddd xmm1, xmm6
- db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
- db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
+ db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
+ db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11
+ db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
+ db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
+ db $66, $44, $0F, $6F, $D5 // movdqa xmm10, xmm5
+ pslld xmm5, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $EA // por xmm5, xmm10
+ movdqa xmm10, [rsp + 368]
+ db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12
+ db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
pslld xmm6, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $F3 // por xmm6, xmm11
+ paddd xmm0, xmm5
+ db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0
+ db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10
paddd xmm1, xmm6
- db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
- db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11
- db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12
- db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8
- db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6
+ db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1
+ db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10
+ db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15
+ db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8
+ db $66, $44, $0F, $6F, $DD // movdqa xmm11, xmm5
+ pslld xmm5, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $EB // por xmm5, xmm11
+ movdqa xmm11, [rsp + 352]
+ db $66, $45, $0F, $FE, $CC // paddd xmm9, xmm12
+ db $66, $41, $0F, $EF, $F1 // pxor xmm6, xmm9
+ db $66, $44, $0F, $6F, $D6 // movdqa xmm10, xmm6
pslld xmm6, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9
- movdqu [rsp + 48], xmm8
- movdqu xmm8, [rsp]
- paddd xmm2, xmm7
- db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
- db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
- pslld xmm7, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $F2 // por xmm6, xmm10
+ movdqa [rsp + 32], xmm8
+ movdqa [rsp + 48], xmm9
+ movdqa xmm8, [rsp]
+ movdqa xmm9, [rsp + 16]
paddd xmm2, xmm7
- db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
- db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11
- db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
- db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
- db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7
- pslld xmm7, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9
- movdqu [rsp], xmm8
- movdqu xmm8, [rsp + 16]
+ db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
+ db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11
paddd xmm3, xmm4
- db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
- db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
+ db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
+ db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11
+ db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
+ db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
+ db $66, $44, $0F, $6F, $D7 // movdqa xmm10, xmm7
+ pslld xmm7, 12
+ db $66, $41, $0F, $72, $D2, $14 // psrld xmm10, 20
+ db $66, $41, $0F, $EB, $FA // por xmm7, xmm10
+ movdqa xmm10, [rsp + 368]
+ db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14
+ db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9
+ db $66, $44, $0F, $6F, $DC // movdqa xmm11, xmm4
pslld xmm4, 12
- db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
+ db $66, $41, $0F, $72, $D3, $14 // psrld xmm11, 20
+ db $66, $41, $0F, $EB, $E3 // por xmm4, xmm11
+ paddd xmm2, xmm7
+ db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2
+ db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10
paddd xmm3, xmm4
- db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
- db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11
- db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14
- db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8
- db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4
+ db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3
+ db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10
+ db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13
+ db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8
+ db $66, $44, $0F, $6F, $DF // movdqa xmm11, xmm7
+ pslld xmm7, 7
+ db $66, $41, $0F, $72, $D3, $19 // psrld xmm11, 25
+ db $66, $41, $0F, $EB, $FB // por xmm7, xmm11
+ movdqa xmm11, [rsp + 352]
+ db $66, $45, $0F, $FE, $CE // paddd xmm9, xmm14
+ db $66, $41, $0F, $EF, $E1 // pxor xmm4, xmm9
+ db $66, $44, $0F, $6F, $D4 // movdqa xmm10, xmm4
pslld xmm4, 7
- db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25
- db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9
- movdqu [rsp + 16], xmm8
- sub r10, 2
+ db $66, $41, $0F, $72, $D2, $19 // psrld xmm10, 25
+ db $66, $41, $0F, $EB, $E2 // por xmm4, xmm10
+ sub eax, 2
jg @@c7539_ssse3_round
- movdqu xmm8, [rsp + 64]
- db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8
- movdqu [rsp + 64], xmm0
- movdqu xmm8, [rsp + 80]
- db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8
- movdqu [rsp + 80], xmm1
- movdqu xmm8, [rsp + 96]
- db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8
- movdqu [rsp + 96], xmm2
- movdqu xmm8, [rsp + 112]
- db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8
- movdqu [rsp + 112], xmm3
- movdqu xmm8, [rsp + 128]
- db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8
- movdqu [rsp + 128], xmm4
- movdqu xmm8, [rsp + 144]
- db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8
- movdqu [rsp + 144], xmm5
- movdqu xmm8, [rsp + 160]
- db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8
- movdqu [rsp + 160], xmm6
- movdqu xmm8, [rsp + 176]
- db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8
- movdqu [rsp + 176], xmm7
- movdqu xmm8, [rsp + 256]
- db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8
- movdqu [rsp + 256], xmm12
- movdqu xmm8, [rsp + 272]
- db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8
- movdqu [rsp + 272], xmm13
- movdqu xmm8, [rsp + 288]
- db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8
- movdqu [rsp + 288], xmm14
- movdqu xmm8, [rsp + 304]
- db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8
- movdqu [rsp + 304], xmm15
- movdqu xmm8, [rsp]
- movdqu xmm9, [rsp + 192]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 192], xmm8
- movdqu xmm8, [rsp + 16]
- movdqu xmm9, [rsp + 208]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 208], xmm8
- movdqu xmm8, [rsp + 32]
- movdqu xmm9, [rsp + 224]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 224], xmm8
- movdqu xmm8, [rsp + 48]
- movdqu xmm9, [rsp + 240]
- db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9
- movdqu [rsp + 240], xmm8
add dword ptr [rdx + 48], 4
{$IFDEF CHACHA_CTR64}
adc dword ptr [rdx + 52], 0
{$ENDIF}
- movdqu xmm0, [rsp + 64]
- movdqu xmm1, [rsp + 80]
- movdqu xmm2, [rsp + 96]
- movdqu xmm3, [rsp + 112]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
+ db $66, $0F, $FE, $44, $24, $40 // paddd xmm0, [rsp + 64]
+ db $66, $0F, $FE, $4C, $24, $50 // paddd xmm1, [rsp + 80]
+ db $66, $0F, $FE, $54, $24, $60 // paddd xmm2, [rsp + 96]
+ db $66, $0F, $FE, $5C, $24, $70 // paddd xmm3, [rsp + 112]
+ db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $44, $0F, $6A, $D1 // punpckhdq xmm10, xmm1
+ db $66, $44, $0F, $6F, $DA // movdqa xmm11, xmm2
+ db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
+ db $66, $44, $0F, $6A, $DB // punpckhdq xmm11, xmm3
movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
+ db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
+ db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
+ db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
+ db $66, $45, $0F, $6C, $D3 // punpcklqdq xmm10, xmm11
+ db $66, $41, $0F, $6D, $DB // punpckhqdq xmm3, xmm11
+ db $66, $0F, $FE, $A4, $24, $80, $00, $00, $00 // paddd xmm4, [rsp + 128]
+ db $66, $0F, $FE, $AC, $24, $90, $00, $00, $00 // paddd xmm5, [rsp + 144]
+ db $66, $0F, $FE, $B4, $24, $A0, $00, $00, $00 // paddd xmm6, [rsp + 160]
+ db $66, $0F, $FE, $BC, $24, $B0, $00, $00, $00 // paddd xmm7, [rsp + 176]
+ movdqa xmm2, xmm4
+ db $66, $0F, $62, $E5 // punpckldq xmm4, xmm5
+ db $66, $0F, $6A, $D5 // punpckhdq xmm2, xmm5
+ db $66, $44, $0F, $6F, $DE // movdqa xmm11, xmm6
+ db $66, $0F, $62, $F7 // punpckldq xmm6, xmm7
+ db $66, $44, $0F, $6A, $DF // punpckhdq xmm11, xmm7
+ movdqa xmm5, xmm4
+ db $66, $0F, $6C, $E6 // punpcklqdq xmm4, xmm6
+ db $66, $0F, $6D, $EE // punpckhqdq xmm5, xmm6
+ movdqa xmm7, xmm2
+ db $66, $41, $0F, $6C, $D3 // punpcklqdq xmm2, xmm11
+ db $66, $41, $0F, $6D, $FB // punpckhqdq xmm7, xmm11
movdqu xmm6, [r8]
pxor xmm0, xmm6
movdqu [r9], xmm0
@@ -328,95 +326,85 @@
pxor xmm1, xmm6
movdqu [r9 + 64], xmm1
movdqu xmm6, [r8 + 128]
- pxor xmm4, xmm6
- movdqu [r9 + 128], xmm4
+ db $66, $44, $0F, $EF, $D6 // pxor xmm10, xmm6
+ movdqu [r9 + 128], xmm10
movdqu xmm6, [r8 + 192]
pxor xmm3, xmm6
movdqu [r9 + 192], xmm3
- movdqu xmm0, [rsp + 128]
- movdqu xmm1, [rsp + 144]
- movdqu xmm2, [rsp + 160]
- movdqu xmm3, [rsp + 176]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
movdqu xmm6, [r8 + 16]
- pxor xmm0, xmm6
- movdqu [r9 + 16], xmm0
+ pxor xmm4, xmm6
+ movdqu [r9 + 16], xmm4
movdqu xmm6, [r8 + 80]
- pxor xmm1, xmm6
- movdqu [r9 + 80], xmm1
+ pxor xmm5, xmm6
+ movdqu [r9 + 80], xmm5
movdqu xmm6, [r8 + 144]
- pxor xmm4, xmm6
- movdqu [r9 + 144], xmm4
+ pxor xmm2, xmm6
+ movdqu [r9 + 144], xmm2
movdqu xmm6, [r8 + 208]
- pxor xmm3, xmm6
- movdqu [r9 + 208], xmm3
- movdqu xmm0, [rsp + 192]
- movdqu xmm1, [rsp + 208]
- movdqu xmm2, [rsp + 224]
- movdqu xmm3, [rsp + 240]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
- movdqu xmm6, [r8 + 32]
- pxor xmm0, xmm6
- movdqu [r9 + 32], xmm0
- movdqu xmm6, [r8 + 96]
- pxor xmm1, xmm6
- movdqu [r9 + 96], xmm1
- movdqu xmm6, [r8 + 160]
- pxor xmm4, xmm6
- movdqu [r9 + 160], xmm4
- movdqu xmm6, [r8 + 224]
- pxor xmm3, xmm6
- movdqu [r9 + 224], xmm3
- movdqu xmm0, [rsp + 256]
- movdqu xmm1, [rsp + 272]
- movdqu xmm2, [rsp + 288]
- movdqu xmm3, [rsp + 304]
- movdqa xmm4, xmm0
- db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
- db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1
- movdqa xmm5, xmm2
- db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3
- db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3
- movdqa xmm1, xmm0
- db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2
- db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2
- movdqa xmm3, xmm4
- db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5
- db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5
- movdqu xmm6, [r8 + 48]
- pxor xmm0, xmm6
- movdqu [r9 + 48], xmm0
- movdqu xmm6, [r8 + 112]
- pxor xmm1, xmm6
- movdqu [r9 + 112], xmm1
- movdqu xmm6, [r8 + 176]
- pxor xmm4, xmm6
- movdqu [r9 + 176], xmm4
- movdqu xmm6, [r8 + 240]
- pxor xmm3, xmm6
- movdqu [r9 + 240], xmm3
- add rsp, 368
+ pxor xmm7, xmm6
+ movdqu [r9 + 208], xmm7
+ movdqa xmm0, [rsp + 32]
+ movdqa xmm1, [rsp + 48]
+ db $66, $44, $0F, $FE, $84, $24, $C0, $00, $00, $00 // paddd xmm8, [rsp + 192]
+ db $66, $44, $0F, $FE, $8C, $24, $D0, $00, $00, $00 // paddd xmm9, [rsp + 208]
+ db $66, $0F, $FE, $84, $24, $E0, $00, $00, $00 // paddd xmm0, [rsp + 224]
+ db $66, $0F, $FE, $8C, $24, $F0, $00, $00, $00 // paddd xmm1, [rsp + 240]
+ db $66, $41, $0F, $6F, $D0 // movdqa xmm2, xmm8
+ db $66, $45, $0F, $62, $C1 // punpckldq xmm8, xmm9
+ db $66, $41, $0F, $6A, $D1 // punpckhdq xmm2, xmm9
+ movdqa xmm3, xmm0
+ db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1
+ db $66, $0F, $6A, $D9 // punpckhdq xmm3, xmm1
+ db $66, $45, $0F, $6F, $C8 // movdqa xmm9, xmm8
+ db $66, $44, $0F, $6C, $C0 // punpcklqdq xmm8, xmm0
+ db $66, $44, $0F, $6D, $C8 // punpckhqdq xmm9, xmm0
+ movdqa xmm1, xmm2
+ db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3
+ db $66, $0F, $6D, $CB // punpckhqdq xmm1, xmm3
+ db $66, $44, $0F, $FE, $A4, $24, $00, $01, $00, $00 // paddd xmm12, [rsp + 256]
+ db $66, $44, $0F, $FE, $AC, $24, $10, $01, $00, $00 // paddd xmm13, [rsp + 272]
+ db $66, $44, $0F, $FE, $B4, $24, $20, $01, $00, $00 // paddd xmm14, [rsp + 288]
+ db $66, $44, $0F, $FE, $BC, $24, $30, $01, $00, $00 // paddd xmm15, [rsp + 304]
+ db $66, $41, $0F, $6F, $C4 // movdqa xmm0, xmm12
+ db $66, $45, $0F, $62, $E5 // punpckldq xmm12, xmm13
+ db $66, $41, $0F, $6A, $C5 // punpckhdq xmm0, xmm13
+ db $66, $41, $0F, $6F, $DE // movdqa xmm3, xmm14
+ db $66, $45, $0F, $62, $F7 // punpckldq xmm14, xmm15
+ db $66, $41, $0F, $6A, $DF // punpckhdq xmm3, xmm15
+ db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
+ db $66, $45, $0F, $6C, $E6 // punpcklqdq xmm12, xmm14
+ db $66, $45, $0F, $6D, $EE // punpckhqdq xmm13, xmm14
+ db $66, $44, $0F, $6F, $F8 // movdqa xmm15, xmm0
+ db $66, $0F, $6C, $C3 // punpcklqdq xmm0, xmm3
+ db $66, $44, $0F, $6D, $FB // punpckhqdq xmm15, xmm3
+ movdqu xmm3, [r8 + 32]
+ db $66, $44, $0F, $EF, $C3 // pxor xmm8, xmm3
+ movdqu [r9 + 32], xmm8
+ movdqu xmm3, [r8 + 96]
+ db $66, $44, $0F, $EF, $CB // pxor xmm9, xmm3
+ movdqu [r9 + 96], xmm9
+ movdqu xmm3, [r8 + 160]
+ pxor xmm2, xmm3
+ movdqu [r9 + 160], xmm2
+ movdqu xmm3, [r8 + 224]
+ pxor xmm1, xmm3
+ movdqu [r9 + 224], xmm1
+ movdqu xmm3, [r8 + 48]
+ db $66, $44, $0F, $EF, $E3 // pxor xmm12, xmm3
+ movdqu [r9 + 48], xmm12
+ movdqu xmm3, [r8 + 112]
+ db $66, $44, $0F, $EF, $EB // pxor xmm13, xmm3
+ movdqu [r9 + 112], xmm13
+ movdqu xmm3, [r8 + 176]
+ pxor xmm0, xmm3
+ movdqu [r9 + 176], xmm0
+ movdqu xmm3, [r8 + 240]
+ db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3
+ movdqu [r9 + 240], xmm15
+ add r8, 256
+ add r9, 256
+ dec r10
+ jnz @@c7539_ssse3_outer
+ mov rsp, r11
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc
new file mode 100644
index 00000000..e20d288a
--- /dev/null
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_i386.inc
@@ -0,0 +1,500 @@
+// ChaCha7539 (IETF) 8-way AVX2 keystream kernel, vertical layout (i386).
+// -------------------------------------------------------------------
+//
+// Eight 64-byte ChaCha20 blocks per group, state words held vertically (one
+// block per ymm lane). i386 has only 8 ymm, so the 16 state rows live in a
+// 64B-aligned stack frame (original esp parked inside it); each quarter-round
+// loads its 4 rows into ymm0-3 (temp ymm4), with the rot16/rot8 vpshufb masks
+// resident in ymm6/ymm7. rot12/rot7 use vpslld/vpsrld/vpor.
+//
+// Streams AGroups x 512B in one call: prologue once, per-group ORIG->STATE
+// reload (counter lanes +8), fused accumulate/transpose/xor tail to AOut
+// (vextracti128 lifts each row's high block piece).
+//
+// On entry (after ClpSimdProc5Begin_i386.inc):
+// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut,
+// ecx = AGroups (8-block groups, >= 1).
+//
+// On exit:
+// AState word-12 counter (byte 48) advanced by 8*AGroups; the keystream is
+// xored with AIn into AOut (512*AGroups bytes).
+//
+// VEX-encoded instructions are db-encoded for broad assembler compatibility.
+ mov edx, esp
+ sub esp, 1168
+ and esp, -64
+ mov dword ptr [esp + 1152], edx
+ mov dword ptr [esp + 1024], 0
+ mov dword ptr [esp + 1028], 1
+ mov dword ptr [esp + 1032], 2
+ mov dword ptr [esp + 1036], 3
+ mov dword ptr [esp + 1040], 4
+ mov dword ptr [esp + 1044], 5
+ mov dword ptr [esp + 1048], 6
+ mov dword ptr [esp + 1052], 7
+ mov dword ptr [esp + 1056], 8
+ mov dword ptr [esp + 1060], 8
+ mov dword ptr [esp + 1064], 8
+ mov dword ptr [esp + 1068], 8
+ mov dword ptr [esp + 1072], 8
+ mov dword ptr [esp + 1076], 8
+ mov dword ptr [esp + 1080], 8
+ mov dword ptr [esp + 1084], 8
+ mov dword ptr [esp + 1088], $01000302
+ mov dword ptr [esp + 1092], $05040706
+ mov dword ptr [esp + 1096], $09080B0A
+ mov dword ptr [esp + 1100], $0D0C0F0E
+ mov dword ptr [esp + 1104], $01000302
+ mov dword ptr [esp + 1108], $05040706
+ mov dword ptr [esp + 1112], $09080B0A
+ mov dword ptr [esp + 1116], $0D0C0F0E
+ mov dword ptr [esp + 1120], $02010003
+ mov dword ptr [esp + 1124], $06050407
+ mov dword ptr [esp + 1128], $0A09080B
+ mov dword ptr [esp + 1132], $0E0D0C0F
+ mov dword ptr [esp + 1136], $02010003
+ mov dword ptr [esp + 1140], $06050407
+ mov dword ptr [esp + 1144], $0A09080B
+ mov dword ptr [esp + 1148], $0E0D0C0F
+ db $C5, $FD, $6F, $B4, $24, $40, $04, $00, $00 // vmovdqa ymm6, [esp + 1088]
+ db $C5, $FD, $6F, $BC, $24, $60, $04, $00, $00 // vmovdqa ymm7, [esp + 1120]
+ db $C4, $E2, $7D, $58, $06 // vpbroadcastd ymm0, [esi + 0]
+ db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0
+ db $C5, $FD, $7F, $84, $24, $00, $02, $00, $00 // vmovdqa [esp + 512], ymm0
+ db $C4, $E2, $7D, $58, $46, $04 // vpbroadcastd ymm0, [esi + 4]
+ db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0
+ db $C5, $FD, $7F, $84, $24, $20, $02, $00, $00 // vmovdqa [esp + 544], ymm0
+ db $C4, $E2, $7D, $58, $46, $08 // vpbroadcastd ymm0, [esi + 8]
+ db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0
+ db $C5, $FD, $7F, $84, $24, $40, $02, $00, $00 // vmovdqa [esp + 576], ymm0
+ db $C4, $E2, $7D, $58, $46, $0C // vpbroadcastd ymm0, [esi + 12]
+ db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0
+ db $C5, $FD, $7F, $84, $24, $60, $02, $00, $00 // vmovdqa [esp + 608], ymm0
+ db $C4, $E2, $7D, $58, $46, $10 // vpbroadcastd ymm0, [esi + 16]
+ db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm0
+ db $C5, $FD, $7F, $84, $24, $80, $02, $00, $00 // vmovdqa [esp + 640], ymm0
+ db $C4, $E2, $7D, $58, $46, $14 // vpbroadcastd ymm0, [esi + 20]
+ db $C5, $FD, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm0
+ db $C5, $FD, $7F, $84, $24, $A0, $02, $00, $00 // vmovdqa [esp + 672], ymm0
+ db $C4, $E2, $7D, $58, $46, $18 // vpbroadcastd ymm0, [esi + 24]
+ db $C5, $FD, $7F, $84, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm0
+ db $C5, $FD, $7F, $84, $24, $C0, $02, $00, $00 // vmovdqa [esp + 704], ymm0
+ db $C4, $E2, $7D, $58, $46, $1C // vpbroadcastd ymm0, [esi + 28]
+ db $C5, $FD, $7F, $84, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm0
+ db $C5, $FD, $7F, $84, $24, $E0, $02, $00, $00 // vmovdqa [esp + 736], ymm0
+ db $C4, $E2, $7D, $58, $46, $20 // vpbroadcastd ymm0, [esi + 32]
+ db $C5, $FD, $7F, $84, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm0
+ db $C5, $FD, $7F, $84, $24, $00, $03, $00, $00 // vmovdqa [esp + 768], ymm0
+ db $C4, $E2, $7D, $58, $46, $24 // vpbroadcastd ymm0, [esi + 36]
+ db $C5, $FD, $7F, $84, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm0
+ db $C5, $FD, $7F, $84, $24, $20, $03, $00, $00 // vmovdqa [esp + 800], ymm0
+ db $C4, $E2, $7D, $58, $46, $28 // vpbroadcastd ymm0, [esi + 40]
+ db $C5, $FD, $7F, $84, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm0
+ db $C5, $FD, $7F, $84, $24, $40, $03, $00, $00 // vmovdqa [esp + 832], ymm0
+ db $C4, $E2, $7D, $58, $46, $2C // vpbroadcastd ymm0, [esi + 44]
+ db $C5, $FD, $7F, $84, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm0
+ db $C5, $FD, $7F, $84, $24, $60, $03, $00, $00 // vmovdqa [esp + 864], ymm0
+ db $C4, $E2, $7D, $58, $46, $30 // vpbroadcastd ymm0, [esi + 48]
+ db $C5, $FD, $FE, $84, $24, $00, $04, $00, $00 // vpaddd ymm0, ymm0, [esp + 1024]
+ db $C5, $FD, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm0
+ db $C5, $FD, $7F, $84, $24, $80, $03, $00, $00 // vmovdqa [esp + 896], ymm0
+ db $C4, $E2, $7D, $58, $46, $34 // vpbroadcastd ymm0, [esi + 52]
+ db $C5, $FD, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm0
+ db $C5, $FD, $7F, $84, $24, $A0, $03, $00, $00 // vmovdqa [esp + 928], ymm0
+ db $C4, $E2, $7D, $58, $46, $38 // vpbroadcastd ymm0, [esi + 56]
+ db $C5, $FD, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm0
+ db $C5, $FD, $7F, $84, $24, $C0, $03, $00, $00 // vmovdqa [esp + 960], ymm0
+ db $C4, $E2, $7D, $58, $46, $3C // vpbroadcastd ymm0, [esi + 60]
+ db $C5, $FD, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm0
+ db $C5, $FD, $7F, $84, $24, $E0, $03, $00, $00 // vmovdqa [esp + 992], ymm0
+ jmp @@c7539_avx2_8x_i386_enter
+@@c7539_avx2_8x_i386_outer:
+ db $C5, $FD, $6F, $84, $24, $00, $02, $00, $00 // vmovdqa ymm0, [esp + 512]
+ db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0
+ db $C5, $FD, $6F, $84, $24, $20, $02, $00, $00 // vmovdqa ymm0, [esp + 544]
+ db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0
+ db $C5, $FD, $6F, $84, $24, $40, $02, $00, $00 // vmovdqa ymm0, [esp + 576]
+ db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0
+ db $C5, $FD, $6F, $84, $24, $60, $02, $00, $00 // vmovdqa ymm0, [esp + 608]
+ db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0
+ db $C5, $FD, $6F, $84, $24, $80, $02, $00, $00 // vmovdqa ymm0, [esp + 640]
+ db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm0
+ db $C5, $FD, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqa ymm0, [esp + 672]
+ db $C5, $FD, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm0
+ db $C5, $FD, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqa ymm0, [esp + 704]
+ db $C5, $FD, $7F, $84, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm0
+ db $C5, $FD, $6F, $84, $24, $E0, $02, $00, $00 // vmovdqa ymm0, [esp + 736]
+ db $C5, $FD, $7F, $84, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm0
+ db $C5, $FD, $6F, $84, $24, $00, $03, $00, $00 // vmovdqa ymm0, [esp + 768]
+ db $C5, $FD, $7F, $84, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm0
+ db $C5, $FD, $6F, $84, $24, $20, $03, $00, $00 // vmovdqa ymm0, [esp + 800]
+ db $C5, $FD, $7F, $84, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm0
+ db $C5, $FD, $6F, $84, $24, $40, $03, $00, $00 // vmovdqa ymm0, [esp + 832]
+ db $C5, $FD, $7F, $84, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm0
+ db $C5, $FD, $6F, $84, $24, $60, $03, $00, $00 // vmovdqa ymm0, [esp + 864]
+ db $C5, $FD, $7F, $84, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm0
+ db $C5, $FD, $6F, $84, $24, $80, $03, $00, $00 // vmovdqa ymm0, [esp + 896]
+ db $C5, $FD, $FE, $84, $24, $20, $04, $00, $00 // vpaddd ymm0, ymm0, [esp + 1056]
+ db $C5, $FD, $7F, $84, $24, $80, $03, $00, $00 // vmovdqa [esp + 896], ymm0
+ db $C5, $FD, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm0
+ db $C5, $FD, $6F, $84, $24, $A0, $03, $00, $00 // vmovdqa ymm0, [esp + 928]
+ db $C5, $FD, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm0
+ db $C5, $FD, $6F, $84, $24, $C0, $03, $00, $00 // vmovdqa ymm0, [esp + 960]
+ db $C5, $FD, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm0
+ db $C5, $FD, $6F, $84, $24, $E0, $03, $00, $00 // vmovdqa ymm0, [esp + 992]
+ db $C5, $FD, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm0
+@@c7539_avx2_8x_i386_enter:
+ mov edx, ebx
+@@c7539_avx2_8x_i386_round:
+ db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0]
+ db $C5, $FD, $6F, $8C, $24, $80, $00, $00, $00 // vmovdqa ymm1, [esp + 128]
+ db $C5, $FD, $6F, $94, $24, $00, $01, $00, $00 // vmovdqa ymm2, [esp + 256]
+ db $C5, $FD, $6F, $9C, $24, $80, $01, $00, $00 // vmovdqa ymm3, [esp + 384]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0
+ db $C5, $FD, $7F, $8C, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm1
+ db $C5, $FD, $7F, $94, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm2
+ db $C5, $FD, $7F, $9C, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm3
+ db $C5, $FD, $6F, $44, $24, $20 // vmovdqa ymm0, [esp + 32]
+ db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160]
+ db $C5, $FD, $6F, $94, $24, $20, $01, $00, $00 // vmovdqa ymm2, [esp + 288]
+ db $C5, $FD, $6F, $9C, $24, $A0, $01, $00, $00 // vmovdqa ymm3, [esp + 416]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0
+ db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm1
+ db $C5, $FD, $7F, $94, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm2
+ db $C5, $FD, $7F, $9C, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm3
+ db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [esp + 64]
+ db $C5, $FD, $6F, $8C, $24, $C0, $00, $00, $00 // vmovdqa ymm1, [esp + 192]
+ db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320]
+ db $C5, $FD, $6F, $9C, $24, $C0, $01, $00, $00 // vmovdqa ymm3, [esp + 448]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0
+ db $C5, $FD, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm1
+ db $C5, $FD, $7F, $94, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm2
+ db $C5, $FD, $7F, $9C, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm3
+ db $C5, $FD, $6F, $44, $24, $60 // vmovdqa ymm0, [esp + 96]
+ db $C5, $FD, $6F, $8C, $24, $E0, $00, $00, $00 // vmovdqa ymm1, [esp + 224]
+ db $C5, $FD, $6F, $94, $24, $60, $01, $00, $00 // vmovdqa ymm2, [esp + 352]
+ db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0
+ db $C5, $FD, $7F, $8C, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm1
+ db $C5, $FD, $7F, $94, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm2
+ db $C5, $FD, $7F, $9C, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm3
+ db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0]
+ db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160]
+ db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320]
+ db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $04, $24 // vmovdqa [esp + 0], ymm0
+ db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm1
+ db $C5, $FD, $7F, $94, $24, $40, $01, $00, $00 // vmovdqa [esp + 320], ymm2
+ db $C5, $FD, $7F, $9C, $24, $E0, $01, $00, $00 // vmovdqa [esp + 480], ymm3
+ db $C5, $FD, $6F, $44, $24, $20 // vmovdqa ymm0, [esp + 32]
+ db $C5, $FD, $6F, $8C, $24, $C0, $00, $00, $00 // vmovdqa ymm1, [esp + 192]
+ db $C5, $FD, $6F, $94, $24, $60, $01, $00, $00 // vmovdqa ymm2, [esp + 352]
+ db $C5, $FD, $6F, $9C, $24, $80, $01, $00, $00 // vmovdqa ymm3, [esp + 384]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $20 // vmovdqa [esp + 32], ymm0
+ db $C5, $FD, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm1
+ db $C5, $FD, $7F, $94, $24, $60, $01, $00, $00 // vmovdqa [esp + 352], ymm2
+ db $C5, $FD, $7F, $9C, $24, $80, $01, $00, $00 // vmovdqa [esp + 384], ymm3
+ db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [esp + 64]
+ db $C5, $FD, $6F, $8C, $24, $E0, $00, $00, $00 // vmovdqa ymm1, [esp + 224]
+ db $C5, $FD, $6F, $94, $24, $00, $01, $00, $00 // vmovdqa ymm2, [esp + 256]
+ db $C5, $FD, $6F, $9C, $24, $A0, $01, $00, $00 // vmovdqa ymm3, [esp + 416]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $40 // vmovdqa [esp + 64], ymm0
+ db $C5, $FD, $7F, $8C, $24, $E0, $00, $00, $00 // vmovdqa [esp + 224], ymm1
+ db $C5, $FD, $7F, $94, $24, $00, $01, $00, $00 // vmovdqa [esp + 256], ymm2
+ db $C5, $FD, $7F, $9C, $24, $A0, $01, $00, $00 // vmovdqa [esp + 416], ymm3
+ db $C5, $FD, $6F, $44, $24, $60 // vmovdqa ymm0, [esp + 96]
+ db $C5, $FD, $6F, $8C, $24, $80, $00, $00, $00 // vmovdqa ymm1, [esp + 128]
+ db $C5, $FD, $6F, $94, $24, $20, $01, $00, $00 // vmovdqa ymm2, [esp + 288]
+ db $C5, $FD, $6F, $9C, $24, $C0, $01, $00, $00 // vmovdqa ymm3, [esp + 448]
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DE // vpshufb ymm3, ymm3, ymm6
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $0C // vpslld ymm4, ymm1, 12
+ db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 20
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1
+ db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0
+ db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7
+ db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3
+ db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2
+ db $C5, $DD, $72, $F1, $07 // vpslld ymm4, ymm1, 7
+ db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 25
+ db $C5, $DD, $EB, $C9 // vpor ymm1, ymm4, ymm1
+ db $C5, $FD, $7F, $44, $24, $60 // vmovdqa [esp + 96], ymm0
+ db $C5, $FD, $7F, $8C, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm1
+ db $C5, $FD, $7F, $94, $24, $20, $01, $00, $00 // vmovdqa [esp + 288], ymm2
+ db $C5, $FD, $7F, $9C, $24, $C0, $01, $00, $00 // vmovdqa [esp + 448], ymm3
+ sub edx, 2
+ jg @@c7539_avx2_8x_i386_round
+ add dword ptr [esi + 48], 8
+{$IFDEF CHACHA_CTR64}
+ adc dword ptr [esi + 52], 0
+{$ENDIF}
+ db $C5, $FD, $6F, $04, $24 // vmovdqa ymm0, [esp + 0]
+ db $C5, $FD, $FE, $84, $24, $00, $02, $00, $00 // vpaddd ymm0, ymm0, [esp + 512]
+ db $C5, $FD, $6F, $4C, $24, $20 // vmovdqa ymm1, [esp + 32]
+ db $C5, $F5, $FE, $8C, $24, $20, $02, $00, $00 // vpaddd ymm1, ymm1, [esp + 544]
+ db $C5, $FD, $6F, $54, $24, $40 // vmovdqa ymm2, [esp + 64]
+ db $C5, $ED, $FE, $94, $24, $40, $02, $00, $00 // vpaddd ymm2, ymm2, [esp + 576]
+ db $C5, $FD, $6F, $5C, $24, $60 // vmovdqa ymm3, [esp + 96]
+ db $C5, $E5, $FE, $9C, $24, $60, $02, $00, $00 // vpaddd ymm3, ymm3, [esp + 608]
+ db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1
+ db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3
+ db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5
+ db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5
+ db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2
+ db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2
+ db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1
+ db $C5, $F1, $EF, $0F // vpxor xmm1, xmm1, [edi]
+ db $C5, $FA, $7F, $08 // vmovdqu [eax], xmm1
+ db $C5, $D1, $EF, $AF, $00, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 256]
+ db $C5, $FA, $7F, $A8, $00, $01, $00, $00 // vmovdqu [eax + 256], xmm5
+ db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1
+ db $C5, $D9, $EF, $67, $40 // vpxor xmm4, xmm4, [edi + 64]
+ db $C5, $FA, $7F, $60, $40 // vmovdqu [eax + 64], xmm4
+ db $C5, $D1, $EF, $AF, $40, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 320]
+ db $C5, $FA, $7F, $A8, $40, $01, $00, $00 // vmovdqu [eax + 320], xmm5
+ db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1
+ db $C5, $E1, $EF, $9F, $80, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 128]
+ db $C5, $FA, $7F, $98, $80, $00, $00, $00 // vmovdqu [eax + 128], xmm3
+ db $C5, $D1, $EF, $AF, $80, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 384]
+ db $C5, $FA, $7F, $A8, $80, $01, $00, $00 // vmovdqu [eax + 384], xmm5
+ db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1
+ db $C5, $F9, $EF, $87, $C0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 192]
+ db $C5, $FA, $7F, $80, $C0, $00, $00, $00 // vmovdqu [eax + 192], xmm0
+ db $C5, $D1, $EF, $AF, $C0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 448]
+ db $C5, $FA, $7F, $A8, $C0, $01, $00, $00 // vmovdqu [eax + 448], xmm5
+ db $C5, $FD, $6F, $84, $24, $80, $00, $00, $00 // vmovdqa ymm0, [esp + 128]
+ db $C5, $FD, $FE, $84, $24, $80, $02, $00, $00 // vpaddd ymm0, ymm0, [esp + 640]
+ db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [esp + 160]
+ db $C5, $F5, $FE, $8C, $24, $A0, $02, $00, $00 // vpaddd ymm1, ymm1, [esp + 672]
+ db $C5, $FD, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqa ymm2, [esp + 192]
+ db $C5, $ED, $FE, $94, $24, $C0, $02, $00, $00 // vpaddd ymm2, ymm2, [esp + 704]
+ db $C5, $FD, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqa ymm3, [esp + 224]
+ db $C5, $E5, $FE, $9C, $24, $E0, $02, $00, $00 // vpaddd ymm3, ymm3, [esp + 736]
+ db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1
+ db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3
+ db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5
+ db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5
+ db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2
+ db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2
+ db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1
+ db $C5, $F1, $EF, $4F, $10 // vpxor xmm1, xmm1, [edi + 16]
+ db $C5, $FA, $7F, $48, $10 // vmovdqu [eax + 16], xmm1
+ db $C5, $D1, $EF, $AF, $10, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 272]
+ db $C5, $FA, $7F, $A8, $10, $01, $00, $00 // vmovdqu [eax + 272], xmm5
+ db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1
+ db $C5, $D9, $EF, $67, $50 // vpxor xmm4, xmm4, [edi + 80]
+ db $C5, $FA, $7F, $60, $50 // vmovdqu [eax + 80], xmm4
+ db $C5, $D1, $EF, $AF, $50, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 336]
+ db $C5, $FA, $7F, $A8, $50, $01, $00, $00 // vmovdqu [eax + 336], xmm5
+ db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1
+ db $C5, $E1, $EF, $9F, $90, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 144]
+ db $C5, $FA, $7F, $98, $90, $00, $00, $00 // vmovdqu [eax + 144], xmm3
+ db $C5, $D1, $EF, $AF, $90, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 400]
+ db $C5, $FA, $7F, $A8, $90, $01, $00, $00 // vmovdqu [eax + 400], xmm5
+ db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1
+ db $C5, $F9, $EF, $87, $D0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 208]
+ db $C5, $FA, $7F, $80, $D0, $00, $00, $00 // vmovdqu [eax + 208], xmm0
+ db $C5, $D1, $EF, $AF, $D0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 464]
+ db $C5, $FA, $7F, $A8, $D0, $01, $00, $00 // vmovdqu [eax + 464], xmm5
+ db $C5, $FD, $6F, $84, $24, $00, $01, $00, $00 // vmovdqa ymm0, [esp + 256]
+ db $C5, $FD, $FE, $84, $24, $00, $03, $00, $00 // vpaddd ymm0, ymm0, [esp + 768]
+ db $C5, $FD, $6F, $8C, $24, $20, $01, $00, $00 // vmovdqa ymm1, [esp + 288]
+ db $C5, $F5, $FE, $8C, $24, $20, $03, $00, $00 // vpaddd ymm1, ymm1, [esp + 800]
+ db $C5, $FD, $6F, $94, $24, $40, $01, $00, $00 // vmovdqa ymm2, [esp + 320]
+ db $C5, $ED, $FE, $94, $24, $40, $03, $00, $00 // vpaddd ymm2, ymm2, [esp + 832]
+ db $C5, $FD, $6F, $9C, $24, $60, $01, $00, $00 // vmovdqa ymm3, [esp + 352]
+ db $C5, $E5, $FE, $9C, $24, $60, $03, $00, $00 // vpaddd ymm3, ymm3, [esp + 864]
+ db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1
+ db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3
+ db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5
+ db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5
+ db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2
+ db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2
+ db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1
+ db $C5, $F1, $EF, $4F, $20 // vpxor xmm1, xmm1, [edi + 32]
+ db $C5, $FA, $7F, $48, $20 // vmovdqu [eax + 32], xmm1
+ db $C5, $D1, $EF, $AF, $20, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 288]
+ db $C5, $FA, $7F, $A8, $20, $01, $00, $00 // vmovdqu [eax + 288], xmm5
+ db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1
+ db $C5, $D9, $EF, $67, $60 // vpxor xmm4, xmm4, [edi + 96]
+ db $C5, $FA, $7F, $60, $60 // vmovdqu [eax + 96], xmm4
+ db $C5, $D1, $EF, $AF, $60, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 352]
+ db $C5, $FA, $7F, $A8, $60, $01, $00, $00 // vmovdqu [eax + 352], xmm5
+ db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1
+ db $C5, $E1, $EF, $9F, $A0, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 160]
+ db $C5, $FA, $7F, $98, $A0, $00, $00, $00 // vmovdqu [eax + 160], xmm3
+ db $C5, $D1, $EF, $AF, $A0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 416]
+ db $C5, $FA, $7F, $A8, $A0, $01, $00, $00 // vmovdqu [eax + 416], xmm5
+ db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1
+ db $C5, $F9, $EF, $87, $E0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 224]
+ db $C5, $FA, $7F, $80, $E0, $00, $00, $00 // vmovdqu [eax + 224], xmm0
+ db $C5, $D1, $EF, $AF, $E0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 480]
+ db $C5, $FA, $7F, $A8, $E0, $01, $00, $00 // vmovdqu [eax + 480], xmm5
+ db $C5, $FD, $6F, $84, $24, $80, $01, $00, $00 // vmovdqa ymm0, [esp + 384]
+ db $C5, $FD, $FE, $84, $24, $80, $03, $00, $00 // vpaddd ymm0, ymm0, [esp + 896]
+ db $C5, $FD, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqa ymm1, [esp + 416]
+ db $C5, $F5, $FE, $8C, $24, $A0, $03, $00, $00 // vpaddd ymm1, ymm1, [esp + 928]
+ db $C5, $FD, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqa ymm2, [esp + 448]
+ db $C5, $ED, $FE, $94, $24, $C0, $03, $00, $00 // vpaddd ymm2, ymm2, [esp + 960]
+ db $C5, $FD, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqa ymm3, [esp + 480]
+ db $C5, $E5, $FE, $9C, $24, $E0, $03, $00, $00 // vpaddd ymm3, ymm3, [esp + 992]
+ db $C5, $FD, $62, $E1 // vpunpckldq ymm4, ymm0, ymm1
+ db $C5, $ED, $62, $EB // vpunpckldq ymm5, ymm2, ymm3
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3
+ db $C5, $DD, $6C, $CD // vpunpcklqdq ymm1, ymm4, ymm5
+ db $C5, $DD, $6D, $E5 // vpunpckhqdq ymm4, ymm4, ymm5
+ db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2
+ db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2
+ db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, 1
+ db $C5, $F1, $EF, $4F, $30 // vpxor xmm1, xmm1, [edi + 48]
+ db $C5, $FA, $7F, $48, $30 // vmovdqu [eax + 48], xmm1
+ db $C5, $D1, $EF, $AF, $30, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 304]
+ db $C5, $FA, $7F, $A8, $30, $01, $00, $00 // vmovdqu [eax + 304], xmm5
+ db $C4, $E3, $7D, $39, $E5, $01 // vextracti128 xmm5, ymm4, 1
+ db $C5, $D9, $EF, $67, $70 // vpxor xmm4, xmm4, [edi + 112]
+ db $C5, $FA, $7F, $60, $70 // vmovdqu [eax + 112], xmm4
+ db $C5, $D1, $EF, $AF, $70, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 368]
+ db $C5, $FA, $7F, $A8, $70, $01, $00, $00 // vmovdqu [eax + 368], xmm5
+ db $C4, $E3, $7D, $39, $DD, $01 // vextracti128 xmm5, ymm3, 1
+ db $C5, $E1, $EF, $9F, $B0, $00, $00, $00 // vpxor xmm3, xmm3, [edi + 176]
+ db $C5, $FA, $7F, $98, $B0, $00, $00, $00 // vmovdqu [eax + 176], xmm3
+ db $C5, $D1, $EF, $AF, $B0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 432]
+ db $C5, $FA, $7F, $A8, $B0, $01, $00, $00 // vmovdqu [eax + 432], xmm5
+ db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, 1
+ db $C5, $F9, $EF, $87, $F0, $00, $00, $00 // vpxor xmm0, xmm0, [edi + 240]
+ db $C5, $FA, $7F, $80, $F0, $00, $00, $00 // vmovdqu [eax + 240], xmm0
+ db $C5, $D1, $EF, $AF, $F0, $01, $00, $00 // vpxor xmm5, xmm5, [edi + 496]
+ db $C5, $FA, $7F, $A8, $F0, $01, $00, $00 // vmovdqu [eax + 496], xmm5
+ add edi, 512
+ add eax, 512
+ dec ecx
+ jnz @@c7539_avx2_8x_i386_outer
+ db $C5, $F8, $77 // vzeroupper
+ mov esp, dword ptr [esp + 1152]
+ pop edi
+ pop esi
+ pop ebx
+
diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc
index 0fd47d0c..5c51ad83 100644
--- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc
@@ -3,21 +3,33 @@
//
// Eight 64-byte ChaCha20 blocks, state words held vertically (one block per
// ymm lane). ymm0-3 = words a (0-3), ymm4-7 = words b (4-7), ymm12-15 = words
-// d (12-15); words c (8-11) live in a stack slot; ymm8-11 are scratch. The four
-// quarter-rounds of each round are step-interleaved; rot16/rot8 use vpshufb with
-// the mask read from memory; rot12/rot7 use vpslld/vpsrld/vpor.
+// d (12-15). Words c (8-11): two are resident in ymm8/ymm9 while the other two
+// wait in stack slots; quarter-rounds run as two resident pairs and the pair a
+// round ends with is the pair the next round starts with, so a round costs one
+// pair swap (2 stores + 2 loads). ymm10/ymm11 are the rotate temps; rot16/rot8
+// use vpshufb (mask reloaded into the just-freed temp), rot12/rot7 use
+// vpslld/vpsrld/vpor. The frame is 64B-aligned (r11 = frame register), so all
+// stack traffic is aligned vmovdqa.
//
-// On entry (after ClpSimdProc4Begin_x86_64.inc):
-// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut.
+// Streams AGroups x 512B in one call: frame, broadcasts and constants are set
+// up once; each group re-enters via a light state reload (counter lanes +8)
+// and ends with a fused accumulate/transpose/xor written straight to AOut
+// (vperm2i128 pairs whole 32-byte block halves; no bounce buffer).
+//
+// On entry (after ClpSimdProc5Begin_x86_64.inc):
+// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut,
+// r10 = AGroups (8-block groups, >= 1).
//
// On exit:
-// AState word-12 counter (byte 48) advanced by 8; the keystream is xored
-// with AIn into AOut (512 bytes).
+// AState word-12 counter (byte 48) advanced by 8*AGroups; the keystream is
+// xored with AIn into AOut (512*AGroups bytes).
//
// VEX-encoded instructions are db-encoded for broad assembler compatibility.
- mov r10, rcx
+ mov r10d, r10d
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- sub rsp, 1248
+ mov r11, rsp
+ sub rsp, 832
+ and rsp, -64
mov dword ptr [rsp + 640], 0
mov dword ptr [rsp + 644], 1
mov dword ptr [rsp + 648], 2
@@ -26,427 +38,349 @@
mov dword ptr [rsp + 660], 5
mov dword ptr [rsp + 664], 6
mov dword ptr [rsp + 668], 7
- mov dword ptr [rsp + 672], $01000302
- mov dword ptr [rsp + 676], $05040706
- mov dword ptr [rsp + 680], $09080B0A
- mov dword ptr [rsp + 684], $0D0C0F0E
- mov dword ptr [rsp + 688], $01000302
- mov dword ptr [rsp + 692], $05040706
- mov dword ptr [rsp + 696], $09080B0A
- mov dword ptr [rsp + 700], $0D0C0F0E
- mov dword ptr [rsp + 704], $02010003
- mov dword ptr [rsp + 708], $06050407
- mov dword ptr [rsp + 712], $0A09080B
- mov dword ptr [rsp + 716], $0E0D0C0F
- mov dword ptr [rsp + 720], $02010003
- mov dword ptr [rsp + 724], $06050407
- mov dword ptr [rsp + 728], $0A09080B
- mov dword ptr [rsp + 732], $0E0D0C0F
- db $C4, $E2, $7D, $58, $02 // vpbroadcastd ymm0, [rdx]
- db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0
- db $C4, $E2, $7D, $58, $4A, $04 // vpbroadcastd ymm1, [rdx + 4]
- db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1
- db $C4, $E2, $7D, $58, $52, $08 // vpbroadcastd ymm2, [rdx + 8]
- db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2
- db $C4, $E2, $7D, $58, $5A, $0C // vpbroadcastd ymm3, [rdx + 12]
- db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3
- db $C4, $E2, $7D, $58, $62, $10 // vpbroadcastd ymm4, [rdx + 16]
- db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4
- db $C4, $E2, $7D, $58, $6A, $14 // vpbroadcastd ymm5, [rdx + 20]
- db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5
- db $C4, $E2, $7D, $58, $72, $18 // vpbroadcastd ymm6, [rdx + 24]
- db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6
- db $C4, $E2, $7D, $58, $7A, $1C // vpbroadcastd ymm7, [rdx + 28]
- db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7
- db $C4, $62, $7D, $58, $42, $20 // vpbroadcastd ymm8, [rdx + 32]
- db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8
- db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8
- db $C4, $62, $7D, $58, $42, $24 // vpbroadcastd ymm8, [rdx + 36]
- db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8
- db $C5, $7E, $7F, $44, $24, $20 // vmovdqu [rsp + 32], ymm8
- db $C4, $62, $7D, $58, $42, $28 // vpbroadcastd ymm8, [rdx + 40]
- db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8
- db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8
- db $C4, $62, $7D, $58, $42, $2C // vpbroadcastd ymm8, [rdx + 44]
- db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8
- db $C5, $7E, $7F, $44, $24, $60 // vmovdqu [rsp + 96], ymm8
- db $C4, $62, $7D, $58, $62, $30 // vpbroadcastd ymm12, [rdx + 48]
- db $C5, $1D, $FE, $A4, $24, $80, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 640]
- db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12
- db $C4, $62, $7D, $58, $6A, $34 // vpbroadcastd ymm13, [rdx + 52]
- db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13
- db $C4, $62, $7D, $58, $72, $38 // vpbroadcastd ymm14, [rdx + 56]
- db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14
- db $C4, $62, $7D, $58, $7A, $3C // vpbroadcastd ymm15, [rdx + 60]
- db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15
+ mov dword ptr [rsp + 672], 8
+ mov dword ptr [rsp + 676], 8
+ mov dword ptr [rsp + 680], 8
+ mov dword ptr [rsp + 684], 8
+ mov dword ptr [rsp + 688], 8
+ mov dword ptr [rsp + 692], 8
+ mov dword ptr [rsp + 696], 8
+ mov dword ptr [rsp + 700], 8
+ mov dword ptr [rsp + 704], $01000302
+ mov dword ptr [rsp + 708], $05040706
+ mov dword ptr [rsp + 712], $09080B0A
+ mov dword ptr [rsp + 716], $0D0C0F0E
+ mov dword ptr [rsp + 720], $01000302
+ mov dword ptr [rsp + 724], $05040706
+ mov dword ptr [rsp + 728], $09080B0A
+ mov dword ptr [rsp + 732], $0D0C0F0E
+ mov dword ptr [rsp + 736], $02010003
+ mov dword ptr [rsp + 740], $06050407
+ mov dword ptr [rsp + 744], $0A09080B
+ mov dword ptr [rsp + 748], $0E0D0C0F
+ mov dword ptr [rsp + 752], $02010003
+ mov dword ptr [rsp + 756], $06050407
+ mov dword ptr [rsp + 760], $0A09080B
+ mov dword ptr [rsp + 764], $0E0D0C0F
+ db $C4, $E2, $7D, $58, $02 // vpbroadcastd ymm0, [rdx]
+ db $C5, $FD, $7F, $84, $24, $80, $00, $00, $00 // vmovdqa [rsp + 128], ymm0
+ db $C4, $E2, $7D, $58, $4A, $04 // vpbroadcastd ymm1, [rdx + 4]
+ db $C5, $FD, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqa [rsp + 160], ymm1
+ db $C4, $E2, $7D, $58, $52, $08 // vpbroadcastd ymm2, [rdx + 8]
+ db $C5, $FD, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqa [rsp + 192], ymm2
+ db $C4, $E2, $7D, $58, $5A, $0C // vpbroadcastd ymm3, [rdx + 12]
+ db $C5, $FD, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqa [rsp + 224], ymm3
+ db $C4, $E2, $7D, $58, $62, $10 // vpbroadcastd ymm4, [rdx + 16]
+ db $C5, $FD, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqa [rsp + 256], ymm4
+ db $C4, $E2, $7D, $58, $6A, $14 // vpbroadcastd ymm5, [rdx + 20]
+ db $C5, $FD, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqa [rsp + 288], ymm5
+ db $C4, $E2, $7D, $58, $72, $18 // vpbroadcastd ymm6, [rdx + 24]
+ db $C5, $FD, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqa [rsp + 320], ymm6
+ db $C4, $E2, $7D, $58, $7A, $1C // vpbroadcastd ymm7, [rdx + 28]
+ db $C5, $FD, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqa [rsp + 352], ymm7
+ db $C4, $62, $7D, $58, $42, $20 // vpbroadcastd ymm8, [rdx + 32]
+ db $C5, $7D, $7F, $84, $24, $80, $01, $00, $00 // vmovdqa [rsp + 384], ymm8
+ db $C4, $62, $7D, $58, $4A, $24 // vpbroadcastd ymm9, [rdx + 36]
+ db $C5, $7D, $7F, $8C, $24, $A0, $01, $00, $00 // vmovdqa [rsp + 416], ymm9
+ db $C4, $62, $7D, $58, $52, $28 // vpbroadcastd ymm10, [rdx + 40]
+ db $C5, $7D, $7F, $94, $24, $C0, $01, $00, $00 // vmovdqa [rsp + 448], ymm10
+ db $C5, $7D, $7F, $54, $24, $40 // vmovdqa [rsp + 64], ymm10
+ db $C4, $62, $7D, $58, $52, $2C // vpbroadcastd ymm10, [rdx + 44]
+ db $C5, $7D, $7F, $94, $24, $E0, $01, $00, $00 // vmovdqa [rsp + 480], ymm10
+ db $C5, $7D, $7F, $54, $24, $60 // vmovdqa [rsp + 96], ymm10
+ db $C4, $62, $7D, $58, $62, $30 // vpbroadcastd ymm12, [rdx + 48]
+ db $C5, $1D, $FE, $A4, $24, $80, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 640]
+ db $C5, $7D, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqa [rsp + 512], ymm12
+ db $C4, $62, $7D, $58, $6A, $34 // vpbroadcastd ymm13, [rdx + 52]
+ db $C5, $7D, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqa [rsp + 544], ymm13
+ db $C4, $62, $7D, $58, $72, $38 // vpbroadcastd ymm14, [rdx + 56]
+ db $C5, $7D, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqa [rsp + 576], ymm14
+ db $C4, $62, $7D, $58, $7A, $3C // vpbroadcastd ymm15, [rdx + 60]
+ db $C5, $7D, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqa [rsp + 608], ymm15
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+ jmp @@c7539_avx2_8x_enter
+@@c7539_avx2_8x_outer:
+ db $C5, $FD, $6F, $84, $24, $80, $00, $00, $00 // vmovdqa ymm0, [rsp + 128]
+ db $C5, $FD, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqa ymm1, [rsp + 160]
+ db $C5, $FD, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqa ymm2, [rsp + 192]
+ db $C5, $FD, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqa ymm3, [rsp + 224]
+ db $C5, $FD, $6F, $A4, $24, $00, $01, $00, $00 // vmovdqa ymm4, [rsp + 256]
+ db $C5, $FD, $6F, $AC, $24, $20, $01, $00, $00 // vmovdqa ymm5, [rsp + 288]
+ db $C5, $FD, $6F, $B4, $24, $40, $01, $00, $00 // vmovdqa ymm6, [rsp + 320]
+ db $C5, $FD, $6F, $BC, $24, $60, $01, $00, $00 // vmovdqa ymm7, [rsp + 352]
+ db $C5, $7D, $6F, $84, $24, $80, $01, $00, $00 // vmovdqa ymm8, [rsp + 384]
+ db $C5, $7D, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqa ymm9, [rsp + 416]
+ db $C5, $7D, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqa ymm10, [rsp + 448]
+ db $C5, $7D, $7F, $54, $24, $40 // vmovdqa [rsp + 64], ymm10
+ db $C5, $7D, $6F, $94, $24, $E0, $01, $00, $00 // vmovdqa ymm10, [rsp + 480]
+ db $C5, $7D, $7F, $54, $24, $60 // vmovdqa [rsp + 96], ymm10
+ db $C5, $7D, $6F, $A4, $24, $00, $02, $00, $00 // vmovdqa ymm12, [rsp + 512]
+ db $C5, $1D, $FE, $A4, $24, $A0, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 672]
+ db $C5, $7D, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqa [rsp + 512], ymm12
+ db $C5, $7D, $6F, $AC, $24, $20, $02, $00, $00 // vmovdqa ymm13, [rsp + 544]
+ db $C5, $7D, $6F, $B4, $24, $40, $02, $00, $00 // vmovdqa ymm14, [rsp + 576]
+ db $C5, $7D, $6F, $BC, $24, $60, $02, $00, $00 // vmovdqa ymm15, [rsp + 608]
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+@@c7539_avx2_8x_enter:
+ mov eax, ecx
@@c7539_avx2_8x_round:
- db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4
- db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5
- db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6
- db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7
- db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0
- db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1
- db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2
- db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3
- db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672]
- db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8
- db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8
- db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8
- db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8
- db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp]
- db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32]
- db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64]
- db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96]
- db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12
- db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13
- db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14
- db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15
- db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8
- db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9
- db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10
- db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11
- db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8
- db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9
- db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10
- db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11
- db $C5, $BD, $72, $F4, $0C // vpslld ymm8, ymm4, 12
- db $C5, $B5, $72, $F5, $0C // vpslld ymm9, ymm5, 12
- db $C5, $AD, $72, $F6, $0C // vpslld ymm10, ymm6, 12
- db $C5, $A5, $72, $F7, $0C // vpslld ymm11, ymm7, 12
- db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20
- db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20
- db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20
- db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20
- db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4
- db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5
- db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6
- db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7
- db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4
- db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5
- db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6
- db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7
- db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0
- db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1
- db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2
- db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3
- db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704]
- db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8
- db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8
- db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8
- db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8
- db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp]
- db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32]
- db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64]
- db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96]
- db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12
- db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13
- db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14
- db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15
- db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8
- db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9
- db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10
- db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11
- db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8
- db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9
- db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10
- db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11
- db $C5, $BD, $72, $F4, $07 // vpslld ymm8, ymm4, 7
- db $C5, $B5, $72, $F5, $07 // vpslld ymm9, ymm5, 7
- db $C5, $AD, $72, $F6, $07 // vpslld ymm10, ymm6, 7
- db $C5, $A5, $72, $F7, $07 // vpslld ymm11, ymm7, 7
- db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25
- db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25
- db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25
- db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25
- db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4
- db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5
- db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6
- db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7
- db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5
- db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6
- db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7
- db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4
- db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0
- db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1
- db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2
- db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3
- db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672]
- db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8
- db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8
- db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8
- db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8
- db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64]
- db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96]
- db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp]
- db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32]
- db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15
- db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12
- db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13
- db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14
- db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8
- db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9
- db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10
- db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11
- db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8
- db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9
- db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10
- db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11
- db $C5, $BD, $72, $F5, $0C // vpslld ymm8, ymm5, 12
- db $C5, $B5, $72, $F6, $0C // vpslld ymm9, ymm6, 12
- db $C5, $AD, $72, $F7, $0C // vpslld ymm10, ymm7, 12
- db $C5, $A5, $72, $F4, $0C // vpslld ymm11, ymm4, 12
- db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20
- db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20
- db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20
- db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20
- db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5
- db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6
- db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7
- db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4
- db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5
- db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6
- db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7
- db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4
- db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0
- db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1
- db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2
- db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3
- db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704]
- db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8
- db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8
- db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8
- db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8
- db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64]
- db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96]
- db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp]
- db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32]
- db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15
- db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12
- db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13
- db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14
- db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8
- db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9
- db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10
- db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11
- db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8
- db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9
- db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10
- db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11
- db $C5, $BD, $72, $F5, $07 // vpslld ymm8, ymm5, 7
- db $C5, $B5, $72, $F6, $07 // vpslld ymm9, ymm6, 7
- db $C5, $AD, $72, $F7, $07 // vpslld ymm10, ymm7, 7
- db $C5, $A5, $72, $F4, $07 // vpslld ymm11, ymm4, 7
- db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25
- db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25
- db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25
- db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25
- db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5
- db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6
- db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7
- db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4
- sub r10, 2
+ db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4
+ db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0
+ db $C4, $42, $1D, $00, $E3 // vpshufb ymm12, ymm12, ymm11
+ db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5
+ db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1
+ db $C4, $42, $15, $00, $EB // vpshufb ymm13, ymm13, ymm11
+ db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12
+ db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8
+ db $C5, $AD, $72, $F4, $0C // vpslld ymm10, ymm4, 12
+ db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20
+ db $C5, $AD, $EB, $E4 // vpor ymm4, ymm10, ymm4
+ db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736]
+ db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13
+ db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9
+ db $C5, $A5, $72, $F5, $0C // vpslld ymm11, ymm5, 12
+ db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20
+ db $C5, $A5, $EB, $ED // vpor ymm5, ymm11, ymm5
+ db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4
+ db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0
+ db $C4, $42, $1D, $00, $E2 // vpshufb ymm12, ymm12, ymm10
+ db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5
+ db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1
+ db $C4, $42, $15, $00, $EA // vpshufb ymm13, ymm13, ymm10
+ db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12
+ db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8
+ db $C5, $A5, $72, $F4, $07 // vpslld ymm11, ymm4, 7
+ db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25
+ db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+ db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13
+ db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9
+ db $C5, $AD, $72, $F5, $07 // vpslld ymm10, ymm5, 7
+ db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25
+ db $C5, $AD, $EB, $ED // vpor ymm5, ymm10, ymm5
+ db $C5, $7D, $7F, $04, $24 // vmovdqa [rsp], ymm8
+ db $C5, $7D, $7F, $4C, $24, $20 // vmovdqa [rsp + 32], ymm9
+ db $C5, $7D, $6F, $44, $24, $40 // vmovdqa ymm8, [rsp + 64]
+ db $C5, $7D, $6F, $4C, $24, $60 // vmovdqa ymm9, [rsp + 96]
+ db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6
+ db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2
+ db $C4, $42, $0D, $00, $F3 // vpshufb ymm14, ymm14, ymm11
+ db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7
+ db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3
+ db $C4, $42, $05, $00, $FB // vpshufb ymm15, ymm15, ymm11
+ db $C4, $41, $3D, $FE, $C6 // vpaddd ymm8, ymm8, ymm14
+ db $C4, $C1, $4D, $EF, $F0 // vpxor ymm6, ymm6, ymm8
+ db $C5, $AD, $72, $F6, $0C // vpslld ymm10, ymm6, 12
+ db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20
+ db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6
+ db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736]
+ db $C4, $41, $35, $FE, $CF // vpaddd ymm9, ymm9, ymm15
+ db $C4, $C1, $45, $EF, $F9 // vpxor ymm7, ymm7, ymm9
+ db $C5, $A5, $72, $F7, $0C // vpslld ymm11, ymm7, 12
+ db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20
+ db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7
+ db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6
+ db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2
+ db $C4, $42, $0D, $00, $F2 // vpshufb ymm14, ymm14, ymm10
+ db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7
+ db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3
+ db $C4, $42, $05, $00, $FA // vpshufb ymm15, ymm15, ymm10
+ db $C4, $41, $3D, $FE, $C6 // vpaddd ymm8, ymm8, ymm14
+ db $C4, $C1, $4D, $EF, $F0 // vpxor ymm6, ymm6, ymm8
+ db $C5, $A5, $72, $F6, $07 // vpslld ymm11, ymm6, 7
+ db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25
+ db $C5, $A5, $EB, $F6 // vpor ymm6, ymm11, ymm6
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+ db $C4, $41, $35, $FE, $CF // vpaddd ymm9, ymm9, ymm15
+ db $C4, $C1, $45, $EF, $F9 // vpxor ymm7, ymm7, ymm9
+ db $C5, $AD, $72, $F7, $07 // vpslld ymm10, ymm7, 7
+ db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25
+ db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7
+ db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5
+ db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0
+ db $C4, $42, $05, $00, $FB // vpshufb ymm15, ymm15, ymm11
+ db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6
+ db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1
+ db $C4, $42, $1D, $00, $E3 // vpshufb ymm12, ymm12, ymm11
+ db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15
+ db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8
+ db $C5, $AD, $72, $F5, $0C // vpslld ymm10, ymm5, 12
+ db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20
+ db $C5, $AD, $EB, $ED // vpor ymm5, ymm10, ymm5
+ db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736]
+ db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12
+ db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9
+ db $C5, $A5, $72, $F6, $0C // vpslld ymm11, ymm6, 12
+ db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20
+ db $C5, $A5, $EB, $F6 // vpor ymm6, ymm11, ymm6
+ db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5
+ db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0
+ db $C4, $42, $05, $00, $FA // vpshufb ymm15, ymm15, ymm10
+ db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6
+ db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1
+ db $C4, $42, $1D, $00, $E2 // vpshufb ymm12, ymm12, ymm10
+ db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15
+ db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8
+ db $C5, $A5, $72, $F5, $07 // vpslld ymm11, ymm5, 7
+ db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25
+ db $C5, $A5, $EB, $ED // vpor ymm5, ymm11, ymm5
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+ db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12
+ db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9
+ db $C5, $AD, $72, $F6, $07 // vpslld ymm10, ymm6, 7
+ db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25
+ db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6
+ db $C5, $7D, $7F, $44, $24, $40 // vmovdqa [rsp + 64], ymm8
+ db $C5, $7D, $7F, $4C, $24, $60 // vmovdqa [rsp + 96], ymm9
+ db $C5, $7D, $6F, $04, $24 // vmovdqa ymm8, [rsp]
+ db $C5, $7D, $6F, $4C, $24, $20 // vmovdqa ymm9, [rsp + 32]
+ db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7
+ db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2
+ db $C4, $42, $15, $00, $EB // vpshufb ymm13, ymm13, ymm11
+ db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4
+ db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3
+ db $C4, $42, $0D, $00, $F3 // vpshufb ymm14, ymm14, ymm11
+ db $C4, $41, $3D, $FE, $C5 // vpaddd ymm8, ymm8, ymm13
+ db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8
+ db $C5, $AD, $72, $F7, $0C // vpslld ymm10, ymm7, 12
+ db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20
+ db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7
+ db $C5, $7D, $6F, $94, $24, $E0, $02, $00, $00 // vmovdqa ymm10, [rsp + 736]
+ db $C4, $41, $35, $FE, $CE // vpaddd ymm9, ymm9, ymm14
+ db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9
+ db $C5, $A5, $72, $F4, $0C // vpslld ymm11, ymm4, 12
+ db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20
+ db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4
+ db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7
+ db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2
+ db $C4, $42, $15, $00, $EA // vpshufb ymm13, ymm13, ymm10
+ db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4
+ db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3
+ db $C4, $42, $0D, $00, $F2 // vpshufb ymm14, ymm14, ymm10
+ db $C4, $41, $3D, $FE, $C5 // vpaddd ymm8, ymm8, ymm13
+ db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8
+ db $C5, $A5, $72, $F7, $07 // vpslld ymm11, ymm7, 7
+ db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25
+ db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7
+ db $C5, $7D, $6F, $9C, $24, $C0, $02, $00, $00 // vmovdqa ymm11, [rsp + 704]
+ db $C4, $41, $35, $FE, $CE // vpaddd ymm9, ymm9, ymm14
+ db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9
+ db $C5, $AD, $72, $F4, $07 // vpslld ymm10, ymm4, 7
+ db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25
+ db $C5, $AD, $EB, $E4 // vpor ymm4, ymm10, ymm4
+ sub eax, 2
jg @@c7539_avx2_8x_round
- db $C5, $7E, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm8, [rsp + 128]
- db $C4, $C1, $7D, $FE, $C0 // vpaddd ymm0, ymm0, ymm8
- db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0
- db $C5, $7E, $6F, $84, $24, $A0, $00, $00, $00 // vmovdqu ymm8, [rsp + 160]
- db $C4, $C1, $75, $FE, $C8 // vpaddd ymm1, ymm1, ymm8
- db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1
- db $C5, $7E, $6F, $84, $24, $C0, $00, $00, $00 // vmovdqu ymm8, [rsp + 192]
- db $C4, $C1, $6D, $FE, $D0 // vpaddd ymm2, ymm2, ymm8
- db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2
- db $C5, $7E, $6F, $84, $24, $E0, $00, $00, $00 // vmovdqu ymm8, [rsp + 224]
- db $C4, $C1, $65, $FE, $D8 // vpaddd ymm3, ymm3, ymm8
- db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3
- db $C5, $7E, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm8, [rsp + 256]
- db $C4, $C1, $5D, $FE, $E0 // vpaddd ymm4, ymm4, ymm8
- db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4
- db $C5, $7E, $6F, $84, $24, $20, $01, $00, $00 // vmovdqu ymm8, [rsp + 288]
- db $C4, $C1, $55, $FE, $E8 // vpaddd ymm5, ymm5, ymm8
- db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5
- db $C5, $7E, $6F, $84, $24, $40, $01, $00, $00 // vmovdqu ymm8, [rsp + 320]
- db $C4, $C1, $4D, $FE, $F0 // vpaddd ymm6, ymm6, ymm8
- db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6
- db $C5, $7E, $6F, $84, $24, $60, $01, $00, $00 // vmovdqu ymm8, [rsp + 352]
- db $C4, $C1, $45, $FE, $F8 // vpaddd ymm7, ymm7, ymm8
- db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7
- db $C5, $7E, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm8, [rsp + 512]
- db $C4, $41, $1D, $FE, $E0 // vpaddd ymm12, ymm12, ymm8
- db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12
- db $C5, $7E, $6F, $84, $24, $20, $02, $00, $00 // vmovdqu ymm8, [rsp + 544]
- db $C4, $41, $15, $FE, $E8 // vpaddd ymm13, ymm13, ymm8
- db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13
- db $C5, $7E, $6F, $84, $24, $40, $02, $00, $00 // vmovdqu ymm8, [rsp + 576]
- db $C4, $41, $0D, $FE, $F0 // vpaddd ymm14, ymm14, ymm8
- db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14
- db $C5, $7E, $6F, $84, $24, $60, $02, $00, $00 // vmovdqu ymm8, [rsp + 608]
- db $C4, $41, $05, $FE, $F8 // vpaddd ymm15, ymm15, ymm8
- db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15
- db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp]
- db $C5, $7E, $6F, $8C, $24, $80, $01, $00, $00 // vmovdqu ymm9, [rsp + 384]
- db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9
- db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8
- db $C5, $7E, $6F, $44, $24, $20 // vmovdqu ymm8, [rsp + 32]
- db $C5, $7E, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm9, [rsp + 416]
- db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9
- db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8
- db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64]
- db $C5, $7E, $6F, $8C, $24, $C0, $01, $00, $00 // vmovdqu ymm9, [rsp + 448]
- db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9
- db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8
- db $C5, $7E, $6F, $44, $24, $60 // vmovdqu ymm8, [rsp + 96]
- db $C5, $7E, $6F, $8C, $24, $E0, $01, $00, $00 // vmovdqu ymm9, [rsp + 480]
- db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9
- db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8
add dword ptr [rdx + 48], 8
{$IFDEF CHACHA_CTR64}
adc dword ptr [rdx + 52], 0
{$ENDIF}
- db $C5, $FE, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm0, [rsp + 128]
- db $C5, $FE, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqu ymm1, [rsp + 160]
- db $C5, $FE, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqu ymm2, [rsp + 192]
- db $C5, $FE, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqu ymm3, [rsp + 224]
- db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0
- db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1
- db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1
- db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2
- db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3
- db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3
- db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0
- db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2
- db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2
- db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4
- db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5
- db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5
- db $C5, $FA, $7F, $84, $24, $E0, $02, $00, $00 // vmovdqu [rsp + 736], xmm0
- db $C4, $E3, $7D, $39, $84, $24, $E0, $03, $00, $00, $01 // vextracti128 [rsp + 992], ymm0, 1
- db $C5, $FA, $7F, $8C, $24, $20, $03, $00, $00 // vmovdqu [rsp + 800], xmm1
- db $C4, $E3, $7D, $39, $8C, $24, $20, $04, $00, $00, $01 // vextracti128 [rsp + 1056], ymm1, 1
- db $C5, $FA, $7F, $A4, $24, $60, $03, $00, $00 // vmovdqu [rsp + 864], xmm4
- db $C4, $E3, $7D, $39, $A4, $24, $60, $04, $00, $00, $01 // vextracti128 [rsp + 1120], ymm4, 1
- db $C5, $FA, $7F, $9C, $24, $A0, $03, $00, $00 // vmovdqu [rsp + 928], xmm3
- db $C4, $E3, $7D, $39, $9C, $24, $A0, $04, $00, $00, $01 // vextracti128 [rsp + 1184], ymm3, 1
- db $C5, $FE, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm0, [rsp + 256]
- db $C5, $FE, $6F, $8C, $24, $20, $01, $00, $00 // vmovdqu ymm1, [rsp + 288]
- db $C5, $FE, $6F, $94, $24, $40, $01, $00, $00 // vmovdqu ymm2, [rsp + 320]
- db $C5, $FE, $6F, $9C, $24, $60, $01, $00, $00 // vmovdqu ymm3, [rsp + 352]
- db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0
- db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1
- db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1
- db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2
- db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3
- db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3
- db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0
- db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2
- db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2
- db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4
- db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5
- db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5
- db $C5, $FA, $7F, $84, $24, $F0, $02, $00, $00 // vmovdqu [rsp + 752], xmm0
- db $C4, $E3, $7D, $39, $84, $24, $F0, $03, $00, $00, $01 // vextracti128 [rsp + 1008], ymm0, 1
- db $C5, $FA, $7F, $8C, $24, $30, $03, $00, $00 // vmovdqu [rsp + 816], xmm1
- db $C4, $E3, $7D, $39, $8C, $24, $30, $04, $00, $00, $01 // vextracti128 [rsp + 1072], ymm1, 1
- db $C5, $FA, $7F, $A4, $24, $70, $03, $00, $00 // vmovdqu [rsp + 880], xmm4
- db $C4, $E3, $7D, $39, $A4, $24, $70, $04, $00, $00, $01 // vextracti128 [rsp + 1136], ymm4, 1
- db $C5, $FA, $7F, $9C, $24, $B0, $03, $00, $00 // vmovdqu [rsp + 944], xmm3
- db $C4, $E3, $7D, $39, $9C, $24, $B0, $04, $00, $00, $01 // vextracti128 [rsp + 1200], ymm3, 1
- db $C5, $FE, $6F, $84, $24, $80, $01, $00, $00 // vmovdqu ymm0, [rsp + 384]
- db $C5, $FE, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm1, [rsp + 416]
- db $C5, $FE, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqu ymm2, [rsp + 448]
- db $C5, $FE, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqu ymm3, [rsp + 480]
- db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0
- db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1
- db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1
- db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2
- db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3
- db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3
- db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0
- db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2
- db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2
- db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4
- db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5
- db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5
- db $C5, $FA, $7F, $84, $24, $00, $03, $00, $00 // vmovdqu [rsp + 768], xmm0
- db $C4, $E3, $7D, $39, $84, $24, $00, $04, $00, $00, $01 // vextracti128 [rsp + 1024], ymm0, 1
- db $C5, $FA, $7F, $8C, $24, $40, $03, $00, $00 // vmovdqu [rsp + 832], xmm1
- db $C4, $E3, $7D, $39, $8C, $24, $40, $04, $00, $00, $01 // vextracti128 [rsp + 1088], ymm1, 1
- db $C5, $FA, $7F, $A4, $24, $80, $03, $00, $00 // vmovdqu [rsp + 896], xmm4
- db $C4, $E3, $7D, $39, $A4, $24, $80, $04, $00, $00, $01 // vextracti128 [rsp + 1152], ymm4, 1
- db $C5, $FA, $7F, $9C, $24, $C0, $03, $00, $00 // vmovdqu [rsp + 960], xmm3
- db $C4, $E3, $7D, $39, $9C, $24, $C0, $04, $00, $00, $01 // vextracti128 [rsp + 1216], ymm3, 1
- db $C5, $FE, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm0, [rsp + 512]
- db $C5, $FE, $6F, $8C, $24, $20, $02, $00, $00 // vmovdqu ymm1, [rsp + 544]
- db $C5, $FE, $6F, $94, $24, $40, $02, $00, $00 // vmovdqu ymm2, [rsp + 576]
- db $C5, $FE, $6F, $9C, $24, $60, $02, $00, $00 // vmovdqu ymm3, [rsp + 608]
- db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0
- db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1
- db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1
- db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2
- db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3
- db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3
- db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0
- db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2
- db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2
- db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4
- db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5
- db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5
- db $C5, $FA, $7F, $84, $24, $10, $03, $00, $00 // vmovdqu [rsp + 784], xmm0
- db $C4, $E3, $7D, $39, $84, $24, $10, $04, $00, $00, $01 // vextracti128 [rsp + 1040], ymm0, 1
- db $C5, $FA, $7F, $8C, $24, $50, $03, $00, $00 // vmovdqu [rsp + 848], xmm1
- db $C4, $E3, $7D, $39, $8C, $24, $50, $04, $00, $00, $01 // vextracti128 [rsp + 1104], ymm1, 1
- db $C5, $FA, $7F, $A4, $24, $90, $03, $00, $00 // vmovdqu [rsp + 912], xmm4
- db $C4, $E3, $7D, $39, $A4, $24, $90, $04, $00, $00, $01 // vextracti128 [rsp + 1168], ymm4, 1
- db $C5, $FA, $7F, $9C, $24, $D0, $03, $00, $00 // vmovdqu [rsp + 976], xmm3
- db $C4, $E3, $7D, $39, $9C, $24, $D0, $04, $00, $00, $01 // vextracti128 [rsp + 1232], ymm3, 1
- db $C5, $FE, $6F, $84, $24, $E0, $02, $00, $00 // vmovdqu ymm0, [rsp + 736]
- db $C4, $C1, $7D, $EF, $00 // vpxor ymm0, ymm0, [r8 + 0]
- db $C4, $C1, $7E, $7F, $01 // vmovdqu [r9], ymm0
- db $C5, $FE, $6F, $84, $24, $00, $03, $00, $00 // vmovdqu ymm0, [rsp + 768]
- db $C4, $C1, $7D, $EF, $40, $20 // vpxor ymm0, ymm0, [r8 + 32]
- db $C4, $C1, $7E, $7F, $41, $20 // vmovdqu [r9 + 32], ymm0
- db $C5, $FE, $6F, $84, $24, $20, $03, $00, $00 // vmovdqu ymm0, [rsp + 800]
- db $C4, $C1, $7D, $EF, $40, $40 // vpxor ymm0, ymm0, [r8 + 64]
- db $C4, $C1, $7E, $7F, $41, $40 // vmovdqu [r9 + 64], ymm0
- db $C5, $FE, $6F, $84, $24, $40, $03, $00, $00 // vmovdqu ymm0, [rsp + 832]
- db $C4, $C1, $7D, $EF, $40, $60 // vpxor ymm0, ymm0, [r8 + 96]
- db $C4, $C1, $7E, $7F, $41, $60 // vmovdqu [r9 + 96], ymm0
- db $C5, $FE, $6F, $84, $24, $60, $03, $00, $00 // vmovdqu ymm0, [rsp + 864]
- db $C4, $C1, $7D, $EF, $80, $80, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 128]
- db $C4, $C1, $7E, $7F, $81, $80, $00, $00, $00 // vmovdqu [r9 + 128], ymm0
- db $C5, $FE, $6F, $84, $24, $80, $03, $00, $00 // vmovdqu ymm0, [rsp + 896]
- db $C4, $C1, $7D, $EF, $80, $A0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 160]
- db $C4, $C1, $7E, $7F, $81, $A0, $00, $00, $00 // vmovdqu [r9 + 160], ymm0
- db $C5, $FE, $6F, $84, $24, $A0, $03, $00, $00 // vmovdqu ymm0, [rsp + 928]
- db $C4, $C1, $7D, $EF, $80, $C0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 192]
- db $C4, $C1, $7E, $7F, $81, $C0, $00, $00, $00 // vmovdqu [r9 + 192], ymm0
- db $C5, $FE, $6F, $84, $24, $C0, $03, $00, $00 // vmovdqu ymm0, [rsp + 960]
- db $C4, $C1, $7D, $EF, $80, $E0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 224]
- db $C4, $C1, $7E, $7F, $81, $E0, $00, $00, $00 // vmovdqu [r9 + 224], ymm0
- db $C5, $FE, $6F, $84, $24, $E0, $03, $00, $00 // vmovdqu ymm0, [rsp + 992]
- db $C4, $C1, $7D, $EF, $80, $00, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 256]
- db $C4, $C1, $7E, $7F, $81, $00, $01, $00, $00 // vmovdqu [r9 + 256], ymm0
- db $C5, $FE, $6F, $84, $24, $00, $04, $00, $00 // vmovdqu ymm0, [rsp + 1024]
- db $C4, $C1, $7D, $EF, $80, $20, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 288]
- db $C4, $C1, $7E, $7F, $81, $20, $01, $00, $00 // vmovdqu [r9 + 288], ymm0
- db $C5, $FE, $6F, $84, $24, $20, $04, $00, $00 // vmovdqu ymm0, [rsp + 1056]
- db $C4, $C1, $7D, $EF, $80, $40, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 320]
- db $C4, $C1, $7E, $7F, $81, $40, $01, $00, $00 // vmovdqu [r9 + 320], ymm0
- db $C5, $FE, $6F, $84, $24, $40, $04, $00, $00 // vmovdqu ymm0, [rsp + 1088]
- db $C4, $C1, $7D, $EF, $80, $60, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 352]
- db $C4, $C1, $7E, $7F, $81, $60, $01, $00, $00 // vmovdqu [r9 + 352], ymm0
- db $C5, $FE, $6F, $84, $24, $60, $04, $00, $00 // vmovdqu ymm0, [rsp + 1120]
- db $C4, $C1, $7D, $EF, $80, $80, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 384]
- db $C4, $C1, $7E, $7F, $81, $80, $01, $00, $00 // vmovdqu [r9 + 384], ymm0
- db $C5, $FE, $6F, $84, $24, $80, $04, $00, $00 // vmovdqu ymm0, [rsp + 1152]
- db $C4, $C1, $7D, $EF, $80, $A0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 416]
- db $C4, $C1, $7E, $7F, $81, $A0, $01, $00, $00 // vmovdqu [r9 + 416], ymm0
- db $C5, $FE, $6F, $84, $24, $A0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1184]
- db $C4, $C1, $7D, $EF, $80, $C0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 448]
- db $C4, $C1, $7E, $7F, $81, $C0, $01, $00, $00 // vmovdqu [r9 + 448], ymm0
- db $C5, $FE, $6F, $84, $24, $C0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1216]
- db $C4, $C1, $7D, $EF, $80, $E0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 480]
- db $C4, $C1, $7E, $7F, $81, $E0, $01, $00, $00 // vmovdqu [r9 + 480], ymm0
- db $C5, $F8, $77 // vzeroupper
- add rsp, 1248
+ db $C5, $FD, $FE, $84, $24, $80, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp + 128]
+ db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp + 160]
+ db $C5, $ED, $FE, $94, $24, $C0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp + 192]
+ db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp + 224]
+ db $C5, $7D, $62, $D1 // vpunpckldq ymm10, ymm0, ymm1
+ db $C5, $6D, $62, $DB // vpunpckldq ymm11, ymm2, ymm3
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C5, $ED, $6A, $D3 // vpunpckhdq ymm2, ymm2, ymm3
+ db $C4, $C1, $2D, $6C, $CB // vpunpcklqdq ymm1, ymm10, ymm11
+ db $C4, $41, $2D, $6D, $D3 // vpunpckhqdq ymm10, ymm10, ymm11
+ db $C5, $FD, $6C, $DA // vpunpcklqdq ymm3, ymm0, ymm2
+ db $C5, $FD, $6D, $C2 // vpunpckhqdq ymm0, ymm0, ymm2
+ db $C5, $DD, $FE, $A4, $24, $00, $01, $00, $00 // vpaddd ymm4, ymm4, [rsp + 256]
+ db $C5, $D5, $FE, $AC, $24, $20, $01, $00, $00 // vpaddd ymm5, ymm5, [rsp + 288]
+ db $C5, $CD, $FE, $B4, $24, $40, $01, $00, $00 // vpaddd ymm6, ymm6, [rsp + 320]
+ db $C5, $C5, $FE, $BC, $24, $60, $01, $00, $00 // vpaddd ymm7, ymm7, [rsp + 352]
+ db $C5, $DD, $62, $D5 // vpunpckldq ymm2, ymm4, ymm5
+ db $C5, $4D, $62, $DF // vpunpckldq ymm11, ymm6, ymm7
+ db $C5, $DD, $6A, $E5 // vpunpckhdq ymm4, ymm4, ymm5
+ db $C5, $CD, $6A, $F7 // vpunpckhdq ymm6, ymm6, ymm7
+ db $C4, $C1, $6D, $6C, $EB // vpunpcklqdq ymm5, ymm2, ymm11
+ db $C4, $C1, $6D, $6D, $D3 // vpunpckhqdq ymm2, ymm2, ymm11
+ db $C5, $DD, $6C, $FE // vpunpcklqdq ymm7, ymm4, ymm6
+ db $C5, $DD, $6D, $E6 // vpunpckhqdq ymm4, ymm4, ymm6
+ db $C4, $E3, $75, $46, $F5, $20 // vperm2i128 ymm6, ymm1, ymm5, $20
+ db $C4, $E3, $75, $46, $ED, $31 // vperm2i128 ymm5, ymm1, ymm5, $31
+ db $C4, $E3, $2D, $46, $CA, $20 // vperm2i128 ymm1, ymm10, ymm2, $20
+ db $C4, $E3, $2D, $46, $D2, $31 // vperm2i128 ymm2, ymm10, ymm2, $31
+ db $C4, $63, $65, $46, $D7, $20 // vperm2i128 ymm10, ymm3, ymm7, $20
+ db $C4, $E3, $65, $46, $FF, $31 // vperm2i128 ymm7, ymm3, ymm7, $31
+ db $C4, $E3, $7D, $46, $DC, $20 // vperm2i128 ymm3, ymm0, ymm4, $20
+ db $C4, $E3, $7D, $46, $E4, $31 // vperm2i128 ymm4, ymm0, ymm4, $31
+ db $C5, $FD, $7F, $B4, $24, $00, $03, $00, $00 // vmovdqa [rsp + 768], ymm6
+ db $C5, $FD, $7F, $8C, $24, $20, $03, $00, $00 // vmovdqa [rsp + 800], ymm1
+ db $C5, $FD, $6F, $44, $24, $40 // vmovdqa ymm0, [rsp + 64]
+ db $C5, $FD, $6F, $4C, $24, $60 // vmovdqa ymm1, [rsp + 96]
+ db $C5, $3D, $FE, $84, $24, $80, $01, $00, $00 // vpaddd ymm8, ymm8, [rsp + 384]
+ db $C5, $35, $FE, $8C, $24, $A0, $01, $00, $00 // vpaddd ymm9, ymm9, [rsp + 416]
+ db $C5, $FD, $FE, $84, $24, $C0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp + 448]
+ db $C5, $F5, $FE, $8C, $24, $E0, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp + 480]
+ db $C4, $C1, $3D, $62, $F1 // vpunpckldq ymm6, ymm8, ymm9
+ db $C5, $7D, $62, $D9 // vpunpckldq ymm11, ymm0, ymm1
+ db $C4, $41, $3D, $6A, $C1 // vpunpckhdq ymm8, ymm8, ymm9
+ db $C5, $FD, $6A, $C1 // vpunpckhdq ymm0, ymm0, ymm1
+ db $C4, $41, $4D, $6C, $CB // vpunpcklqdq ymm9, ymm6, ymm11
+ db $C4, $C1, $4D, $6D, $F3 // vpunpckhqdq ymm6, ymm6, ymm11
+ db $C5, $BD, $6C, $C8 // vpunpcklqdq ymm1, ymm8, ymm0
+ db $C5, $3D, $6D, $C0 // vpunpckhqdq ymm8, ymm8, ymm0
+ db $C5, $1D, $FE, $A4, $24, $00, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 512]
+ db $C5, $15, $FE, $AC, $24, $20, $02, $00, $00 // vpaddd ymm13, ymm13, [rsp + 544]
+ db $C5, $0D, $FE, $B4, $24, $40, $02, $00, $00 // vpaddd ymm14, ymm14, [rsp + 576]
+ db $C5, $05, $FE, $BC, $24, $60, $02, $00, $00 // vpaddd ymm15, ymm15, [rsp + 608]
+ db $C4, $C1, $1D, $62, $C5 // vpunpckldq ymm0, ymm12, ymm13
+ db $C4, $41, $0D, $62, $DF // vpunpckldq ymm11, ymm14, ymm15
+ db $C4, $41, $1D, $6A, $E5 // vpunpckhdq ymm12, ymm12, ymm13
+ db $C4, $41, $0D, $6A, $F7 // vpunpckhdq ymm14, ymm14, ymm15
+ db $C4, $41, $7D, $6C, $EB // vpunpcklqdq ymm13, ymm0, ymm11
+ db $C4, $C1, $7D, $6D, $C3 // vpunpckhqdq ymm0, ymm0, ymm11
+ db $C4, $41, $1D, $6C, $FE // vpunpcklqdq ymm15, ymm12, ymm14
+ db $C4, $41, $1D, $6D, $E6 // vpunpckhqdq ymm12, ymm12, ymm14
+ db $C4, $43, $35, $46, $DD, $20 // vperm2i128 ymm11, ymm9, ymm13, $20
+ db $C4, $43, $35, $46, $ED, $31 // vperm2i128 ymm13, ymm9, ymm13, $31
+ db $C4, $63, $4D, $46, $C8, $20 // vperm2i128 ymm9, ymm6, ymm0, $20
+ db $C4, $E3, $4D, $46, $C0, $31 // vperm2i128 ymm0, ymm6, ymm0, $31
+ db $C4, $C3, $75, $46, $F7, $20 // vperm2i128 ymm6, ymm1, ymm15, $20
+ db $C4, $43, $75, $46, $FF, $31 // vperm2i128 ymm15, ymm1, ymm15, $31
+ db $C4, $C3, $3D, $46, $CC, $20 // vperm2i128 ymm1, ymm8, ymm12, $20
+ db $C4, $43, $3D, $46, $E4, $31 // vperm2i128 ymm12, ymm8, ymm12, $31
+ db $C5, $7D, $6F, $84, $24, $00, $03, $00, $00 // vmovdqa ymm8, [rsp + 768]
+ db $C4, $41, $3D, $EF, $00 // vpxor ymm8, ymm8, [r8 + 0]
+ db $C4, $41, $7E, $7F, $01 // vmovdqu [r9], ymm8
+ db $C4, $41, $25, $EF, $58, $20 // vpxor ymm11, ymm11, [r8 + 32]
+ db $C4, $41, $7E, $7F, $59, $20 // vmovdqu [r9 + 32], ymm11
+ db $C5, $7D, $6F, $B4, $24, $20, $03, $00, $00 // vmovdqa ymm14, [rsp + 800]
+ db $C4, $41, $0D, $EF, $70, $40 // vpxor ymm14, ymm14, [r8 + 64]
+ db $C4, $41, $7E, $7F, $71, $40 // vmovdqu [r9 + 64], ymm14
+ db $C4, $41, $35, $EF, $48, $60 // vpxor ymm9, ymm9, [r8 + 96]
+ db $C4, $41, $7E, $7F, $49, $60 // vmovdqu [r9 + 96], ymm9
+ db $C4, $41, $2D, $EF, $90, $80, $00, $00, $00 // vpxor ymm10, ymm10, [r8 + 128]
+ db $C4, $41, $7E, $7F, $91, $80, $00, $00, $00 // vmovdqu [r9 + 128], ymm10
+ db $C4, $C1, $4D, $EF, $B0, $A0, $00, $00, $00 // vpxor ymm6, ymm6, [r8 + 160]
+ db $C4, $C1, $7E, $7F, $B1, $A0, $00, $00, $00 // vmovdqu [r9 + 160], ymm6
+ db $C4, $C1, $65, $EF, $98, $C0, $00, $00, $00 // vpxor ymm3, ymm3, [r8 + 192]
+ db $C4, $C1, $7E, $7F, $99, $C0, $00, $00, $00 // vmovdqu [r9 + 192], ymm3
+ db $C4, $C1, $75, $EF, $88, $E0, $00, $00, $00 // vpxor ymm1, ymm1, [r8 + 224]
+ db $C4, $C1, $7E, $7F, $89, $E0, $00, $00, $00 // vmovdqu [r9 + 224], ymm1
+ db $C4, $C1, $55, $EF, $A8, $00, $01, $00, $00 // vpxor ymm5, ymm5, [r8 + 256]
+ db $C4, $C1, $7E, $7F, $A9, $00, $01, $00, $00 // vmovdqu [r9 + 256], ymm5
+ db $C4, $41, $15, $EF, $A8, $20, $01, $00, $00 // vpxor ymm13, ymm13, [r8 + 288]
+ db $C4, $41, $7E, $7F, $A9, $20, $01, $00, $00 // vmovdqu [r9 + 288], ymm13
+ db $C4, $C1, $6D, $EF, $90, $40, $01, $00, $00 // vpxor ymm2, ymm2, [r8 + 320]
+ db $C4, $C1, $7E, $7F, $91, $40, $01, $00, $00 // vmovdqu [r9 + 320], ymm2
+ db $C4, $C1, $7D, $EF, $80, $60, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 352]
+ db $C4, $C1, $7E, $7F, $81, $60, $01, $00, $00 // vmovdqu [r9 + 352], ymm0
+ db $C4, $C1, $45, $EF, $B8, $80, $01, $00, $00 // vpxor ymm7, ymm7, [r8 + 384]
+ db $C4, $C1, $7E, $7F, $B9, $80, $01, $00, $00 // vmovdqu [r9 + 384], ymm7
+ db $C4, $41, $05, $EF, $B8, $A0, $01, $00, $00 // vpxor ymm15, ymm15, [r8 + 416]
+ db $C4, $41, $7E, $7F, $B9, $A0, $01, $00, $00 // vmovdqu [r9 + 416], ymm15
+ db $C4, $C1, $5D, $EF, $A0, $C0, $01, $00, $00 // vpxor ymm4, ymm4, [r8 + 448]
+ db $C4, $C1, $7E, $7F, $A1, $C0, $01, $00, $00 // vmovdqu [r9 + 448], ymm4
+ db $C4, $41, $1D, $EF, $A0, $E0, $01, $00, $00 // vpxor ymm12, ymm12, [r8 + 480]
+ db $C4, $41, $7E, $7F, $A1, $E0, $01, $00, $00 // vmovdqu [r9 + 480], ymm12
+ add r8, 512
+ add r9, 512
+ dec r10
+ jnz @@c7539_avx2_8x_outer
+ db $C5, $F8, $77 // vzeroupper
+ mov rsp, r11
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc
index fa5b49ed..5fd64011 100644
--- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc
+++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc
@@ -1,18 +1,20 @@
-// SSE2 + PCLMULQDQ fused N-way GHASH batch kernel (i386).
+// SSE2 + PCLMULQDQ N-way GHASH batch kernel (i386).
// -------------------------------------------------------------------
-// One monolithic body that performs a full GHASH batch fold: for each
-// block loads the (byte-reversed) ciphertext, XORs the running state
-// into block 0, multiplies by the matching H power via four PCLMULQDQ
-// products (Karatsuba), accumulates across blocks into three 128-bit
-// limbs Z0/Z1/Z2, then folds Z1 and reduces (Z0, Z2) modulo the GCM
-// polynomial x^128 + x^7 + x^2 + x + 1 to a single field element,
-// byte-reverses, and stores back to PFS. i386 has only 8 XMM
-// registers so the PSHUFB mask cannot stay pinned; it is reloaded
-// from [eax] on every use.
+// Loops over ABatchCount batches inside the asm body. Per batch: for each
+// block loads the (byte-reversed) data, XORs the running state into
+// block 0, multiplies by the matching H power via four PCLMULQDQ products,
+// accumulates across blocks into three 128-bit limbs Z0/Z1/Z2, then folds
+// Z1 and reduces with two carry-less multiplies by the folded constant
+// 0xC2000000_00000000 (PHPow holds the powers PRE-MULTIPLIED BY x in the
+// reflected representation - see TGcmUtilities.InitEightWayHPowFromH - so
+// no bit-shift correction is needed). The running state stays reflected
+// across batches; the canonical byte order is only touched at entry/exit,
+// and the nonvolatile save, mask load and reduction constant are paid
+// once per call instead of once per batch.
//
// Variant selection (set EXACTLY ONE of the block counts):
-// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1.
-// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1.
+// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batches, H^4..H^1.
+// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batches, H^8..H^1.
// Optional:
// GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so
// skip the per-block input byte-reverse. Used by the
@@ -21,261 +23,232 @@
// still byte-reversed. Unset for GCM (canonical
// input), leaving that path byte-identical.
//
-// On entry (after ClpSimdProc4Begin_i386.inc):
+// On entry (after ClpSimdProc5Begin_i386.inc):
// ebx = PFS (16-byte running GHASH state, canonical; in/out)
-// esi = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT)
-// edi = PHPow (H^N..H^1 as 16-byte limbs, field form)
+// esi = PC0 (data blocks; canonical, or field form with RAW_INPUT; advances)
+// edi = PHPow (H^N..H^1 as 16-byte limbs, x-pre-multiplied)
// eax = PMask (16-byte PSHUFB byte-reverse control)
+// ecx = ABatchCount (> 0)
//
-// XMM (8/8 used):
-// xmm0..xmm2 : Z0 / Z1 / Z2 accumulators (persistent across blocks)
-// xmm3 : current byte-reversed data block
-// xmm4 : current H-power limb
-// xmm5..xmm7 : pclmul product destinations (per-block); xmm7 is also
-// reused as the byte-reverse mask (reloaded from [eax]
-// before each PSHUFB)
+// XMM (8/8 used): xmm0..xmm2 = Z0/Z1/Z2, xmm3 = data block / reduction
+// constant, xmm4..xmm7 = scratch. The byte-reverse mask, the reflected
+// running state and the reduction constant live in a 16-aligned frame
+// (PSHUFB memory operands); the caller esp is parked inside it.
//
-// PCLMULQDQ instructions are db-encoded for broad assembler compatibility.
+// PCLMULQDQ / PSHUFD and the PSHUFB memory-operand form are db-encoded
+// for broad assembler compatibility.
//
- pxor xmm0, xmm0
- pxor xmm1, xmm1
- pxor xmm2, xmm2
+ mov edx, esp
+ sub esp, 72
+ and esp, -16 // align the frame
+ mov [esp + 48], edx // park the caller esp
- // Block 0: byte-reverse + fold running state
+ mov dword ptr [esp + 32], 0
+ mov dword ptr [esp + 36], $C2000000 // reduction constant (low qword)
+ mov dword ptr [esp + 40], 0
+ mov dword ptr [esp + 44], 0
+ movdqu xmm0, [eax] // byte-reverse mask
+ movdqa [esp + 16], xmm0
+ movdqu xmm1, [ebx]
+ pshufb xmm1, xmm0
+ movdqa [esp + 0], xmm1 // reflected running state
+
+@@GhashFullLoop:
+ pxor xmm0, xmm0 // Z0 = 0
+ pxor xmm1, xmm1 // Z1 = 0
+ pxor xmm2, xmm2 // Z2 = 0
+
+ // Block 0 (folds the carried state)
movdqu xmm3, [esi]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
- movdqu xmm5, [ebx]
- pshufb xmm5, xmm7
- pxor xmm3, xmm5
+ pxor xmm3, [esp + 0] // fold carried state
movdqu xmm4, [edi]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 1
+ // Block 1 (folds the carried state)
movdqu xmm3, [esi + 16]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 16]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 2
+ // Block 2 (folds the carried state)
movdqu xmm3, [esi + 32]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 32]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 3
+ // Block 3 (folds the carried state)
movdqu xmm3, [esi + 48]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 48]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
-
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
{$IFDEF GCM_GHASH_FULL_BLOCKS_8}
- // Block 4
+
+ // Block 4 (folds the carried state)
movdqu xmm3, [esi + 64]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 64]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 5
+ // Block 5 (folds the carried state)
movdqu xmm3, [esi + 80]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 80]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 6
+ // Block 6 (folds the carried state)
movdqu xmm3, [esi + 96]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 96]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 7
+ // Block 7 (folds the carried state)
movdqu xmm3, [esi + 112]
- movdqu xmm7, [eax]
{$IFNDEF GCM_GHASH_RAW_INPUT}
- pshufb xmm3, xmm7
+ db $66, $0F, $38, $00, $5C, $24, $10 // pshufb xmm3, [esp+16] (byte-reverse)
{$ENDIF}
movdqu xmm4, [edi + 112]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
{$ENDIF GCM_GHASH_FULL_BLOCKS_8}
- // Inline 3-limb GHASH reduction (Z0=xmm0, Z1=xmm1, Z2=xmm2): fold Z1 into
- // (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1.
+ // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into
+ // the 256-bit product, then two carry-less multiplies by the
+ // folded constant 0xC2000000_00000000 (the H powers are
+ // pre-multiplied by x, so no bit-shift correction is needed). ---
+ movdqa xmm3, [esp + 32]
movdqa xmm4, xmm1
- movdqa xmm5, xmm1
pslldq xmm4, 8
+ pxor xmm0, xmm4 // ZLo ^= Z1 << 64
+ movdqa xmm5, xmm1
psrldq xmm5, 8
- pxor xmm0, xmm4
- pxor xmm2, xmm5
- movq xmm3, xmm0
- movq xmm4, xmm2
- movdqa xmm5, xmm3
- psrlq xmm5, 1
- movdqa xmm6, xmm3
- psrlq xmm6, 2
- movdqa xmm7, xmm3
- psrlq xmm7, 7
- pxor xmm4, xmm3
- pxor xmm4, xmm5
- pxor xmm4, xmm6
- pxor xmm4, xmm7
- movdqa xmm1, xmm0
- psrldq xmm1, 8
- movdqa xmm5, xmm3
- psllq xmm5, 63
- movdqa xmm6, xmm3
- psllq xmm6, 62
- movdqa xmm7, xmm3
- psllq xmm7, 57
- pxor xmm1, xmm5
- pxor xmm1, xmm6
- pxor xmm1, xmm7
- movdqa xmm5, xmm2
- psrldq xmm5, 8
- movdqa xmm6, xmm5
- psllq xmm6, 1
- movdqa xmm7, xmm4
- psrlq xmm7, 63
- por xmm6, xmm7
- movdqa xmm7, xmm4
- psllq xmm7, 1
- movdqa xmm3, xmm1
- psrlq xmm3, 63
- por xmm7, xmm3
- movdqa xmm0, xmm1
- psllq xmm0, 1
- movdqa xmm3, xmm0
- psrlq xmm3, 1
- movdqa xmm2, xmm0
- psrlq xmm2, 2
- movdqa xmm5, xmm0
- psrlq xmm5, 7
- pxor xmm6, xmm0
- pxor xmm6, xmm3
- pxor xmm6, xmm2
- pxor xmm6, xmm5
- movdqa xmm3, xmm1
- psllq xmm3, 63
- movdqa xmm2, xmm1
- psllq xmm2, 58
- pxor xmm7, xmm3
- pxor xmm7, xmm2
+ pxor xmm2, xmm5 // ZHi ^= Z1 >> 64
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 1
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 2
+ pxor xmm0, xmm2 // ^= ZHi -> reflected state
+ movdqa [esp + 0], xmm0 // carry state to the next batch
- // Pack and byte-reverse.
- movq xmm0, xmm7
- pslldq xmm6, 8
- por xmm0, xmm6
- movdqu xmm7, [eax]
- pshufb xmm0, xmm7
+{$IFDEF GCM_GHASH_FULL_BLOCKS_8}
+ add esi, 128
+{$ELSE}
+ add esi, 64
+{$ENDIF}
+ dec ecx
+ jnz @@GhashFullLoop
+
+ movdqa xmm0, [esp + 0]
+ db $66, $0F, $38, $00, $44, $24, $10 // pshufb xmm0, [esp+16] (-> canonical)
movdqu [ebx], xmm0
- pop edi
- pop esi
- pop ebx
+ mov esp, [esp + 48] // unwind the frame
+ pop edi
+ pop esi
+ pop ebx
+
diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc
index 8a88bd6e..bbfeb6f1 100644
--- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc
@@ -1,19 +1,20 @@
-// SSE2 + PCLMULQDQ fused N-way GHASH batch kernel (x86-64).
+// SSE2 + PCLMULQDQ N-way GHASH batch kernel (x86-64).
// -------------------------------------------------------------------
-// One monolithic body that performs a full GHASH batch fold: for each
-// block loads the (byte-reversed) ciphertext, XORs the running state
-// into block 0, multiplies by the matching H power via four PCLMULQDQ
-// products (Karatsuba), accumulates across blocks into three 128-bit
-// limbs Z0/Z1/Z2, then folds Z1 and reduces (Z0, Z2) modulo the GCM
-// polynomial x^128 + x^7 + x^2 + x + 1 to a single field element,
-// byte-reverses, and stores back to PFS. Replaces the previous chain
-// of 11 Pascal -> ASM calls (one byte-reverse per block, one XOR fold,
-// multiply-accumulate, 3-limb reduction, byte-reverse back) with a
-// single kernel.
+// Loops over ABatchCount batches inside the asm body. Per batch: for each
+// block loads the (byte-reversed) data, XORs the running state into
+// block 0, multiplies by the matching H power via four PCLMULQDQ products,
+// accumulates across blocks into three 128-bit limbs Z0/Z1/Z2, then folds
+// Z1 and reduces with two carry-less multiplies by the folded constant
+// 0xC2000000_00000000 (PHPow holds the powers PRE-MULTIPLIED BY x in the
+// reflected representation - see TGcmUtilities.InitEightWayHPowFromH - so
+// no bit-shift correction is needed). The running state stays reflected
+// across batches; the canonical byte order is only touched at entry/exit,
+// and the nonvolatile save, mask load and reduction constant are paid
+// once per call instead of once per batch.
//
// Variant selection (set EXACTLY ONE of the block counts):
-// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1.
-// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1.
+// GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batches, H^4..H^1.
+// GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batches, H^8..H^1.
// Optional:
// GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so
// skip the per-block input byte-reverse. Used by the
@@ -22,253 +23,223 @@
// still byte-reversed. Unset for GCM (canonical
// input), leaving that path byte-identical.
//
-// On entry (after ClpSimdProc4Begin_x86_64.inc):
+// On entry (after ClpSimdProc5Begin_x86_64.inc):
// rcx = PFS (16-byte running GHASH state, canonical; in/out)
-// rdx = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT)
-// r8 = PHPow (H^N..H^1 as 16-byte limbs, field form)
+// rdx = PC0 (data blocks; canonical, or field form with RAW_INPUT; advances)
+// r8 = PHPow (H^N..H^1 as 16-byte limbs, x-pre-multiplied)
// r9 = PMask (16-byte PSHUFB byte-reverse control)
+// r10 = ABatchCount (> 0)
//
-// XMM (9/16 used):
-// xmm0..xmm2 : Z0 / Z1 / Z2 accumulators (persistent across blocks)
-// xmm3 : current byte-reversed data block
-// xmm4 : current H-power limb
-// xmm5..xmm7 : pclmul product destinations (per-block)
-// xmm8 : byte-reverse mask (held for the whole kernel)
+// XMM (10/16 used): xmm0..xmm2 = Z0/Z1/Z2, xmm3 = data block / reduction
+// constant, xmm4..xmm7 = scratch, xmm8 = mask (whole call), xmm9 =
+// reflected running state (whole call). [rsp] (16-aligned) holds the
+// reduction constant.
//
-// PCLMULQDQ instructions are db-encoded for broad assembler compatibility.
+// PCLMULQDQ / PSHUFD and pxor forms touching xmm8/xmm9 are db-encoded
+// for broad assembler compatibility.
//
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- movdqu xmm8, [r9]
- pxor xmm0, xmm0
- pxor xmm1, xmm1
- pxor xmm2, xmm2
+ sub rsp, 24 // [rsp] 16-aligned
+ mov rax, $C200000000000000
+ mov [rsp], rax // reduction constant (low qword)
+ xor eax, eax
+ mov [rsp + 8], rax
+ movdqu xmm8, [r9] // byte-reverse mask
+ movdqu xmm9, [rcx]
+ pshufb xmm9, xmm8 // reflected running state
- // Block 0: byte-reverse + fold running state
+@@GhashFullLoop:
+ pxor xmm0, xmm0 // Z0 = 0
+ pxor xmm1, xmm1 // Z1 = 0
+ pxor xmm2, xmm2 // Z2 = 0
+
+ // Block 0 (folds the carried state)
movdqu xmm3, [rdx]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
- movdqu xmm5, [rcx]
- pshufb xmm5, xmm8
- pxor xmm3, xmm5
+ db $66, $41, $0F, $EF, $D9 // pxor xmm3, xmm9 (fold carried state)
movdqu xmm4, [r8]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 1
+ // Block 1 (folds the carried state)
movdqu xmm3, [rdx + 16]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 16]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 2
+ // Block 2 (folds the carried state)
movdqu xmm3, [rdx + 32]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 32]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 3
+ // Block 3 (folds the carried state)
movdqu xmm3, [rdx + 48]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 48]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
-
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
{$IFDEF GCM_GHASH_FULL_BLOCKS_8}
- // Block 4
+
+ // Block 4 (folds the carried state)
movdqu xmm3, [rdx + 64]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 64]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 5
+ // Block 5 (folds the carried state)
movdqu xmm3, [rdx + 80]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 80]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 6
+ // Block 6 (folds the carried state)
movdqu xmm3, [rdx + 96]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 96]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
- // Block 7
+ // Block 7 (folds the carried state)
movdqu xmm3, [rdx + 112]
{$IFNDEF GCM_GHASH_RAW_INPUT}
pshufb xmm3, xmm8
{$ENDIF}
movdqu xmm4, [r8 + 112]
movdqa xmm5, xmm3
- db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0
+ db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0x00
movdqa xmm6, xmm3
- db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 1
+ db $66, $0F, $3A, $44, $F4, $01 // pclmulqdq xmm6, xmm4, 0x01
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 16
+ db $66, $0F, $3A, $44, $FC, $10 // pclmulqdq xmm7, xmm4, 0x10
pxor xmm6, xmm7
movdqa xmm7, xmm3
- db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 17
- pxor xmm0, xmm5
- pxor xmm1, xmm6
- pxor xmm2, xmm7
+ db $66, $0F, $3A, $44, $FC, $11 // pclmulqdq xmm7, xmm4, 0x11
+ pxor xmm0, xmm5 // Z0 +=
+ pxor xmm1, xmm6 // Z1 +=
+ pxor xmm2, xmm7 // Z2 +=
{$ENDIF GCM_GHASH_FULL_BLOCKS_8}
- // Inline 3-limb GHASH reduction (Z0=xmm0, Z1=xmm1, Z2=xmm2): fold Z1 into
- // (Z0, Z2), then reduce modulo x^128 + x^7 + x^2 + x + 1.
- // Step 1 -- fold Z1 into Z0/Z2.
+ // --- Reduce [Z2:Z1:Z0] modulo the field polynomial: fold Z1 into
+ // the 256-bit product, then two carry-less multiplies by the
+ // folded constant 0xC2000000_00000000 (the H powers are
+ // pre-multiplied by x, so no bit-shift correction is needed). ---
+ movdqa xmm3, [rsp]
movdqa xmm4, xmm1
- movdqa xmm5, xmm1
pslldq xmm4, 8
+ pxor xmm0, xmm4 // ZLo ^= Z1 << 64
+ movdqa xmm5, xmm1
psrldq xmm5, 8
- pxor xmm0, xmm4
- pxor xmm2, xmm5
- // Extract T3, T1 as lower-64 halves; T2, T0 as upper-64 halves.
- movq xmm3, xmm0 // xmm3 = T3 (low 64 of folded Z0)
- movq xmm4, xmm2 // xmm4 = T1 (low 64 of folded Z2)
- movdqa xmm5, xmm3
- psrlq xmm5, 1
- movdqa xmm6, xmm3
- psrlq xmm6, 2
- movdqa xmm7, xmm3
- psrlq xmm7, 7
- pxor xmm4, xmm3
- pxor xmm4, xmm5
- pxor xmm4, xmm6
- pxor xmm4, xmm7 // xmm4 = T1 ^ T3 ^ (T3>>1) ^ (T3>>2) ^ (T3>>7)
- movdqa xmm1, xmm0
- psrldq xmm1, 8 // xmm1 = T2 (upper 64 of folded Z0, now in lower)
- movdqa xmm5, xmm3
- psllq xmm5, 63
- movdqa xmm6, xmm3
- psllq xmm6, 62
- movdqa xmm7, xmm3
- psllq xmm7, 57
- pxor xmm1, xmm5
- pxor xmm1, xmm6
- pxor xmm1, xmm7 // xmm1 = T2 ^ (T3<<63) ^ (T3<<62) ^ (T3<<57)
- movdqa xmm5, xmm2
- psrldq xmm5, 8 // xmm5 = T0 (upper 64 of folded Z2)
- movdqa xmm6, xmm5
- psllq xmm6, 1
- movdqa xmm7, xmm4
- psrlq xmm7, 63
- por xmm6, xmm7 // xmm6 = LZ0 initial = (T0<<1) | (T1>>63)
- movdqa xmm7, xmm4
- psllq xmm7, 1
- movdqa xmm3, xmm1
- psrlq xmm3, 63
- por xmm7, xmm3 // xmm7 = LZ1 initial = (T1<<1) | (T2>>63)
- movdqa xmm0, xmm1
- psllq xmm0, 1 // xmm0 = LZ2 = T2<<1
- movdqa xmm3, xmm0
- psrlq xmm3, 1
- movdqa xmm2, xmm0
- psrlq xmm2, 2
- movdqa xmm5, xmm0
- psrlq xmm5, 7
- pxor xmm6, xmm0
- pxor xmm6, xmm3
- pxor xmm6, xmm2
- pxor xmm6, xmm5 // xmm6 (low 64) = final LZ0
- movdqa xmm3, xmm1
- psllq xmm3, 63
- movdqa xmm2, xmm1
- psllq xmm2, 58
- pxor xmm7, xmm3
- pxor xmm7, xmm2 // xmm7 (low 64) = final LZ1
+ pxor xmm2, xmm5 // ZHi ^= Z1 >> 64
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 1
+ movdqa xmm4, xmm0
+ db $66, $0F, $3A, $44, $E3, $00 // pclmulqdq xmm4, xmm3, 0x00
+ db $66, $0F, $70, $C0, $4E // pshufd xmm0, xmm0, 0x4E (swap halves)
+ pxor xmm0, xmm4 // fold 2
+ pxor xmm0, xmm2 // ^= ZHi -> reflected state
+ movdqa xmm9, xmm0 // carry state to the next batch
- // Pack 16-byte result: bytes 0..7 = xmm7.q0, bytes 8..15 = xmm6.q0.
- movq xmm0, xmm7
- pslldq xmm6, 8
- por xmm0, xmm6
- // Byte-reverse to canonical form and store to PFS.
- pshufb xmm0, xmm8
- movdqu [rcx], xmm0
+{$IFDEF GCM_GHASH_FULL_BLOCKS_8}
+ add rdx, 128
+{$ELSE}
+ add rdx, 64
+{$ENDIF}
+ dec r10
+ jnz @@GhashFullLoop
+
+ pshufb xmm9, xmm8 // reflected -> canonical
+ movdqu [rcx], xmm9
+ add rsp, 24
{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
+
diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc
index 89324967..b0b12226 100644
--- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc
+++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc
@@ -18,283 +18,224 @@
// (5*r), row 9 pad. Bulk reads [+32*j] (broadcast r^4); tail reads
// [+32*j+4] ([r^4,r^3,r^2,r^1]).
- sub esp, 320
- // ---- Prologue: mask26 (ymm5), padbit (ymm6) ----
+ // ---- 64B-aligned frame; mask26/padbit stored as mem operands ----
+ mov edx, esp
+ sub esp, 256
+ and esp, -64
+ mov dword ptr [esp + 224], edx
db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5
db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38
- db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6
- db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63
- db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24
+ db $C5, $FD, $7F, $AC, $24, $A0, $00, $00, $00 // vmovdqa [esp + 160], ymm5
+ db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5
+ db $C5, $D5, $73, $D5, $3F // vpsrlq ymm5, ymm5, 63
+ db $C5, $D5, $73, $F5, $18 // vpsllq ymm5, ymm5, 24
+ db $C5, $FD, $7F, $AC, $24, $C0, $00, $00, $00 // vmovdqa [esp + 192], ymm5
// ---- Load H0..H4 into lane 0 ----
db $C5, $F9, $6E, $43, $24 // vmovd xmm0, dword ptr [ebx + 36]
db $C5, $F9, $6E, $4B, $28 // vmovd xmm1, dword ptr [ebx + 40]
db $C5, $F9, $6E, $53, $2C // vmovd xmm2, dword ptr [ebx + 44]
db $C5, $F9, $6E, $5B, $30 // vmovd xmm3, dword ptr [ebx + 48]
db $C5, $F9, $6E, $63, $34 // vmovd xmm4, dword ptr [ebx + 52]
- // ---- Pre-loop: spill H, absorb first 4 blocks ----
- db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0
- db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1
- db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2
- db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3
- db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4
- db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi]
- db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16]
- db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1
- db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1
+ // ---- Pre-loop: absorb first 4 blocks ----
+ db $C5, $FA, $6F, $2F // vmovdqu xmm5, [edi]
+ db $C5, $FA, $6F, $77, $10 // vmovdqu xmm6, [edi + 16]
+ db $C4, $E3, $55, $38, $6F, $20, $01 // vinserti128 ymm5, ymm5, [edi + 32], 1
+ db $C4, $E3, $4D, $38, $77, $30, $01 // vinserti128 ymm6, ymm6, [edi + 48], 1
add edi, 64
- db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6
- db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6
- db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1
- db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1
- db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3
- db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30
- db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4
- db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26
- db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5
- db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5
- db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5
- db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6
- db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160]
- db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192]
- db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224]
- db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256]
- db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288]
+ db $C5, $D5, $6D, $FE // vpunpckhqdq ymm7, ymm5, ymm6
+ db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40
+ db $C5, $C5, $EB, $BC, $24, $C0, $00, $00, $00 // vpor ymm7, ymm7, [esp + 192]
+ db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
+ db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6
+ db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6
+ db $C5, $CD, $73, $DE, $06 // vpsrldq ymm6, ymm6, 6
+ db $C5, $C5, $6C, $FE // vpunpcklqdq ymm7, ymm7, ymm6
+ db $C5, $CD, $73, $D7, $1E // vpsrlq ymm6, ymm7, 30
+ db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160]
+ db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6
+ db $C5, $C5, $73, $D7, $04 // vpsrlq ymm7, ymm7, 4
+ db $C5, $C5, $DB, $BC, $24, $A0, $00, $00, $00 // vpand ymm7, ymm7, [esp + 160]
+ db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
+ db $C5, $D5, $DB, $AC, $24, $A0, $00, $00, $00 // vpand ymm5, ymm5, [esp + 160]
+ db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5
sub eax, 64
jz @@poly1305_avx2_i386_tail
@@poly1305_avx2_i386_loop:
- // ---- Bulk: H *= r^4 (broadcast), D0..D4 spilled ----
- db $C5, $FD, $F4, $BE, $80, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $F5, $F4, $7E, $60 // vpmuludq ymm7, ymm1, [esi + 96]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $ED, $F4, $7E, $40 // vpmuludq ymm7, ymm2, [esi + 64]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $E5, $F4, $7E, $20 // vpmuludq ymm7, ymm3, [esi + 32]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $DD, $F4, $3E // vpmuludq ymm7, ymm4, [esi]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $FD, $F4, $7E, $60 // vpmuludq ymm7, ymm0, [esi + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $F5, $F4, $7E, $40 // vpmuludq ymm7, ymm1, [esi + 64]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $ED, $F4, $7E, $20 // vpmuludq ymm7, ymm2, [esi + 32]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $E5, $F4, $3E // vpmuludq ymm7, ymm3, [esi]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $DD, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 256]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $FD, $F4, $7E, $40 // vpmuludq ymm7, ymm0, [esi + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $F5, $F4, $7E, $20 // vpmuludq ymm7, ymm1, [esi + 32]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $ED, $F4, $3E // vpmuludq ymm7, ymm2, [esi]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $E5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 256]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $DD, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 224]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $FD, $F4, $7E, $20 // vpmuludq ymm7, ymm0, [esi + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $F5, $F4, $3E // vpmuludq ymm7, ymm1, [esi]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $ED, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 256]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $E5, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 224]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $DD, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 192]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $FD, $F4, $3E // vpmuludq ymm7, ymm0, [esi]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $F5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 256]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $ED, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 224]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $E5, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 192]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $DD, $F4, $BE, $A0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 160]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- // ---- Lazy reduce ----
- db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp]
- db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32]
- db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64]
- db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96]
- db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128]
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
- db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26
- db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5
- db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
- db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26
- db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5
- db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7
- db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26
- db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5
- db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
- db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26
- db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
+ // ---- Bulk: H *= r^4 (broadcast), k-major resident accumulator ----
+ db $C5, $FD, $F4, $AE, $80, $00, $00, $00 // vpmuludq ymm5, ymm0, [esi + 128]
+ db $C5, $F5, $F4, $76, $60 // vpmuludq ymm6, ymm1, [esi + 96]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $76, $40 // vpmuludq ymm6, ymm2, [esi + 64]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $76, $20 // vpmuludq ymm6, ymm3, [esi + 32]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $36 // vpmuludq ymm6, ymm4, [esi]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $AC, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm5
+ db $C5, $FD, $F4, $6E, $60 // vpmuludq ymm5, ymm0, [esi + 96]
+ db $C5, $F5, $F4, $76, $40 // vpmuludq ymm6, ymm1, [esi + 64]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $76, $20 // vpmuludq ymm6, ymm2, [esi + 32]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $36 // vpmuludq ymm6, ymm3, [esi]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm4, [esi + 256]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $60 // vmovdqa [esp + 96], ymm5
+ db $C5, $FD, $F4, $6E, $40 // vpmuludq ymm5, ymm0, [esi + 64]
+ db $C5, $F5, $F4, $76, $20 // vpmuludq ymm6, ymm1, [esi + 32]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $36 // vpmuludq ymm6, ymm2, [esi]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm3, [esi + 256]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 224]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $40 // vmovdqa [esp + 64], ymm5
+ db $C5, $FD, $F4, $6E, $20 // vpmuludq ymm5, ymm0, [esi + 32]
+ db $C5, $F5, $F4, $36 // vpmuludq ymm6, ymm1, [esi]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm2, [esi + 256]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 224]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $C0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 192]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $20 // vmovdqa [esp + 32], ymm5
+ db $C5, $FD, $F4, $2E // vpmuludq ymm5, ymm0, [esi]
+ db $C5, $F5, $F4, $B6, $00, $01, $00, $00 // vpmuludq ymm6, ymm1, [esi + 256]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $B6, $E0, $00, $00, $00 // vpmuludq ymm6, ymm2, [esi + 224]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $C0, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 192]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $A0, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 160]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ // ---- Lazy reduce (D0 from ymm5) ----
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C5, $D5, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm5, [esp + 160]
+ db $C5, $CD, $D4, $4C, $24, $20 // vpaddq ymm1, ymm6, [esp + 32]
+ db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26
+ db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160]
+ db $C5, $CD, $D4, $54, $24, $40 // vpaddq ymm2, ymm6, [esp + 64]
+ db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26
+ db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160]
+ db $C5, $CD, $D4, $5C, $24, $60 // vpaddq ymm3, ymm6, [esp + 96]
+ db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26
+ db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160]
+ db $C5, $CD, $D4, $A4, $24, $80, $00, $00, $00 // vpaddq ymm4, ymm6, [esp + 128]
+ db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26
+ db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160]
+ db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2
+ db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7
+ db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6
+ db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26
+ db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
// ---- Absorb next 4 blocks ----
- db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0
- db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1
- db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2
- db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3
- db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4
- db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi]
- db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16]
- db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1
- db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1
+ db $C5, $FA, $6F, $2F // vmovdqu xmm5, [edi]
+ db $C5, $FA, $6F, $77, $10 // vmovdqu xmm6, [edi + 16]
+ db $C4, $E3, $55, $38, $6F, $20, $01 // vinserti128 ymm5, ymm5, [edi + 32], 1
+ db $C4, $E3, $4D, $38, $77, $30, $01 // vinserti128 ymm6, ymm6, [edi + 48], 1
add edi, 64
- db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6
- db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6
- db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1
- db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1
- db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3
- db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30
- db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4
- db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26
- db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5
- db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5
- db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5
- db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6
- db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160]
- db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192]
- db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224]
- db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256]
- db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288]
+ db $C5, $D5, $6D, $FE // vpunpckhqdq ymm7, ymm5, ymm6
+ db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40
+ db $C5, $C5, $EB, $BC, $24, $C0, $00, $00, $00 // vpor ymm7, ymm7, [esp + 192]
+ db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
+ db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6
+ db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6
+ db $C5, $CD, $73, $DE, $06 // vpsrldq ymm6, ymm6, 6
+ db $C5, $C5, $6C, $FE // vpunpcklqdq ymm7, ymm7, ymm6
+ db $C5, $CD, $73, $D7, $1E // vpsrlq ymm6, ymm7, 30
+ db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160]
+ db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6
+ db $C5, $C5, $73, $D7, $04 // vpsrlq ymm7, ymm7, 4
+ db $C5, $C5, $DB, $BC, $24, $A0, $00, $00, $00 // vpand ymm7, ymm7, [esp + 160]
+ db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C5, $CD, $DB, $B4, $24, $A0, $00, $00, $00 // vpand ymm6, ymm6, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
+ db $C5, $D5, $DB, $AC, $24, $A0, $00, $00, $00 // vpand ymm5, ymm5, [esp + 160]
+ db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5
sub eax, 64
jnz @@poly1305_avx2_i386_loop
@@poly1305_avx2_i386_tail:
// ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ----
- db $C5, $FD, $F4, $BE, $84, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 132]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $F5, $F4, $7E, $64 // vpmuludq ymm7, ymm1, [esi + 100]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $ED, $F4, $7E, $44 // vpmuludq ymm7, ymm2, [esi + 68]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $E5, $F4, $7E, $24 // vpmuludq ymm7, ymm3, [esi + 36]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $DD, $F4, $7E, $04 // vpmuludq ymm7, ymm4, [esi + 4]
- db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128]
- db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7
- db $C5, $FD, $F4, $7E, $64 // vpmuludq ymm7, ymm0, [esi + 100]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $F5, $F4, $7E, $44 // vpmuludq ymm7, ymm1, [esi + 68]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $ED, $F4, $7E, $24 // vpmuludq ymm7, ymm2, [esi + 36]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $E5, $F4, $7E, $04 // vpmuludq ymm7, ymm3, [esi + 4]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $DD, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 260]
- db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96]
- db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7
- db $C5, $FD, $F4, $7E, $44 // vpmuludq ymm7, ymm0, [esi + 68]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $F5, $F4, $7E, $24 // vpmuludq ymm7, ymm1, [esi + 36]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $ED, $F4, $7E, $04 // vpmuludq ymm7, ymm2, [esi + 4]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $E5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 260]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $DD, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 228]
- db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64]
- db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7
- db $C5, $FD, $F4, $7E, $24 // vpmuludq ymm7, ymm0, [esi + 36]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $F5, $F4, $7E, $04 // vpmuludq ymm7, ymm1, [esi + 4]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $ED, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 260]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $E5, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 228]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $DD, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 196]
- db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32]
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7
- db $C5, $FD, $F4, $7E, $04 // vpmuludq ymm7, ymm0, [esi + 4]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $F5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 260]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $ED, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 228]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $E5, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 196]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $DD, $F4, $BE, $A4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 164]
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
+ db $C5, $FD, $F4, $AE, $84, $00, $00, $00 // vpmuludq ymm5, ymm0, [esi + 132]
+ db $C5, $F5, $F4, $76, $64 // vpmuludq ymm6, ymm1, [esi + 100]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $76, $44 // vpmuludq ymm6, ymm2, [esi + 68]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $76, $24 // vpmuludq ymm6, ymm3, [esi + 36]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $76, $04 // vpmuludq ymm6, ymm4, [esi + 4]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $AC, $24, $80, $00, $00, $00 // vmovdqa [esp + 128], ymm5
+ db $C5, $FD, $F4, $6E, $64 // vpmuludq ymm5, ymm0, [esi + 100]
+ db $C5, $F5, $F4, $76, $44 // vpmuludq ymm6, ymm1, [esi + 68]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $76, $24 // vpmuludq ymm6, ymm2, [esi + 36]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $76, $04 // vpmuludq ymm6, ymm3, [esi + 4]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm4, [esi + 260]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $60 // vmovdqa [esp + 96], ymm5
+ db $C5, $FD, $F4, $6E, $44 // vpmuludq ymm5, ymm0, [esi + 68]
+ db $C5, $F5, $F4, $76, $24 // vpmuludq ymm6, ymm1, [esi + 36]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $76, $04 // vpmuludq ymm6, ymm2, [esi + 4]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm3, [esi + 260]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 228]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $40 // vmovdqa [esp + 64], ymm5
+ db $C5, $FD, $F4, $6E, $24 // vpmuludq ymm5, ymm0, [esi + 36]
+ db $C5, $F5, $F4, $76, $04 // vpmuludq ymm6, ymm1, [esi + 4]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm2, [esi + 260]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 228]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $C4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 196]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $FD, $7F, $6C, $24, $20 // vmovdqa [esp + 32], ymm5
+ db $C5, $FD, $F4, $6E, $04 // vpmuludq ymm5, ymm0, [esi + 4]
+ db $C5, $F5, $F4, $B6, $04, $01, $00, $00 // vpmuludq ymm6, ymm1, [esi + 260]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $ED, $F4, $B6, $E4, $00, $00, $00 // vpmuludq ymm6, ymm2, [esi + 228]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $E5, $F4, $B6, $C4, $00, $00, $00 // vpmuludq ymm6, ymm3, [esi + 196]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
+ db $C5, $DD, $F4, $B6, $A4, $00, $00, $00 // vpmuludq ymm6, ymm4, [esi + 164]
+ db $C5, $D5, $D4, $EE // vpaddq ymm5, ymm5, ymm6
// ---- Tail lazy reduce ----
- db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp]
- db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32]
- db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64]
- db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96]
- db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128]
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
- db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26
- db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5
- db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
- db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26
- db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5
- db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7
- db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26
- db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5
- db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
- db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26
- db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C5, $D5, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm5, [esp + 160]
+ db $C5, $CD, $D4, $4C, $24, $20 // vpaddq ymm1, ymm6, [esp + 32]
+ db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26
+ db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160]
+ db $C5, $CD, $D4, $54, $24, $40 // vpaddq ymm2, ymm6, [esp + 64]
+ db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26
+ db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160]
+ db $C5, $CD, $D4, $5C, $24, $60 // vpaddq ymm3, ymm6, [esp + 96]
+ db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26
+ db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160]
+ db $C5, $CD, $D4, $A4, $24, $80, $00, $00, $00 // vpaddq ymm4, ymm6, [esp + 128]
+ db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26
+ db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160]
+ db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2
+ db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7
+ db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6
+ db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26
+ db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
// ---- Horizontal sum 4 lanes -> lane 0 ----
db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8
db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7
@@ -317,27 +258,26 @@
db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E
db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
// ---- Final lazy reduce (lane 0 scalars) ----
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
- db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26
- db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5
- db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
- db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26
- db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5
- db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7
- db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26
- db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5
- db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
- db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26
- db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5
- db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7
- db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2
- db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp]
- db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7
- db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7
+ db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26
+ db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
+ db $C5, $CD, $73, $D1, $1A // vpsrlq ymm6, ymm1, 26
+ db $C5, $F5, $DB, $8C, $24, $A0, $00, $00, $00 // vpand ymm1, ymm1, [esp + 160]
+ db $C5, $ED, $D4, $D6 // vpaddq ymm2, ymm2, ymm6
+ db $C5, $CD, $73, $D2, $1A // vpsrlq ymm6, ymm2, 26
+ db $C5, $ED, $DB, $94, $24, $A0, $00, $00, $00 // vpand ymm2, ymm2, [esp + 160]
+ db $C5, $E5, $D4, $DE // vpaddq ymm3, ymm3, ymm6
+ db $C5, $CD, $73, $D3, $1A // vpsrlq ymm6, ymm3, 26
+ db $C5, $E5, $DB, $9C, $24, $A0, $00, $00, $00 // vpand ymm3, ymm3, [esp + 160]
+ db $C5, $DD, $D4, $E6 // vpaddq ymm4, ymm4, ymm6
+ db $C5, $CD, $73, $D4, $1A // vpsrlq ymm6, ymm4, 26
+ db $C5, $DD, $DB, $A4, $24, $A0, $00, $00, $00 // vpand ymm4, ymm4, [esp + 160]
+ db $C5, $C5, $73, $F6, $02 // vpsllq ymm7, ymm6, 2
+ db $C5, $CD, $D4, $F7 // vpaddq ymm6, ymm6, ymm7
+ db $C5, $FD, $D4, $C6 // vpaddq ymm0, ymm0, ymm6
+ db $C5, $CD, $73, $D0, $1A // vpsrlq ymm6, ymm0, 26
+ db $C5, $FD, $DB, $84, $24, $A0, $00, $00, $00 // vpand ymm0, ymm0, [esp + 160]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
// ---- Store H0..H4 back to ctx ----
db $C5, $F9, $7E, $43, $24 // vmovd dword ptr [ebx + 36], xmm0
db $C5, $F9, $7E, $4B, $28 // vmovd dword ptr [ebx + 40], xmm1
@@ -345,7 +285,7 @@
db $C5, $F9, $7E, $5B, $30 // vmovd dword ptr [ebx + 48], xmm3
db $C5, $F9, $7E, $63, $34 // vmovd dword ptr [ebx + 52], xmm4
db $C5, $F8, $77 // vzeroupper
- add esp, 320
+ mov esp, dword ptr [esp + 224]
pop edi
pop esi
pop ebx
diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc
index 50df2c65..0d41bd35 100644
--- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc
@@ -17,146 +17,158 @@
// [+32*j+4] ([r^4,r^3,r^2,r^1]).
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
- // ---- Prologue: mask26 (ymm5), padbit (ymm6) ----
- db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5
- db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38
- db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6
- db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63
- db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24
+ // ---- Prologue: mask26 -> ymm15 + [rsp], padbit -> [rsp+32] ----
+ mov r11, rsp
+ sub rsp, 64
+ and rsp, -64
+ db $C4, $41, $05, $76, $FF // vpcmpeqd ymm15, ymm15, ymm15
+ db $C4, $C1, $05, $73, $D7, $26 // vpsrlq ymm15, ymm15, 38
+ db $C5, $7D, $7F, $3C, $24 // vmovdqa [rsp + 0], ymm15
+ db $C5, $C5, $76, $FF // vpcmpeqd ymm7, ymm7, ymm7
+ db $C5, $C5, $73, $D7, $3F // vpsrlq ymm7, ymm7, 63
+ db $C5, $C5, $73, $F7, $18 // vpsllq ymm7, ymm7, 24
+ db $C5, $FD, $7F, $7C, $24, $20 // vmovdqa [rsp + 32], ymm7
// ---- Load H0..H4 into lane 0 ----
db $C5, $F9, $6E, $41, $24 // vmovd xmm0, dword ptr [rcx + 36]
db $C5, $F9, $6E, $49, $28 // vmovd xmm1, dword ptr [rcx + 40]
db $C5, $F9, $6E, $51, $2C // vmovd xmm2, dword ptr [rcx + 44]
db $C5, $F9, $6E, $59, $30 // vmovd xmm3, dword ptr [rcx + 48]
db $C5, $F9, $6E, $61, $34 // vmovd xmm4, dword ptr [rcx + 52]
- // ---- Pre-loop: absorb first 4 blocks ----
- db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8]
- db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16]
- db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1
- db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1
+ // ---- Pre-loop: split first 4 blocks into T; h2 absorbs early ----
+ db $C4, $C1, $7A, $6F, $28 // vmovdqu xmm5, [r8]
+ db $C4, $C3, $55, $38, $68, $20, $01 // vinserti128 ymm5, ymm5, [r8 + 32], 1
+ db $C4, $C1, $7A, $6F, $70, $10 // vmovdqu xmm6, [r8 + 16]
+ db $C4, $C3, $4D, $38, $70, $30, $01 // vinserti128 ymm6, ymm6, [r8 + 48], 1
add r8, 64
- db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6
- db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6
- db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13
- db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13
- db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15
- db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30
- db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4
- db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26
- db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40
- db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5
- db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5
- db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5
- db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5
- db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6
- db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12
- db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13
- db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14
- db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15
- db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
+ db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6
+ db $C5, $BD, $73, $DE, $06 // vpsrldq ymm8, ymm6, 6
+ db $C5, $55, $6D, $CE // vpunpckhqdq ymm9, ymm5, ymm6
+ db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6
+ db $C4, $41, $45, $6C, $C0 // vpunpcklqdq ymm8, ymm7, ymm8
+ db $C4, $C1, $45, $73, $D0, $04 // vpsrlq ymm7, ymm8, 4
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C4, $C1, $3D, $73, $D0, $1E // vpsrlq ymm8, ymm8, 30
+ db $C4, $C1, $35, $73, $D1, $28 // vpsrlq ymm9, ymm9, 40
+ db $C4, $C1, $55, $DB, $EF // vpand ymm5, ymm5, ymm15
+ db $C4, $C1, $4D, $DB, $F7 // vpand ymm6, ymm6, ymm15
+ db $C4, $C1, $45, $DB, $FF // vpand ymm7, ymm7, ymm15
+ db $C4, $41, $3D, $DB, $C7 // vpand ymm8, ymm8, ymm15
+ db $C5, $35, $EB, $4C, $24, $20 // vpor ymm9, ymm9, [rsp + 32]
+ db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
sub r9, 64
jz @@poly1305_avx2_tail
@@poly1305_avx2_loop:
- // ---- Bulk: H *= r^4 (broadcast) ----
- db $C5, $7D, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 128]
- db $C5, $75, $F4, $6A, $60 // vpmuludq ymm13, ymm1, [rdx + 96]
- db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13
- db $C5, $6D, $F4, $6A, $40 // vpmuludq ymm13, ymm2, [rdx + 64]
- db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13
- db $C5, $65, $F4, $6A, $20 // vpmuludq ymm13, ymm3, [rdx + 32]
- db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13
- db $C5, $5D, $F4, $2A // vpmuludq ymm13, ymm4, [rdx]
- db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13
- db $C5, $7D, $F4, $5A, $60 // vpmuludq ymm11, ymm0, [rdx + 96]
- db $C5, $75, $F4, $6A, $40 // vpmuludq ymm13, ymm1, [rdx + 64]
- db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13
- db $C5, $6D, $F4, $6A, $20 // vpmuludq ymm13, ymm2, [rdx + 32]
- db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13
- db $C5, $65, $F4, $2A // vpmuludq ymm13, ymm3, [rdx]
- db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13
- db $C5, $5D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 256]
- db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13
- db $C5, $7D, $F4, $52, $40 // vpmuludq ymm10, ymm0, [rdx + 64]
- db $C5, $75, $F4, $6A, $20 // vpmuludq ymm13, ymm1, [rdx + 32]
- db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13
- db $C5, $6D, $F4, $2A // vpmuludq ymm13, ymm2, [rdx]
- db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13
- db $C5, $65, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 256]
- db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13
- db $C5, $5D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 224]
- db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13
- db $C5, $7D, $F4, $4A, $20 // vpmuludq ymm9, ymm0, [rdx + 32]
- db $C5, $75, $F4, $2A // vpmuludq ymm13, ymm1, [rdx]
- db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13
- db $C5, $6D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 256]
- db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13
- db $C5, $65, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 224]
- db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13
- db $C5, $5D, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 192]
- db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13
- db $C5, $7D, $F4, $02 // vpmuludq ymm8, ymm0, [rdx]
- db $C5, $75, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm1, [rdx + 256]
- db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13
- db $C5, $6D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 224]
- db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13
- db $C5, $65, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 192]
- db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13
- db $C5, $5D, $F4, $AA, $A0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 160]
- db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13
- // ---- Lazy reduce (two interleaved carry chains) ----
- db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26
- db $C4, $C1, $0D, $73, $D3, $1A // vpsrlq ymm14, ymm11, 26
- db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5
- db $C5, $25, $DB, $DD // vpand ymm11, ymm11, ymm5
- db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13
- db $C4, $41, $1D, $D4, $E6 // vpaddq ymm12, ymm12, ymm14
- db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26
- db $C4, $C1, $0D, $73, $D4, $1A // vpsrlq ymm14, ymm12, 26
- db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5
- db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5
- db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13
- db $C4, $C1, $05, $73, $F6, $02 // vpsllq ymm15, ymm14, 2
- db $C4, $41, $0D, $D4, $F7 // vpaddq ymm14, ymm14, ymm15
+ db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5
+ db $C5, $FE, $6F, $2A // vmovdqu ymm5, [rdx]
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
+ db $C5, $FE, $6F, $72, $20 // vmovdqu ymm6, [rdx + 32]
+ db $C4, $C1, $65, $D4, $D8 // vpaddq ymm3, ymm3, ymm8
+ db $C5, $FE, $6F, $7A, $40 // vmovdqu ymm7, [rdx + 64]
+ db $C4, $C1, $5D, $D4, $E1 // vpaddq ymm4, ymm4, ymm9
+ db $C5, $7E, $6F, $82, $E0, $00, $00, $00 // vmovdqu ymm8, [rdx + 224]
+ db $C5, $7E, $6F, $BA, $00, $01, $00, $00 // vmovdqu ymm15, [rdx + 256]
+ db $C5, $6D, $F4, $E5 // vpmuludq ymm12, ymm2, ymm5
+ db $C5, $6D, $F4, $EE // vpmuludq ymm13, ymm2, ymm6
+ db $C5, $6D, $F4, $F7 // vpmuludq ymm14, ymm2, ymm7
+ db $C4, $41, $6D, $F4, $D0 // vpmuludq ymm10, ymm2, ymm8
+ db $C4, $41, $6D, $F4, $DF // vpmuludq ymm11, ymm2, ymm15
+ db $C5, $7D, $F4, $CE // vpmuludq ymm9, ymm0, ymm6
+ db $C5, $F5, $F4, $D6 // vpmuludq ymm2, ymm1, ymm6
+ db $C4, $41, $25, $D4, $D9 // vpaddq ymm11, ymm11, ymm9
+ db $C5, $1D, $D4, $E2 // vpaddq ymm12, ymm12, ymm2
+ db $C5, $65, $F4, $CE // vpmuludq ymm9, ymm3, ymm6
+ db $C5, $DD, $F4, $92, $A0, $00, $00, $00 // vpmuludq ymm2, ymm4, [rdx + 160]
+ db $C4, $41, $0D, $D4, $F1 // vpaddq ymm14, ymm14, ymm9
+ db $C5, $2D, $D4, $D2 // vpaddq ymm10, ymm10, ymm2
+ db $C5, $FE, $6F, $B2, $C0, $00, $00, $00 // vmovdqu ymm6, [rdx + 192]
+ db $C5, $7D, $F4, $CD // vpmuludq ymm9, ymm0, ymm5
+ db $C5, $F5, $F4, $D5 // vpmuludq ymm2, ymm1, ymm5
+ db $C4, $41, $2D, $D4, $D1 // vpaddq ymm10, ymm10, ymm9
+ db $C5, $25, $D4, $DA // vpaddq ymm11, ymm11, ymm2
+ db $C5, $65, $F4, $CD // vpmuludq ymm9, ymm3, ymm5
+ db $C5, $DD, $F4, $D5 // vpmuludq ymm2, ymm4, ymm5
+ db $C4, $C1, $7A, $6F, $28 // vmovdqu xmm5, [r8]
+ db $C4, $41, $15, $D4, $E9 // vpaddq ymm13, ymm13, ymm9
+ db $C5, $0D, $D4, $F2 // vpaddq ymm14, ymm14, ymm2
+ db $C4, $C3, $55, $38, $68, $20, $01 // vinserti128 ymm5, ymm5, [r8 + 32], 1
+ db $C5, $65, $F4, $CE // vpmuludq ymm9, ymm3, ymm6
+ db $C5, $DD, $F4, $D6 // vpmuludq ymm2, ymm4, ymm6
+ db $C4, $C1, $7A, $6F, $70, $10 // vmovdqu xmm6, [r8 + 16]
+ db $C4, $41, $2D, $D4, $D1 // vpaddq ymm10, ymm10, ymm9
+ db $C5, $25, $D4, $DA // vpaddq ymm11, ymm11, ymm2
+ db $C5, $FE, $6F, $52, $60 // vmovdqu ymm2, [rdx + 96]
+ db $C5, $75, $F4, $CF // vpmuludq ymm9, ymm1, ymm7
+ db $C5, $FD, $F4, $FF // vpmuludq ymm7, ymm0, ymm7
+ db $C4, $41, $15, $D4, $E9 // vpaddq ymm13, ymm13, ymm9
+ db $C5, $1D, $D4, $E7 // vpaddq ymm12, ymm12, ymm7
+ db $C4, $C3, $4D, $38, $70, $30, $01 // vinserti128 ymm6, ymm6, [r8 + 48], 1
+ add r8, 64
+ db $C5, $75, $F4, $CA // vpmuludq ymm9, ymm1, ymm2
+ db $C5, $FD, $F4, $D2 // vpmuludq ymm2, ymm0, ymm2
+ db $C5, $C5, $73, $DD, $06 // vpsrldq ymm7, ymm5, 6
+ db $C4, $41, $0D, $D4, $F1 // vpaddq ymm14, ymm14, ymm9
+ db $C5, $15, $D4, $EA // vpaddq ymm13, ymm13, ymm2
+ db $C4, $41, $65, $F4, $C8 // vpmuludq ymm9, ymm3, ymm8
+ db $C4, $C1, $5D, $F4, $D0 // vpmuludq ymm2, ymm4, ymm8
+ db $C5, $BD, $73, $DE, $06 // vpsrldq ymm8, ymm6, 6
+ db $C4, $41, $25, $D4, $D9 // vpaddq ymm11, ymm11, ymm9
+ db $C5, $1D, $D4, $E2 // vpaddq ymm12, ymm12, ymm2
+ db $C5, $55, $6D, $CE // vpunpckhqdq ymm9, ymm5, ymm6
+ db $C4, $C1, $65, $F4, $DF // vpmuludq ymm3, ymm3, ymm15
+ db $C4, $C1, $5D, $F4, $E7 // vpmuludq ymm4, ymm4, ymm15
+ db $C5, $D5, $6C, $EE // vpunpcklqdq ymm5, ymm5, ymm6
+ db $C5, $9D, $D4, $D3 // vpaddq ymm2, ymm12, ymm3
+ db $C5, $95, $D4, $DC // vpaddq ymm3, ymm13, ymm4
+ db $C4, $41, $45, $6C, $C0 // vpunpcklqdq ymm8, ymm7, ymm8
+ db $C5, $FD, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm4, ymm0, [rdx + 128]
+ db $C4, $C1, $75, $F4, $C7 // vpmuludq ymm0, ymm1, ymm15
+ db $C5, $7D, $6F, $3C, $24 // vmovdqa ymm15, [rsp + 0]
+ db $C4, $C1, $5D, $D4, $E6 // vpaddq ymm4, ymm4, ymm14
+ db $C4, $C1, $7D, $D4, $C2 // vpaddq ymm0, ymm0, ymm10
+ db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26
+ db $C4, $C1, $65, $DB, $DF // vpand ymm3, ymm3, ymm15
+ db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13
+ db $C5, $AD, $73, $D0, $1A // vpsrlq ymm10, ymm0, 26
+ db $C4, $C1, $7D, $DB, $C7 // vpand ymm0, ymm0, ymm15
+ db $C4, $C1, $25, $D4, $CA // vpaddq ymm1, ymm11, ymm10
+ db $C5, $8D, $73, $D4, $1A // vpsrlq ymm14, ymm4, 26
+ db $C4, $C1, $5D, $DB, $E7 // vpand ymm4, ymm4, ymm15
+ db $C4, $C1, $45, $73, $D0, $04 // vpsrlq ymm7, ymm8, 4
+ db $C5, $A5, $73, $D1, $1A // vpsrlq ymm11, ymm1, 26
+ db $C4, $C1, $75, $DB, $CF // vpand ymm1, ymm1, ymm15
+ db $C4, $C1, $6D, $D4, $D3 // vpaddq ymm2, ymm2, ymm11
db $C4, $C1, $7D, $D4, $C6 // vpaddq ymm0, ymm0, ymm14
- db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26
- db $C5, $8D, $73, $D0, $1A // vpsrlq ymm14, ymm0, 26
- db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5
- db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5
- db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13
- db $C4, $C1, $75, $D4, $CE // vpaddq ymm1, ymm1, ymm14
- db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26
- db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5
+ db $C4, $C1, $0D, $73, $F6, $02 // vpsllq ymm14, ymm14, 2
+ db $C4, $C1, $7D, $D4, $C6 // vpaddq ymm0, ymm0, ymm14
+ db $C4, $C1, $45, $DB, $FF // vpand ymm7, ymm7, ymm15
+ db $C5, $CD, $73, $D5, $1A // vpsrlq ymm6, ymm5, 26
+ db $C5, $9D, $73, $D2, $1A // vpsrlq ymm12, ymm2, 26
+ db $C4, $C1, $6D, $DB, $D7 // vpand ymm2, ymm2, ymm15
+ db $C4, $C1, $65, $D4, $DC // vpaddq ymm3, ymm3, ymm12
+ db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7
+ db $C4, $C1, $3D, $73, $D0, $1E // vpsrlq ymm8, ymm8, 30
+ db $C5, $AD, $73, $D0, $1A // vpsrlq ymm10, ymm0, 26
+ db $C4, $C1, $7D, $DB, $C7 // vpand ymm0, ymm0, ymm15
+ db $C4, $C1, $75, $D4, $CA // vpaddq ymm1, ymm1, ymm10
+ db $C4, $C1, $35, $73, $D1, $28 // vpsrlq ymm9, ymm9, 40
+ db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26
+ db $C4, $C1, $65, $DB, $DF // vpand ymm3, ymm3, ymm15
db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13
- // ---- Absorb next 4 blocks ----
- db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8]
- db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16]
- db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1
- db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1
- add r8, 64
- db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6
- db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6
- db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13
- db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13
- db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15
- db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30
- db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4
- db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26
- db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40
- db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5
- db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5
- db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5
- db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5
- db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6
- db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12
- db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13
- db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14
- db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15
- db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7
+ db $C4, $C1, $55, $DB, $EF // vpand ymm5, ymm5, ymm15
+ db $C4, $C1, $4D, $DB, $F7 // vpand ymm6, ymm6, ymm15
+ db $C4, $41, $3D, $DB, $C7 // vpand ymm8, ymm8, ymm15
+ db $C5, $35, $EB, $4C, $24, $20 // vpor ymm9, ymm9, [rsp + 32]
sub r9, 64
jnz @@poly1305_avx2_loop
@@poly1305_avx2_tail:
+ // ---- Absorb the final pending limbs (T2 already folded) ----
+ db $C5, $FD, $D4, $C5 // vpaddq ymm0, ymm0, ymm5
+ db $C5, $F5, $D4, $CE // vpaddq ymm1, ymm1, ymm6
+ db $C4, $C1, $65, $D4, $D8 // vpaddq ymm3, ymm3, ymm8
+ db $C4, $C1, $5D, $D4, $E1 // vpaddq ymm4, ymm4, ymm9
+ db $C5, $FD, $6F, $2C, $24 // vmovdqa ymm5, [rsp + 0]
// ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ----
db $C5, $7D, $F4, $A2, $84, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 132]
db $C5, $75, $F4, $6A, $64 // vpmuludq ymm13, ymm1, [rdx + 100]
@@ -273,4 +285,5 @@
db $C5, $F9, $7E, $59, $30 // vmovd dword ptr [rcx + 48], xmm3
db $C5, $F9, $7E, $61, $34 // vmovd dword ptr [rcx + 52], xmm4
db $C5, $F8, $77 // vzeroupper
+ mov rsp, r11
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}
diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc
index 7c288564..adf29b51 100644
--- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc
+++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc
@@ -5,11 +5,11 @@
// 2 blocks per iteration against an r/r^2 power table, then a single
// pshufd-based horizontal fold of the 2 lanes.
//
-// i386 has only xmm0..7 (no callee-saved xmm), so the accumulator
-// H0..H4 and the transient products D0..D4 live in a 160-byte esp
-// stack frame; xmm7 = mask26 and xmm6 = padbit stay resident. Every
-// table/state access is an unaligned movdqu load/store (SSE memory-
-// operand arithmetic would #GP on the non-16-aligned power table).
+// i386 has only xmm0..7 (no callee-saved xmm): H0..H4 stay RESIDENT
+// in xmm0..4, xmm5..7 are the accumulator/product temps, and the
+// 64B-aligned esp frame holds a one-time copy of the bulk table rows
+// plus mask26/padbit, making SSE memory-operand arithmetic against
+// [esp+off] legal and every hot access aligned.
//
// On entry (after ClpSimdProc5Begin_i386.inc):
// ebx = ACtx (72-byte R/S/H/K state); esi = APowTable (160-byte r/r^2)
@@ -19,490 +19,388 @@
// rows 5..8 s-limbs (5*r), row 9 pad. Bulk reads [+16*j] (broadcast r^2);
// tail reads [+16*j+4] ([r^2,r^1]).
- sub esp, 160
- // ---- Prologue: mask26 (xmm7), padbit (xmm6) ----
- db $66, $0F, $76, $FF // pcmpeqd xmm7, xmm7
- db $66, $0F, $73, $D7, $26 // psrlq xmm7, 38
- db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6
- db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63
- db $66, $0F, $73, $F6, $18 // psllq xmm6, 24
- // ---- Load H0..H4 (lane 0) into the stack frame ----
+ // ---- 64B-aligned frame: table copy + D slots + mask/pad ----
+ mov edx, esp
+ sub esp, 256
+ and esp, -64
+ mov dword ptr [esp + 240], edx
+ db $F3, $0F, $6F, $2E // movdqu xmm5, [esi]
+ db $66, $0F, $7F, $2C, $24 // movdqa [esp + 0], xmm5
+ db $F3, $0F, $6F, $6E, $10 // movdqu xmm5, [esi + 16]
+ db $66, $0F, $7F, $6C, $24, $10 // movdqa [esp + 16], xmm5
+ db $F3, $0F, $6F, $6E, $20 // movdqu xmm5, [esi + 32]
+ db $66, $0F, $7F, $6C, $24, $20 // movdqa [esp + 32], xmm5
+ db $F3, $0F, $6F, $6E, $30 // movdqu xmm5, [esi + 48]
+ db $66, $0F, $7F, $6C, $24, $30 // movdqa [esp + 48], xmm5
+ db $F3, $0F, $6F, $6E, $40 // movdqu xmm5, [esi + 64]
+ db $66, $0F, $7F, $6C, $24, $40 // movdqa [esp + 64], xmm5
+ db $F3, $0F, $6F, $6E, $50 // movdqu xmm5, [esi + 80]
+ db $66, $0F, $7F, $6C, $24, $50 // movdqa [esp + 80], xmm5
+ db $F3, $0F, $6F, $6E, $60 // movdqu xmm5, [esi + 96]
+ db $66, $0F, $7F, $6C, $24, $60 // movdqa [esp + 96], xmm5
+ db $F3, $0F, $6F, $6E, $70 // movdqu xmm5, [esi + 112]
+ db $66, $0F, $7F, $6C, $24, $70 // movdqa [esp + 112], xmm5
+ db $F3, $0F, $6F, $AE, $80, $00, $00, $00 // movdqu xmm5, [esi + 128]
+ db $66, $0F, $7F, $AC, $24, $80, $00, $00, $00 // movdqa [esp + 128], xmm5
+ db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5
+ db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38
+ db $66, $0F, $7F, $AC, $24, $D0, $00, $00, $00 // movdqa [esp + 208], xmm5
+ db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5
+ db $66, $0F, $73, $D5, $3F // psrlq xmm5, 63
+ db $66, $0F, $73, $F5, $18 // psllq xmm5, 24
+ db $66, $0F, $7F, $AC, $24, $E0, $00, $00, $00 // movdqa [esp + 224], xmm5
+ // ---- Load H0..H4 into lane 0 (resident) ----
db $66, $0F, $6E, $43, $24 // movd xmm0, dword ptr [ebx + 36]
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $66, $0F, $6E, $43, $28 // movd xmm0, dword ptr [ebx + 40]
- db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0
- db $66, $0F, $6E, $43, $2C // movd xmm0, dword ptr [ebx + 44]
- db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0
- db $66, $0F, $6E, $43, $30 // movd xmm0, dword ptr [ebx + 48]
- db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0
- db $66, $0F, $6E, $43, $34 // movd xmm0, dword ptr [ebx + 52]
- db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0
+ db $66, $0F, $6E, $4B, $28 // movd xmm1, dword ptr [ebx + 40]
+ db $66, $0F, $6E, $53, $2C // movd xmm2, dword ptr [ebx + 44]
+ db $66, $0F, $6E, $5B, $30 // movd xmm3, dword ptr [ebx + 48]
+ db $66, $0F, $6E, $63, $34 // movd xmm4, dword ptr [ebx + 52]
// ---- Pre-loop: absorb first 2 blocks ----
- db $F3, $0F, $6F, $07 // movdqu xmm0, [edi]
- db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16]
+ db $F3, $0F, $6F, $2F // movdqu xmm5, [edi]
+ db $F3, $0F, $6F, $77, $10 // movdqu xmm6, [edi + 16]
add edi, 32
- db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0
- db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6
- db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1
- db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6
- db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0
- db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1
- db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1
- db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3
- db $66, $0F, $6F, $DA // movdqa xmm3, xmm2
- db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30
- db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $66, $0F, $DB, $CF // pand xmm1, xmm7
- db $66, $0F, $DB, $D7 // pand xmm2, xmm7
- db $66, $0F, $DB, $DF // pand xmm3, xmm7
- db $66, $0F, $EB, $E6 // por xmm4, xmm6
- db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp]
- db $66, $0F, $D4, $E8 // paddq xmm5, xmm0
- db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5
- db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16]
- db $66, $0F, $D4, $E9 // paddq xmm5, xmm1
- db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5
- db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32]
- db $66, $0F, $D4, $EA // paddq xmm5, xmm2
- db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5
- db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48]
- db $66, $0F, $D4, $EB // paddq xmm5, xmm3
- db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5
- db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64]
- db $66, $0F, $D4, $EC // paddq xmm5, xmm4
- db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5
+ db $66, $0F, $6F, $FD // movdqa xmm7, xmm5
+ db $66, $0F, $6D, $FE // punpckhqdq xmm7, xmm6
+ db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
+ db $66, $0F, $EB, $BC, $24, $E0, $00, $00, $00 // por xmm7, [esp + 224]
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $0F, $6F, $FD // movdqa xmm7, xmm5
+ db $66, $0F, $73, $DF, $06 // psrldq xmm7, 6
+ db $66, $0F, $6C, $EE // punpcklqdq xmm5, xmm6
+ db $66, $0F, $73, $DE, $06 // psrldq xmm6, 6
+ db $66, $0F, $6C, $FE // punpcklqdq xmm7, xmm6
+ db $66, $0F, $6F, $F7 // movdqa xmm6, xmm7
+ db $66, $0F, $73, $D6, $1E // psrlq xmm6, 30
+ db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208]
+ db $66, $0F, $D4, $DE // paddq xmm3, xmm6
+ db $66, $0F, $73, $D7, $04 // psrlq xmm7, 4
+ db $66, $0F, $DB, $BC, $24, $D0, $00, $00, $00 // pand xmm7, [esp + 208]
+ db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
+ db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
+ db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208]
+ db $66, $0F, $D4, $C5 // paddq xmm0, xmm5
sub eax, 32
jz @@poly1305_sse2_tail
@@poly1305_sse2_loop:
- // ---- Bulk: H *= r^2 (broadcast) ----
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $40 // movdqu xmm1, [esi + 64]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $30 // movdqu xmm2, [esi + 48]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $16 // movdqu xmm2, [esi]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $30 // movdqu xmm1, [esi + 48]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $16 // movdqu xmm2, [esi]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $20 // movdqu xmm1, [esi + 32]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $16 // movdqu xmm2, [esi]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $10 // movdqu xmm1, [esi + 16]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $16 // movdqu xmm2, [esi]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $0E // movdqu xmm1, [esi]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $50 // movdqu xmm2, [esi + 80]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0
- // ---- Lazy reduce D -> H ----
- db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2
- db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0
- db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2
- db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0
- db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2
- db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0
- db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2
- db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0
- db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1
- db $66, $0F, $73, $F2, $02 // psllq xmm2, 2
- db $66, $0F, $D4, $CA // paddq xmm1, xmm2
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2
+ // ---- Bulk: H *= r^2 (broadcast), k-major resident accumulator ----
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $66, $0F, $F4, $6C, $24, $40 // pmuludq xmm5, [esp + 64]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $F4, $74, $24, $30 // pmuludq xmm6, [esp + 48]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $F4, $74, $24, $20 // pmuludq xmm6, [esp + 32]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $C0, $00, $00, $00 // movdqa [esp + 192], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $66, $0F, $F4, $6C, $24, $30 // pmuludq xmm5, [esp + 48]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $F4, $74, $24, $20 // pmuludq xmm6, [esp + 32]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $B0, $00, $00, $00 // movdqa [esp + 176], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $66, $0F, $F4, $6C, $24, $20 // pmuludq xmm5, [esp + 32]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $F4, $74, $24, $10 // pmuludq xmm6, [esp + 16]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $A0, $00, $00, $00 // movdqa [esp + 160], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $66, $0F, $F4, $6C, $24, $10 // pmuludq xmm5, [esp + 16]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $F4, $34, $24 // pmuludq xmm6, [esp + 0]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $F4, $74, $24, $60 // pmuludq xmm6, [esp + 96]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $90, $00, $00, $00 // movdqa [esp + 144], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $66, $0F, $F4, $2C, $24 // pmuludq xmm5, [esp + 0]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $F4, $B4, $24, $80, $00, $00, $00 // pmuludq xmm6, [esp + 128]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $F4, $74, $24, $70 // pmuludq xmm6, [esp + 112]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $F4, $74, $24, $60 // pmuludq xmm6, [esp + 96]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $F4, $74, $24, $50 // pmuludq xmm6, [esp + 80]
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ // ---- Lazy reduce (D0 from xmm5) ----
+ db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208]
+ db $66, $0F, $6F, $C5 // movdqa xmm0, xmm5
+ db $66, $0F, $6F, $CE // movdqa xmm1, xmm6
+ db $66, $0F, $D4, $8C, $24, $90, $00, $00, $00 // paddq xmm1, [esp + 144]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208]
+ db $66, $0F, $6F, $D6 // movdqa xmm2, xmm6
+ db $66, $0F, $D4, $94, $24, $A0, $00, $00, $00 // paddq xmm2, [esp + 160]
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208]
+ db $66, $0F, $6F, $DE // movdqa xmm3, xmm6
+ db $66, $0F, $D4, $9C, $24, $B0, $00, $00, $00 // paddq xmm3, [esp + 176]
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208]
+ db $66, $0F, $6F, $E6 // movdqa xmm4, xmm6
+ db $66, $0F, $D4, $A4, $24, $C0, $00, $00, $00 // paddq xmm4, [esp + 192]
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208]
+ db $66, $0F, $6F, $FE // movdqa xmm7, xmm6
+ db $66, $0F, $73, $F7, $02 // psllq xmm7, 2
+ db $66, $0F, $D4, $F7 // paddq xmm6, xmm7
+ db $66, $0F, $D4, $C6 // paddq xmm0, xmm6
+ db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
// ---- Absorb next 2 blocks ----
- db $F3, $0F, $6F, $07 // movdqu xmm0, [edi]
- db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16]
+ db $F3, $0F, $6F, $2F // movdqu xmm5, [edi]
+ db $F3, $0F, $6F, $77, $10 // movdqu xmm6, [edi + 16]
add edi, 32
- db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0
- db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6
- db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1
- db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6
- db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0
- db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1
- db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1
- db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3
- db $66, $0F, $6F, $DA // movdqa xmm3, xmm2
- db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30
- db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $66, $0F, $DB, $CF // pand xmm1, xmm7
- db $66, $0F, $DB, $D7 // pand xmm2, xmm7
- db $66, $0F, $DB, $DF // pand xmm3, xmm7
- db $66, $0F, $EB, $E6 // por xmm4, xmm6
- db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp]
- db $66, $0F, $D4, $E8 // paddq xmm5, xmm0
- db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5
- db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16]
- db $66, $0F, $D4, $E9 // paddq xmm5, xmm1
- db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5
- db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32]
- db $66, $0F, $D4, $EA // paddq xmm5, xmm2
- db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5
- db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48]
- db $66, $0F, $D4, $EB // paddq xmm5, xmm3
- db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5
- db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64]
- db $66, $0F, $D4, $EC // paddq xmm5, xmm4
- db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5
+ db $66, $0F, $6F, $FD // movdqa xmm7, xmm5
+ db $66, $0F, $6D, $FE // punpckhqdq xmm7, xmm6
+ db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
+ db $66, $0F, $EB, $BC, $24, $E0, $00, $00, $00 // por xmm7, [esp + 224]
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $0F, $6F, $FD // movdqa xmm7, xmm5
+ db $66, $0F, $73, $DF, $06 // psrldq xmm7, 6
+ db $66, $0F, $6C, $EE // punpcklqdq xmm5, xmm6
+ db $66, $0F, $73, $DE, $06 // psrldq xmm6, 6
+ db $66, $0F, $6C, $FE // punpcklqdq xmm7, xmm6
+ db $66, $0F, $6F, $F7 // movdqa xmm6, xmm7
+ db $66, $0F, $73, $D6, $1E // psrlq xmm6, 30
+ db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208]
+ db $66, $0F, $D4, $DE // paddq xmm3, xmm6
+ db $66, $0F, $73, $D7, $04 // psrlq xmm7, 4
+ db $66, $0F, $DB, $BC, $24, $D0, $00, $00, $00 // pand xmm7, [esp + 208]
+ db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
+ db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $B4, $24, $D0, $00, $00, $00 // pand xmm6, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
+ db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208]
+ db $66, $0F, $D4, $C5 // paddq xmm0, xmm5
sub eax, 32
jnz @@poly1305_sse2_loop
@@poly1305_sse2_tail:
// ---- Tail: H *= [r^2, r^1] (shifted table read) ----
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $44 // movdqu xmm1, [esi + 68]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $34 // movdqu xmm2, [esi + 52]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $34 // movdqu xmm1, [esi + 52]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $24 // movdqu xmm1, [esi + 36]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $14 // movdqu xmm1, [esi + 20]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $F3, $0F, $6F, $4E, $04 // movdqu xmm1, [esi + 4]
- db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16]
- db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32]
- db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48]
- db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64]
- db $F3, $0F, $6F, $56, $54 // movdqu xmm2, [esi + 84]
- db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0
- db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2
- db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0
- db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2
- db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0
- db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2
- db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0
- db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2
- db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0
- db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1
- db $66, $0F, $73, $F2, $02 // psllq xmm2, 2
- db $66, $0F, $D4, $CA // paddq xmm1, xmm2
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $F3, $0F, $6F, $7E, $44 // movdqu xmm7, [esi + 68]
+ db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $F3, $0F, $6F, $7E, $34 // movdqu xmm7, [esi + 52]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $C0, $00, $00, $00 // movdqa [esp + 192], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $F3, $0F, $6F, $7E, $34 // movdqu xmm7, [esi + 52]
+ db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $B0, $00, $00, $00 // movdqa [esp + 176], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $F3, $0F, $6F, $7E, $24 // movdqu xmm7, [esi + 36]
+ db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $A0, $00, $00, $00 // movdqa [esp + 160], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $F3, $0F, $6F, $7E, $14 // movdqu xmm7, [esi + 20]
+ db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $F3, $0F, $6F, $7E, $64 // movdqu xmm7, [esi + 100]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $7F, $AC, $24, $90, $00, $00, $00 // movdqa [esp + 144], xmm5
+ db $66, $0F, $6F, $E8 // movdqa xmm5, xmm0
+ db $F3, $0F, $6F, $7E, $04 // movdqu xmm7, [esi + 4]
+ db $66, $0F, $F4, $EF // pmuludq xmm5, xmm7
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $F3, $0F, $6F, $BE, $84, $00, $00, $00 // movdqu xmm7, [esi + 132]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $F3, $0F, $6F, $7E, $74 // movdqu xmm7, [esi + 116]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $F3, $0F, $6F, $7E, $64 // movdqu xmm7, [esi + 100]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $F3, $0F, $6F, $7E, $54 // movdqu xmm7, [esi + 84]
+ db $66, $0F, $F4, $F7 // pmuludq xmm6, xmm7
+ db $66, $0F, $D4, $EE // paddq xmm5, xmm6
+ db $66, $0F, $6F, $F5 // movdqa xmm6, xmm5
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $AC, $24, $D0, $00, $00, $00 // pand xmm5, [esp + 208]
+ db $66, $0F, $6F, $C5 // movdqa xmm0, xmm5
+ db $66, $0F, $6F, $CE // movdqa xmm1, xmm6
+ db $66, $0F, $D4, $8C, $24, $90, $00, $00, $00 // paddq xmm1, [esp + 144]
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208]
+ db $66, $0F, $6F, $D6 // movdqa xmm2, xmm6
+ db $66, $0F, $D4, $94, $24, $A0, $00, $00, $00 // paddq xmm2, [esp + 160]
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208]
+ db $66, $0F, $6F, $DE // movdqa xmm3, xmm6
+ db $66, $0F, $D4, $9C, $24, $B0, $00, $00, $00 // paddq xmm3, [esp + 176]
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208]
+ db $66, $0F, $6F, $E6 // movdqa xmm4, xmm6
+ db $66, $0F, $D4, $A4, $24, $C0, $00, $00, $00 // paddq xmm4, [esp + 192]
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208]
+ db $66, $0F, $6F, $FE // movdqa xmm7, xmm6
+ db $66, $0F, $73, $F7, $02 // psllq xmm7, 2
+ db $66, $0F, $D4, $F7 // paddq xmm6, xmm7
+ db $66, $0F, $D4, $C6 // paddq xmm0, xmm6
+ db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
// ---- Horizontal fold 2 lanes -> 1, final reduce, store ----
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16]
- db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0
- db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32]
- db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0
- db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48]
- db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0
- db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64]
- db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2
- db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0
- db $F3, $0F, $6F, $54, $24, $20 // movdqu xmm2, [esp + 32]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $20 // movdqu [esp + 32], xmm2
- db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0
- db $F3, $0F, $6F, $54, $24, $30 // movdqu xmm2, [esp + 48]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $30 // movdqu [esp + 48], xmm2
- db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0
- db $F3, $0F, $6F, $54, $24, $40 // movdqu xmm2, [esp + 64]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $40 // movdqu [esp + 64], xmm2
- db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64]
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0
- db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1
- db $66, $0F, $73, $F2, $02 // psllq xmm2, 2
- db $66, $0F, $D4, $CA // paddq xmm1, xmm2
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
- db $66, $0F, $D4, $C1 // paddq xmm0, xmm1
- db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0
- db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26
- db $66, $0F, $DB, $C7 // pand xmm0, xmm7
- db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0
- db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16]
- db $66, $0F, $D4, $D1 // paddq xmm2, xmm1
- db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2
- db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp]
+ db $66, $0F, $70, $F0, $4E // pshufd xmm6, xmm0, 0x4E
+ db $66, $0F, $D4, $C6 // paddq xmm0, xmm6
+ db $66, $0F, $70, $F1, $4E // pshufd xmm6, xmm1, 0x4E
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
+ db $66, $0F, $70, $F2, $4E // pshufd xmm6, xmm2, 0x4E
+ db $66, $0F, $D4, $D6 // paddq xmm2, xmm6
+ db $66, $0F, $70, $F3, $4E // pshufd xmm6, xmm3, 0x4E
+ db $66, $0F, $D4, $DE // paddq xmm3, xmm6
+ db $66, $0F, $70, $F4, $4E // pshufd xmm6, xmm4, 0x4E
+ db $66, $0F, $D4, $E6 // paddq xmm4, xmm6
+ db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
+ db $66, $0F, $6F, $F1 // movdqa xmm6, xmm1
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $8C, $24, $D0, $00, $00, $00 // pand xmm1, [esp + 208]
+ db $66, $0F, $D4, $D6 // paddq xmm2, xmm6
+ db $66, $0F, $6F, $F2 // movdqa xmm6, xmm2
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $94, $24, $D0, $00, $00, $00 // pand xmm2, [esp + 208]
+ db $66, $0F, $D4, $DE // paddq xmm3, xmm6
+ db $66, $0F, $6F, $F3 // movdqa xmm6, xmm3
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $9C, $24, $D0, $00, $00, $00 // pand xmm3, [esp + 208]
+ db $66, $0F, $D4, $E6 // paddq xmm4, xmm6
+ db $66, $0F, $6F, $F4 // movdqa xmm6, xmm4
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $A4, $24, $D0, $00, $00, $00 // pand xmm4, [esp + 208]
+ db $66, $0F, $6F, $FE // movdqa xmm7, xmm6
+ db $66, $0F, $73, $F7, $02 // psllq xmm7, 2
+ db $66, $0F, $D4, $F7 // paddq xmm6, xmm7
+ db $66, $0F, $D4, $C6 // paddq xmm0, xmm6
+ db $66, $0F, $6F, $F0 // movdqa xmm6, xmm0
+ db $66, $0F, $73, $D6, $1A // psrlq xmm6, 26
+ db $66, $0F, $DB, $84, $24, $D0, $00, $00, $00 // pand xmm0, [esp + 208]
+ db $66, $0F, $D4, $CE // paddq xmm1, xmm6
db $66, $0F, $7E, $43, $24 // movd dword ptr [ebx + 36], xmm0
- db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16]
- db $66, $0F, $7E, $43, $28 // movd dword ptr [ebx + 40], xmm0
- db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32]
- db $66, $0F, $7E, $43, $2C // movd dword ptr [ebx + 44], xmm0
- db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48]
- db $66, $0F, $7E, $43, $30 // movd dword ptr [ebx + 48], xmm0
- db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64]
- db $66, $0F, $7E, $43, $34 // movd dword ptr [ebx + 52], xmm0
- add esp, 160
+ db $66, $0F, $7E, $4B, $28 // movd dword ptr [ebx + 40], xmm1
+ db $66, $0F, $7E, $53, $2C // movd dword ptr [ebx + 44], xmm2
+ db $66, $0F, $7E, $5B, $30 // movd dword ptr [ebx + 48], xmm3
+ db $66, $0F, $7E, $63, $34 // movd dword ptr [ebx + 52], xmm4
+ mov esp, dword ptr [esp + 240]
pop edi
pop esi
pop ebx
diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc
index e8172d0b..ebb4aae0 100644
--- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc
+++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc
@@ -15,380 +15,378 @@
// tail reads [+16*j+4] ([r^2,r^1]).
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc}
+ // ---- 64B-aligned frame: bulk table rows copied in once ----
+ mov r11, rsp
+ sub rsp, 192
+ and rsp, -64
+ db $F3, $44, $0F, $6F, $22 // movdqu xmm12, [rdx]
+ db $66, $44, $0F, $7F, $24, $24 // movdqa [rsp + 0], xmm12
+ db $F3, $44, $0F, $6F, $62, $10 // movdqu xmm12, [rdx + 16]
+ db $66, $44, $0F, $7F, $64, $24, $10 // movdqa [rsp + 16], xmm12
+ db $F3, $44, $0F, $6F, $62, $20 // movdqu xmm12, [rdx + 32]
+ db $66, $44, $0F, $7F, $64, $24, $20 // movdqa [rsp + 32], xmm12
+ db $F3, $44, $0F, $6F, $62, $30 // movdqu xmm12, [rdx + 48]
+ db $66, $44, $0F, $7F, $64, $24, $30 // movdqa [rsp + 48], xmm12
+ db $F3, $44, $0F, $6F, $62, $40 // movdqu xmm12, [rdx + 64]
+ db $66, $44, $0F, $7F, $64, $24, $40 // movdqa [rsp + 64], xmm12
+ db $F3, $44, $0F, $6F, $62, $50 // movdqu xmm12, [rdx + 80]
+ db $66, $44, $0F, $7F, $64, $24, $50 // movdqa [rsp + 80], xmm12
+ db $F3, $44, $0F, $6F, $62, $60 // movdqu xmm12, [rdx + 96]
+ db $66, $44, $0F, $7F, $64, $24, $60 // movdqa [rsp + 96], xmm12
+ db $F3, $44, $0F, $6F, $62, $70 // movdqu xmm12, [rdx + 112]
+ db $66, $44, $0F, $7F, $64, $24, $70 // movdqa [rsp + 112], xmm12
+ db $F3, $44, $0F, $6F, $A2, $80, $00, $00, $00 // movdqu xmm12, [rdx + 128]
+ db $66, $44, $0F, $7F, $A4, $24, $80, $00, $00, $00 // movdqa [rsp + 128], xmm12
// ---- Prologue: mask26 (xmm5), padbit (xmm6) ----
- db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5
- db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38
- db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6
- db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63
- db $66, $0F, $73, $F6, $18 // psllq xmm6, 24
+ db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5
+ db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38
+ db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6
+ db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63
+ db $66, $0F, $73, $F6, $18 // psllq xmm6, 24
// ---- Load H0..H4 into lane 0 ----
- db $66, $0F, $6E, $41, $24 // movd xmm0, dword ptr [rcx + 36]
- db $66, $0F, $6E, $49, $28 // movd xmm1, dword ptr [rcx + 40]
- db $66, $0F, $6E, $51, $2C // movd xmm2, dword ptr [rcx + 44]
- db $66, $0F, $6E, $59, $30 // movd xmm3, dword ptr [rcx + 48]
- db $66, $0F, $6E, $61, $34 // movd xmm4, dword ptr [rcx + 52]
+ db $66, $0F, $6E, $41, $24 // movd xmm0, dword ptr [rcx + 36]
+ db $66, $0F, $6E, $49, $28 // movd xmm1, dword ptr [rcx + 40]
+ db $66, $0F, $6E, $51, $2C // movd xmm2, dword ptr [rcx + 44]
+ db $66, $0F, $6E, $59, $30 // movd xmm3, dword ptr [rcx + 48]
+ db $66, $0F, $6E, $61, $34 // movd xmm4, dword ptr [rcx + 52]
// ---- Pre-loop: absorb first 2 blocks ----
- db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8]
- db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16]
+ db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8]
+ db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16]
add r8, 32
- db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12
- db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6
- db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13
- db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6
- db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12
- db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13
- db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13
- db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15
- db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14
- db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30
- db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4
- db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
- db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5
- db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5
- db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5
- db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5
- db $66, $0F, $EB, $FE // por xmm7, xmm6
- db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12
- db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
- db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14
- db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15
- db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12
+ db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6
+ db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13
+ db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6
+ db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12
+ db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13
+ db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13
+ db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15
+ db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14
+ db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30
+ db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4
+ db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
+ db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5
+ db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5
+ db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5
+ db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5
+ db $66, $0F, $EB, $FE // por xmm7, xmm6
+ db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12
+ db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
+ db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14
+ db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
sub r9, 32
jz @@poly1305_sse2_tail
@@poly1305_sse2_loop:
// ---- Bulk: H *= r^2 (broadcast) ----
- db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0
- db $F3, $44, $0F, $6F, $6A, $40 // movdqu xmm13, [rdx + 64]
- db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
- db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48]
- db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0
- db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32]
- db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0
- db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16]
- db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
- db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx]
- db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $50 // movdqu xmm13, [rdx + 80]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0
+ db $66, $44, $0F, $F4, $5C, $24, $40 // pmuludq xmm11, [rsp + 64]
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $66, $44, $0F, $F4, $64, $24, $30 // pmuludq xmm12, [rsp + 48]
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $66, $44, $0F, $F4, $64, $24, $20 // pmuludq xmm12, [rsp + 32]
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16]
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0]
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
+ db $66, $44, $0F, $F4, $54, $24, $30 // pmuludq xmm10, [rsp + 48]
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $66, $44, $0F, $F4, $64, $24, $20 // pmuludq xmm12, [rsp + 32]
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16]
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0]
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128]
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0
+ db $66, $44, $0F, $F4, $4C, $24, $20 // pmuludq xmm9, [rsp + 32]
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $66, $44, $0F, $F4, $64, $24, $10 // pmuludq xmm12, [rsp + 16]
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0]
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128]
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112]
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0
+ db $66, $44, $0F, $F4, $44, $24, $10 // pmuludq xmm8, [rsp + 16]
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $66, $44, $0F, $F4, $24, $24 // pmuludq xmm12, [rsp + 0]
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128]
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112]
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $66, $44, $0F, $F4, $64, $24, $60 // pmuludq xmm12, [rsp + 96]
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
+ db $66, $0F, $F4, $3C, $24 // pmuludq xmm7, [rsp + 0]
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $66, $44, $0F, $F4, $A4, $24, $80, $00, $00, $00 // pmuludq xmm12, [rsp + 128]
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $66, $44, $0F, $F4, $64, $24, $70 // pmuludq xmm12, [rsp + 112]
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $66, $44, $0F, $F4, $64, $24, $60 // pmuludq xmm12, [rsp + 96]
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $66, $44, $0F, $F4, $64, $24, $50 // pmuludq xmm12, [rsp + 80]
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
// ---- Lazy reduce D -> H ----
- db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $DB, $FD // pand xmm7, xmm5
- db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7
- db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13
- db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5
- db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8
- db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13
- db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5
- db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9
- db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13
- db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5
- db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
- db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13
- db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5
- db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11
- db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13
- db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
- db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12
- db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13
- db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $DB, $C5 // pand xmm0, xmm5
- db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
+ db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $DB, $FD // pand xmm7, xmm5
+ db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7
+ db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13
+ db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5
+ db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8
+ db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13
+ db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5
+ db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9
+ db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13
+ db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5
+ db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
+ db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13
+ db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5
+ db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11
+ db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13
+ db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
+ db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12
+ db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13
+ db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $DB, $C5 // pand xmm0, xmm5
+ db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
// ---- Absorb next 2 blocks ----
- db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8]
- db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16]
+ db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8]
+ db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16]
add r8, 32
- db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12
- db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6
- db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13
- db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6
- db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12
- db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13
- db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13
- db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15
- db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14
- db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30
- db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4
- db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
- db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5
- db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5
- db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5
- db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5
- db $66, $0F, $EB, $FE // por xmm7, xmm6
- db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12
- db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
- db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14
- db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15
- db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12
+ db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6
+ db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13
+ db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6
+ db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12
+ db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13
+ db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13
+ db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15
+ db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14
+ db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30
+ db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4
+ db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40
+ db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5
+ db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5
+ db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5
+ db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5
+ db $66, $0F, $EB, $FE // por xmm7, xmm6
+ db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12
+ db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
+ db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14
+ db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
sub r9, 32
jnz @@poly1305_sse2_loop
@@poly1305_sse2_tail:
// ---- Tail: H *= [r^2, r^1] (shifted table read) ----
- db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0
- db $F3, $44, $0F, $6F, $6A, $44 // movdqu xmm13, [rdx + 68]
- db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
- db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
- db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52]
- db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
- db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0
- db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
- db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
- db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0
- db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
- db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
- db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
- db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
- db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13
- db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
- db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
- db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
- db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
- db $F3, $44, $0F, $6F, $6A, $54 // movdqu xmm13, [rdx + 84]
- db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $DB, $FD // pand xmm7, xmm5
- db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7
- db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13
- db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5
- db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8
- db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13
- db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5
- db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9
- db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13
- db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5
- db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
- db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13
- db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5
- db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11
- db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13
- db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
- db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12
- db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13
- db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0
- db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
- db $66, $0F, $DB, $C5 // pand xmm0, xmm5
- db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
+ db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0
+ db $F3, $44, $0F, $6F, $6A, $44 // movdqu xmm13, [rdx + 68]
+ db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12
+ db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0
+ db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52]
+ db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12
+ db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0
+ db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36]
+ db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12
+ db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0
+ db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20]
+ db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12
+ db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
+ db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4]
+ db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13
+ db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1
+ db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2
+ db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3
+ db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4
+ db $F3, $44, $0F, $6F, $6A, $54 // movdqu xmm13, [rdx + 84]
+ db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $DB, $FD // pand xmm7, xmm5
+ db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7
+ db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13
+ db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5
+ db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8
+ db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13
+ db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5
+ db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9
+ db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13
+ db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5
+ db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10
+ db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13
+ db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5
+ db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11
+ db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13
+ db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
+ db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12
+ db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13
+ db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0
+ db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26
+ db $66, $0F, $DB, $C5 // pand xmm0, xmm5
+ db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13
// ---- Horizontal fold 2 lanes -> 1, final reduce, store ----
- db $66, $0F, $70, $F8, $4E // pshufd xmm7, xmm0, 0x4E
- db $66, $0F, $D4, $C7 // paddq xmm0, xmm7
- db $66, $0F, $70, $F9, $4E // pshufd xmm7, xmm1, 0x4E
- db $66, $0F, $D4, $CF // paddq xmm1, xmm7
- db $66, $0F, $70, $FA, $4E // pshufd xmm7, xmm2, 0x4E
- db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
- db $66, $0F, $70, $FB, $4E // pshufd xmm7, xmm3, 0x4E
- db $66, $0F, $D4, $DF // paddq xmm3, xmm7
- db $66, $0F, $70, $FC, $4E // pshufd xmm7, xmm4, 0x4E
- db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
- db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $C5 // pand xmm0, xmm5
- db $66, $0F, $D4, $CF // paddq xmm1, xmm7
- db $66, $0F, $6F, $F9 // movdqa xmm7, xmm1
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $CD // pand xmm1, xmm5
- db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
- db $66, $0F, $6F, $FA // movdqa xmm7, xmm2
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $D5 // pand xmm2, xmm5
- db $66, $0F, $D4, $DF // paddq xmm3, xmm7
- db $66, $0F, $6F, $FB // movdqa xmm7, xmm3
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $DD // pand xmm3, xmm5
- db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
- db $66, $0F, $6F, $FC // movdqa xmm7, xmm4
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $E5 // pand xmm4, xmm5
- db $66, $44, $0F, $6F, $E7 // movdqa xmm12, xmm7
- db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
- db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
- db $66, $0F, $D4, $C7 // paddq xmm0, xmm7
- db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
- db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
- db $66, $0F, $DB, $C5 // pand xmm0, xmm5
- db $66, $0F, $D4, $CF // paddq xmm1, xmm7
- db $66, $0F, $7E, $41, $24 // movd dword ptr [rcx + 36], xmm0
- db $66, $0F, $7E, $49, $28 // movd dword ptr [rcx + 40], xmm1
- db $66, $0F, $7E, $51, $2C // movd dword ptr [rcx + 44], xmm2
- db $66, $0F, $7E, $59, $30 // movd dword ptr [rcx + 48], xmm3
- db $66, $0F, $7E, $61, $34 // movd dword ptr [rcx + 52], xmm4
+ db $66, $0F, $70, $F8, $4E // pshufd xmm7, xmm0, 0x4E
+ db $66, $0F, $D4, $C7 // paddq xmm0, xmm7
+ db $66, $0F, $70, $F9, $4E // pshufd xmm7, xmm1, 0x4E
+ db $66, $0F, $D4, $CF // paddq xmm1, xmm7
+ db $66, $0F, $70, $FA, $4E // pshufd xmm7, xmm2, 0x4E
+ db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
+ db $66, $0F, $70, $FB, $4E // pshufd xmm7, xmm3, 0x4E
+ db $66, $0F, $D4, $DF // paddq xmm3, xmm7
+ db $66, $0F, $70, $FC, $4E // pshufd xmm7, xmm4, 0x4E
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $C5 // pand xmm0, xmm5
+ db $66, $0F, $D4, $CF // paddq xmm1, xmm7
+ db $66, $0F, $6F, $F9 // movdqa xmm7, xmm1
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $CD // pand xmm1, xmm5
+ db $66, $0F, $D4, $D7 // paddq xmm2, xmm7
+ db $66, $0F, $6F, $FA // movdqa xmm7, xmm2
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $D5 // pand xmm2, xmm5
+ db $66, $0F, $D4, $DF // paddq xmm3, xmm7
+ db $66, $0F, $6F, $FB // movdqa xmm7, xmm3
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $DD // pand xmm3, xmm5
+ db $66, $0F, $D4, $E7 // paddq xmm4, xmm7
+ db $66, $0F, $6F, $FC // movdqa xmm7, xmm4
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $E5 // pand xmm4, xmm5
+ db $66, $44, $0F, $6F, $E7 // movdqa xmm12, xmm7
+ db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2
+ db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12
+ db $66, $0F, $D4, $C7 // paddq xmm0, xmm7
+ db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0
+ db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26
+ db $66, $0F, $DB, $C5 // pand xmm0, xmm5
+ db $66, $0F, $D4, $CF // paddq xmm1, xmm7
+ db $66, $0F, $7E, $41, $24 // movd dword ptr [rcx + 36], xmm0
+ db $66, $0F, $7E, $49, $28 // movd dword ptr [rcx + 40], xmm1
+ db $66, $0F, $7E, $51, $2C // movd dword ptr [rcx + 44], xmm2
+ db $66, $0F, $7E, $59, $30 // movd dword ptr [rcx + 48], xmm3
+ db $66, $0F, $7E, $61, $34 // movd dword ptr [rcx + 52], xmm4
+ mov rsp, r11
{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc}