SIMD kernel throughput optimization: streaming ChaCha/Poly1305, fused AES AEAD rework, and bulk GHASH AAD

CryptoLib.Tests/src/Crypto/CcmTests.pas

@@ -668,9 +668,11 @@ procedure TTestCcm.DoTestNoUnverifiedPlaintextOnFailure;
   except
     on E: EInvalidCipherTextCryptoLibException do
     begin
+      // The tentative plaintext must be wiped before the exception leaves:
+      // every byte of the output region has to read back as zero.
       for LI := 0 to High(LOutput) do
       begin
-        if LOutput[LI] <> $55 then
+        if LOutput[LI] <> $00 then
           Fail('CCM left unverified plaintext in the output buffer on tag failure');
       end;
     end;

CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiGcmKernel.pas

@@ -22,6 +22,7 @@ interface
 
 uses
   SysUtils,
+  ClpCryptoLibTypes,
   ClpIBlockCipher,
   ClpIAesEngineX86,
   ClpCipherKernelTypes,
@@ -55,6 +56,10 @@   TAesNiGcmKernel = class sealed(TInterfacedObject, IGcmKernel)
     FRounds: Int32;
     FHPow128: Pointer;
     FMask: Pointer;
+    // Kernel-owned copy of the mode's H^8..H^1 table (already x-pre-multiplied
+    // by TGcmUtilities.InitEightWayHPowFromH, matching the carry-less-multiply
+    // folding reduction inside the fused kernel). FHPow128 points at it.
+    FHPowShifted: TCryptoLibByteArray;
   public
     constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer;
       ARounds: Int32; AHPow128, AMask: Pointer);
@@ -82,11 +87,12 @@ implementation
   // Natural pointer-sized alignment, no padding: the field offsets match the
   // [rcx + N] / [ebx + N] accesses in AesGcmFusedCtrGhashEight_x86_64.inc and
   // AesGcmFusedCtrGhashEight_i386.inc (they differ with pointer / NativeUInt
-  // width: 8-byte fields on x86_64, 4-byte on i386). The kernel advances
-  // PXorIn/POut/PPrevCipher and Counter32 in place across the batch run and
+  // width: 8-byte fields on x86_64, 4-byte on i386). The kernel holds the loop
+  // state in registers across the batch run, writing only Counter32 back, and
   // builds each batch's counter blocks in-register from Counter32 + PJ0Template
   // (both arches); the GHASH-from-output flag is 1 for encrypt (fold the output
-  // ciphertext) and 0 for decrypt (fold the input).
+  // ciphertext) and 0 for decrypt (fold the input). PHPow128 points at the
+  // kernel-owned x-pre-multiplied H-power table (see TAesNiGcmKernel.Create).
   TGcmFusedCtx = record
     PXorIn: Pointer;
     POut: Pointer;
@@ -157,8 +163,13 @@ constructor TAesNiGcmKernel.Create(const AEngine: IAesEngineX86;
   FEngine := AEngine;
   FKeys := AKeys;
   FRounds := ARounds;
-  FHPow128 := AHPow128;
   FMask := AMask;
+  // The mode's table already holds the powers pre-multiplied by x (see
+  // TGcmUtilities.InitEightWayHPowFromH); keep a kernel-owned copy so the
+  // pointer stays valid for the kernel's lifetime.
+  System.SetLength(FHPowShifted, 128);
+  System.Move(AHPow128^, FHPowShifted[0], 128);
+  FHPow128 := @FHPowShifted[0];
 end;
 
 function TAesNiGcmKernel.MinimumBlockCount: Int32;

CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiOcbKernel.pas

@@ -49,9 +49,9 @@   TAesNiOcbKernel = class sealed(TInterfacedObject, IOcbKernel)
   strict private
   const
 {$IFDEF CRYPTOLIB_X86_64_ASM}
-    FUSED_OCB_MIN_BLOCKS = 8;
+    FUSED_OCB_MIN_BLOCKS = 16;
 {$ELSE}
-    FUSED_OCB_MIN_BLOCKS = 4;
+    FUSED_OCB_MIN_BLOCKS = 12;
 {$ENDIF}
   strict private
     FEngine: IAesEngineX86;

CryptoLib/src/Crypto/CipherKernels/Block/ClpPclmulGcmSivKernel.pas

@@ -85,9 +85,11 @@ function TPclmulGcmSivKernel.MinimumBlockCount: Int32;
 procedure TPclmulGcmSivKernel.ProcessPolyvalBatch(AInPtr, AAccumulator: Pointer;
   ABlockCount: Int32);
 begin
-  if ABlockCount <> FUSED_POLYVAL_MIN_BLOCKS then
+  if (ABlockCount < FUSED_POLYVAL_MIN_BLOCKS) or
+    (ABlockCount mod FUSED_POLYVAL_MIN_BLOCKS <> 0) then
     Exit;
-  TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask);
+  TGcmSivSimd.ProcessPolyvalBatch(AAccumulator, AInPtr, FHPow128, FMask,
+    ABlockCount div FUSED_POLYVAL_MIN_BLOCKS);
 end;
 
 { TPclmulGcmSivKernelFactory }

CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas

@@ -57,21 +57,25 @@ interface
   TChaChaBaseEngine = class(TSalsa20Engine)
 
   strict private
-    // Pointer-form, unvalidated inner helpers (1 / 2 / 4 / 8 blocks).
+    // Pointer-form, unvalidated inner helpers (1 / 2 blocks).
     procedure ProcessBlockFast(AIn, AOut: PByte);
     procedure ProcessBlocks2Fast(AIn, AOut: PByte);
-    procedure ProcessBlocks4Fast(AIn, AOut: PByte);
-    procedure ProcessBlocks8Fast(AIn, AOut: PByte);
+    // One streaming SIMD tier (AGroupBlocks = 8 or 4): clamps to the wrap-safe
+    // span, runs the kernel once over every whole group, advances the cursors.
+    // False = no SIMD kernel for that width or no safe groups (degrade a tier).
+    function TryProcessWide(AGroupBlocks: Int32; var AIn, AOut: PByte;
+      var ABlockCount: Int32): Boolean;
 
   strict protected
     // 8 -> 4 -> 2 -> 1 bulk ladder (widest kernel first).
     function DoProcessBlocks(AIn, AOut: PByte; ABlockCount: Int32): Int32; override;
 
     // True for the DJB 64-bit counter (words 12-13); False (IETF).
     function CounterIs64Bit: Boolean; virtual;
-    // Wide kernels broadcast word 13; for the 64-bit counter that holds only while
-    // word 12 does not wrap over the N-block span (else the caller degrades a tier).
-    function WideBlocksSafe(ABlocks: Int32): Boolean; inline;
+    // Wide kernels broadcast word 13, valid only while word 12 does not wrap
+    // inside the processed span: clamps a span of AGroups x AGroupBlocks
+    // blocks to the groups fully before the wrap (identity for IETF).
+    function WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32;
 
     procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override;
 
@@ -133,12 +137,20 @@ function TChaChaBaseEngine.CounterIs64Bit: Boolean;
   Result := False;
 end;
 
-function TChaChaBaseEngine.WideBlocksSafe(ABlocks: Int32): Boolean;
+function TChaChaBaseEngine.WideGroupsSafe(AGroups, AGroupBlocks: Int32): Int32;
+var
+  LSafeBlocks: UInt64;
 begin
-  Result := (not CounterIs64Bit) or
-    (FEngineState[12] <= (UInt32($FFFFFFFF) - UInt32(ABlocks - 1)));
+  Result := AGroups;
+  if CounterIs64Bit then
+  begin
+    LSafeBlocks := UInt64($100000000) - FEngineState[12];
+    if (UInt64(Result) * UInt32(AGroupBlocks)) > LSafeBlocks then
+      Result := Int32(LSafeBlocks div UInt32(AGroupBlocks));
+  end;
 end;
 
+
 procedure TChaChaBaseEngine.GenerateKeyStream(const AOutput: TCryptoLibByteArray);
 begin
   ChaChaCore(FRounds, FEngineState, AOutput);
@@ -299,26 +311,29 @@ procedure TChaChaBaseEngine.ProcessBlocks2Fast(AIn, AOut: PByte);
   ProcessBlockFast(AIn + 64, AOut + 64);
 end;
 
-procedure TChaChaBaseEngine.ProcessBlocks4Fast(AIn, AOut: PByte);
-begin
-  if WideBlocksSafe(4) then
-    if TChaChaSimd.TryProcessBlocks4(FRounds, PByte(@FEngineState[0]),
-      AIn, AOut, CounterIs64Bit) then
-      Exit;
-
-  ProcessBlocks2Fast(AIn, AOut);
-  ProcessBlocks2Fast(AIn + 128, AOut + 128);
-end;
-
-procedure TChaChaBaseEngine.ProcessBlocks8Fast(AIn, AOut: PByte);
+function TChaChaBaseEngine.TryProcessWide(AGroupBlocks: Int32;
+  var AIn, AOut: PByte; var ABlockCount: Int32): Boolean;
+var
+  LGroups: Int32;
 begin
-  if WideBlocksSafe(8) then
-    if TChaChaSimd.TryProcessBlocks8(FRounds, PByte(@FEngineState[0]),
-      AIn, AOut, CounterIs64Bit) then
-      Exit;
-
-  ProcessBlocks4Fast(AIn, AOut);
-  ProcessBlocks4Fast(AIn + 256, AOut + 256);
+  Result := False;
+  LGroups := WideGroupsSafe(ABlockCount div AGroupBlocks, AGroupBlocks);
+  if LGroups = 0 then
+    Exit;
+  case AGroupBlocks of
+    8:
+      Result := TChaChaSimd.TryProcessBlocks8(FRounds,
+        PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit);
+  else
+    Result := TChaChaSimd.TryProcessBlocks4(FRounds,
+      PByte(@FEngineState[0]), AIn, AOut, LGroups, CounterIs64Bit);
+  end;
+  if Result then
+  begin
+    AIn := AIn + LGroups * (AGroupBlocks * 64);
+    AOut := AOut + LGroups * (AGroupBlocks * 64);
+    System.Dec(ABlockCount, LGroups * AGroupBlocks);
+  end;
 end;
 
 function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte;
@@ -327,14 +342,24 @@ function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte;
   Result := ABlockCount * 64;
   while ABlockCount >= 8 do
   begin
-    ProcessBlocks8Fast(AIn, AOut);
+    // Widest streaming tier that lands, over all safe groups at once.
+    if TryProcessWide(8, AIn, AOut, ABlockCount) then
+      continue;
+    if TryProcessWide(4, AIn, AOut, ABlockCount) then
+      continue;
+    // No wide SIMD (or at the DJB wrap boundary): one 8-block group via 2/1.
+    ProcessBlocks2Fast(AIn, AOut);
+    ProcessBlocks2Fast(AIn + 128, AOut + 128);
+    ProcessBlocks2Fast(AIn + 256, AOut + 256);
+    ProcessBlocks2Fast(AIn + 384, AOut + 384);
     AIn := AIn + 512;
     AOut := AOut + 512;
     System.Dec(ABlockCount, 8);
   end;
-  if ABlockCount >= 4 then
+  if (ABlockCount >= 4) and (not TryProcessWide(4, AIn, AOut, ABlockCount)) then
   begin
-    ProcessBlocks4Fast(AIn, AOut);
+    ProcessBlocks2Fast(AIn, AOut);
+    ProcessBlocks2Fast(AIn + 128, AOut + 128);
     AIn := AIn + 256;
     AOut := AOut + 256;
     System.Dec(ABlockCount, 4);

CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas

@@ -42,10 +42,14 @@   TChaChaSimd = class sealed
     /// <summary>Two-block ChaCha keystream (128 bytes). ACtr64 = DJB 64-bit
     /// counter (no SIMD 2-block body; returns False).</summary>
     class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
-    /// <summary>Four-block ChaCha keystream (256 bytes). ACtr64 = DJB variant.</summary>
-    class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
-    /// <summary>Eight-block ChaCha keystream (512 bytes). ACtr64 = DJB variant.</summary>
-    class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static;
+    /// <summary>Four-block ChaCha keystream, streaming AGroups consecutive
+    /// 256-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the
+    /// caller guarantees no counter low-word wrap inside the span.</summary>
+    class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
+    /// <summary>Eight-block ChaCha keystream, streaming AGroups consecutive
+    /// 512-byte groups (AGroups >= 1) in one call. ACtr64 = DJB variant; the
+    /// caller guarantees no counter low-word wrap inside the span.</summary>
+    class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean = False): Boolean; static;
   end;
 
 implementation
@@ -70,19 +74,19 @@ class function TChaChaSimd.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut:
 {$IFEND}
 end;
 
-class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
 begin
 {$IF DEFINED(CRYPTOLIB_X86_SIMD)}
-  Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, ACtr64);
+  Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, AGroups, ACtr64);
 {$ELSE}
   Result := False;
 {$IFEND}
 end;
 
-class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean;
+class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; AGroups: Int32; ACtr64: Boolean): Boolean;
 begin
 {$IF DEFINED(CRYPTOLIB_X86_SIMD)}
-  Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, ACtr64);
+  Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, AGroups, ACtr64);
 {$ELSE}
   Result := False;
 {$IFEND}