Skip to content

Add parallel AES-NI CBC-decrypt kernel (single fused pass) #144

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Xor-el merged 1 commit into from
Jul 10, 2026
Merged

Add parallel AES-NI CBC-decrypt kernel (single fused pass) #144

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CryptoLib.Tests/src/Asn1/InputStreamTests.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ function TDummyStreamWithoutKnownLimit.ReadByte: Int32;

function TDummyStreamWithoutKnownLimit.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64;
begin
Result := 0; // unreachable; the raise below never returns
// CanSeek is already False, so callers should not reach here; a genuine
// non-seekable stream raises if seeked anyway.
raise ENotSupportedCryptoLibException.Create('stream is not seekable');
Expand Down
121 changes: 97 additions & 24 deletions CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiCbcKernel.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,17 @@ interface

type
/// <summary>
/// AES-NI implementation of ICbcKernel: the serial CBC-encrypt chain
/// C_i = E_K(P_i xor C_{i-1}) applied over a whole run in one call, with the
/// chaining value held in a register between blocks (kernel body in
/// Include\Simd\Aes\Cbc). Reuses the shared 1-wide AES round chain
/// (AesNiOneRoundsOnly). Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386
/// AES-NI implementation of ICbcKernel, applied over a whole run in one
/// call (kernel bodies in Include\Simd\Aes\Cbc). Encrypt runs the serial
/// chain C_i = E_K(P_i xor C_{i-1}) 1-wide with the chaining value held in
/// a register (reusing AesNiOneRoundsOnly). Decrypt runs P_i = D_K(C_i) xor
/// C_{i-1}: the inverse transforms are independent, so it decrypts 8-wide
/// (x86_64) / 4-wide (i386) and folds the chain XOR into that pass, turning
/// the mode's decrypt-then-XOR two passes into one. Direction is fixed at
/// construction. Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386
/// (CRYPTOLIB_I386_ASM), gated collectively by CRYPTOLIB_X86_SIMD. When
/// unavailable the factory returns nil and TCbcBlockCipher keeps its existing
/// per-block bulk path.
/// unavailable the factory returns nil and TCbcBlockCipher keeps its
/// existing per-block / two-pass bulk paths.
/// </summary>
TAesNiCbcKernel = class sealed(TInterfacedObject, ICbcKernel)
strict private
Expand All @@ -48,9 +51,10 @@ TAesNiCbcKernel = class sealed(TInterfacedObject, ICbcKernel)
FEngine: IAesEngineX86;
FKeys: Pointer;
FRounds: Int32;
FDirection: TCipherKernelDirection;
public
constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer;
ARounds: Int32);
ARounds: Int32; ADirection: TCipherKernelDirection);
procedure ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer;
ABlockCount: NativeInt);
end;
Expand All @@ -68,12 +72,13 @@ implementation
{$IFDEF CRYPTOLIB_X86_SIMD}

type
// Context handed to the fused AES-NI CBC-encrypt kernel. Field offsets match
// the [rcx + N] / [ebx + N] accesses in AesNiCbcEncryptSerial_x86_64.inc and
// AesNiCbcEncryptSerial_i386.inc (8-byte fields on x86_64, 4-byte on i386). The
// kernel loops internally over BlockCount blocks, advancing InPtr/OutPtr and
// updating the 16-byte chaining value at IvPtr in place.
TCbcEncryptFusedCtx = record
// Context handed to the fused AES-NI CBC kernels (both directions share the
// same shape). Field offsets match the [rcx + N] / [ebx + N] accesses in
// AesNiCbc{Encrypt,Decrypt}*_{x86_64,i386}.inc (8-byte fields on x86_64,
// 4-byte on i386). Each kernel loops internally over BlockCount blocks,
// advancing InPtr/OutPtr and updating the 16-byte chaining value at IvPtr in
// place.
TCbcFusedCtx = record
InPtr: Pointer;
OutPtr: Pointer;
KeysPtr: Pointer;
Expand Down Expand Up @@ -123,24 +128,74 @@ procedure AesNiCbcEncrypt256(PCtx: Pointer);
{$UNDEF CRYPTOLIB_AESNI_KEY256}
end;

// Fused AES-NI CBC decryption (P_i = D_K(C_i) xor C_{i-1}). Each proc processes
// BlockCount blocks in 8-wide (x86_64) / 4-wide (i386) batches with a 1-wide
// tail, looping internally, and writes the final ciphertext block back to the
// chaining slot at IvPtr. Consumes the inverse-MixColumns (decrypt) schedule.
procedure AesNiCbcDecrypt128(PCtx: Pointer);
{$DEFINE CRYPTOLIB_AESNI_KEY128}
{$DEFINE CRYPTOLIB_AESNI_DECRYPT}
{$IFDEF CRYPTOLIB_X86_64_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc}
{$ENDIF}
{$UNDEF CRYPTOLIB_AESNI_DECRYPT}
{$UNDEF CRYPTOLIB_AESNI_KEY128}
end;

procedure AesNiCbcDecrypt192(PCtx: Pointer);
{$DEFINE CRYPTOLIB_AESNI_KEY192}
{$DEFINE CRYPTOLIB_AESNI_DECRYPT}
{$IFDEF CRYPTOLIB_X86_64_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc}
{$ENDIF}
{$UNDEF CRYPTOLIB_AESNI_DECRYPT}
{$UNDEF CRYPTOLIB_AESNI_KEY192}
end;

procedure AesNiCbcDecrypt256(PCtx: Pointer);
{$DEFINE CRYPTOLIB_AESNI_KEY256}
{$DEFINE CRYPTOLIB_AESNI_DECRYPT}
{$IFDEF CRYPTOLIB_X86_64_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc}
{$ENDIF}
{$IFDEF CRYPTOLIB_I386_ASM}
{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc}
{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc}
{$ENDIF}
{$UNDEF CRYPTOLIB_AESNI_DECRYPT}
{$UNDEF CRYPTOLIB_AESNI_KEY256}
end;

{$ENDIF CRYPTOLIB_X86_SIMD}

{ TAesNiCbcKernel }

constructor TAesNiCbcKernel.Create(const AEngine: IAesEngineX86;
AKeys: Pointer; ARounds: Int32);
AKeys: Pointer; ARounds: Int32; ADirection: TCipherKernelDirection);
begin
inherited Create;
FEngine := AEngine;
FKeys := AKeys;
FRounds := ARounds;
FDirection := ADirection;
end;

procedure TAesNiCbcKernel.ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer;
ABlockCount: NativeInt);
{$IFDEF CRYPTOLIB_X86_SIMD}
var
LCtx: TCbcEncryptFusedCtx;
LCtx: TCbcFusedCtx;
{$ENDIF CRYPTOLIB_X86_SIMD}
begin
{$IFDEF CRYPTOLIB_X86_SIMD}
Expand All @@ -151,11 +206,23 @@ procedure TAesNiCbcKernel.ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer;
LCtx.KeysPtr := FKeys;
LCtx.IvPtr := AIvPtr;
LCtx.BlockCount := NativeUInt(ABlockCount);
case FRounds of
10: AesNiCbcEncrypt128(@LCtx);
12: AesNiCbcEncrypt192(@LCtx);
if FDirection = TCipherKernelDirection.Decrypt then
begin
case FRounds of
10: AesNiCbcDecrypt128(@LCtx);
12: AesNiCbcDecrypt192(@LCtx);
else
AesNiCbcDecrypt256(@LCtx);
end;
end
else
AesNiCbcEncrypt256(@LCtx);
begin
case FRounds of
10: AesNiCbcEncrypt128(@LCtx);
12: AesNiCbcEncrypt192(@LCtx);
else
AesNiCbcEncrypt256(@LCtx);
end;
end;
{$ENDIF CRYPTOLIB_X86_SIMD}
end;
Expand All @@ -176,17 +243,23 @@ function TAesNiCbcKernelFactory.TryCreate(const ACipher: IBlockCipher;
begin
AKernel := nil;
Result := False;
if ADirection <> TCipherKernelDirection.Encrypt then
Exit; // only encrypt is implemented; decrypt not yet supported
try
{$IFDEF CRYPTOLIB_X86_SIMD}
if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then
Exit;
if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then
// Encrypt consumes the forward schedule; decrypt the inverse-MixColumns
// schedule. The relevant accessor only succeeds when the engine is keyed
// for that direction, so this also validates state.
if ADirection = TCipherKernelDirection.Decrypt then
begin
if not LEngine.TryGetDecKeysPtr(LKeys, LRounds) then
Exit;
end
else if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then
Exit;
if not (LRounds in [10, 12, 14]) then
Exit;
AKernel := TAesNiCbcKernel.Create(LEngine, LKeys, LRounds);
AKernel := TAesNiCbcKernel.Create(LEngine, LKeys, LRounds, ADirection);
Result := True;
{$ENDIF CRYPTOLIB_X86_SIMD}
except
Expand Down
33 changes: 24 additions & 9 deletions CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,12 @@ TCbcBlockCipher = class sealed(TInterfacedObject, ICbcBlockCipher,
// one interface dispatch per block. Any bulk-capable block cipher
// lights up both paths automatically by implementing the interface.
FBulkCipher: IBulkBlockCipher;
// Cached on encrypt Init only. Non-nil when a registered accelerator
// claims the underlying cipher. Runs the whole serial CBC chain
// C[i] = E(P[i] xor C[i-1]) in one call with the chaining value held in a
// register, replacing the per-block dispatch loop. nil otherwise --
// CbcEncryptBulk keeps its per-block path.
// Cached on Init, direction-matched to FEncrypting. Non-nil when a
// registered accelerator claims the underlying cipher. Encrypt runs the
// whole serial chain C[i] = E(P[i] xor C[i-1]) in one register-held call;
// decrypt runs P[i] = DEC(C[i]) xor C[i-1] as a single fused pass (parallel
// aesdec + chain XOR). Both update FCbcV in place, matching the per-block
// post-state exactly. nil otherwise -- the bulk fallbacks below take over.
FCbcKernel: ICbcKernel;

function EncryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32;
Expand Down Expand Up @@ -239,6 +240,18 @@ procedure TCbcBlockCipher.CbcDecryptBulk(const AInBuf: TCryptoLibByteArray;
// carried in FCbcV). DEC(C[i]) are all independent (8-wide parallel), and
// every XOR feed C[i-1] is *input* ciphertext -- so there is no serial
// dependency; the whole run parallelises.

// Fast path: a registered accelerator fuses the parallel aesdec and the chain
// XOR into a single pass over memory (vs the decrypt-then-XOR two passes
// below), handling any block count and in-place aliasing internally, and
// updates FCbcV in place to the last ciphertext block.
if FCbcKernel <> nil then
begin
FCbcKernel.ProcessCbcBlocks(@AInBuf[AInOff], @AOutBuf[AOutOff],
@FCbcV[0], ABlockCount);
Exit;
end;

LPIn := @AInBuf[AInOff];
LPOut := @AOutBuf[AOutOff];
LTotal := ABlockCount * 16;
Expand Down Expand Up @@ -343,14 +356,16 @@ procedure TCbcBlockCipher.Init(AForEncryption: Boolean;
// engine ever surfaces.
TBlockCipherBulkUtilities.TryResolveBulkCipher(FCipher, FBulkCipher);

// CBC encrypt is serial; acquire the in-register-chain accelerator when a
// provider claims FCipher. Encrypt-only: CBC decrypt is parallel and already
// served by the bulk engine's 8-wide path.
// Acquire the direction-matched fused kernel when a provider claims FCipher.
// Encrypt folds the serial chain into one register-held pass; decrypt folds
// the chain XOR into the parallel aesdec pass (one pass over memory instead
// of the bulk-decrypt-then-XOR two passes). nil -> the bulk fallbacks below.
if FEncrypting then
TCipherKernelRegistry.TryAcquireCbc(FCipher, TCipherKernelDirection.Encrypt,
FCbcKernel)
else
FCbcKernel := nil;
TCipherKernelRegistry.TryAcquireCbc(FCipher, TCipherKernelDirection.Decrypt,
FCbcKernel);
end;

function TCbcBlockCipher.ProcessBlock(const AInput: TCryptoLibByteArray;
Expand Down
94 changes: 94 additions & 0 deletions CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// AES-NI CBC decryption, parallel 4-wide + fused chain XOR kernel (i386).
// -------------------------------------------------------------------
//
// i386 counterpart of AesNiCbcDecryptWide_x86_64.inc: P_i = D_K(C_i) xor
// C_{i-1} (C_{-1} = IV). The D_K(C_i) are independent, so blocks run through
// the shared aesdec chain in parallel; the trailing XOR (fed by input
// ciphertext) folds into the same pass, replacing the mode's decrypt-then-XOR
// two passes with one. i386 has only xmm0..7 (all volatile) and the 4-wide
// chain (AesNiFourRoundsOnly_i386) uses xmm4 as its round-key register, so the
// batch is 4 blocks: xmm0..3 = data, xmm4 = feedback reload scratch (free once
// the chain returns), xmm5 = chaining value (persists across groups). A 1-wide
// tail (AesNiOneRoundsOnly_i386) drains 1..3 residual blocks; the kernel
// accepts any block count >= 1.
//
// In-place safe: within each 4-block group every input read (4 block loads +
// 3 feedback reloads + the next chain seed) precedes the first output store,
// so OutPtr may alias InPtr.
//
// Entry (after ClpSimdProc1Begin_i386.inc): ebx = pointer to
// TCbcFusedCtx. The kernel pushes esi/edi and unpacks the record:
// [ebx + 0] InPtr -> esi : input (advances +16 per block)
// [ebx + 4] OutPtr -> edx : output (advances +16; may alias InPtr)
// [ebx + 8] KeysPtr -> edi : AES *decrypt* (inverse-MixColumns) schedule
// [ebx + 12] IvPtr : 16-byte chaining value (seed + writeback)
// [ebx + 16] BlockCount -> ecx : number of blocks, caller guarantees >= 1
//
// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined,
// together with CRYPTOLIB_AESNI_DECRYPT (selects aesdec in the shared chains).
//
// AES-NI instructions and register pxor/movdqa forms are db-encoded for broad
// assembler compatibility.
// ---- unpack TCbcFusedCtx (ebx = PCtx) ----
push esi
push edi
mov esi, [ebx + 0] // InPtr
mov edx, [ebx + 4] // OutPtr
mov edi, [ebx + 8] // KeysPtr (decrypt schedule)
mov ecx, [ebx + 16] // BlockCount
mov eax, [ebx + 12] // IvPtr
movdqu xmm5, [eax] // chain = C_{-1} (the IV)

cmp ecx, 4
jb @@CbcDecTailI386

@@CbcDecWideI386:
movdqu xmm0, [esi + 0] // C_0 .. C_3
movdqu xmm1, [esi + 16]
movdqu xmm2, [esi + 32]
movdqu xmm3, [esi + 48]
{$I ..\..\..\Include\Simd\Aes\AesNiFourRoundsOnly_i386.inc} // xmm0..3 = D_K(C_i)
// feedback: P_i = D_K(C_i) xor C_{i-1}. C_0..C_2 reloaded from the (still
// intact) input; block 0 uses the running chain. All reads precede stores.
movdqu xmm4, [esi + 0]
db $66, $0F, $EF, $CC // pxor xmm1, xmm4 (^ C_0)
movdqu xmm4, [esi + 16]
db $66, $0F, $EF, $D4 // pxor xmm2, xmm4 (^ C_1)
movdqu xmm4, [esi + 32]
db $66, $0F, $EF, $DC // pxor xmm3, xmm4 (^ C_2)
db $66, $0F, $EF, $C5 // pxor xmm0, xmm5 (^ chain = C_{-1})
movdqu xmm5, [esi + 48] // next chain = C_3 (before any store)
movdqu [edx + 0], xmm0 // P_0 .. P_3
movdqu [edx + 16], xmm1
movdqu [edx + 32], xmm2
movdqu [edx + 48], xmm3
add esi, 64
add edx, 64
sub ecx, 4
cmp ecx, 4
jae @@CbcDecWideI386

@@CbcDecTailI386:
test ecx, ecx
jz @@CbcDecDoneI386
@@CbcDecTailLoopI386:
movdqu xmm0, [esi] // C_i
db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 (save C_i)
{$I ..\..\..\Include\Simd\Aes\AesNiOneRoundsOnly_i386.inc} // xmm0 = D_K(C_i)
db $66, $0F, $EF, $C5 // pxor xmm0, xmm5 (^ chain)
db $66, $0F, $6F, $E9 // movdqa xmm5, xmm1 (chain = C_i)
movdqu [edx], xmm0 // P_i
add esi, 16
add edx, 16
sub ecx, 1
jnz @@CbcDecTailLoopI386

@@CbcDecDoneI386:
mov eax, [ebx + 12] // IvPtr
movdqu [eax], xmm5 // write final chain (= last C) back
db $66, $0F, $EF, $C0 // pxor xmm0, xmm0 (scrub last plaintext)
db $66, $0F, $EF, $ED // pxor xmm5, xmm5 (scrub chain)

pop edi
pop esi
pop ebx
Loading
Loading