From 1016211fc99082c2395e401106c0947406678b10 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Fri, 10 Jul 2026 22:32:02 +0100 Subject: [PATCH] Add parallel AES-NI CBC-decrypt kernel (single fused pass) CBC decrypt is P[i] = DEC(C[i]) xor C[i-1]: the inverse transforms are independent, so decrypt the run 8-wide (x86_64) / 4-wide (i386) and fold the chain XOR into that same pass. This replaces the mode's two passes over memory (bulk aesdec write, then a separate XOR re-read) with one, which lifts the DRAM-bandwidth ceiling that bounded large buffers. TAesNiCbcKernel now serves both directions (branch on FDirection, as the CCM kernel does); the factory probes TryGetDecKeysPtr for the inverse- MixColumns schedule. New AesNiCbcDecryptWide_{x86_64,i386}.inc are kept separate from the serial encrypt include -- the two share no structure (serial 1-wide leaf vs parallel wide with an xmm save frame), only the shared round chains. In-place safe with no stack spill: every input read completes before the first output store. A 1-wide tail drains the 1..7 (1..3 on i386) residual blocks, so the kernel takes any block count. TCbcBlockCipher resolves a direction-matched kernel at Init and routes CbcDecryptBulk through it, falling back to the existing path when nil. --- CryptoLib.Tests/src/Asn1/InputStreamTests.pas | 1 - .../CipherKernels/Block/ClpAesNiCbcKernel.pas | 121 ++++++++++++++---- .../src/Crypto/Modes/ClpCbcBlockCipher.pas | 33 +++-- .../Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc | 94 ++++++++++++++ .../Aes/Cbc/AesNiCbcDecryptWide_x86_64.inc | 110 ++++++++++++++++ .../Aes/Cbc/AesNiCbcEncryptSerial_i386.inc | 4 +- .../Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc | 4 +- 7 files changed, 329 insertions(+), 38 deletions(-) create mode 100644 CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc create mode 100644 CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_x86_64.inc diff --git a/CryptoLib.Tests/src/Asn1/InputStreamTests.pas b/CryptoLib.Tests/src/Asn1/InputStreamTests.pas index e84732ff..e126df7c 100644 --- a/CryptoLib.Tests/src/Asn1/InputStreamTests.pas +++ b/CryptoLib.Tests/src/Asn1/InputStreamTests.pas @@ -84,7 +84,6 @@ function TDummyStreamWithoutKnownLimit.ReadByte: Int32; function TDummyStreamWithoutKnownLimit.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; begin - Result := 0; // unreachable; the raise below never returns // CanSeek is already False, so callers should not reach here; a genuine // non-seekable stream raises if seeked anyway. raise ENotSupportedCryptoLibException.Create('stream is not seekable'); diff --git a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiCbcKernel.pas b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiCbcKernel.pas index ada78f49..b38cd1b4 100644 --- a/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiCbcKernel.pas +++ b/CryptoLib/src/Crypto/CipherKernels/Block/ClpAesNiCbcKernel.pas @@ -32,14 +32,17 @@ interface type /// - /// AES-NI implementation of ICbcKernel: the serial CBC-encrypt chain - /// C_i = E_K(P_i xor C_{i-1}) applied over a whole run in one call, with the - /// chaining value held in a register between blocks (kernel body in - /// Include\Simd\Aes\Cbc). Reuses the shared 1-wide AES round chain - /// (AesNiOneRoundsOnly). Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386 + /// AES-NI implementation of ICbcKernel, applied over a whole run in one + /// call (kernel bodies in Include\Simd\Aes\Cbc). Encrypt runs the serial + /// chain C_i = E_K(P_i xor C_{i-1}) 1-wide with the chaining value held in + /// a register (reusing AesNiOneRoundsOnly). Decrypt runs P_i = D_K(C_i) xor + /// C_{i-1}: the inverse transforms are independent, so it decrypts 8-wide + /// (x86_64) / 4-wide (i386) and folds the chain XOR into that pass, turning + /// the mode's decrypt-then-XOR two passes into one. Direction is fixed at + /// construction. Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386 /// (CRYPTOLIB_I386_ASM), gated collectively by CRYPTOLIB_X86_SIMD. When - /// unavailable the factory returns nil and TCbcBlockCipher keeps its existing - /// per-block bulk path. + /// unavailable the factory returns nil and TCbcBlockCipher keeps its + /// existing per-block / two-pass bulk paths. /// TAesNiCbcKernel = class sealed(TInterfacedObject, ICbcKernel) strict private @@ -48,9 +51,10 @@ TAesNiCbcKernel = class sealed(TInterfacedObject, ICbcKernel) FEngine: IAesEngineX86; FKeys: Pointer; FRounds: Int32; + FDirection: TCipherKernelDirection; public constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; - ARounds: Int32); + ARounds: Int32; ADirection: TCipherKernelDirection); procedure ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; ABlockCount: NativeInt); end; @@ -68,12 +72,13 @@ implementation {$IFDEF CRYPTOLIB_X86_SIMD} type - // Context handed to the fused AES-NI CBC-encrypt kernel. Field offsets match - // the [rcx + N] / [ebx + N] accesses in AesNiCbcEncryptSerial_x86_64.inc and - // AesNiCbcEncryptSerial_i386.inc (8-byte fields on x86_64, 4-byte on i386). The - // kernel loops internally over BlockCount blocks, advancing InPtr/OutPtr and - // updating the 16-byte chaining value at IvPtr in place. - TCbcEncryptFusedCtx = record + // Context handed to the fused AES-NI CBC kernels (both directions share the + // same shape). Field offsets match the [rcx + N] / [ebx + N] accesses in + // AesNiCbc{Encrypt,Decrypt}*_{x86_64,i386}.inc (8-byte fields on x86_64, + // 4-byte on i386). Each kernel loops internally over BlockCount blocks, + // advancing InPtr/OutPtr and updating the 16-byte chaining value at IvPtr in + // place. + TCbcFusedCtx = record InPtr: Pointer; OutPtr: Pointer; KeysPtr: Pointer; @@ -123,24 +128,74 @@ procedure AesNiCbcEncrypt256(PCtx: Pointer); {$UNDEF CRYPTOLIB_AESNI_KEY256} end; +// Fused AES-NI CBC decryption (P_i = D_K(C_i) xor C_{i-1}). Each proc processes +// BlockCount blocks in 8-wide (x86_64) / 4-wide (i386) batches with a 1-wide +// tail, looping internally, and writes the final ciphertext block back to the +// chaining slot at IvPtr. Consumes the inverse-MixColumns (decrypt) schedule. +procedure AesNiCbcDecrypt128(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY128} +{$DEFINE CRYPTOLIB_AESNI_DECRYPT} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_DECRYPT} +{$UNDEF CRYPTOLIB_AESNI_KEY128} +end; + +procedure AesNiCbcDecrypt192(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY192} +{$DEFINE CRYPTOLIB_AESNI_DECRYPT} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_DECRYPT} +{$UNDEF CRYPTOLIB_AESNI_KEY192} +end; + +procedure AesNiCbcDecrypt256(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY256} +{$DEFINE CRYPTOLIB_AESNI_DECRYPT} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\ClpSimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcDecryptWide_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_DECRYPT} +{$UNDEF CRYPTOLIB_AESNI_KEY256} +end; + {$ENDIF CRYPTOLIB_X86_SIMD} { TAesNiCbcKernel } constructor TAesNiCbcKernel.Create(const AEngine: IAesEngineX86; - AKeys: Pointer; ARounds: Int32); + AKeys: Pointer; ARounds: Int32; ADirection: TCipherKernelDirection); begin inherited Create; FEngine := AEngine; FKeys := AKeys; FRounds := ARounds; + FDirection := ADirection; end; procedure TAesNiCbcKernel.ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; ABlockCount: NativeInt); {$IFDEF CRYPTOLIB_X86_SIMD} var - LCtx: TCbcEncryptFusedCtx; + LCtx: TCbcFusedCtx; {$ENDIF CRYPTOLIB_X86_SIMD} begin {$IFDEF CRYPTOLIB_X86_SIMD} @@ -151,11 +206,23 @@ procedure TAesNiCbcKernel.ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; LCtx.KeysPtr := FKeys; LCtx.IvPtr := AIvPtr; LCtx.BlockCount := NativeUInt(ABlockCount); - case FRounds of - 10: AesNiCbcEncrypt128(@LCtx); - 12: AesNiCbcEncrypt192(@LCtx); + if FDirection = TCipherKernelDirection.Decrypt then + begin + case FRounds of + 10: AesNiCbcDecrypt128(@LCtx); + 12: AesNiCbcDecrypt192(@LCtx); + else + AesNiCbcDecrypt256(@LCtx); + end; + end else - AesNiCbcEncrypt256(@LCtx); + begin + case FRounds of + 10: AesNiCbcEncrypt128(@LCtx); + 12: AesNiCbcEncrypt192(@LCtx); + else + AesNiCbcEncrypt256(@LCtx); + end; end; {$ENDIF CRYPTOLIB_X86_SIMD} end; @@ -176,17 +243,23 @@ function TAesNiCbcKernelFactory.TryCreate(const ACipher: IBlockCipher; begin AKernel := nil; Result := False; - if ADirection <> TCipherKernelDirection.Encrypt then - Exit; // only encrypt is implemented; decrypt not yet supported try {$IFDEF CRYPTOLIB_X86_SIMD} if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then Exit; - if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then + // Encrypt consumes the forward schedule; decrypt the inverse-MixColumns + // schedule. The relevant accessor only succeeds when the engine is keyed + // for that direction, so this also validates state. + if ADirection = TCipherKernelDirection.Decrypt then + begin + if not LEngine.TryGetDecKeysPtr(LKeys, LRounds) then + Exit; + end + else if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then Exit; if not (LRounds in [10, 12, 14]) then Exit; - AKernel := TAesNiCbcKernel.Create(LEngine, LKeys, LRounds); + AKernel := TAesNiCbcKernel.Create(LEngine, LKeys, LRounds, ADirection); Result := True; {$ENDIF CRYPTOLIB_X86_SIMD} except diff --git a/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas index f9f674c9..2cde01f8 100644 --- a/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas @@ -71,11 +71,12 @@ TCbcBlockCipher = class sealed(TInterfacedObject, ICbcBlockCipher, // one interface dispatch per block. Any bulk-capable block cipher // lights up both paths automatically by implementing the interface. FBulkCipher: IBulkBlockCipher; - // Cached on encrypt Init only. Non-nil when a registered accelerator - // claims the underlying cipher. Runs the whole serial CBC chain - // C[i] = E(P[i] xor C[i-1]) in one call with the chaining value held in a - // register, replacing the per-block dispatch loop. nil otherwise -- - // CbcEncryptBulk keeps its per-block path. + // Cached on Init, direction-matched to FEncrypting. Non-nil when a + // registered accelerator claims the underlying cipher. Encrypt runs the + // whole serial chain C[i] = E(P[i] xor C[i-1]) in one register-held call; + // decrypt runs P[i] = DEC(C[i]) xor C[i-1] as a single fused pass (parallel + // aesdec + chain XOR). Both update FCbcV in place, matching the per-block + // post-state exactly. nil otherwise -- the bulk fallbacks below take over. FCbcKernel: ICbcKernel; function EncryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; @@ -239,6 +240,18 @@ procedure TCbcBlockCipher.CbcDecryptBulk(const AInBuf: TCryptoLibByteArray; // carried in FCbcV). DEC(C[i]) are all independent (8-wide parallel), and // every XOR feed C[i-1] is *input* ciphertext -- so there is no serial // dependency; the whole run parallelises. + + // Fast path: a registered accelerator fuses the parallel aesdec and the chain + // XOR into a single pass over memory (vs the decrypt-then-XOR two passes + // below), handling any block count and in-place aliasing internally, and + // updates FCbcV in place to the last ciphertext block. + if FCbcKernel <> nil then + begin + FCbcKernel.ProcessCbcBlocks(@AInBuf[AInOff], @AOutBuf[AOutOff], + @FCbcV[0], ABlockCount); + Exit; + end; + LPIn := @AInBuf[AInOff]; LPOut := @AOutBuf[AOutOff]; LTotal := ABlockCount * 16; @@ -343,14 +356,16 @@ procedure TCbcBlockCipher.Init(AForEncryption: Boolean; // engine ever surfaces. TBlockCipherBulkUtilities.TryResolveBulkCipher(FCipher, FBulkCipher); - // CBC encrypt is serial; acquire the in-register-chain accelerator when a - // provider claims FCipher. Encrypt-only: CBC decrypt is parallel and already - // served by the bulk engine's 8-wide path. + // Acquire the direction-matched fused kernel when a provider claims FCipher. + // Encrypt folds the serial chain into one register-held pass; decrypt folds + // the chain XOR into the parallel aesdec pass (one pass over memory instead + // of the bulk-decrypt-then-XOR two passes). nil -> the bulk fallbacks below. if FEncrypting then TCipherKernelRegistry.TryAcquireCbc(FCipher, TCipherKernelDirection.Encrypt, FCbcKernel) else - FCbcKernel := nil; + TCipherKernelRegistry.TryAcquireCbc(FCipher, TCipherKernelDirection.Decrypt, + FCbcKernel); end; function TCbcBlockCipher.ProcessBlock(const AInput: TCryptoLibByteArray; diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc new file mode 100644 index 00000000..c19275eb --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_i386.inc @@ -0,0 +1,94 @@ +// AES-NI CBC decryption, parallel 4-wide + fused chain XOR kernel (i386). +// ------------------------------------------------------------------- +// +// i386 counterpart of AesNiCbcDecryptWide_x86_64.inc: P_i = D_K(C_i) xor +// C_{i-1} (C_{-1} = IV). The D_K(C_i) are independent, so blocks run through +// the shared aesdec chain in parallel; the trailing XOR (fed by input +// ciphertext) folds into the same pass, replacing the mode's decrypt-then-XOR +// two passes with one. i386 has only xmm0..7 (all volatile) and the 4-wide +// chain (AesNiFourRoundsOnly_i386) uses xmm4 as its round-key register, so the +// batch is 4 blocks: xmm0..3 = data, xmm4 = feedback reload scratch (free once +// the chain returns), xmm5 = chaining value (persists across groups). A 1-wide +// tail (AesNiOneRoundsOnly_i386) drains 1..3 residual blocks; the kernel +// accepts any block count >= 1. +// +// In-place safe: within each 4-block group every input read (4 block loads + +// 3 feedback reloads + the next chain seed) precedes the first output store, +// so OutPtr may alias InPtr. +// +// Entry (after ClpSimdProc1Begin_i386.inc): ebx = pointer to +// TCbcFusedCtx. The kernel pushes esi/edi and unpacks the record: +// [ebx + 0] InPtr -> esi : input (advances +16 per block) +// [ebx + 4] OutPtr -> edx : output (advances +16; may alias InPtr) +// [ebx + 8] KeysPtr -> edi : AES *decrypt* (inverse-MixColumns) schedule +// [ebx + 12] IvPtr : 16-byte chaining value (seed + writeback) +// [ebx + 16] BlockCount -> ecx : number of blocks, caller guarantees >= 1 +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined, +// together with CRYPTOLIB_AESNI_DECRYPT (selects aesdec in the shared chains). +// +// AES-NI instructions and register pxor/movdqa forms are db-encoded for broad +// assembler compatibility. + // ---- unpack TCbcFusedCtx (ebx = PCtx) ---- + push esi + push edi + mov esi, [ebx + 0] // InPtr + mov edx, [ebx + 4] // OutPtr + mov edi, [ebx + 8] // KeysPtr (decrypt schedule) + mov ecx, [ebx + 16] // BlockCount + mov eax, [ebx + 12] // IvPtr + movdqu xmm5, [eax] // chain = C_{-1} (the IV) + + cmp ecx, 4 + jb @@CbcDecTailI386 + +@@CbcDecWideI386: + movdqu xmm0, [esi + 0] // C_0 .. C_3 + movdqu xmm1, [esi + 16] + movdqu xmm2, [esi + 32] + movdqu xmm3, [esi + 48] +{$I ..\..\..\Include\Simd\Aes\AesNiFourRoundsOnly_i386.inc} // xmm0..3 = D_K(C_i) + // feedback: P_i = D_K(C_i) xor C_{i-1}. C_0..C_2 reloaded from the (still + // intact) input; block 0 uses the running chain. All reads precede stores. + movdqu xmm4, [esi + 0] + db $66, $0F, $EF, $CC // pxor xmm1, xmm4 (^ C_0) + movdqu xmm4, [esi + 16] + db $66, $0F, $EF, $D4 // pxor xmm2, xmm4 (^ C_1) + movdqu xmm4, [esi + 32] + db $66, $0F, $EF, $DC // pxor xmm3, xmm4 (^ C_2) + db $66, $0F, $EF, $C5 // pxor xmm0, xmm5 (^ chain = C_{-1}) + movdqu xmm5, [esi + 48] // next chain = C_3 (before any store) + movdqu [edx + 0], xmm0 // P_0 .. P_3 + movdqu [edx + 16], xmm1 + movdqu [edx + 32], xmm2 + movdqu [edx + 48], xmm3 + add esi, 64 + add edx, 64 + sub ecx, 4 + cmp ecx, 4 + jae @@CbcDecWideI386 + +@@CbcDecTailI386: + test ecx, ecx + jz @@CbcDecDoneI386 +@@CbcDecTailLoopI386: + movdqu xmm0, [esi] // C_i + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 (save C_i) +{$I ..\..\..\Include\Simd\Aes\AesNiOneRoundsOnly_i386.inc} // xmm0 = D_K(C_i) + db $66, $0F, $EF, $C5 // pxor xmm0, xmm5 (^ chain) + db $66, $0F, $6F, $E9 // movdqa xmm5, xmm1 (chain = C_i) + movdqu [edx], xmm0 // P_i + add esi, 16 + add edx, 16 + sub ecx, 1 + jnz @@CbcDecTailLoopI386 + +@@CbcDecDoneI386: + mov eax, [ebx + 12] // IvPtr + movdqu [eax], xmm5 // write final chain (= last C) back + db $66, $0F, $EF, $C0 // pxor xmm0, xmm0 (scrub last plaintext) + db $66, $0F, $EF, $ED // pxor xmm5, xmm5 (scrub chain) + + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_x86_64.inc new file mode 100644 index 00000000..7a762306 --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcDecryptWide_x86_64.inc @@ -0,0 +1,110 @@ +// AES-NI CBC decryption, parallel 8-wide + fused chain XOR kernel (x86-64). +// ------------------------------------------------------------------- +// +// Fused CBC-decrypt body for TCbcBlockCipher's bulk path: +// P_i = D_K(C_i) xor C_{i-1} (C_{-1} = IV). +// Unlike CBC encrypt, the D_K(C_i) are mutually independent, so 8 blocks +// run through one shared 8-wide aesdec chain (AesNiEightRoundsOnly, decrypt +// variant); the only serial part is the trailing XOR, and every feed is an +// *input* ciphertext block. Folding that XOR into the decrypt pass turns the +// mode's decrypt-then-XOR (two passes over memory) into a single pass. A +// 1-wide tail (AesNiOneRoundsOnly) drains the 1..7 residual blocks, so the +// kernel accepts any block count >= 1. +// +// In-place safe: within each 8-block group every input read (8 block loads + +// 7 feedback reloads + the next chain seed) completes before the first output +// store, so OutPtr may alias InPtr. +// +// Entry (after ClpSimdProc1Begin_x86_64.inc): rcx = pointer to +// TCbcFusedCtx. The kernel unpacks it (InPtr loaded into r11 last): +// [rcx + 0] InPtr -> r11 : input (advances +16 per block) +// [rcx + 8] OutPtr -> rdx : output (advances +16; may alias InPtr) +// [rcx + 16] KeysPtr -> r8 : AES *decrypt* (inverse-MixColumns) schedule +// [rcx + 24] IvPtr -> r9 : 16-byte chaining value (seed + writeback) +// [rcx + 32] BlockCount -> r10 : number of blocks, caller guarantees >= 1 +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined, +// together with CRYPTOLIB_AESNI_DECRYPT (selects aesdec in the shared chains). +// +// XMM: xmm0..xmm7 = 8 data lanes; xmm8 = shared round-key scratch (owned by +// AesNiEightRoundsOnly); xmm9 = chaining value (persists across groups); +// xmm10 = feedback reload scratch. xmm6..xmm10 are nonvolatile on Win64, so +// the ClpSimdNonVolatile{Save,Restore} frame brackets the body. +// +// AES-NI instructions and pxor/movdqa forms touching xmm8..xmm15 are +// db-encoded for broad assembler compatibility. +{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + // ---- unpack TCbcFusedCtx (rcx = PCtx); load InPtr into r11 last ---- + mov r8, [rcx + 16] // KeysPtr (decrypt schedule) + mov r9, [rcx + 24] // IvPtr + mov r10, [rcx + 32] // BlockCount + mov rdx, [rcx + 8] // OutPtr + mov r11, [rcx + 0] // InPtr + + movdqu xmm9, [r9] // chain = C_{-1} (the IV) + + cmp r10, 8 + jb @@CbcDecTail + +@@CbcDecWide: + movdqu xmm0, [r11 + 0] // C_0 .. C_7 + movdqu xmm1, [r11 + 16] + movdqu xmm2, [r11 + 32] + movdqu xmm3, [r11 + 48] + movdqu xmm4, [r11 + 64] + movdqu xmm5, [r11 + 80] + movdqu xmm6, [r11 + 96] + movdqu xmm7, [r11 + 112] +{$I ..\..\..\Include\Simd\Aes\AesNiEightRoundsOnly_x86_64.inc} // xmm0..7 = D_K(C_i) + // feedback: P_i = D_K(C_i) xor C_{i-1}. C_0..C_6 reloaded from the (still + // intact) input; block 0 uses the running chain. All reads precede stores. + movdqu xmm10, [r11 + 0] + db $66, $41, $0F, $EF, $CA // pxor xmm1, xmm10 (^ C_0) + movdqu xmm10, [r11 + 16] + db $66, $41, $0F, $EF, $D2 // pxor xmm2, xmm10 (^ C_1) + movdqu xmm10, [r11 + 32] + db $66, $41, $0F, $EF, $DA // pxor xmm3, xmm10 (^ C_2) + movdqu xmm10, [r11 + 48] + db $66, $41, $0F, $EF, $E2 // pxor xmm4, xmm10 (^ C_3) + movdqu xmm10, [r11 + 64] + db $66, $41, $0F, $EF, $EA // pxor xmm5, xmm10 (^ C_4) + movdqu xmm10, [r11 + 80] + db $66, $41, $0F, $EF, $F2 // pxor xmm6, xmm10 (^ C_5) + movdqu xmm10, [r11 + 96] + db $66, $41, $0F, $EF, $FA // pxor xmm7, xmm10 (^ C_6) + db $66, $41, $0F, $EF, $C1 // pxor xmm0, xmm9 (^ chain = C_{-1}) + movdqu xmm9, [r11 + 112] // next chain = C_7 (before any store) + movdqu [rdx + 0], xmm0 // P_0 .. P_7 + movdqu [rdx + 16], xmm1 + movdqu [rdx + 32], xmm2 + movdqu [rdx + 48], xmm3 + movdqu [rdx + 64], xmm4 + movdqu [rdx + 80], xmm5 + movdqu [rdx + 96], xmm6 + movdqu [rdx + 112], xmm7 + add r11, 128 + add rdx, 128 + sub r10, 8 + cmp r10, 8 + jae @@CbcDecWide + +@@CbcDecTail: + test r10, r10 + jz @@CbcDecDone +@@CbcDecTailLoop: + movdqu xmm0, [r11] // C_i + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 (save C_i) +{$I ..\..\..\Include\Simd\Aes\AesNiOneRoundsOnly_x86_64.inc} // xmm0 = D_K(C_i) + db $66, $41, $0F, $EF, $C1 // pxor xmm0, xmm9 (^ chain) + db $66, $45, $0F, $6F, $CA // movdqa xmm9, xmm10 (chain = C_i) + movdqu [rdx], xmm0 // P_i + add r11, 16 + add rdx, 16 + sub r10, 1 + jnz @@CbcDecTailLoop + +@@CbcDecDone: + movdqu [r9], xmm9 // write final chain (= last C) back + db $66, $0F, $EF, $C0 // pxor xmm0, xmm0 (scrub last plaintext) + db $66, $0F, $EF, $C9 // pxor xmm1, xmm1 (scrub) +{$I ..\..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc index b64f838f..533b9508 100644 --- a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc @@ -12,7 +12,7 @@ // i386 has only xmm0..7 (all volatile); the chain uses xmm1. ebx (pushed by the // prologue), esi and edi are callee-saved and restored before return. // -// Entry (after ClpSimdProc1Begin_i386.inc): ebx = pointer to TCbcEncryptFusedCtx. The +// Entry (after ClpSimdProc1Begin_i386.inc): ebx = pointer to TCbcFusedCtx. The // kernel pushes esi/edi and unpacks the record: // [ebx + 0] InPtr -> esi : input (advances +16 per block) // [ebx + 4] OutPtr -> edx : output (advances +16) @@ -25,7 +25,7 @@ // // AES-NI instructions and the register pxor forms are db-encoded for broad // assembler compatibility. - // ---- unpack TCbcEncryptFusedCtx (ebx = PCtx) ---- + // ---- unpack TCbcFusedCtx (ebx = PCtx) ---- push esi push edi mov esi, [ebx + 0] // InPtr diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc index 1dac946e..356b1b2c 100644 --- a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc @@ -12,7 +12,7 @@ // Every working register (rcx,rdx,r8,r9,r10,r11,xmm0,xmm1) is volatile on the // MS x64 ABI, so this stays a true leaf: no xmm6-15 save, no callee-saved push. // -// Entry (after ClpSimdProc1Begin_x86_64.inc): rcx = pointer to TCbcEncryptFusedCtx. +// Entry (after ClpSimdProc1Begin_x86_64.inc): rcx = pointer to TCbcFusedCtx. // The kernel unpacks it (InPtr loaded into r11): // [rcx + 0] InPtr -> r11 : input (advances +16 per block) // [rcx + 8] OutPtr -> rdx : output (advances +16) @@ -25,7 +25,7 @@ // // AES-NI instructions and the register pxor forms are db-encoded for broad // assembler compatibility. - // ---- unpack TCbcEncryptFusedCtx (rcx = PCtx); load InPtr into r11 last ---- + // ---- unpack TCbcFusedCtx (rcx = PCtx); load InPtr into r11 last ---- mov r8, [rcx + 16] // KeysPtr mov r9, [rcx + 24] // IvPtr mov r10, [rcx + 32] // BlockCount