diff --git a/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr b/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr index f1ccdfd7..ad1ad510 100644 --- a/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr +++ b/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr @@ -658,6 +658,7 @@ uses ClpIBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBlockCipherMode.pas', ClpIBulkBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBulkBlockCipherMode.pas', ClpIBulkBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkBlockCipher.pas', + ClpIBulkStreamCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkStreamCipher.pas', ClpIEcbBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIEcbBlockCipher.pas', ClpEcbBlockCipher in '..\..\CryptoLib\src\Crypto\Modes\ClpEcbBlockCipher.pas', ClpGnuObjectIdentifiers in '..\..\CryptoLib\src\Asn1\Gnu\ClpGnuObjectIdentifiers.pas', diff --git a/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr b/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr index a370cb5d..a0be813e 100644 --- a/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr +++ b/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr @@ -674,6 +674,7 @@ uses ClpIBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBlockCipherMode.pas', ClpIBulkBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBulkBlockCipherMode.pas', ClpIBulkBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkBlockCipher.pas', + ClpIBulkStreamCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkStreamCipher.pas', ClpIEcbBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIEcbBlockCipher.pas', ClpEcbBlockCipher in '..\..\CryptoLib\src\Crypto\Modes\ClpEcbBlockCipher.pas', ClpGnuObjectIdentifiers in '..\..\CryptoLib\src\Asn1\Gnu\ClpGnuObjectIdentifiers.pas', diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr index 575fd462..35178dd1 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr @@ -655,6 +655,7 @@ uses ClpIBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBlockCipherMode.pas', ClpIBulkBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBulkBlockCipherMode.pas', ClpIBulkBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkBlockCipher.pas', + ClpIBulkStreamCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkStreamCipher.pas', ClpIEcbBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIEcbBlockCipher.pas', ClpEcbBlockCipher in '..\..\CryptoLib\src\Crypto\Modes\ClpEcbBlockCipher.pas', ClpGnuObjectIdentifiers in '..\..\CryptoLib\src\Asn1\Gnu\ClpGnuObjectIdentifiers.pas', @@ -890,7 +891,8 @@ uses SecP256R1FieldTests in '..\src\Math\EC\Custom\Sec\SecP256R1FieldTests.pas', Salsa20Tests in '..\src\Crypto\Salsa20Tests.pas', ChaChaTests in '..\src\Crypto\ChaChaTests.pas', - ChaCha7539ProcessBlocks2Tests in '..\src\Crypto\ChaCha7539ProcessBlocks2Tests.pas', + ChaCha7539Tests in '..\src\Crypto\ChaCha7539Tests.pas', + StreamCipherTestBase in '..\src\Crypto\StreamCipherTestBase.pas', HChaCha20Tests in '..\src\Crypto\HChaCha20Tests.pas', XChaCha20Tests in '..\src\Crypto\XChaCha20Tests.pas', XChaCha20Poly1305Tests in '..\src\Crypto\XChaCha20Poly1305Tests.pas', diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj index f0c8319e..9c815f7a 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj @@ -998,6 +998,7 @@ + @@ -1228,7 +1229,8 @@ - + + diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr index 261747d0..c7e20866 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr @@ -674,6 +674,7 @@ uses ClpIBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBlockCipherMode.pas', ClpIBulkBlockCipherMode in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIBulkBlockCipherMode.pas', ClpIBulkBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkBlockCipher.pas', + ClpIBulkStreamCipher in '..\..\CryptoLib\src\Interfaces\Crypto\ClpIBulkStreamCipher.pas', ClpIEcbBlockCipher in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\ClpIEcbBlockCipher.pas', ClpEcbBlockCipher in '..\..\CryptoLib\src\Crypto\Modes\ClpEcbBlockCipher.pas', ClpGnuObjectIdentifiers in '..\..\CryptoLib\src\Asn1\Gnu\ClpGnuObjectIdentifiers.pas', @@ -909,7 +910,8 @@ uses SecP256R1FieldTests in '..\src\Math\EC\Custom\Sec\SecP256R1FieldTests.pas', Salsa20Tests in '..\src\Crypto\Salsa20Tests.pas', ChaChaTests in '..\src\Crypto\ChaChaTests.pas', - ChaCha7539ProcessBlocks2Tests in '..\src\Crypto\ChaCha7539ProcessBlocks2Tests.pas', + ChaCha7539Tests in '..\src\Crypto\ChaCha7539Tests.pas', + StreamCipherTestBase in '..\src\Crypto\StreamCipherTestBase.pas', HChaCha20Tests in '..\src\Crypto\HChaCha20Tests.pas', XChaCha20Tests in '..\src\Crypto\XChaCha20Tests.pas', XChaCha20Poly1305Tests in '..\src\Crypto\XChaCha20Poly1305Tests.pas', diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi index 4ec355e6..abd65594 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi @@ -79,7 +79,7 @@ - + @@ -618,7 +618,7 @@ - + @@ -845,6 +845,10 @@ + + + + diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr index cf4d0936..16ccb173 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr @@ -18,7 +18,7 @@ IESCipherTests, HMacTests, Pkcs5Tests, HkdfGeneratorTests, ECIESTests, PascalCoinECIESTests, ECNRTests, PrimesTests, ECEncodingTests, GF256AesTests, PaddingTests, DSATests, DeterministicDsaTests, Salsa20Tests, XSalsa20Tests, - ChaChaTests, ChaCha7539ProcessBlocks2Tests, HChaCha20Tests, XChaCha20Tests, + ChaChaTests, ChaCha7539Tests, StreamCipherTestBase, HChaCha20Tests, XChaCha20Tests, XChaCha20Poly1305Tests, StreamCipherResetTests, CTSTests, X25519Tests, X448Tests, Ed25519Tests, Ed448Tests, X25519HigherLevelTests, Ed25519HigherLevelTests, ShortenedDigestTests, Kdf1GeneratorTests, diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi index ce438b02..a7f291f4 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi @@ -39,7 +39,7 @@ - + @@ -577,7 +577,7 @@ - + @@ -804,6 +804,10 @@ + + + + diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr index 059fb17a..35b33c6c 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr @@ -19,7 +19,7 @@ IESCipherTests, HMacTests, Pkcs5Tests, HkdfGeneratorTests, ECIESTests, PascalCoinECIESTests, ECNRTests, PrimesTests, ECEncodingTests, GF256AesTests, PaddingTests, DSATests, DeterministicDsaTests, Salsa20Tests, XSalsa20Tests, - ChaChaTests, ChaCha7539ProcessBlocks2Tests, HChaCha20Tests, XChaCha20Tests, + ChaChaTests, ChaCha7539Tests, StreamCipherTestBase, HChaCha20Tests, XChaCha20Tests, XChaCha20Poly1305Tests, StreamCipherResetTests, CTSTests, X25519Tests, X448Tests, Ed25519Tests, Ed448Tests, X25519HigherLevelTests, Ed25519HigherLevelTests, Curve25519KeyUtilitiesTests, ShortenedDigestTests, diff --git a/CryptoLib.Tests/src/Crypto/AesHardwareEngineTests.pas b/CryptoLib.Tests/src/Crypto/AesHardwareEngineTests.pas index f2a0b90e..95a33532 100644 --- a/CryptoLib.Tests/src/Crypto/AesHardwareEngineTests.pas +++ b/CryptoLib.Tests/src/Crypto/AesHardwareEngineTests.pas @@ -43,9 +43,10 @@ interface type /// /// Engine-agnostic test suite for hardware-accelerated AES engines. All the - /// vector / Monte-Carlo / engine-check coverage plus the SIMD batch (4-/8-wide) - /// parity and memory-layout coverage lives here, written against - /// . A concrete per-architecture suite only + /// vector / Monte-Carlo / engine-check coverage plus the bulk batch parity + /// (ProcessBlocks 4-/8-wide vs single-block) and single-block memory-layout + /// coverage lives here, written against (bulk + /// via its inherited IBulkBlockCipher). A concrete per-architecture suite only /// has to supply four hooks (CreateHwEngine / EngineSupported / EngineLabel / /// GetEngineFactory) and register itself under its architecture guard; the /// published tests are inherited and discovered automatically. @@ -62,8 +63,6 @@ TAesHardwareEngineTestBase = class abstract(TAesBlockCipherTestBase) procedure ImplTestEightBlocks(AForEncryption: Boolean; AKeySizeBytes: Int32); procedure ImplTestPByteOverloadParity(AKeySizeBytes: Int32); procedure ImplTestProcessBlockMemoryLayouts(AKeySizeBytes: Int32); - procedure ImplTestProcessFourBlocksMemoryLayouts(AKeySizeBytes: Int32); - procedure ImplTestProcessEightBlocksMemoryLayouts(AKeySizeBytes: Int32); published // TestBlockCipherVector / TestMonteCarloAES / TestBadParameters are // inherited from TAesBlockCipherTestBase (guarded by EngineSupported). @@ -88,12 +87,6 @@ TAesHardwareEngineTestBase = class abstract(TAesBlockCipherTestBase) procedure TestProcessBlockMemoryLayouts128; procedure TestProcessBlockMemoryLayouts192; procedure TestProcessBlockMemoryLayouts256; - procedure TestProcessFourBlocksMemoryLayouts128; - procedure TestProcessFourBlocksMemoryLayouts192; - procedure TestProcessFourBlocksMemoryLayouts256; - procedure TestProcessEightBlocksMemoryLayouts128; - procedure TestProcessEightBlocksMemoryLayouts192; - procedure TestProcessEightBlocksMemoryLayouts256; end; {$IFDEF CRYPTOLIB_X86_SIMD} @@ -143,13 +136,13 @@ procedure TAesHardwareEngineTestBase.ImplTestFourBlocks(AForEncryption: Boolean; LRnd.NextBytes(LKey); LEngine := CreateHwEngine; LEngine.Init(AForEncryption, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessFourBlocks(LData, 0, LFourOut, 0); + LEngine.ProcessBlocks(LData, 0, 4, LFourOut, 0); for LJ := 0 to 3 do LEngine.ProcessBlock(LData, LJ * 16, LSingleOut, LJ * 16); if not AreEqual(LFourOut, LSingleOut) then begin Fail(Format( - 'ProcessFourBlocks vs ProcessBlock mismatch (key %d bytes, iteration %d)', + 'ProcessBlocks(4) vs ProcessBlock mismatch (key %d bytes, iteration %d)', [AKeySizeBytes, LI])); end; end; @@ -175,13 +168,13 @@ procedure TAesHardwareEngineTestBase.ImplTestEightBlocks(AForEncryption: Boolean LRnd.NextBytes(LKey); LEngine := CreateHwEngine; LEngine.Init(AForEncryption, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessEightBlocks(LData, 0, LEightOut, 0); + LEngine.ProcessBlocks(LData, 0, 8, LEightOut, 0); for LJ := 0 to 7 do LEngine.ProcessBlock(LData, LJ * 16, LSingleOut, LJ * 16); if not AreEqual(LEightOut, LSingleOut) then begin Fail(Format( - 'ProcessEightBlocks vs ProcessBlock mismatch (key %d bytes, iteration %d)', + 'ProcessBlocks(8) vs ProcessBlock mismatch (key %d bytes, iteration %d)', [AKeySizeBytes, LI])); end; end; @@ -231,31 +224,31 @@ procedure TAesHardwareEngineTestBase.ImplTestPByteOverloadParity(AKeySizeBytes: LRnd.NextBytes(LIn64); LFourArr := System.Copy(LIn64); LFourPtr := System.Copy(LIn64); - LEngine.ProcessFourBlocks(LIn64, 0, LFourArr, 0); - LEngine.ProcessFourBlocks(@LIn64[0], @LFourPtr[0]); + LEngine.ProcessBlocks(LIn64, 0, 4, LFourArr, 0); + LEngine.ProcessBlocks(@LIn64[0], @LFourPtr[0], 4); if not AreEqual(LFourArr, LFourPtr) then - Fail(Format('ProcessFourBlocks PByte disjoint mismatch (key %d, iter %d)', + Fail(Format('ProcessBlocks(4) PByte disjoint mismatch (key %d, iter %d)', [AKeySizeBytes, LI])); LFourPtr := System.Copy(LIn64); - LEngine.ProcessFourBlocks(@LFourPtr[0], @LFourPtr[0]); + LEngine.ProcessBlocks(@LFourPtr[0], @LFourPtr[0], 4); if not AreEqual(LFourArr, LFourPtr) then - Fail(Format('ProcessFourBlocks PByte in-place mismatch (key %d, iter %d)', + Fail(Format('ProcessBlocks(4) PByte in-place mismatch (key %d, iter %d)', [AKeySizeBytes, LI])); LRnd.NextBytes(LIn128); LEightArr := System.Copy(LIn128); LEightPtr := System.Copy(LIn128); - LEngine.ProcessEightBlocks(LIn128, 0, LEightArr, 0); - LEngine.ProcessEightBlocks(@LIn128[0], @LEightPtr[0]); + LEngine.ProcessBlocks(LIn128, 0, 8, LEightArr, 0); + LEngine.ProcessBlocks(@LIn128[0], @LEightPtr[0], 8); if not AreEqual(LEightArr, LEightPtr) then - Fail(Format('ProcessEightBlocks PByte disjoint mismatch (key %d, iter %d)', + Fail(Format('ProcessBlocks(8) PByte disjoint mismatch (key %d, iter %d)', [AKeySizeBytes, LI])); LEightPtr := System.Copy(LIn128); - LEngine.ProcessEightBlocks(@LEightPtr[0], @LEightPtr[0]); + LEngine.ProcessBlocks(@LEightPtr[0], @LEightPtr[0], 8); if not AreEqual(LEightArr, LEightPtr) then - Fail(Format('ProcessEightBlocks PByte in-place mismatch (key %d, iter %d)', + Fail(Format('ProcessBlocks(8) PByte in-place mismatch (key %d, iter %d)', [AKeySizeBytes, LI])); end; end; @@ -365,180 +358,6 @@ procedure TAesHardwareEngineTestBase.ImplTestProcessBlockMemoryLayouts(AKeySizeB end; end; -procedure TAesHardwareEngineTestBase.ImplTestProcessFourBlocksMemoryLayouts(AKeySizeBytes: Int32); -var - LRnd: ISecureRandom; - LKey, LPlain, LCipher, LScratch: TBytes; - LI: Int32; - LEngine: IAesHardwareEngine; - - function MemEq(const A: TBytes; AOff: Int32; const B: TBytes; BOff, ALen: Int32): Boolean; - begin - Result := CompareMem(@A[AOff], @B[BOff], ALen); - end; - -begin - LRnd := TSecureRandom.Create(); - System.SetLength(LKey, AKeySizeBytes); - System.SetLength(LPlain, 64); - System.SetLength(LCipher, 64); - - for LI := 0 to 24 do - begin - LRnd.NextBytes(LKey); - LRnd.NextBytes(LPlain); - LEngine := CreateHwEngine; - LEngine.Init(True, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessFourBlocks(LPlain, 0, LCipher, 0); - - LScratch := System.Copy(LPlain); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[0]); - if not AreEqual(LCipher, LScratch) then - Fail(Format('ProcessFourBlocks enc in-place (key %d, iter %d)', [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 192); - System.Move(LPlain[0], LScratch[0], 64); - System.FillChar(LScratch[64], 128, $5A); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[64]); - if not MemEq(LCipher, 0, LScratch, 64, 64) then - Fail(Format('ProcessFourBlocks enc disjoint same allocation (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 128); - System.Move(LPlain[0], LScratch[0], 64); - System.FillChar(LScratch[64], 64, 0); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[32]); - if not MemEq(LCipher, 0, LScratch, 32, 64) then - Fail(Format('ProcessFourBlocks enc overlap dst=src+32 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.FillChar(LScratch[0], 128, 0); - System.Move(LPlain[0], LScratch[32], 64); - LEngine.ProcessFourBlocks(@LScratch[32], @LScratch[0]); - if not MemEq(LCipher, 0, LScratch, 0, 64) then - Fail(Format('ProcessFourBlocks enc overlap dst=src-32 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - LEngine.Init(False, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessFourBlocks(LCipher, 0, LPlain, 0); - - LScratch := System.Copy(LCipher); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[0]); - if not AreEqual(LPlain, LScratch) then - Fail(Format('ProcessFourBlocks dec in-place (key %d, iter %d)', [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 192); - System.Move(LCipher[0], LScratch[0], 64); - System.FillChar(LScratch[64], 128, $4B); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[64]); - if not MemEq(LPlain, 0, LScratch, 64, 64) then - Fail(Format('ProcessFourBlocks dec disjoint same allocation (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 128); - System.Move(LCipher[0], LScratch[0], 64); - System.FillChar(LScratch[64], 64, $33); - LEngine.ProcessFourBlocks(@LScratch[0], @LScratch[32]); - if not MemEq(LPlain, 0, LScratch, 32, 64) then - Fail(Format('ProcessFourBlocks dec overlap dst=src+32 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.FillChar(LScratch[0], 128, $44); - System.Move(LCipher[0], LScratch[32], 64); - LEngine.ProcessFourBlocks(@LScratch[32], @LScratch[0]); - if not MemEq(LPlain, 0, LScratch, 0, 64) then - Fail(Format('ProcessFourBlocks dec overlap dst=src-32 (key %d, iter %d)', - [AKeySizeBytes, LI])); - end; -end; - -procedure TAesHardwareEngineTestBase.ImplTestProcessEightBlocksMemoryLayouts(AKeySizeBytes: Int32); -var - LRnd: ISecureRandom; - LKey, LPlain, LCipher, LScratch: TBytes; - LI: Int32; - LEngine: IAesHardwareEngine; - - function MemEq(const A: TBytes; AOff: Int32; const B: TBytes; BOff, ALen: Int32): Boolean; - begin - Result := CompareMem(@A[AOff], @B[BOff], ALen); - end; - -begin - LRnd := TSecureRandom.Create(); - System.SetLength(LKey, AKeySizeBytes); - System.SetLength(LPlain, 128); - System.SetLength(LCipher, 128); - - for LI := 0 to 14 do - begin - LRnd.NextBytes(LKey); - LRnd.NextBytes(LPlain); - LEngine := CreateHwEngine; - LEngine.Init(True, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessEightBlocks(LPlain, 0, LCipher, 0); - - LScratch := System.Copy(LPlain); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[0]); - if not AreEqual(LCipher, LScratch) then - Fail(Format('ProcessEightBlocks enc in-place (key %d, iter %d)', [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 384); - System.Move(LPlain[0], LScratch[0], 128); - System.FillChar(LScratch[128], 256, $5A); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[128]); - if not MemEq(LCipher, 0, LScratch, 128, 128) then - Fail(Format('ProcessEightBlocks enc disjoint same allocation (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 256); - System.Move(LPlain[0], LScratch[0], 128); - System.FillChar(LScratch[128], 128, 0); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[64]); - if not MemEq(LCipher, 0, LScratch, 64, 128) then - Fail(Format('ProcessEightBlocks enc overlap dst=src+64 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.FillChar(LScratch[0], 256, 0); - System.Move(LPlain[0], LScratch[64], 128); - LEngine.ProcessEightBlocks(@LScratch[64], @LScratch[0]); - if not MemEq(LCipher, 0, LScratch, 0, 128) then - Fail(Format('ProcessEightBlocks enc overlap dst=src-64 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - LEngine.Init(False, TKeyParameter.Create(LKey) as IKeyParameter); - LEngine.ProcessEightBlocks(LCipher, 0, LPlain, 0); - - LScratch := System.Copy(LCipher); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[0]); - if not AreEqual(LPlain, LScratch) then - Fail(Format('ProcessEightBlocks dec in-place (key %d, iter %d)', [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 384); - System.Move(LCipher[0], LScratch[0], 128); - System.FillChar(LScratch[128], 256, $4B); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[128]); - if not MemEq(LPlain, 0, LScratch, 128, 128) then - Fail(Format('ProcessEightBlocks dec disjoint same allocation (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.SetLength(LScratch, 256); - System.Move(LCipher[0], LScratch[0], 128); - System.FillChar(LScratch[128], 128, $33); - LEngine.ProcessEightBlocks(@LScratch[0], @LScratch[64]); - if not MemEq(LPlain, 0, LScratch, 64, 128) then - Fail(Format('ProcessEightBlocks dec overlap dst=src+64 (key %d, iter %d)', - [AKeySizeBytes, LI])); - - System.FillChar(LScratch[0], 256, $44); - System.Move(LCipher[0], LScratch[64], 128); - LEngine.ProcessEightBlocks(@LScratch[64], @LScratch[0]); - if not MemEq(LPlain, 0, LScratch, 0, 128) then - Fail(Format('ProcessEightBlocks dec overlap dst=src-64 (key %d, iter %d)', - [AKeySizeBytes, LI])); - end; -end; - procedure TAesHardwareEngineTestBase.TestEngineChecks128; var LKey: TBytes; @@ -707,48 +526,6 @@ procedure TAesHardwareEngineTestBase.TestProcessBlockMemoryLayouts256; ImplTestProcessBlockMemoryLayouts(32); end; -procedure TAesHardwareEngineTestBase.TestProcessFourBlocksMemoryLayouts128; -begin - if not EngineSupported then - Exit; - ImplTestProcessFourBlocksMemoryLayouts(16); -end; - -procedure TAesHardwareEngineTestBase.TestProcessFourBlocksMemoryLayouts192; -begin - if not EngineSupported then - Exit; - ImplTestProcessFourBlocksMemoryLayouts(24); -end; - -procedure TAesHardwareEngineTestBase.TestProcessFourBlocksMemoryLayouts256; -begin - if not EngineSupported then - Exit; - ImplTestProcessFourBlocksMemoryLayouts(32); -end; - -procedure TAesHardwareEngineTestBase.TestProcessEightBlocksMemoryLayouts128; -begin - if not EngineSupported then - Exit; - ImplTestProcessEightBlocksMemoryLayouts(16); -end; - -procedure TAesHardwareEngineTestBase.TestProcessEightBlocksMemoryLayouts192; -begin - if not EngineSupported then - Exit; - ImplTestProcessEightBlocksMemoryLayouts(24); -end; - -procedure TAesHardwareEngineTestBase.TestProcessEightBlocksMemoryLayouts256; -begin - if not EngineSupported then - Exit; - ImplTestProcessEightBlocksMemoryLayouts(32); -end; - {$IFDEF CRYPTOLIB_X86_SIMD} function CreateAesX86Engine: IBlockCipher; diff --git a/CryptoLib.Tests/src/Crypto/ChaCha7539ProcessBlocks2Tests.pas b/CryptoLib.Tests/src/Crypto/ChaCha7539ProcessBlocks2Tests.pas deleted file mode 100644 index 526c8e65..00000000 --- a/CryptoLib.Tests/src/Crypto/ChaCha7539ProcessBlocks2Tests.pas +++ /dev/null @@ -1,184 +0,0 @@ -{ *********************************************************************************** } -{ * CryptoLib Library * } -{ * Author - Ugochukwu Mmaduekwe * } -{ * Github Repository * } -{ * * } -{ * Distributed under the MIT software license, see the accompanying file LICENSE * } -{ * or visit http://www.opensource.org/licenses/mit-license.php. * } -{ * * } -{ * Acknowledgements: * } -{ * * } -{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } -{ * the development of this library * } -{ * ******************************************************************************* * } - -(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) - -unit ChaCha7539ProcessBlocks2Tests; - -interface - -{$IFDEF FPC} -{$MODE DELPHI} -{$ENDIF FPC} - -uses - SysUtils, -{$IFDEF FPC} - fpcunit, - testregistry, -{$ELSE} - TestFramework, -{$ENDIF FPC} - ClpChaCha7539Engine, - ClpIKeyParameter, - ClpKeyParameter, - ClpParametersWithIV, - ClpIParametersWithIV, - CryptoLibTestBase; - -type - - TTestChaCha7539ProcessBlocks2 = class(TCryptoLibAlgorithmTestCase) - published - procedure Test128ByteProcessBlocks2VsTwoBlockCalls; - procedure Test256ByteProcessBlocks4VsTwoProcessBlocks2; - procedure TestProcessBytesVsReturnByte; - end; - -implementation - -{ TTestChaCha7539ProcessBlocks2 } - -procedure TTestChaCha7539ProcessBlocks2.Test128ByteProcessBlocks2VsTwoBlockCalls; -var - LKey, LNonce, LIn, LOutA, LOutB: TBytes; - LParams: IParametersWithIV; - LA, LB: TChaCha7539Engine; - LIdx: Int32; -begin - LKey := DecodeHex('000102030405060708090A0B0C0D0E0F' + - '101112131415161718191A1B1C1D1E1F'); - LNonce := DecodeHex('00000000000000000000000A'); - LParams := TParametersWithIV.Create( - TKeyParameter.Create(LKey) as IKeyParameter, LNonce); - SetLength(LIn, 128); - for LIdx := 0 to 127 do - begin - LIn[LIdx] := Byte(LIdx); - end; - SetLength(LOutA, 128); - SetLength(LOutB, 128); - LA := TChaCha7539Engine.Create; - try - LA.Init(True, LParams); - LA.ProcessBlocks2(LIn, 0, LOutA, 0); - finally - LA.Free; - end; - LB := TChaCha7539Engine.Create; - try - LB.Init(True, LParams); - LB.ProcessBlock(LIn, 0, LOutB, 0); - LB.ProcessBlock(LIn, 64, LOutB, 64); - finally - LB.Free; - end; - if not AreEqual(LOutA, LOutB) then - begin - Fail('ChaCha7539 128B stream mismatch'); - end; -end; - -procedure TTestChaCha7539ProcessBlocks2.Test256ByteProcessBlocks4VsTwoProcessBlocks2; -var - LKey, LNonce, LIn, LOutA, LOutB: TBytes; - LParams: IParametersWithIV; - LA, LB: TChaCha7539Engine; - LIdx: Int32; -begin - LKey := DecodeHex('000102030405060708090A0B0C0D0E0F' + - '101112131415161718191A1B1C1D1E1F'); - LNonce := DecodeHex('00000000000000000000000A'); - LParams := TParametersWithIV.Create( - TKeyParameter.Create(LKey) as IKeyParameter, LNonce); - SetLength(LIn, 256); - for LIdx := 0 to 255 do - begin - LIn[LIdx] := Byte(LIdx); - end; - SetLength(LOutA, 256); - SetLength(LOutB, 256); - LA := TChaCha7539Engine.Create; - try - LA.Init(True, LParams); - LA.ProcessBlocks4(LIn, 0, LOutA, 0); - finally - LA.Free; - end; - LB := TChaCha7539Engine.Create; - try - LB.Init(True, LParams); - LB.ProcessBlocks2(LIn, 0, LOutB, 0); - LB.ProcessBlocks2(LIn, 128, LOutB, 128); - finally - LB.Free; - end; - if not AreEqual(LOutA, LOutB) then - begin - Fail('ChaCha7539 256B ProcessBlocks4 vs two ProcessBlocks2 mismatch'); - end; -end; - -procedure TTestChaCha7539ProcessBlocks2.TestProcessBytesVsReturnByte; -const - CLen = 300; -var - LKey, LNonce, LIn, LOutA, LOutB: TBytes; - LParams: IParametersWithIV; - LA, LB: TChaCha7539Engine; - LIdx: Int32; -begin - LKey := DecodeHex('000102030405060708090A0B0C0D0E0F' + - '101112131415161718191A1B1C1D1E1F'); - LNonce := DecodeHex('00000000000000000000000A'); - LParams := TParametersWithIV.Create( - TKeyParameter.Create(LKey) as IKeyParameter, LNonce); - SetLength(LIn, CLen); - for LIdx := 0 to CLen - 1 do - begin - LIn[LIdx] := Byte(LIdx * 3 + 17); - end; - SetLength(LOutA, CLen); - SetLength(LOutB, CLen); - LA := TChaCha7539Engine.Create; - try - LA.Init(True, LParams); - LA.ProcessBytes(LIn, 0, CLen, LOutA, 0); - finally - LA.Free; - end; - LB := TChaCha7539Engine.Create; - try - LB.Init(True, LParams); - for LIdx := 0 to CLen - 1 do - begin - LOutB[LIdx] := Byte(LB.ReturnByte(LIn[LIdx])); - end; - finally - LB.Free; - end; - if not AreEqual(LOutA, LOutB) then - begin - Fail('ChaCha7539 ProcessBytes vs ReturnByte stream mismatch'); - end; -end; - -initialization -{$IFDEF FPC} - RegisterTest(TTestChaCha7539ProcessBlocks2); -{$ELSE} - RegisterTest(TTestChaCha7539ProcessBlocks2.Suite); -{$ENDIF FPC} - -end. diff --git a/CryptoLib.Tests/src/Crypto/ChaCha7539Tests.pas b/CryptoLib.Tests/src/Crypto/ChaCha7539Tests.pas new file mode 100644 index 00000000..9d287684 --- /dev/null +++ b/CryptoLib.Tests/src/Crypto/ChaCha7539Tests.pas @@ -0,0 +1,84 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ChaCha7539Tests; + +interface + +{$IFDEF FPC} +{$MODE DELPHI} +{$ENDIF FPC} + +uses +{$IFDEF FPC} + testregistry, +{$ELSE} + TestFramework, +{$ENDIF FPC} + ClpIStreamCipher, + ClpChaCha7539Engine, + StreamCipherTestBase; + +type + // RFC 7539 keystream is validated against fixed vectors through the AEAD suites; + // here the shared base covers the engine-agnostic properties (whole-vs-chunked, + // ProcessBytes-vs-ReturnByte, bulk-vs-single-block, SIMD-vs-scalar). + TTestChaCha7539 = class(TStreamCipherTestBase) + strict protected + function GetEngineFactory: TStreamCipherFactory; override; + function EngineLabel: String; override; + function KeySizeInBytes: Int32; override; + function NonceSizeInBytes: Int32; override; + end; + +implementation + +function CreateChaCha7539Engine: IStreamCipher; +begin + Result := TChaCha7539Engine.Create() as IStreamCipher; +end; + +{ TTestChaCha7539 } + +function TTestChaCha7539.GetEngineFactory: TStreamCipherFactory; +begin + Result := CreateChaCha7539Engine; +end; + +function TTestChaCha7539.EngineLabel: String; +begin + Result := 'ChaCha7539'; +end; + +function TTestChaCha7539.KeySizeInBytes: Int32; +begin + Result := 32; +end; + +function TTestChaCha7539.NonceSizeInBytes: Int32; +begin + Result := 12; +end; + +initialization + +{$IFDEF FPC} + RegisterTest(TTestChaCha7539); +{$ELSE} + RegisterTest(TTestChaCha7539.Suite); +{$ENDIF FPC} + +end. diff --git a/CryptoLib.Tests/src/Crypto/ChaChaTests.pas b/CryptoLib.Tests/src/Crypto/ChaChaTests.pas index 62675214..d020089c 100644 --- a/CryptoLib.Tests/src/Crypto/ChaChaTests.pas +++ b/CryptoLib.Tests/src/Crypto/ChaChaTests.pas @@ -32,12 +32,14 @@ interface {$ENDIF FPC} ClpChaChaEngine, ClpIChaChaEngine, + ClpIStreamCipher, ClpICipherParameters, ClpKeyParameter, ClpIKeyParameter, ClpParametersWithIV, ClpIParametersWithIV, ClpCryptoLibTypes, + StreamCipherTestBase, CryptoLibTestBase, ChaChaPoly1305Vectors; @@ -52,7 +54,12 @@ interface /// estreambench-20080905. /// /// - TTestChaCha = class(TCryptoLibAlgorithmTestCase) + TTestChaCha = class(TStreamCipherTestBase) + strict protected + function GetEngineFactory: TStreamCipherFactory; override; + function EngineLabel: String; override; + function KeySizeInBytes: Int32; override; + function NonceSizeInBytes: Int32; override; private var FZeroes: TBytes; @@ -79,8 +86,33 @@ TTestChaCha = class(TCryptoLibAlgorithmTestCase) implementation +function CreateChaChaEngine: IStreamCipher; +begin + Result := TChaChaEngine.Create() as IStreamCipher; +end; + { TTestChaCha } +function TTestChaCha.GetEngineFactory: TStreamCipherFactory; +begin + Result := CreateChaChaEngine; +end; + +function TTestChaCha.EngineLabel: String; +begin + Result := 'ChaCha20'; +end; + +function TTestChaCha.KeySizeInBytes: Int32; +begin + Result := 32; +end; + +function TTestChaCha.NonceSizeInBytes: Int32; +begin + Result := 8; +end; + function TTestChaCha.GetCheckpointHex(const ASet: TChaChaKeystreamSet; AByteOffset: Integer): string; var diff --git a/CryptoLib.Tests/src/Crypto/Salsa20Tests.pas b/CryptoLib.Tests/src/Crypto/Salsa20Tests.pas index f56c8752..0778cb09 100644 --- a/CryptoLib.Tests/src/Crypto/Salsa20Tests.pas +++ b/CryptoLib.Tests/src/Crypto/Salsa20Tests.pas @@ -32,17 +32,24 @@ interface {$ENDIF FPC} ClpSalsa20Engine, ClpISalsa20Engine, + ClpIStreamCipher, ClpICipherParameters, ClpKeyParameter, ClpIKeyParameter, ClpParametersWithIV, ClpIParametersWithIV, ClpCryptoLibTypes, + StreamCipherTestBase, CryptoLibTestBase; type - TTestSalsa20 = class(TCryptoLibAlgorithmTestCase) + TTestSalsa20 = class(TStreamCipherTestBase) + strict protected + function GetEngineFactory: TStreamCipherFactory; override; + function EngineLabel: String; override; + function KeySizeInBytes: Int32; override; + function NonceSizeInBytes: Int32; override; private var FZeroes: TBytes; @@ -68,14 +75,38 @@ TTestSalsa20 = class(TCryptoLibAlgorithmTestCase) procedure TestSalsa20Test1; procedure TestSalsa20Test2; procedure TestReInitBug; - procedure TestProcessBytesVsReturnByte; end; implementation +function CreateSalsa20Engine: IStreamCipher; +begin + Result := TSalsa20Engine.Create() as IStreamCipher; +end; + { TTestSalsa20 } +function TTestSalsa20.GetEngineFactory: TStreamCipherFactory; +begin + Result := CreateSalsa20Engine; +end; + +function TTestSalsa20.EngineLabel: String; +begin + Result := 'Salsa20'; +end; + +function TTestSalsa20.KeySizeInBytes: Int32; +begin + Result := 32; +end; + +function TTestSalsa20.NonceSizeInBytes: Int32; +begin + Result := 8; +end; + procedure TTestSalsa20.SetUp; begin inherited; @@ -354,42 +385,6 @@ procedure TTestSalsa20.TestReInitBug; end; end; -procedure TTestSalsa20.TestProcessBytesVsReturnByte; -const - CLen = 400; -var - LKey, LNonce, LIn, LOutA, LOutB: TBytes; - LParams: IParametersWithIV; - LA, LB: ISalsa20Engine; - LIdx: Int32; -begin - LKey := DecodeHex('000102030405060708090A0B0C0D0E0F' + - '101112131415161718191A1B1C1D1E1F'); - LNonce := DecodeHex('0000000000000000'); - LParams := TParametersWithIV.Create( - TKeyParameter.Create(LKey) as IKeyParameter, LNonce); - SetLength(LIn, CLen); - for LIdx := 0 to CLen - 1 do - begin - LIn[LIdx] := Byte(LIdx * 5 + 11); - end; - SetLength(LOutA, CLen); - SetLength(LOutB, CLen); - LA := TSalsa20Engine.Create(); - LA.Init(True, LParams); - LA.ProcessBytes(LIn, 0, CLen, LOutA, 0); - LB := TSalsa20Engine.Create(); - LB.Init(True, LParams); - for LIdx := 0 to CLen - 1 do - begin - LOutB[LIdx] := Byte(LB.ReturnByte(LIn[LIdx])); - end; - if not AreEqual(LOutA, LOutB) then - begin - Fail('Salsa20 ProcessBytes vs ReturnByte stream mismatch'); - end; -end; - initialization // Register any test cases with the test runner diff --git a/CryptoLib.Tests/src/Crypto/StreamCipherTestBase.pas b/CryptoLib.Tests/src/Crypto/StreamCipherTestBase.pas new file mode 100644 index 00000000..a92f6fdb --- /dev/null +++ b/CryptoLib.Tests/src/Crypto/StreamCipherTestBase.pas @@ -0,0 +1,213 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit StreamCipherTestBase; + +interface + +{$IFDEF FPC} +{$MODE DELPHI} +{$ENDIF FPC} + +uses + SysUtils, +{$IFDEF FPC} + fpcunit, +{$ELSE} + TestFramework, +{$ENDIF FPC} + ClpIStreamCipher, + ClpIBulkStreamCipher, + ClpICipherParameters, + ClpKeyParameter, + ClpIKeyParameter, + ClpParametersWithIV, + ClpIParametersWithIV, + ClpCryptoLibTypes, + CryptoLibTestBase; + +type + TStreamCipherFactory = function: IStreamCipher; + + /// + /// Base for the Salsa/ChaCha stream-cipher suites. A concrete suite supplies an + /// engine factory + label + key/nonce sizes; the reusable runners below cover the + /// engine-agnostic properties (whole-vs-chunked, ProcessBytes-vs-ReturnByte, + /// bulk-vs-single-block). Concrete suites add their own KAT vectors. + /// + TStreamCipherTestBase = class(TCryptoLibAlgorithmTestCase) + strict protected + function GetEngineFactory: TStreamCipherFactory; virtual; abstract; + function EngineLabel: String; virtual; abstract; + function KeySizeInBytes: Int32; virtual; abstract; + function NonceSizeInBytes: Int32; virtual; abstract; + + function SampleBytes(ALen, ASeed: Int32): TBytes; + function InitEngine(const ACreateEngine: TStreamCipherFactory; + const AKey, ANonce: TBytes): IStreamCipher; + + // Reusable property checks, each taking the factory + label (house pattern). + procedure RunWholeVsChunked(const ACreateEngine: TStreamCipherFactory; + const AEngineLabel: String); + procedure RunProcessBytesVsReturnByte(const ACreateEngine: TStreamCipherFactory; + const AEngineLabel: String); + procedure RunBulkBlocksVsSingleBlock(const ACreateEngine: TStreamCipherFactory; + const AEngineLabel: String); + published + procedure TestWholeVsChunked; + procedure TestProcessBytesVsReturnByte; + procedure TestBulkBlocksVsSingleBlock; + end; + +implementation + +{ TStreamCipherTestBase } + +function TStreamCipherTestBase.SampleBytes(ALen, ASeed: Int32): TBytes; +var + LI: Int32; +begin + System.SetLength(Result, ALen); + for LI := 0 to ALen - 1 do + Result[LI] := Byte((LI * ASeed + (ASeed shr 1)) and $FF); +end; + +function TStreamCipherTestBase.InitEngine( + const ACreateEngine: TStreamCipherFactory; const AKey, ANonce: TBytes): IStreamCipher; +begin + Result := ACreateEngine(); + Result.Init(True, TParametersWithIV.Create( + TKeyParameter.Create(AKey) as IKeyParameter, ANonce) as IParametersWithIV); +end; + +// Processing a buffer in one call must equal processing it in ragged chunks. +procedure TStreamCipherTestBase.RunWholeVsChunked( + const ACreateEngine: TStreamCipherFactory; const AEngineLabel: String); +const + LTotal = 1500; +var + LKey, LNonce, LIn, LWhole, LChunk: TBytes; + LEng: IStreamCipher; + LOff, LStep: Int32; +begin + LKey := SampleBytes(KeySizeInBytes, 7); + LNonce := SampleBytes(NonceSizeInBytes, 11); + LIn := SampleBytes(LTotal, 31); + System.SetLength(LWhole, LTotal); + System.SetLength(LChunk, LTotal); + + LEng := InitEngine(ACreateEngine, LKey, LNonce); + LEng.ProcessBytes(LIn, 0, LTotal, LWhole, 0); + + LEng := InitEngine(ACreateEngine, LKey, LNonce); + LOff := 0; + LStep := 1; + while LOff < LTotal do + begin + if LStep > (LTotal - LOff) then + LStep := LTotal - LOff; + LEng.ProcessBytes(LIn, LOff, LStep, LChunk, LOff); + LOff := LOff + LStep; + LStep := LStep + 7; // ragged, block-straddling chunk sizes + end; + + if not AreEqual(LWhole, LChunk) then + Fail(Format('%s: whole vs chunked ProcessBytes mismatch', [AEngineLabel])); +end; + +// ProcessBytes over the whole buffer must equal ReturnByte one byte at a time. +procedure TStreamCipherTestBase.RunProcessBytesVsReturnByte( + const ACreateEngine: TStreamCipherFactory; const AEngineLabel: String); +const + LTotal = 517; +var + LKey, LNonce, LIn, LBulk, LByByte: TBytes; + LEng: IStreamCipher; + LI: Int32; +begin + LKey := SampleBytes(KeySizeInBytes, 7); + LNonce := SampleBytes(NonceSizeInBytes, 11); + LIn := SampleBytes(LTotal, 31); + System.SetLength(LBulk, LTotal); + System.SetLength(LByByte, LTotal); + + LEng := InitEngine(ACreateEngine, LKey, LNonce); + LEng.ProcessBytes(LIn, 0, LTotal, LBulk, 0); + + LEng := InitEngine(ACreateEngine, LKey, LNonce); + for LI := 0 to LTotal - 1 do + LByByte[LI] := LEng.ReturnByte(LIn[LI]); + + if not AreEqual(LBulk, LByByte) then + Fail(Format('%s: ProcessBytes vs ReturnByte mismatch', [AEngineLabel])); +end; + +// IBulkStreamCipher.ProcessBlocks(N) must equal N single 64B block transforms. +procedure TStreamCipherTestBase.RunBulkBlocksVsSingleBlock( + const ACreateEngine: TStreamCipherFactory; const AEngineLabel: String); +const + LCounts: array [0 .. 9] of Int32 = (1, 2, 3, 4, 5, 7, 8, 9, 16, 33); +var + LKey, LNonce, LIn, LBulkOut, LRefOut: TBytes; + LBulk: IBulkStreamCipher; + LRef: IStreamCipher; + LCount, LLen, LOff: Int32; +begin + if not Supports(ACreateEngine(), IBulkStreamCipher) then + Exit; // engine has no bulk fast path; nothing to compare + + LKey := SampleBytes(KeySizeInBytes, 7); + LNonce := SampleBytes(NonceSizeInBytes, 11); + for LCount in LCounts do + begin + LLen := LCount * 64; + LIn := SampleBytes(LLen, 31); + System.SetLength(LBulkOut, LLen); + System.SetLength(LRefOut, LLen); + + LBulk := InitEngine(ACreateEngine, LKey, LNonce) as IBulkStreamCipher; + LBulk.ProcessBlocks(LIn, 0, LCount, LBulkOut, 0); + + LRef := InitEngine(ACreateEngine, LKey, LNonce); + LOff := 0; + while LOff < LLen do + begin + LRef.ProcessBytes(LIn, LOff, 64, LRefOut, LOff); + LOff := LOff + 64; + end; + + if not AreEqual(LBulkOut, LRefOut) then + Fail(Format('%s: ProcessBlocks(%d) vs single-block mismatch', + [AEngineLabel, LCount])); + end; +end; + +procedure TStreamCipherTestBase.TestWholeVsChunked; +begin + RunWholeVsChunked(GetEngineFactory(), EngineLabel); +end; + +procedure TStreamCipherTestBase.TestProcessBytesVsReturnByte; +begin + RunProcessBytesVsReturnByte(GetEngineFactory(), EngineLabel); +end; + +procedure TStreamCipherTestBase.TestBulkBlocksVsSingleBlock; +begin + RunBulkBlocksVsSingleBlock(GetEngineFactory(), EngineLabel); +end; + +end. diff --git a/CryptoLib.Tests/src/Crypto/XChaCha20Tests.pas b/CryptoLib.Tests/src/Crypto/XChaCha20Tests.pas index 3bf215d0..a5822d79 100644 --- a/CryptoLib.Tests/src/Crypto/XChaCha20Tests.pas +++ b/CryptoLib.Tests/src/Crypto/XChaCha20Tests.pas @@ -45,11 +45,17 @@ interface ClpSecureRandom, ClpISecureRandom, ClpCryptoLibTypes, + StreamCipherTestBase, CryptoLibTestBase, ChaChaPoly1305Vectors; type - TTestXChaCha20 = class(TCryptoLibAlgorithmTestCase) + TTestXChaCha20 = class(TStreamCipherTestBase) + strict protected + function GetEngineFactory: TStreamCipherFactory; override; + function EngineLabel: String; override; + function KeySizeInBytes: Int32; override; + function NonceSizeInBytes: Int32; override; published procedure TestAppendixA2_1KeystreamFirst288; procedure TestAppendixA3_2_1KeystreamFull304; @@ -61,7 +67,6 @@ TTestXChaCha20 = class(TCryptoLibAlgorithmTestCase) procedure TestHChaCha20Indirect; procedure TestDraftStreamVector; procedure TestRandomRoundTrip; - procedure TestChunkedProcessing; procedure TestRejectShortNonce64Bits; procedure TestRejectShortNonce96Bits; procedure TestRejectShortNonce23Bytes; @@ -70,8 +75,33 @@ TTestXChaCha20 = class(TCryptoLibAlgorithmTestCase) implementation +function CreateXChaCha20Engine: IStreamCipher; +begin + Result := TXChaCha20Engine.Create() as IStreamCipher; +end; + { TTestXChaCha20 } +function TTestXChaCha20.GetEngineFactory: TStreamCipherFactory; +begin + Result := CreateXChaCha20Engine; +end; + +function TTestXChaCha20.EngineLabel: String; +begin + Result := 'XChaCha20'; +end; + +function TTestXChaCha20.KeySizeInBytes: Int32; +begin + Result := 32; +end; + +function TTestXChaCha20.NonceSizeInBytes: Int32; +begin + Result := 24; +end; + procedure TTestXChaCha20.TestAppendixA2_1KeystreamFirst288; var LRow: TXChaChaStreamRow; @@ -411,52 +441,6 @@ procedure TTestXChaCha20.TestRandomRoundTrip; end; end; -procedure TTestXChaCha20.TestChunkedProcessing; -const - ChunkSizes: array[0..8] of Int32 = (1, 7, 32, 63, 64, 65, 127, 128, 129); -var - LKey, LNonce, LPlain, LWhole, LPiecewise: TCryptoLibByteArray; - LEng: IXChaCha20Engine; - LParams: IParametersWithIV; - LRandom: ISecureRandom; - LI, LChunk, LOff, LN: Int32; -begin - LKey := THexEncoder.Decode( - '808182838485868788898a8b8c8d8e8f' + - '909192939495969798999a9b9c9d9e9f'); - LNonce := THexEncoder.Decode( - '404142434445464748494a4b4c4d4e4f5051525354555657'); - System.SetLength(LPlain, 1024); - LRandom := TSecureRandom.Create; - LRandom.NextBytes(LPlain); - - System.SetLength(LWhole, System.Length(LPlain)); - LEng := TXChaCha20Engine.Create; - LParams := TParametersWithIV.Create(TKeyParameter.Create(LKey) as IKeyParameter, - LNonce); - LEng.Init(True, LParams); - LEng.ProcessBytes(LPlain, 0, System.Length(LPlain), LWhole, 0); - - for LI := Low(ChunkSizes) to High(ChunkSizes) do - begin - LChunk := ChunkSizes[LI]; - System.SetLength(LPiecewise, System.Length(LPlain)); - LEng := TXChaCha20Engine.Create; - LEng.Init(True, LParams); - LOff := 0; - while LOff < System.Length(LPlain) do - begin - LN := LChunk; - if LN > System.Length(LPlain) - LOff then - LN := System.Length(LPlain) - LOff; - LEng.ProcessBytes(LPlain, LOff, LN, LPiecewise, LOff); - System.Inc(LOff, LN); - end; - if not AreEqual(LWhole, LPiecewise) then - Fail(Format('chunked processing differs from bulk at chunk size %d', [LChunk])); - end; -end; - procedure TTestXChaCha20.TestRejectShortNonce23Bytes; var LKey, LNonce23: TCryptoLibByteArray; diff --git a/CryptoLib.Tests/src/Crypto/XSalsa20Tests.pas b/CryptoLib.Tests/src/Crypto/XSalsa20Tests.pas index edbdcbcc..56b17c7c 100644 --- a/CryptoLib.Tests/src/Crypto/XSalsa20Tests.pas +++ b/CryptoLib.Tests/src/Crypto/XSalsa20Tests.pas @@ -32,17 +32,24 @@ interface {$ENDIF FPC} ClpXSalsa20Engine, ClpIXSalsa20Engine, + ClpIStreamCipher, ClpKeyParameter, ClpIKeyParameter, ClpParametersWithIV, ClpIParametersWithIV, ClpCryptoLibTypes, + StreamCipherTestBase, CryptoLibTestBase; type - TTestXSalsa20 = class(TCryptoLibAlgorithmTestCase) + TTestXSalsa20 = class(TStreamCipherTestBase) + strict protected + function GetEngineFactory: TStreamCipherFactory; override; + function EngineLabel: String; override; + function KeySizeInBytes: Int32; override; + function NonceSizeInBytes: Int32; override; private var FTestVectors: TCryptoLibGenericArray; @@ -61,8 +68,33 @@ TTestXSalsa20 = class(TCryptoLibAlgorithmTestCase) implementation +function CreateXSalsa20Engine: IStreamCipher; +begin + Result := TXSalsa20Engine.Create() as IStreamCipher; +end; + { TTestXSalsa20 } +function TTestXSalsa20.GetEngineFactory: TStreamCipherFactory; +begin + Result := CreateXSalsa20Engine; +end; + +function TTestXSalsa20.EngineLabel: String; +begin + Result := 'XSalsa20'; +end; + +function TTestXSalsa20.KeySizeInBytes: Int32; +begin + Result := 32; +end; + +function TTestXSalsa20.NonceSizeInBytes: Int32; +begin + Result := 24; +end; + procedure TTestXSalsa20.DoXSalsa20Test(number: Int32; const ATestVectorParams: TCryptoLibStringArray); var diff --git a/CryptoLib/src/Crypto/Engines/ClpAesEngineX86.pas b/CryptoLib/src/Crypto/Engines/ClpAesEngineX86.pas index 9636ce45..8461ee63 100644 --- a/CryptoLib/src/Crypto/Engines/ClpAesEngineX86.pas +++ b/CryptoLib/src/Crypto/Engines/ClpAesEngineX86.pas @@ -76,28 +76,16 @@ TAesEngineX86 = class sealed(TInterfacedObject, IAesEngineX86, IAesHardwareEng function GetAlgorithmName: String; // ===== Internal fast-path helpers (engine's own ProcessBlocks ladder) ===== - // Bypass the nil-pointer / overlap safety checks and the stack-buffer - // staging used by the public ProcessBlock / ProcessFourBlocks / - // ProcessEightBlocks variants. Only ProcessBlocks (PByte overload) drives - // these: it has already honoured the IBulkBlockCipher aliasing contract - // (identical-or-fully-disjoint) for the whole batch, so each 4/8-wide - // slice inherits that same contract for free. Hidden behind strict private - // so no external caller can reach them and accidentally skip safety. - /// - /// Four consecutive 16-byte blocks (64 bytes) via pointers. Skips - /// nil/overlap safety checks and the stack-buffer copy used by - /// . Callers MUST pass valid 64-byte - /// buffers that are either identical (in-place) or fully disjoint. - /// + // The 4/8-wide inner steps of the IBulkBlockCipher ProcessBlocks ladder. + // Only ProcessBlocks (PByte overload) drives these: it has already honoured + // the IBulkBlockCipher aliasing contract (identical-or-fully-disjoint) for + // the whole batch, so each slice inherits it. Strict private so no external + // caller can reach them and skip that check. + /// Four consecutive 16-byte blocks (64 bytes) via pointers. + /// Identical-or-fully-disjoint buffers only. function ProcessFourBlocksFast(AInput, AOutput: PByte): Int32; inline; - /// - /// Eight consecutive 16-byte blocks (128 bytes) via pointers. Skips the - /// nil/overlap safety checks and stack-buffer copy used by the safe - /// / / - /// variants. Callers MUST pass valid - /// 128-byte buffers that are either identical (in-place) or fully - /// disjoint. - /// + /// Eight consecutive 16-byte blocks (128 bytes) via pointers. + /// Identical-or-fully-disjoint buffers only. function ProcessEightBlocksFast(AInput, AOutput: PByte): Int32; inline; public class function IsSupported: Boolean; static; @@ -107,18 +95,7 @@ TAesEngineX86 = class sealed(TInterfacedObject, IAesEngineX86, IAesHardwareEng function GetBlockSize(): Int32; function ProcessBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; overload; - function ProcessFourBlocks(const AInput: TCryptoLibByteArray; AInOff: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; overload; - /// - /// Eight consecutive 16-byte blocks (128 bytes). In-place, disjoint, and - /// overlapping ranges are handled safely; overlapping ranges are staged - /// through a 128-byte stack buffer. Returns BlockSize * 8. - /// - function ProcessEightBlocks(const AInput: TCryptoLibByteArray; AInOff: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; overload; function ProcessBlock(AInput, AOutput: PByte): Int32; overload; - function ProcessFourBlocks(AInput, AOutput: PByte): Int32; overload; - function ProcessEightBlocks(AInput, AOutput: PByte): Int32; overload; // ===== IBulkBlockCipher ===== // Generic multi-block fast path exposed as a cipher-agnostic capability @@ -1035,36 +1012,10 @@ function TAesEngineX86.ProcessBlock(const AInput: TCryptoLibByteArray; Result := ProcessBlock(@AInput[AInOff], @AOutput[AOutOff]); end; -function TAesEngineX86.ProcessFourBlocks(const AInput: TCryptoLibByteArray; - AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; -begin - if FKeys = nil then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - - TCheck.DataLength(AInput, AInOff, 64, SInputBufferTooShort); - TCheck.OutputLength(AOutput, AOutOff, 64, SOutputBufferTooShort); - - Result := ProcessFourBlocks(@AInput[AInOff], @AOutput[AOutOff]); -end; - -function TAesEngineX86.ProcessEightBlocks(const AInput: TCryptoLibByteArray; - AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; -begin - if FKeys = nil then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - - TCheck.DataLength(AInput, AInOff, 128, SInputBufferTooShort); - TCheck.OutputLength(AOutput, AOutOff, 128, SOutputBufferTooShort); - - Result := ProcessEightBlocks(@AInput[AInOff], @AOutput[AOutOff]); -end; - // ===================================================================== -// Pointer-overload block processing - defense-in-depth version that -// guards against overlapping in/out ranges by staging through a stack -// buffer. Same validation as the array overloads plus nil-pointer and -// binding checks. Used by higher-level modes (CTR, GCM) that already -// know the buffer shape. +// Pointer-overload single-block processing - defense-in-depth version +// that guards against overlapping in/out ranges by staging through a +// stack buffer, plus nil-pointer and binding checks. // ===================================================================== function TAesEngineX86.ProcessBlock(AInput, AOutput: PByte): Int32; @@ -1099,68 +1050,6 @@ function TAesEngineX86.ProcessBlock(AInput, AOutput: PByte): Int32; Result := 16; end; -function TAesEngineX86.ProcessFourBlocks(AInput, AOutput: PByte): Int32; -var - LWork: array [0 .. 63] of Byte; -begin - if FKeys = nil then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - if not Assigned(FAesNiCipherFour) or not Assigned(FAesNiCipherFourInOut) then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - if (AInput = nil) or (AOutput = nil) then - raise EArgumentCryptoLibException.CreateRes(@SNilPointerBuffer); - - if AInput = AOutput then - begin - FAesNiCipherFour(AOutput, FKeys); - Result := 64; - Exit; - end; - - if ((AOutput >= AInput) and (AOutput < AInput + 64)) or - ((AInput >= AOutput) and (AInput < AOutput + 64)) then - begin - System.Move(AInput^, LWork[0], 64); - FAesNiCipherFour(@LWork[0], FKeys); - System.Move(LWork[0], AOutput^, 64); - FillChar(LWork, SizeOf(LWork), 0); - end - else - FAesNiCipherFourInOut(AInput, AOutput, FKeys); - Result := 64; -end; - -function TAesEngineX86.ProcessEightBlocks(AInput, AOutput: PByte): Int32; -var - LWork: array [0 .. 127] of Byte; -begin - if FKeys = nil then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - if not Assigned(FAesNiCipherEight) or not Assigned(FAesNiCipherEightInOut) then - raise EInvalidOperationCryptoLibException.CreateRes(@SAesEngineX86NotInitialised); - if (AInput = nil) or (AOutput = nil) then - raise EArgumentCryptoLibException.CreateRes(@SNilPointerBuffer); - - if AInput = AOutput then - begin - FAesNiCipherEight(AOutput, FKeys); - Result := 128; - Exit; - end; - - if ((AOutput >= AInput) and (AOutput < AInput + 128)) or - ((AInput >= AOutput) and (AInput < AOutput + 128)) then - begin - System.Move(AInput^, LWork[0], 128); - FAesNiCipherEight(@LWork[0], FKeys); - System.Move(LWork[0], AOutput^, 128); - FillChar(LWork, SizeOf(LWork), 0); - end - else - FAesNiCipherEightInOut(AInput, AOutput, FKeys); - Result := 128; -end; - // ===================================================================== // "Fast" pointer overloads - no validation, no overlap staging. Only // callable from code paths that have already proven their preconditions diff --git a/CryptoLib/src/Crypto/Engines/ClpChaCha7539Engine.pas b/CryptoLib/src/Crypto/Engines/ClpChaCha7539Engine.pas index 2c04da98..d1e5e1f4 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaCha7539Engine.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaCha7539Engine.pas @@ -21,29 +21,21 @@ interface uses - SysUtils, - ClpCheck, ClpIStreamCipher, ClpIChaCha7539Engine, - ClpSalsa20Engine, ClpChaChaEngine, ClpPack, - ClpCryptoLibTypes, - ClpChaChaSimd, - ClpByteUtilities; + ClpCryptoLibTypes; resourcestring SInvalidKeySizeTwoFiftySix = '%s requires 256 bit key'; SCounterExceeded = 'attempt to increase counter past 2^32'; - SNotInitialised = '%s not initialized'; - SNotBlockAligned = '%s not in block-aligned state'; - SMaxByteExceeded = - '2^38 byte limit per IV would be exceeded; change IV'; - SInputBufferTooShort = 'input buffer too short'; - SOutputBufferTooShort = 'output buffer too short'; type - TChaCha7539Engine = class(TSalsa20Engine, IChaCha7539Engine, IStreamCipher) + /// + /// ChaCha20 per RFC 7539 (IETF): 96-bit nonce, 32-bit block counter (word 12). + /// + TChaCha7539Engine = class(TChaChaBaseEngine, IChaCha7539Engine, IStreamCipher) strict protected function GetAlgorithmName: String; override; @@ -52,36 +44,10 @@ TChaCha7539Engine = class(TSalsa20Engine, IChaCha7539Engine, IStreamCipher) procedure AdvanceCounter(); override; procedure ResetCounter(); override; procedure SetKey(const AKeyBytes, AIvBytes: TCryptoLibByteArray); override; - procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override; - - strict private - procedure ImplProcessBlock(const AInBuf: TCryptoLibByteArray; AInOff: Int32; - const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); inline; - // Two-block keystream body (SIMD-accelerated when available, else scalar) with - // no state validation or 2^38 byte-limit accounting - those belong to the - // callers (ProcessBlocks2, ProcessBlocks4), which do them exactly once. - procedure ProcessBlocks2Core(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); public constructor Create(); - procedure DoFinal(const AInBuf: TCryptoLibByteArray; AInOff, AInLen: Int32; - const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); - - procedure ProcessBytes(const AInBytes: TCryptoLibByteArray; - AInOff, ALen: Int32; const AOutBytes: TCryptoLibByteArray; - AOutOff: Int32); override; - - procedure ProcessBlock(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - - procedure ProcessBlocks2(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); override; - - procedure ProcessBlocks4(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - end; implementation @@ -138,249 +104,4 @@ procedure TChaCha7539Engine.SetKey(const AKeyBytes, TPack.LE_To_UInt32(AIvBytes, 0, FEngineState, 13, 3); end; -procedure TChaCha7539Engine.GenerateKeyStream( - const AOutput: TCryptoLibByteArray); -begin - TChaChaEngine.ChaChaCore(FRounds, FEngineState, AOutput); -end; - -procedure TChaCha7539Engine.DoFinal(const AInBuf: TCryptoLibByteArray; - AInOff, AInLen: Int32; const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); -var - LIdx, LQ: Int32; -begin - if (not FInitialised) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, - [AlgorithmName]); - end; - if (FIndex <> 0) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, - [AlgorithmName]); - end; - - TCheck.DataLength(AInBuf, AInOff, AInLen, SInputBufferTooShort); - TCheck.OutputLength(AOutBuf, AOutOff, AInLen, SOutputBufferTooShort); - - while (AInLen >= 256) do - begin - ProcessBlocks4(AInBuf, AInOff, AOutBuf, AOutOff); - AInOff := AInOff + 256; - AInLen := AInLen - 256; - AOutOff := AOutOff + 256; - end; - - while (AInLen >= 128) do - begin - ProcessBlocks2(AInBuf, AInOff, AOutBuf, AOutOff); - AInOff := AInOff + 128; - AInLen := AInLen - 128; - AOutOff := AOutOff + 128; - end; - - if (AInLen >= 64) then - begin - ImplProcessBlock(AInBuf, AInOff, AOutBuf, AOutOff); - AInOff := AInOff + 64; - AInLen := AInLen - 64; - AOutOff := AOutOff + 64; - end; - - if (AInLen > 0) then - begin - GenerateKeyStream(FKeyStream); - AdvanceCounter(); - - LQ := AInLen shr 3; - if LQ > 0 then - TByteUtilities.&Xor(LQ * 8, AInBuf, AInOff, FKeyStream, 0, AOutBuf, AOutOff); - for LIdx := (LQ * 8) to System.Pred(AInLen) do - begin - AOutBuf[AOutOff + LIdx] := Byte( - AInBuf[AInOff + LIdx] xor FKeyStream[LIdx]); - end; - end; - - FEngineState[12] := 0; -end; - -procedure TChaCha7539Engine.ProcessBytes(const AInBytes: TCryptoLibByteArray; - AInOff, ALen: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -var - LIdx, LTake, LQ: Int32; - LInP, LOutP, LKeyP: PByte; -begin - if (not FInitialised) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt - (@SNotInitialised, [AlgorithmName]); - end; - - TCheck.DataLength(AInBytes, AInOff, ALen, SInputBufferTooShort); - TCheck.OutputLength(AOutBytes, AOutOff, ALen, SOutputBufferTooShort); - - if (LimitExceeded(UInt32(ALen))) then - begin - raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); - end; - - while ALen > 0 do - begin - if (FIndex <> 0) then - begin - LTake := ALen; - if LTake > (64 - FIndex) then - begin - LTake := 64 - FIndex; - end; - for LIdx := 0 to System.Pred(LTake) do - begin - AOutBytes[AOutOff + LIdx] := Byte( - FKeyStream[FIndex + LIdx] xor AInBytes[AInOff + LIdx]); - end; - FIndex := (FIndex + LTake) and 63; - AInOff := AInOff + LTake; - AOutOff := AOutOff + LTake; - System.Dec(ALen, LTake); - continue; - end; - if (ALen >= 256) then - begin - ProcessBlocks4(AInBytes, AInOff, AOutBytes, AOutOff); - AInOff := AInOff + 256; - AOutOff := AOutOff + 256; - System.Dec(ALen, 256); - end - else if (ALen >= 128) then - begin - ProcessBlocks2(AInBytes, AInOff, AOutBytes, AOutOff); - AInOff := AInOff + 128; - AOutOff := AOutOff + 128; - System.Dec(ALen, 128); - end - else if (ALen >= 64) then - begin - ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); - AInOff := AInOff + 64; - AOutOff := AOutOff + 64; - System.Dec(ALen, 64); - end - else - begin - GenerateKeyStream(FKeyStream); - AdvanceCounter(); - LTake := ALen; - LInP := PByte(AInBytes) + AInOff; - LOutP := PByte(AOutBytes) + AOutOff; - LKeyP := PByte(FKeyStream); - LQ := LTake shr 3; - if LQ > 0 then - TByteUtilities.&Xor(LQ * 8, LInP, LKeyP, LOutP); - for LIdx := (LQ * 8) to System.Pred(LTake) do - begin - LOutP[LIdx] := LInP[LIdx] xor LKeyP[LIdx]; - end; - FIndex := (FIndex + LTake) and 63; - AInOff := AInOff + LTake; - AOutOff := AOutOff + LTake; - System.Dec(ALen, LTake); - end; - end; -end; - -procedure TChaCha7539Engine.ProcessBlock(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -begin - if (not FInitialised) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, - [AlgorithmName]); - end; - if (FIndex <> 0) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, - [AlgorithmName]); - end; - if (LimitExceeded(UInt32(64))) then - begin - raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); - end; - - ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); -end; - -procedure TChaCha7539Engine.ProcessBlocks2(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -begin - if (not FInitialised) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, - [AlgorithmName]); - end; - if (FIndex <> 0) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, - [AlgorithmName]); - end; - if (LimitExceeded(UInt32(128))) then - begin - raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); - end; - - ProcessBlocks2Core(AInBytes, AInOff, AOutBytes, AOutOff); -end; - -procedure TChaCha7539Engine.ProcessBlocks2Core(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -begin - if TChaChaSimd.TryProcessBlocks2(FRounds, PByte(@FEngineState[0]), - PByte(@AInBytes[AInOff]), PByte(@AOutBytes[AOutOff])) then - Exit; - - ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); - ImplProcessBlock(AInBytes, AInOff + 64, AOutBytes, AOutOff + 64); -end; - -procedure TChaCha7539Engine.ProcessBlocks4(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -begin - if (not FInitialised) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, - [AlgorithmName]); - end; - if (FIndex <> 0) then - begin - raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, - [AlgorithmName]); - end; - if (LimitExceeded(UInt32(256))) then - begin - raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); - end; - - if TChaChaSimd.TryProcessBlocks4(FRounds, PByte(@FEngineState[0]), - PByte(@AInBytes[AInOff]), PByte(@AOutBytes[AOutOff])) then - Exit; - - ProcessBlocks2Core(AInBytes, AInOff, AOutBytes, AOutOff); - ProcessBlocks2Core(AInBytes, AInOff + 128, AOutBytes, AOutOff + 128); -end; - -procedure TChaCha7539Engine.ImplProcessBlock( - const AInBuf: TCryptoLibByteArray; AInOff: Int32; - const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); -var - LInP, LOutP, LKeyP: PByte; -begin - TChaChaEngine.ChaChaCore(FRounds, FEngineState, FKeyStream); - AdvanceCounter(); - - LInP := PByte(AInBuf) + AInOff; - LKeyP := PByte(FKeyStream); - LOutP := PByte(AOutBuf) + AOutOff; - TByteUtilities.&Xor(64, LInP, LKeyP, LOutP); -end; - end. diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas index 9f26e81d..d77e3a1e 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaEngine.pas @@ -22,11 +22,13 @@ interface uses SysUtils, + ClpCheck, ClpIStreamCipher, ClpIChaChaEngine, ClpSalsa20Engine, ClpPack, ClpChaChaSimd, + ClpByteUtilities, ClpCryptoLibTypes; resourcestring @@ -39,21 +41,75 @@ interface SHChaChaNonce128 = 'HChaCha20 nonce must be 128 bits'; SHChaChaOutNil = 'HChaCha20 output buffer cannot be nil'; SHChaChaOutSpace = 'HChaCha20 output buffer too short'; + SNotInitialised = '%s not initialized'; + SNotBlockAligned = '%s not in block-aligned state'; + SMaxByteExceeded = + '2^38 byte limit per IV would be exceeded; change IV'; + SInputBufferTooShort = 'input buffer too short'; + SOutputBufferTooShort = 'output buffer too short'; type + /// + /// Shared ChaCha core (block function + SIMD bulk ladder) for the IETF and DJB + /// engines; they differ only in counter width (CounterIs64Bit) and key/nonce layout. + /// + TChaChaBaseEngine = class(TSalsa20Engine) + + strict private + // Pointer-form, unvalidated inner helpers (1 / 2 / 4 / 8 blocks). + procedure ProcessBlockFast(AIn, AOut: PByte); + procedure ProcessBlocks2Fast(AIn, AOut: PByte); + procedure ProcessBlocks4Fast(AIn, AOut: PByte); + procedure ProcessBlocks8Fast(AIn, AOut: PByte); + + strict protected + // 8 -> 4 -> 2 -> 1 bulk ladder (widest kernel first). + function DoProcessBlocks(AIn, AOut: PByte; ABlockCount: Int32): Int32; override; + + // True for the DJB 64-bit counter (words 12-13); False (IETF). + function CounterIs64Bit: Boolean; virtual; + // Wide kernels broadcast word 13; for the 64-bit counter that holds only while + // word 12 does not wrap over the N-block span (else the caller degrades a tier). + function WideBlocksSafe(ABlocks: Int32): Boolean; inline; + + procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override; + + public + procedure DoFinal(const AInBuf: TCryptoLibByteArray; AInOff, AInLen: Int32; + const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); + + procedure ProcessBytes(const AInBytes: TCryptoLibByteArray; + AInOff, ALen: Int32; const AOutBytes: TCryptoLibByteArray; + AOutOff: Int32); override; + + procedure ProcessBlock(const AInBytes: TCryptoLibByteArray; AInOff: Int32; + const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); + + class procedure ChaChaCore(ARounds: Int32; + const AInput: TCryptoLibUInt32Array; + const AOutput: TCryptoLibByteArray); static; + + /// + /// HChaCha20 subkey derivation (draft-irtf-cfrg-xchacha). Writes 32 bytes to ASubKeyOut at ASubKeyOutOff. + /// + class procedure HChaCha20(const AKey256, ANonce128: TCryptoLibByteArray; + const ASubKeyOut: TCryptoLibByteArray; ASubKeyOutOff: Int32); static; + + end; + /// /// Implementation of Daniel J. Bernstein's ChaCha stream cipher. /// - TChaChaEngine = class(TSalsa20Engine, IChaChaEngine, IStreamCipher) + TChaChaEngine = class(TChaChaBaseEngine, IChaChaEngine, IStreamCipher) strict protected function GetAlgorithmName: String; override; + function CounterIs64Bit: Boolean; override; procedure AdvanceCounter(); override; procedure ResetCounter(); override; procedure SetKey(const AKeyBytes, AIvBytes: TCryptoLibByteArray); override; - procedure GenerateKeyStream(const AOutput: TCryptoLibByteArray); override; public /// @@ -66,44 +122,240 @@ TChaChaEngine = class(TSalsa20Engine, IChaChaEngine, IStreamCipher) /// the number of rounds (must be an even number). constructor Create(ARounds: Int32); overload; - procedure ProcessBlocks2(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); override; + end; + +implementation - class procedure ChaChaCore(ARounds: Int32; - const AInput: TCryptoLibUInt32Array; - const AOutput: TCryptoLibByteArray); static; +{ TChaChaBaseEngine } - /// - /// HChaCha20 subkey derivation (draft-irtf-cfrg-xchacha). Writes 32 bytes to ASubKeyOut at ASubKeyOutOff. - /// - class procedure HChaCha20(const AKey256, ANonce128: TCryptoLibByteArray; - const ASubKeyOut: TCryptoLibByteArray; ASubKeyOutOff: Int32); static; +function TChaChaBaseEngine.CounterIs64Bit: Boolean; +begin + Result := False; +end; +function TChaChaBaseEngine.WideBlocksSafe(ABlocks: Int32): Boolean; +begin + Result := (not CounterIs64Bit) or + (FEngineState[12] <= (UInt32($FFFFFFFF) - UInt32(ABlocks - 1))); +end; + +procedure TChaChaBaseEngine.GenerateKeyStream(const AOutput: TCryptoLibByteArray); +begin + ChaChaCore(FRounds, FEngineState, AOutput); +end; + +procedure TChaChaBaseEngine.DoFinal(const AInBuf: TCryptoLibByteArray; + AInOff, AInLen: Int32; const AOutBuf: TCryptoLibByteArray; AOutOff: Int32); +var + LIdx, LQ, LWholeBytes: Int32; +begin + if (not FInitialised) then + begin + raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, + [AlgorithmName]); + end; + if (FIndex <> 0) then + begin + raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, + [AlgorithmName]); end; -implementation + TCheck.DataLength(AInBuf, AInOff, AInLen, SInputBufferTooShort); + TCheck.OutputLength(AOutBuf, AOutOff, AInLen, SOutputBufferTooShort); -{ TChaChaEngine } + if (AInLen >= 64) then + begin + LWholeBytes := (AInLen shr 6) shl 6; // whole 64B blocks, engine ladders 8/4/2/1 + DoProcessBlocks(PByte(@AInBuf[AInOff]), PByte(@AOutBuf[AOutOff]), AInLen shr 6); + AInOff := AInOff + LWholeBytes; + AInLen := AInLen - LWholeBytes; + AOutOff := AOutOff + LWholeBytes; + end; + + if (AInLen > 0) then + begin + GenerateKeyStream(FKeyStream); + AdvanceCounter(); + + LQ := AInLen shr 3; + if LQ > 0 then + TByteUtilities.&Xor(LQ * 8, AInBuf, AInOff, FKeyStream, 0, AOutBuf, AOutOff); + for LIdx := (LQ * 8) to System.Pred(AInLen) do + begin + AOutBuf[AOutOff + LIdx] := Byte( + AInBuf[AInOff + LIdx] xor FKeyStream[LIdx]); + end; + end; + + ResetCounter(); +end; + +procedure TChaChaBaseEngine.ProcessBytes(const AInBytes: TCryptoLibByteArray; + AInOff, ALen: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); +var + LIdx, LTake, LQ: Int32; + LInP, LOutP, LKeyP: PByte; +begin + if (not FInitialised) then + begin + raise EInvalidOperationCryptoLibException.CreateResFmt + (@SNotInitialised, [AlgorithmName]); + end; + + TCheck.DataLength(AInBytes, AInOff, ALen, SInputBufferTooShort); + TCheck.OutputLength(AOutBytes, AOutOff, ALen, SOutputBufferTooShort); + + if (LimitExceeded(UInt32(ALen))) then + begin + raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); + end; + + while ALen > 0 do + begin + if (FIndex <> 0) then + begin + LTake := ALen; + if LTake > (64 - FIndex) then + begin + LTake := 64 - FIndex; + end; + for LIdx := 0 to System.Pred(LTake) do + begin + AOutBytes[AOutOff + LIdx] := Byte( + FKeyStream[FIndex + LIdx] xor AInBytes[AInOff + LIdx]); + end; + FIndex := (FIndex + LTake) and 63; + AInOff := AInOff + LTake; + AOutOff := AOutOff + LTake; + System.Dec(ALen, LTake); + continue; + end; + if (ALen >= 64) then + begin + LTake := (ALen shr 6) shl 6; // whole 64B blocks; engine ladders 8/4/2/1 + DoProcessBlocks(PByte(@AInBytes[AInOff]), PByte(@AOutBytes[AOutOff]), ALen shr 6); + AInOff := AInOff + LTake; + AOutOff := AOutOff + LTake; + System.Dec(ALen, LTake); + end + else + begin + GenerateKeyStream(FKeyStream); + AdvanceCounter(); + LTake := ALen; + LInP := PByte(AInBytes) + AInOff; + LOutP := PByte(AOutBytes) + AOutOff; + LKeyP := PByte(FKeyStream); + LQ := LTake shr 3; + if LQ > 0 then + TByteUtilities.&Xor(LQ * 8, LInP, LKeyP, LOutP); + for LIdx := (LQ * 8) to System.Pred(LTake) do + begin + LOutP[LIdx] := LInP[LIdx] xor LKeyP[LIdx]; + end; + FIndex := (FIndex + LTake) and 63; + AInOff := AInOff + LTake; + AOutOff := AOutOff + LTake; + System.Dec(ALen, LTake); + end; + end; +end; -procedure TChaChaEngine.ProcessBlocks2( - const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); +procedure TChaChaBaseEngine.ProcessBlock(const AInBytes: TCryptoLibByteArray; + AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); begin - AssertInitialisedAndBlockAligned; + if (not FInitialised) then + begin + raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotInitialised, + [AlgorithmName]); + end; + if (FIndex <> 0) then + begin + raise EInvalidOperationCryptoLibException.CreateResFmt(@SNotBlockAligned, + [AlgorithmName]); + end; + if (LimitExceeded(UInt32(64))) then + begin + raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); + end; + ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); - ImplProcessBlock(AInBytes, AInOff + 64, AOutBytes, AOutOff + 64); end; -procedure TChaChaEngine.AdvanceCounter; +procedure TChaChaBaseEngine.ProcessBlockFast(AIn, AOut: PByte); begin - System.Inc(FEngineState[12]); - if (FEngineState[12] = 0) then + ChaChaCore(FRounds, FEngineState, FKeyStream); + AdvanceCounter(); + TByteUtilities.&Xor(64, AIn, PByte(FKeyStream), AOut); +end; + +procedure TChaChaBaseEngine.ProcessBlocks2Fast(AIn, AOut: PByte); +begin + if TChaChaSimd.TryProcessBlocks2(FRounds, PByte(@FEngineState[0]), + AIn, AOut, CounterIs64Bit) then + Exit; + + ProcessBlockFast(AIn, AOut); + ProcessBlockFast(AIn + 64, AOut + 64); +end; + +procedure TChaChaBaseEngine.ProcessBlocks4Fast(AIn, AOut: PByte); +begin + if WideBlocksSafe(4) then + if TChaChaSimd.TryProcessBlocks4(FRounds, PByte(@FEngineState[0]), + AIn, AOut, CounterIs64Bit) then + Exit; + + ProcessBlocks2Fast(AIn, AOut); + ProcessBlocks2Fast(AIn + 128, AOut + 128); +end; + +procedure TChaChaBaseEngine.ProcessBlocks8Fast(AIn, AOut: PByte); +begin + if WideBlocksSafe(8) then + if TChaChaSimd.TryProcessBlocks8(FRounds, PByte(@FEngineState[0]), + AIn, AOut, CounterIs64Bit) then + Exit; + + ProcessBlocks4Fast(AIn, AOut); + ProcessBlocks4Fast(AIn + 256, AOut + 256); +end; + +function TChaChaBaseEngine.DoProcessBlocks(AIn, AOut: PByte; + ABlockCount: Int32): Int32; +begin + Result := ABlockCount * 64; + while ABlockCount >= 8 do begin - System.Inc(FEngineState[13]); + ProcessBlocks8Fast(AIn, AOut); + AIn := AIn + 512; + AOut := AOut + 512; + System.Dec(ABlockCount, 8); + end; + if ABlockCount >= 4 then + begin + ProcessBlocks4Fast(AIn, AOut); + AIn := AIn + 256; + AOut := AOut + 256; + System.Dec(ABlockCount, 4); + end; + if ABlockCount >= 2 then + begin + ProcessBlocks2Fast(AIn, AOut); + AIn := AIn + 128; + AOut := AOut + 128; + System.Dec(ABlockCount, 2); + end; + while ABlockCount >= 1 do + begin + ProcessBlockFast(AIn, AOut); + AIn := AIn + 64; + AOut := AOut + 64; + System.Dec(ABlockCount); end; end; -class procedure TChaChaEngine.ChaChaCore(ARounds: Int32; +class procedure TChaChaBaseEngine.ChaChaCore(ARounds: Int32; const AInput: TCryptoLibUInt32Array; const AOutput: TCryptoLibByteArray); var LX00, LX01, LX02, LX03, LX04, LX05, LX06, LX07, LX08, LX09, LX10, LX11, LX12, LX13, LX14, @@ -239,54 +491,7 @@ class procedure TChaChaEngine.ChaChaCore(ARounds: Int32; end; -constructor TChaChaEngine.Create; -begin - Inherited Create(); -end; - -constructor TChaChaEngine.Create(ARounds: Int32); -begin - Inherited Create(ARounds); -end; - -procedure TChaChaEngine.GenerateKeyStream(const AOutput: TCryptoLibByteArray); -begin - ChaChaCore(FRounds, FEngineState, AOutput); -end; - -function TChaChaEngine.GetAlgorithmName: String; -begin - Result := Format('ChaCha%d', [FRounds]); -end; - -procedure TChaChaEngine.ResetCounter; -begin - FEngineState[12] := 0; - FEngineState[13] := 0; -end; - -procedure TChaChaEngine.SetKey(const AKeyBytes, AIvBytes: TCryptoLibByteArray); -begin - if (AKeyBytes <> nil) then - begin - if not(System.Length(AKeyBytes) in [16, 32]) then - begin - raise EArgumentCryptoLibException.CreateResFmt(@SInvalidKeySize, - [AlgorithmName]); - end; - - PackTauOrSigma(System.Length(AKeyBytes), FEngineState); - - // Key - TPack.LE_To_UInt32(AKeyBytes, 0, FEngineState, 4, 4); - TPack.LE_To_UInt32(AKeyBytes, System.Length(AKeyBytes) - 16, FEngineState, 8, 4); - end; - - // IV - TPack.LE_To_UInt32(AIvBytes, 0, FEngineState, 14, 2); -end; - -class procedure TChaChaEngine.HChaCha20(const AKey256, ANonce128: TCryptoLibByteArray; +class procedure TChaChaBaseEngine.HChaCha20(const AKey256, ANonce128: TCryptoLibByteArray; const ASubKeyOut: TCryptoLibByteArray; ASubKeyOutOff: Int32); var LState: TCryptoLibUInt32Array; @@ -335,4 +540,62 @@ class procedure TChaChaEngine.HChaCha20(const AKey256, ANonce128: TCryptoLibByte TPack.UInt32_To_LE(TPack.LE_To_UInt32(LOut, 60) - LState[15], ASubKeyOut, ASubKeyOutOff + 28); end; +{ TChaChaEngine } + +constructor TChaChaEngine.Create; +begin + Inherited Create(); +end; + +constructor TChaChaEngine.Create(ARounds: Int32); +begin + Inherited Create(ARounds); +end; + +function TChaChaEngine.CounterIs64Bit: Boolean; +begin + Result := True; +end; + +procedure TChaChaEngine.AdvanceCounter; +begin + System.Inc(FEngineState[12]); + if (FEngineState[12] = 0) then + begin + System.Inc(FEngineState[13]); + end; +end; + +function TChaChaEngine.GetAlgorithmName: String; +begin + Result := Format('ChaCha%d', [FRounds]); +end; + +procedure TChaChaEngine.ResetCounter; +begin + FEngineState[12] := 0; + FEngineState[13] := 0; +end; + +procedure TChaChaEngine.SetKey(const AKeyBytes, AIvBytes: TCryptoLibByteArray); +begin + if (AKeyBytes <> nil) then + begin + if not(System.Length(AKeyBytes) in [16, 32]) then + begin + raise EArgumentCryptoLibException.CreateResFmt(@SInvalidKeySize, + [AlgorithmName]); + end; + + PackTauOrSigma(System.Length(AKeyBytes), FEngineState); + + // Key + TPack.LE_To_UInt32(AKeyBytes, 0, FEngineState, 4, 4); + TPack.LE_To_UInt32(AKeyBytes, System.Length(AKeyBytes) - 16, FEngineState, 8, 4); + end; + + // IV + TPack.LE_To_UInt32(AIvBytes, 0, FEngineState, 14, 2); +end; + end. diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas index 1100839b..bf10519e 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaSimd.pas @@ -39,10 +39,13 @@ TChaChaSimd = class sealed public /// Single-block ChaCha core (ChaCha20 keystream block). class function TryCore(ARounds: Int32; AInput, AOut: PByte): Boolean; static; - /// Two-block ChaCha7539 keystream (128 bytes). - class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; static; - /// Four-block ChaCha7539 keystream (256 bytes). - class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; static; + /// Two-block ChaCha keystream (128 bytes). ACtr64 = DJB 64-bit + /// counter (no SIMD 2-block body; returns False). + class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// Four-block ChaCha keystream (256 bytes). ACtr64 = DJB variant. + class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// Eight-block ChaCha keystream (512 bytes). ACtr64 = DJB variant. + class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; end; implementation @@ -58,19 +61,28 @@ class function TChaChaSimd.TryCore(ARounds: Int32; AInput, AOut: PByte): Boolean {$IFEND} end; -class function TChaChaSimd.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; +class function TChaChaSimd.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TChaChaX86Backend.TryProcessBlocks2(ARounds, AState, AIn, AOut); + Result := TChaChaX86Backend.TryProcessBlocks2(ARounds, AState, AIn, AOut, ACtr64); {$ELSE} Result := False; {$IFEND} end; -class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; +class function TChaChaSimd.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut); + Result := TChaChaX86Backend.TryProcessBlocks4(ARounds, AState, AIn, AOut, ACtr64); +{$ELSE} + Result := False; +{$IFEND} +end; + +class function TChaChaSimd.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +begin +{$IF DEFINED(CRYPTOLIB_X86_SIMD)} + Result := TChaChaX86Backend.TryProcessBlocks8(ARounds, AState, AIn, AOut, ACtr64); {$ELSE} Result := False; {$IFEND} diff --git a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas index 67e98368..04f13d17 100644 --- a/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas +++ b/CryptoLib/src/Crypto/Engines/ClpChaChaX86Backend.pas @@ -41,10 +41,16 @@ TChaChaX86Backend = class sealed public /// SIMD single-block ChaCha core (ChaCha20 keystream block). class function TryCore(ARounds: Int32; AInput, AOut: PByte): Boolean; static; - /// SIMD two-block ChaCha7539 keystream (128 bytes). - class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; static; - /// AVX2 four-block ChaCha7539 keystream (256 bytes). - class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; static; + /// SIMD two-block ChaCha keystream (128 bytes). ACtr64 selects the + /// 64-bit-counter (DJB) variant; no SIMD 2-block kernel exists for it, so it + /// returns False and the caller runs its scalar path. + class function TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// Four-block ChaCha keystream (256 bytes). ACtr64 selects the DJB + /// 64-bit-counter kernel. + class function TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; + /// AVX2 eight-block ChaCha keystream (512 bytes). ACtr64 selects the + /// DJB 64-bit-counter kernel. + class function TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean = False): Boolean; static; end; implementation @@ -77,27 +83,79 @@ procedure ChaCha7539ProcessBlocks2Sse2(ARounds: Int32; AState, AIn, AOut: PByte) {$ENDIF} end; -procedure ChaCha7539ProcessBlocks2Avx2(ARounds: Int32; AState, AIn, AOut: PByte); +procedure ChaCha7539ProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte); {$IFDEF CRYPTOLIB_X86_64_ASM} {$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} -{$I ..\..\Include\Simd\ChaCha\ChaCha7539ProcessBlocks2Avx2_x86_64.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} {$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} -{$I ..\..\Include\Simd\ChaCha\ChaCha7539ProcessBlocks2Avx2_i386.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc} {$ENDIF} end; -procedure ChaCha7539ProcessBlocks4Avx2(ARounds: Int32; AState, AIn, AOut: PByte); +procedure ChaCha7539ProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte); {$IFDEF CRYPTOLIB_X86_64_ASM} {$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} -{$I ..\..\Include\Simd\ChaCha\ChaCha7539ProcessBlocks4Avx2_x86_64.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc} {$ENDIF} {$IFDEF CRYPTOLIB_I386_ASM} {$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} -{$I ..\..\Include\Simd\ChaCha\ChaCha7539ProcessBlocks4Avx2_i386.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc} {$ENDIF} end; + +// AVX2 8-way (512B): x86_64 only (needs 16 ymm; i386 has 8, so it falls back to +// two 256B ProcessBlocks4 in the engine). +{$IFDEF CRYPTOLIB_X86_64_ASM} +procedure ChaCha7539ProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte); +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc} +end; +{$ENDIF} + +// DJB ChaCha (64-bit counter, words 12-13): the vertical kernels are identical to +// the 7539 ones except the counter advance carries into word 13, which the shared +// includes emit under CHACHA_CTR64. Callers guard against a low-word wrap across the +// lanes before invoking these (see the engine's WideBlocksSafe). +procedure ChaChaProcessBlocks4Sse2(ARounds: Int32; AState, AIn, AOut: PByte); +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_x86_64.inc} +{$UNDEF CHACHA_CTR64} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Sse2_i386.inc} +{$UNDEF CHACHA_CTR64} +{$ENDIF} +end; + +procedure ChaChaProcessBlocks4Ssse3(ARounds: Int32; AState, AIn, AOut: PByte); +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_x86_64.inc} +{$UNDEF CHACHA_CTR64} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_i386.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks4Ssse3_i386.inc} +{$UNDEF CHACHA_CTR64} +{$ENDIF} +end; + +{$IFDEF CRYPTOLIB_X86_64_ASM} +procedure ChaChaProcessBlocks8Avx2(ARounds: Int32; AState, AIn, AOut: PByte); +{$I ..\..\Include\Simd\Common\ClpSimdProc4Begin_x86_64.inc} +{$DEFINE CHACHA_CTR64} +{$I ..\..\Include\Simd\ChaCha\ChaCha7539Blocks8Avx2_x86_64.inc} +{$UNDEF CHACHA_CTR64} +end; +{$ENDIF} {$ENDIF CRYPTOLIB_X86_SIMD} { TChaChaX86Backend } @@ -116,18 +174,45 @@ class function TChaChaX86Backend.TryCore(ARounds: Int32; AInput, AOut: PByte): B Result := False; end; -class function TChaChaX86Backend.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; +class function TChaChaX86Backend.TryProcessBlocks2(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: + // 2-block (128B) tier: AVX2 adds nothing at this width, so the SSE2 body serves + // every SIMD CPU (the vertical 4-way/8-way kernels handle the larger tiers). The + // 2-block kernel is 7539-only (32-bit counter with raise-on-wrap), so DJB + // (ACtr64) has no SIMD body here and runs scalar. + if not ACtr64 then + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + begin + ChaCha7539ProcessBlocks2Sse2(ARounds, AState, AIn, AOut); + Exit(True); + end; + end; +{$ENDIF} + Result := False; +end; + +class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; +begin +{$IFDEF CRYPTOLIB_X86_SIMD} + // 4-block (256B) tier: AVX2 gives no extra lanes at 4-way, and AVX2 CPUs run the + // 8-way kernel for the bulk (512B) tier, so SSSE3/SSE2 cover every CPU here. + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSSE3: begin - ChaCha7539ProcessBlocks2Avx2(ARounds, AState, AIn, AOut); + if ACtr64 then + ChaChaProcessBlocks4Ssse3(ARounds, AState, AIn, AOut) + else + ChaCha7539ProcessBlocks4Ssse3(ARounds, AState, AIn, AOut); Exit(True); end; TX86SimdLevel.SSE2: begin - ChaCha7539ProcessBlocks2Sse2(ARounds, AState, AIn, AOut); + if ACtr64 then + ChaChaProcessBlocks4Sse2(ARounds, AState, AIn, AOut) + else + ChaCha7539ProcessBlocks4Sse2(ARounds, AState, AIn, AOut); Exit(True); end; end; @@ -135,13 +220,16 @@ class function TChaChaX86Backend.TryProcessBlocks2(ARounds: Int32; AState, AIn, Result := False; end; -class function TChaChaX86Backend.TryProcessBlocks4(ARounds: Int32; AState, AIn, AOut: PByte): Boolean; +class function TChaChaX86Backend.TryProcessBlocks8(ARounds: Int32; AState, AIn, AOut: PByte; ACtr64: Boolean): Boolean; begin -{$IFDEF CRYPTOLIB_X86_SIMD} +{$IFDEF CRYPTOLIB_X86_64_ASM} case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2]) of TX86SimdLevel.AVX2: begin - ChaCha7539ProcessBlocks4Avx2(ARounds, AState, AIn, AOut); + if ACtr64 then + ChaChaProcessBlocks8Avx2(ARounds, AState, AIn, AOut) + else + ChaCha7539ProcessBlocks8Avx2(ARounds, AState, AIn, AOut); Exit(True); end; end; diff --git a/CryptoLib/src/Crypto/Engines/ClpSalsa20Engine.pas b/CryptoLib/src/Crypto/Engines/ClpSalsa20Engine.pas index 222e579d..24f18204 100644 --- a/CryptoLib/src/Crypto/Engines/ClpSalsa20Engine.pas +++ b/CryptoLib/src/Crypto/Engines/ClpSalsa20Engine.pas @@ -25,6 +25,7 @@ interface ClpBitOperations, ClpCheck, ClpIStreamCipher, + ClpIBulkStreamCipher, ClpISalsa20Engine, ClpIKeyParameter, ClpICipherParameters, @@ -56,7 +57,8 @@ interface /// /// Implementation of Daniel J. Bernstein's Salsa20 stream cipher, Snuffle 2005 /// - TSalsa20Engine = class(TInterfacedObject, ISalsa20Engine, IStreamCipher) + TSalsa20Engine = class(TInterfacedObject, ISalsa20Engine, IStreamCipher, + IBulkStreamCipher) strict private const @@ -72,6 +74,10 @@ TSalsa20Engine = class(TInterfacedObject, ISalsa20Engine, IStreamCipher) class procedure QuarterRound(var A, B, C, D: UInt32); static; inline; + // Pointer-form, unvalidated inner helpers (1 / 2 blocks). + procedure ProcessBlockFast(AIn, AOut: PByte); + procedure ProcessBlocks2Fast(AIn, AOut: PByte); + strict protected var FIndex: Int32; @@ -97,6 +103,9 @@ TSalsa20Engine = class(TInterfacedObject, ISalsa20Engine, IStreamCipher) procedure ImplProcessBlock(const AInBytes: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); inline; + // 2 -> 1 bulk ladder; the ChaCha engine overrides with 8 -> 4 -> 2 -> 1. + function DoProcessBlocks(AIn, AOut: PByte; ABlockCount: Int32): Int32; virtual; + /// /// Rotate left /// @@ -130,8 +139,11 @@ TSalsa20Engine = class(TInterfacedObject, ISalsa20Engine, IStreamCipher) const AParameters: ICipherParameters); virtual; function ReturnByte(AInput: Byte): Byte; virtual; - procedure ProcessBlocks2(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); virtual; + // IBulkStreamCipher: bulk keystream over whole 64B blocks; engine owns the ladder. + function ProcessBlocks(const AInBuf: TCryptoLibByteArray; + AInOff, ABlockCount: Int32; const AOutBuf: TCryptoLibByteArray; + AOutOff: Int32): Int32; overload; + function ProcessBlocks(AInput, AOutput: PByte; ABlockCount: Int32): Int32; overload; procedure ProcessBytes(const AInBytes: TCryptoLibByteArray; AInOff, ALen: Int32; const AOutBytes: TCryptoLibByteArray; @@ -314,18 +326,77 @@ procedure TSalsa20Engine.ImplProcessBlock( TByteUtilities.&Xor(64, LInP, LKeyP, LOutP); end; -procedure TSalsa20Engine.ProcessBlocks2( - const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); +procedure TSalsa20Engine.ProcessBlockFast(AIn, AOut: PByte); begin - AssertInitialisedAndBlockAligned; + GenerateKeyStream(FKeyStream); + AdvanceCounter(); + TByteUtilities.&Xor(64, AIn, PByte(FKeyStream), AOut); +end; +procedure TSalsa20Engine.ProcessBlocks2Fast(AIn, AOut: PByte); +begin if TSalsaSimd.TryProcessBlocks2(FRounds, PByte(@FEngineState[0]), - PByte(@AInBytes[AInOff]), PByte(@AOutBytes[AOutOff])) then + AIn, AOut) then + Exit; + + ProcessBlockFast(AIn, AOut); + ProcessBlockFast(AIn + 64, AOut + 64); +end; + +function TSalsa20Engine.DoProcessBlocks(AIn, AOut: PByte; + ABlockCount: Int32): Int32; +begin + Result := ABlockCount * 64; + while ABlockCount >= 2 do + begin + ProcessBlocks2Fast(AIn, AOut); + AIn := AIn + 128; + AOut := AOut + 128; + System.Dec(ABlockCount, 2); + end; + while ABlockCount >= 1 do + begin + ProcessBlockFast(AIn, AOut); + AIn := AIn + 64; + AOut := AOut + 64; + System.Dec(ABlockCount); + end; +end; + +function TSalsa20Engine.ProcessBlocks(const AInBuf: TCryptoLibByteArray; + AInOff, ABlockCount: Int32; const AOutBuf: TCryptoLibByteArray; + AOutOff: Int32): Int32; +begin + AssertInitialisedAndBlockAligned; + if (ABlockCount <= 0) then + begin + Result := 0; + Exit; + end; + if (LimitExceeded(UInt32(ABlockCount) * 64)) then + begin + raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); + end; + + Result := DoProcessBlocks(PByte(@AInBuf[AInOff]), PByte(@AOutBuf[AOutOff]), + ABlockCount); +end; + +function TSalsa20Engine.ProcessBlocks(AInput, AOutput: PByte; + ABlockCount: Int32): Int32; +begin + AssertInitialisedAndBlockAligned; + if (ABlockCount <= 0) then + begin + Result := 0; Exit; + end; + if (LimitExceeded(UInt32(ABlockCount) * 64)) then + begin + raise EMaxBytesExceededCryptoLibException.CreateRes(@SMaxByteExceeded); + end; - ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); - ImplProcessBlock(AInBytes, AInOff + 64, AOutBytes, AOutOff + 64); + Result := DoProcessBlocks(AInput, AOutput, ABlockCount); end; class procedure TSalsa20Engine.PackTauOrSigma(AKeyLength: Int32; @@ -379,20 +450,13 @@ procedure TSalsa20Engine.ProcessBytes(const AInBytes: TCryptoLibByteArray; continue; end; - if (ALen >= 128) then + if (ALen >= 64) then begin - ProcessBlocks2(AInBytes, AInOff, AOutBytes, AOutOff); - AInOff := AInOff + 128; - AOutOff := AOutOff + 128; - System.Dec(ALen, 128); - continue; - end - else if (ALen >= 64) then - begin - ImplProcessBlock(AInBytes, AInOff, AOutBytes, AOutOff); - AInOff := AInOff + 64; - AOutOff := AOutOff + 64; - System.Dec(ALen, 64); + LTake := (ALen shr 6) shl 6; // whole 64B blocks; engine ladders 2/1 + DoProcessBlocks(PByte(@AInBytes[AInOff]), PByte(@AOutBytes[AOutOff]), ALen shr 6); + AInOff := AInOff + LTake; + AOutOff := AOutOff + LTake; + System.Dec(ALen, LTake); continue; end else diff --git a/CryptoLib/src/Crypto/Macs/ClpPoly1305X86Backend.pas b/CryptoLib/src/Crypto/Macs/ClpPoly1305X86Backend.pas index cdd71d09..f9f49ae3 100644 --- a/CryptoLib/src/Crypto/Macs/ClpPoly1305X86Backend.pas +++ b/CryptoLib/src/Crypto/Macs/ClpPoly1305X86Backend.pas @@ -169,12 +169,56 @@ procedure Poly1305Avx2InitPowerTable(var APowTable: TCryptoLibByteArray; LTbl^[9, LIdx] := 0; end; -// Asm wrapper around the architecture-specific 4-way bulk kernel. The .inc -// files contain pure assembly (db-encoded VEX with mnemonic comments); the -// Pascal layer below is just the procedure header + the ClpSimdProc5Begin ABI -// glue + the kernel body include. ACtx points at the 72-byte R/S/H/K -// portion of TPoly1305State; APowTable points at the separate 320-byte -// power table buffer; the kernel never reads the K limbs. +// (Re)allocate APowTable and pack r^1, r^2 for the 2-way SSE2 bulk kernel. +// 160 bytes = 10 rows x 4 dwords. Rows 0..4 hold limbs of [r^2, r^2, r^2, r^1] +// (so a broadcast load gives r^2 in both 64-bit lanes and a +4 shifted load +// gives [r^2, r^1]); rows 5..8 hold the 5x wraparound multipliers; row 9 is +// padding for the +4 over-read of the last shifted load. +procedure Poly1305Sse2InitPowerTable(var APowTable: TCryptoLibByteArray; + const AState: TPoly1305State); +const + TableSize = Int32(160); +type + TPowTableLayout = array[0..9, 0..3] of UInt32; + PPowTableLayout = ^TPowTableLayout; +var + LTbl: PPowTableLayout; + Lr1, Lr2: array[0..4] of UInt32; + LIdx, LRow, LJ: Int32; +begin + System.SetLength(APowTable, TableSize); + LTbl := PPowTableLayout(APowTable); + + Lr1[0] := AState.R0; + Lr1[1] := AState.R1; + Lr1[2] := AState.R2; + Lr1[3] := AState.R3; + Lr1[4] := AState.R4; + + Poly1305MulLimbs(Lr2, Lr1, Lr1); + + for LIdx := 0 to 4 do + begin + LTbl^[LIdx, 0] := Lr2[LIdx]; + LTbl^[LIdx, 1] := Lr2[LIdx]; + LTbl^[LIdx, 2] := Lr2[LIdx]; + LTbl^[LIdx, 3] := Lr1[LIdx]; + end; + + for LRow := 5 to 8 do + begin + LJ := LRow - 4; // 1..4 + LTbl^[LRow, 0] := Lr2[LJ] * 5; + LTbl^[LRow, 1] := Lr2[LJ] * 5; + LTbl^[LRow, 2] := Lr2[LJ] * 5; + LTbl^[LRow, 3] := Lr1[LJ] * 5; + end; + + for LIdx := 0 to 3 do + LTbl^[9, LIdx] := 0; +end; + +// AVX2 4-way bulk kernel (r^1..r^4 power table, 64-byte stride). procedure Poly1305BlocksBulkAvx2Core(ACtx, APowTable, AInp: PByte; ALen: NativeUInt; APad: Int32); {$IFDEF CRYPTOLIB_X86_64_ASM} @@ -187,6 +231,19 @@ procedure Poly1305BlocksBulkAvx2Core(ACtx, APowTable, AInp: PByte; {$ENDIF} end; +// SSE2 2-way bulk kernel (r/r^2 power table, 32-byte stride). +procedure Poly1305BlocksBulkSse2Core(ACtx, APowTable, AInp: PByte; + ALen: NativeUInt; APad: Int32); +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Poly1305\Poly1305BlocksBulkSse2Core_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\ClpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Poly1305\Poly1305BlocksBulkSse2Core_i386.inc} +{$ENDIF} +end; + {$ENDIF CRYPTOLIB_X86_SIMD} { TPoly1305X86Backend } @@ -195,12 +252,17 @@ class function TPoly1305X86Backend.TryInitPowerTable(var APowTable: TCryptoLibBy const AState: TPoly1305State): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2]) of + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of TX86SimdLevel.AVX2: begin Poly1305Avx2InitPowerTable(APowTable, AState); Exit(True); end; + TX86SimdLevel.SSE2: + begin + Poly1305Sse2InitPowerTable(APowTable, AState); + Exit(True); + end; end; {$ENDIF} Result := False; @@ -209,26 +271,35 @@ class function TPoly1305X86Backend.TryInitPowerTable(var APowTable: TCryptoLibBy class function TPoly1305X86Backend.ProcessBulk(var AState: TPoly1305State; APowTable: PByte; const ABuf: TCryptoLibByteArray; AOff, ANumBlocks: Int32): Int32; {$IFDEF CRYPTOLIB_X86_SIMD} -const - // Minimum number of 16-byte blocks before the AVX2 4-way kernel pays off - // over the scalar block step; smaller batches go straight to the caller's - // scalar tail. - LMinBlocks = Int32(4); - // Number of 16-byte blocks consumed per AVX2 kernel iteration (one block - // per 64-bit lane of a 256-bit ymm); used to round the dispatch count - // down to a multiple supported by the kernel. - LLaneCount = Int32(4); var LSimdBlocks: Int32; {$ENDIF} begin {$IFDEF CRYPTOLIB_X86_SIMD} - if (APowTable <> nil) and (ANumBlocks >= LMinBlocks) then - begin - LSimdBlocks := ANumBlocks and not (LLaneCount - 1); - Poly1305BlocksBulkAvx2Core(@AState, APowTable, @ABuf[AOff], - NativeUInt(LSimdBlocks) * 16, 1); - Exit(LSimdBlocks); + if (APowTable = nil) then + Exit(0); + // The tier is re-selected here; SelectSlot is deterministic, so it matches the + // one TryInitPowerTable built the power table for. AVX2 consumes 4 blocks per + // iteration, SSE2 2; both need at least one lane's worth to pay off. + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + begin + if ANumBlocks < 4 then + Exit(0); + LSimdBlocks := ANumBlocks and not 3; + Poly1305BlocksBulkAvx2Core(@AState, APowTable, @ABuf[AOff], + NativeUInt(LSimdBlocks) * 16, 1); + Exit(LSimdBlocks); + end; + TX86SimdLevel.SSE2: + begin + if ANumBlocks < 2 then + Exit(0); + LSimdBlocks := ANumBlocks and not 1; + Poly1305BlocksBulkSse2Core(@AState, APowTable, @ABuf[AOff], + NativeUInt(LSimdBlocks) * 16, 1); + Exit(LSimdBlocks); + end; end; {$ENDIF} Result := 0; diff --git a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas index d875d4c5..7beb4cdf 100644 --- a/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas +++ b/CryptoLib/src/Crypto/Modes/ClpChaCha20Poly1305.pas @@ -28,6 +28,7 @@ interface ClpICipherParameters, ClpIKeyParameter, ClpCipherModeParameterUtilities, + ClpIBulkStreamCipher, ClpIChaCha7539Engine, ClpChaCha7539Engine, ClpPoly1305, @@ -107,6 +108,9 @@ TChaCha20Poly1305 = class(TInterfacedObject, IChaCha20Poly1305, IAeadCipher) var FChaCha20: IChaCha7539Engine; + // The cipher engine's bulk stream interface when it exposes one (resolved once); + // enables the 512B/8-way tier. nil => the narrower per-tier path is used. + FBulkChaCha: IBulkStreamCipher; FNonceBytes: Int32; FPoly1305: IMac; @@ -130,10 +134,10 @@ TChaCha20Poly1305 = class(TInterfacedObject, IChaCha20Poly1305, IAeadCipher) procedure PadMac(ACount: UInt64); procedure ProcessBlock(const AInBytes: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - procedure ProcessBlocks2(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - procedure ProcessBlocks4(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); + // Bulk tier via the engine's IBulkStreamCipher (ACount whole 64B blocks); keeps + // the same running data-count bookkeeping as the single-block ProcessBlock. + procedure ProcessBlocksBulk(const AInBytes: TCryptoLibByteArray; + AInOff, ACount: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); procedure ProcessData(const AInBytes: TCryptoLibByteArray; AInOff, AInLen: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); procedure Reset(AClearMac, AResetCipher: Boolean); overload; @@ -202,6 +206,8 @@ constructor TChaCha20Poly1305.Create(const APoly1305: IMac; FPoly1305 := APoly1305; FChaCha20 := AEngine; + if not Supports(FChaCha20, IBulkStreamCipher, FBulkChaCha) then + FBulkChaCha := nil; FNonceBytes := ANonceBytes; System.SetLength(FKey, KeySize); @@ -395,7 +401,7 @@ function TChaCha20Poly1305.ProcessByte(AInput: Byte; function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; AInOff, ALen: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LResultLen, LAvailable, LInLimit1, LInLimit2: Int32; + LResultLen, LAvailable, LInLimit1, LInLimit2, LRemBlocks: Int32; begin if (AInput = nil) then raise EArgumentNilCryptoLibException.CreateRes(@SInBytesNil); @@ -451,28 +457,36 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; AInOff := AInOff + LAvailable; LResultLen := LResultLen + BufSize; - while (AInOff <= LInLimit2 - (BufSize * 2)) do - begin - FPoly1305.BlockUpdate(AInput, AInOff, BufSize * 4); - ProcessBlocks4(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 4; - LResultLen := LResultLen + BufSize * 4; - end; - - while (AInOff <= LInLimit2) do + if (FBulkChaCha <> nil) then begin - FPoly1305.BlockUpdate(AInput, AInOff, BufSize * 2); - ProcessBlocks2(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 2; - LResultLen := LResultLen + BufSize * 2; - end; - - if (AInOff <= LInLimit1) then + // Bulk: MAC + decrypt in 512B (8-way) strides, then the remaining whole + // blocks (< 8) in a single engine-laddered call. + while (AInOff <= LInLimit2 - (BufSize * 6)) do + begin + FPoly1305.BlockUpdate(AInput, AInOff, BufSize * 8); + ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen); + AInOff := AInOff + BufSize * 8; + LResultLen := LResultLen + BufSize * 8; + end; + if (AInOff <= LInLimit1) then + begin + LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1; + FPoly1305.BlockUpdate(AInput, AInOff, LRemBlocks * BufSize); + ProcessBlocksBulk(AInput, AInOff, LRemBlocks, AOutput, AOutOff + LResultLen); + AInOff := AInOff + LRemBlocks * BufSize; + LResultLen := LResultLen + LRemBlocks * BufSize; + end; + end + else begin - FPoly1305.BlockUpdate(AInput, AInOff, BufSize); - ProcessBlock(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize; - LResultLen := LResultLen + BufSize; + // No bulk engine: single-block MAC-then-decrypt. + while (AInOff <= LInLimit1) do + begin + FPoly1305.BlockUpdate(AInput, AInOff, BufSize); + ProcessBlock(AInput, AInOff, AOutput, AOutOff + LResultLen); + AInOff := AInOff + BufSize; + LResultLen := LResultLen + BufSize; + end; end; FBufPos := System.Length(FBuf) + LInLimit1 - AInOff; @@ -500,25 +514,32 @@ function TChaCha20Poly1305.ProcessBytes(const AInput: TCryptoLibByteArray; LResultLen := BufSize; end; - while (AInOff <= LInLimit2 - (BufSize * 2)) do - begin - ProcessBlocks4(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 4; - LResultLen := LResultLen + BufSize * 4; - end; - - while (AInOff <= LInLimit2) do + if (FBulkChaCha <> nil) then begin - ProcessBlocks2(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize * 2; - LResultLen := LResultLen + BufSize * 2; - end; - - if (AInOff <= LInLimit1) then + // Bulk: encrypt in 512B (8-way) strides, then the remaining whole blocks + // (< 8) in a single engine-laddered call; MAC over the ciphertext below. + while (AInOff <= LInLimit2 - (BufSize * 6)) do + begin + ProcessBlocksBulk(AInput, AInOff, 8, AOutput, AOutOff + LResultLen); + AInOff := AInOff + BufSize * 8; + LResultLen := LResultLen + BufSize * 8; + end; + if (AInOff <= LInLimit1) then + begin + LRemBlocks := ((LInLimit1 - AInOff) div BufSize) + 1; + ProcessBlocksBulk(AInput, AInOff, LRemBlocks, AOutput, AOutOff + LResultLen); + AInOff := AInOff + LRemBlocks * BufSize; + LResultLen := LResultLen + LRemBlocks * BufSize; + end; + end + else begin - ProcessBlock(AInput, AInOff, AOutput, AOutOff + LResultLen); - AInOff := AInOff + BufSize; - LResultLen := LResultLen + BufSize; + while (AInOff <= LInLimit1) do + begin + ProcessBlock(AInput, AInOff, AOutput, AOutOff + LResultLen); + AInOff := AInOff + BufSize; + LResultLen := LResultLen + BufSize; + end; end; FPoly1305.BlockUpdate(AOutput, AOutOff, LResultLen); @@ -701,24 +722,14 @@ procedure TChaCha20Poly1305.ProcessBlock(const AInBytes: TCryptoLibByteArray; FDataCount := IncrementCount(FDataCount, 64, DataLimit); end; -procedure TChaCha20Poly1305.ProcessBlocks2(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); -begin - TCheck.OutputLength(AOutBytes, AOutOff, 128, SOutputBufferTooShort); - - FChaCha20.ProcessBlocks2(AInBytes, AInOff, AOutBytes, AOutOff); - - FDataCount := IncrementCount(FDataCount, 128, DataLimit); -end; - -procedure TChaCha20Poly1305.ProcessBlocks4(const AInBytes: TCryptoLibByteArray; - AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); +procedure TChaCha20Poly1305.ProcessBlocksBulk(const AInBytes: TCryptoLibByteArray; + AInOff, ACount: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); begin - TCheck.OutputLength(AOutBytes, AOutOff, 256, SOutputBufferTooShort); + TCheck.OutputLength(AOutBytes, AOutOff, ACount * BufSize, SOutputBufferTooShort); - FChaCha20.ProcessBlocks4(AInBytes, AInOff, AOutBytes, AOutOff); + FBulkChaCha.ProcessBlocks(AInBytes, AInOff, ACount, AOutBytes, AOutOff); - FDataCount := IncrementCount(FDataCount, 256, DataLimit); + FDataCount := IncrementCount(FDataCount, ACount * BufSize, DataLimit); end; procedure TChaCha20Poly1305.ProcessData(const AInBytes: TCryptoLibByteArray; diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc new file mode 100644 index 00000000..a5031baf --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_i386.inc @@ -0,0 +1,531 @@ +// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (i386). +// ------------------------------------------------------------------- +// +// Four 64-byte ChaCha20 blocks, state words held vertically (one block per +// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame; +// each quarter-round loads its 4 words into xmm0-3 (temp xmm4). +// rot16/rot8 use shift+or; rot12/rot7 use shift+or. +// +// On entry (after ClpSimdProc4Begin_i386.inc): +// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut. +// +// On exit: +// AState word-12 counter (byte 48) advanced by 4; the keystream is xored +// with AIn into AOut (256 bytes). +// +// punpck* / pshufb are db-encoded for broad assembler compatibility. + sub esp, 528 + mov dword ptr [esp + 512], 0 + mov dword ptr [esp + 516], 1 + mov dword ptr [esp + 520], 2 + mov dword ptr [esp + 524], 3 + movd xmm0, [esi] + pshufd xmm0, xmm0, $00 + movdqu [esp], xmm0 + movdqu [esp + 256], xmm0 + movd xmm0, [esi + 4] + pshufd xmm0, xmm0, $00 + movdqu [esp + 16], xmm0 + movdqu [esp + 272], xmm0 + movd xmm0, [esi + 8] + pshufd xmm0, xmm0, $00 + movdqu [esp + 32], xmm0 + movdqu [esp + 288], xmm0 + movd xmm0, [esi + 12] + pshufd xmm0, xmm0, $00 + movdqu [esp + 48], xmm0 + movdqu [esp + 304], xmm0 + movd xmm0, [esi + 16] + pshufd xmm0, xmm0, $00 + movdqu [esp + 64], xmm0 + movdqu [esp + 320], xmm0 + movd xmm0, [esi + 20] + pshufd xmm0, xmm0, $00 + movdqu [esp + 80], xmm0 + movdqu [esp + 336], xmm0 + movd xmm0, [esi + 24] + pshufd xmm0, xmm0, $00 + movdqu [esp + 96], xmm0 + movdqu [esp + 352], xmm0 + movd xmm0, [esi + 28] + pshufd xmm0, xmm0, $00 + movdqu [esp + 112], xmm0 + movdqu [esp + 368], xmm0 + movd xmm0, [esi + 32] + pshufd xmm0, xmm0, $00 + movdqu [esp + 128], xmm0 + movdqu [esp + 384], xmm0 + movd xmm0, [esi + 36] + pshufd xmm0, xmm0, $00 + movdqu [esp + 144], xmm0 + movdqu [esp + 400], xmm0 + movd xmm0, [esi + 40] + pshufd xmm0, xmm0, $00 + movdqu [esp + 160], xmm0 + movdqu [esp + 416], xmm0 + movd xmm0, [esi + 44] + pshufd xmm0, xmm0, $00 + movdqu [esp + 176], xmm0 + movdqu [esp + 432], xmm0 + movd xmm0, [esi + 48] + pshufd xmm0, xmm0, $00 + movdqu xmm1, [esp + 512] + paddd xmm0, xmm1 + movdqu [esp + 192], xmm0 + movdqu [esp + 448], xmm0 + movd xmm0, [esi + 52] + pshufd xmm0, xmm0, $00 + movdqu [esp + 208], xmm0 + movdqu [esp + 464], xmm0 + movd xmm0, [esi + 56] + pshufd xmm0, xmm0, $00 + movdqu [esp + 224], xmm0 + movdqu [esp + 480], xmm0 + movd xmm0, [esi + 60] + pshufd xmm0, xmm0, $00 + movdqu [esp + 240], xmm0 + movdqu [esp + 496], xmm0 +@@c7539_sse2_round: + movdqu xmm0, [esp] + movdqu xmm1, [esp + 64] + movdqu xmm2, [esp + 128] + movdqu xmm3, [esp + 192] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp], xmm0 + movdqu [esp + 64], xmm1 + movdqu [esp + 128], xmm2 + movdqu [esp + 192], xmm3 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 144] + movdqu xmm3, [esp + 208] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 16], xmm0 + movdqu [esp + 80], xmm1 + movdqu [esp + 144], xmm2 + movdqu [esp + 208], xmm3 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 96] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 224] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 32], xmm0 + movdqu [esp + 96], xmm1 + movdqu [esp + 160], xmm2 + movdqu [esp + 224], xmm3 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 112] + movdqu xmm2, [esp + 176] + movdqu xmm3, [esp + 240] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 48], xmm0 + movdqu [esp + 112], xmm1 + movdqu [esp + 176], xmm2 + movdqu [esp + 240], xmm3 + movdqu xmm0, [esp] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 240] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp], xmm0 + movdqu [esp + 80], xmm1 + movdqu [esp + 160], xmm2 + movdqu [esp + 240], xmm3 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 96] + movdqu xmm2, [esp + 176] + movdqu xmm3, [esp + 192] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 16], xmm0 + movdqu [esp + 96], xmm1 + movdqu [esp + 176], xmm2 + movdqu [esp + 192], xmm3 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 112] + movdqu xmm2, [esp + 128] + movdqu xmm3, [esp + 208] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 32], xmm0 + movdqu [esp + 112], xmm1 + movdqu [esp + 128], xmm2 + movdqu [esp + 208], xmm3 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 64] + movdqu xmm2, [esp + 144] + movdqu xmm3, [esp + 224] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 16 + psrld xmm4, 16 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm4, xmm3 + pslld xmm3, 8 + psrld xmm4, 24 + por xmm3, xmm4 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 48], xmm0 + movdqu [esp + 64], xmm1 + movdqu [esp + 144], xmm2 + movdqu [esp + 224], xmm3 + sub ebx, 2 + jg @@c7539_sse2_round + movdqu xmm0, [esp] + movdqu xmm1, [esp + 256] + paddd xmm0, xmm1 + movdqu [esp], xmm0 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 272] + paddd xmm0, xmm1 + movdqu [esp + 16], xmm0 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 288] + paddd xmm0, xmm1 + movdqu [esp + 32], xmm0 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 304] + paddd xmm0, xmm1 + movdqu [esp + 48], xmm0 + movdqu xmm0, [esp + 64] + movdqu xmm1, [esp + 320] + paddd xmm0, xmm1 + movdqu [esp + 64], xmm0 + movdqu xmm0, [esp + 80] + movdqu xmm1, [esp + 336] + paddd xmm0, xmm1 + movdqu [esp + 80], xmm0 + movdqu xmm0, [esp + 96] + movdqu xmm1, [esp + 352] + paddd xmm0, xmm1 + movdqu [esp + 96], xmm0 + movdqu xmm0, [esp + 112] + movdqu xmm1, [esp + 368] + paddd xmm0, xmm1 + movdqu [esp + 112], xmm0 + movdqu xmm0, [esp + 128] + movdqu xmm1, [esp + 384] + paddd xmm0, xmm1 + movdqu [esp + 128], xmm0 + movdqu xmm0, [esp + 144] + movdqu xmm1, [esp + 400] + paddd xmm0, xmm1 + movdqu [esp + 144], xmm0 + movdqu xmm0, [esp + 160] + movdqu xmm1, [esp + 416] + paddd xmm0, xmm1 + movdqu [esp + 160], xmm0 + movdqu xmm0, [esp + 176] + movdqu xmm1, [esp + 432] + paddd xmm0, xmm1 + movdqu [esp + 176], xmm0 + movdqu xmm0, [esp + 192] + movdqu xmm1, [esp + 448] + paddd xmm0, xmm1 + movdqu [esp + 192], xmm0 + movdqu xmm0, [esp + 208] + movdqu xmm1, [esp + 464] + paddd xmm0, xmm1 + movdqu [esp + 208], xmm0 + movdqu xmm0, [esp + 224] + movdqu xmm1, [esp + 480] + paddd xmm0, xmm1 + movdqu [esp + 224], xmm0 + movdqu xmm0, [esp + 240] + movdqu xmm1, [esp + 496] + paddd xmm0, xmm1 + movdqu [esp + 240], xmm0 + add dword ptr [esi + 48], 4 +{$IFDEF CHACHA_CTR64} + adc dword ptr [esi + 52], 0 +{$ENDIF} + movdqu xmm0, [esp] + movdqu xmm1, [esp + 16] + movdqu xmm2, [esp + 32] + movdqu xmm3, [esp + 48] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi] + pxor xmm0, xmm5 + movdqu [eax], xmm0 + movdqu xmm5, [edi + 64] + pxor xmm1, xmm5 + movdqu [eax + 64], xmm1 + movdqu xmm5, [edi + 128] + pxor xmm4, xmm5 + movdqu [eax + 128], xmm4 + movdqu xmm5, [edi + 192] + pxor xmm3, xmm5 + movdqu [eax + 192], xmm3 + movdqu xmm0, [esp + 64] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 96] + movdqu xmm3, [esp + 112] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 16] + pxor xmm0, xmm5 + movdqu [eax + 16], xmm0 + movdqu xmm5, [edi + 80] + pxor xmm1, xmm5 + movdqu [eax + 80], xmm1 + movdqu xmm5, [edi + 144] + pxor xmm4, xmm5 + movdqu [eax + 144], xmm4 + movdqu xmm5, [edi + 208] + pxor xmm3, xmm5 + movdqu [eax + 208], xmm3 + movdqu xmm0, [esp + 128] + movdqu xmm1, [esp + 144] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 176] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 32] + pxor xmm0, xmm5 + movdqu [eax + 32], xmm0 + movdqu xmm5, [edi + 96] + pxor xmm1, xmm5 + movdqu [eax + 96], xmm1 + movdqu xmm5, [edi + 160] + pxor xmm4, xmm5 + movdqu [eax + 160], xmm4 + movdqu xmm5, [edi + 224] + pxor xmm3, xmm5 + movdqu [eax + 224], xmm3 + movdqu xmm0, [esp + 192] + movdqu xmm1, [esp + 208] + movdqu xmm2, [esp + 224] + movdqu xmm3, [esp + 240] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 48] + pxor xmm0, xmm5 + movdqu [eax + 48], xmm0 + movdqu xmm5, [edi + 112] + pxor xmm1, xmm5 + movdqu [eax + 112], xmm1 + movdqu xmm5, [edi + 176] + pxor xmm4, xmm5 + movdqu [eax + 176], xmm4 + movdqu xmm5, [edi + 240] + pxor xmm3, xmm5 + movdqu [eax + 240], xmm3 + add esp, 528 + pop edi + pop esi + pop ebx + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc new file mode 100644 index 00000000..080a9a32 --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Sse2_x86_64.inc @@ -0,0 +1,460 @@ +// ChaCha7539 (IETF) 4-way (256B) SSE2 keystream kernel, vertical layout (x86-64). +// ------------------------------------------------------------------- +// +// Four 64-byte ChaCha20 blocks, state words held vertically (one block per +// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words +// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch. +// rot16/rot8 use shift+or; rot12/rot7 use shift+or. +// +// On entry (after ClpSimdProc4Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// +// On exit: +// AState word-12 counter (byte 48) advanced by 4; the keystream is xored +// with AIn into AOut (256 bytes). +// +// Ops touching the extended xmm8-15 registers are db-encoded for broad +// assembler compatibility. + mov r10, rcx +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + sub rsp, 336 + mov dword ptr [rsp + 320], 0 + mov dword ptr [rsp + 324], 1 + mov dword ptr [rsp + 328], 2 + mov dword ptr [rsp + 332], 3 + movd xmm0, [rdx] + pshufd xmm0, xmm0, $00 + movdqu [rsp + 64], xmm0 + movd xmm1, [rdx + 4] + pshufd xmm1, xmm1, $00 + movdqu [rsp + 80], xmm1 + movd xmm2, [rdx + 8] + pshufd xmm2, xmm2, $00 + movdqu [rsp + 96], xmm2 + movd xmm3, [rdx + 12] + pshufd xmm3, xmm3, $00 + movdqu [rsp + 112], xmm3 + movd xmm4, [rdx + 16] + pshufd xmm4, xmm4, $00 + movdqu [rsp + 128], xmm4 + movd xmm5, [rdx + 20] + pshufd xmm5, xmm5, $00 + movdqu [rsp + 144], xmm5 + movd xmm6, [rdx + 24] + pshufd xmm6, xmm6, $00 + movdqu [rsp + 160], xmm6 + movd xmm7, [rdx + 28] + pshufd xmm7, xmm7, $00 + movdqu [rsp + 176], xmm7 + movd xmm8, [rdx + 32] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp], xmm8 + movdqu [rsp + 192], xmm8 + movd xmm8, [rdx + 36] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 16], xmm8 + movdqu [rsp + 208], xmm8 + movd xmm8, [rdx + 40] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 32], xmm8 + movdqu [rsp + 224], xmm8 + movd xmm8, [rdx + 44] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 48], xmm8 + movdqu [rsp + 240], xmm8 + movd xmm12, [rdx + 48] + db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 + movdqu xmm8, [rsp + 320] + db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 + movdqu [rsp + 256], xmm12 + movd xmm13, [rdx + 52] + db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 + movdqu [rsp + 272], xmm13 + movd xmm14, [rdx + 56] + db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 + movdqu [rsp + 288], xmm14 + movd xmm15, [rdx + 60] + db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 + movdqu [rsp + 304], xmm15 +@@c7539_sse2_round: + movdqu xmm8, [rsp] + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 + db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 + db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + movdqu [rsp], xmm8 + movdqu xmm8, [rsp + 16] + paddd xmm1, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 + db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + paddd xmm1, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 + db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + movdqu [rsp + 16], xmm8 + movdqu xmm8, [rsp + 32] + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 + db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 + db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + movdqu [rsp + 32], xmm8 + movdqu xmm8, [rsp + 48] + paddd xmm3, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 + db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + paddd xmm3, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 + db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + movdqu [rsp + 48], xmm8 + movdqu xmm8, [rsp + 32] + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 + db $66, $41, $0F, $72, $F7, $10 // pslld xmm15, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $6F, $CF // movdqa xmm9, xmm15 + db $66, $41, $0F, $72, $F7, $08 // pslld xmm15, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $F9 // por xmm15, xmm9 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + movdqu [rsp + 32], xmm8 + movdqu xmm8, [rsp + 48] + paddd xmm1, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 + db $66, $41, $0F, $72, $F4, $10 // pslld xmm12, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + paddd xmm1, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $6F, $CC // movdqa xmm9, xmm12 + db $66, $41, $0F, $72, $F4, $08 // pslld xmm12, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $E1 // por xmm12, xmm9 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + movdqu [rsp + 48], xmm8 + movdqu xmm8, [rsp] + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 + db $66, $41, $0F, $72, $F5, $10 // pslld xmm13, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $6F, $CD // movdqa xmm9, xmm13 + db $66, $41, $0F, $72, $F5, $08 // pslld xmm13, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $E9 // por xmm13, xmm9 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + movdqu [rsp], xmm8 + movdqu xmm8, [rsp + 16] + paddd xmm3, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 + db $66, $41, $0F, $72, $F6, $10 // pslld xmm14, 16 + db $66, $41, $0F, $72, $D1, $10 // psrld xmm9, 16 + db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + paddd xmm3, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $6F, $CE // movdqa xmm9, xmm14 + db $66, $41, $0F, $72, $F6, $08 // pslld xmm14, 8 + db $66, $41, $0F, $72, $D1, $18 // psrld xmm9, 24 + db $66, $45, $0F, $EB, $F1 // por xmm14, xmm9 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + movdqu [rsp + 16], xmm8 + sub r10, 2 + jg @@c7539_sse2_round + movdqu xmm8, [rsp + 64] + db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8 + movdqu [rsp + 64], xmm0 + movdqu xmm8, [rsp + 80] + db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8 + movdqu [rsp + 80], xmm1 + movdqu xmm8, [rsp + 96] + db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8 + movdqu [rsp + 96], xmm2 + movdqu xmm8, [rsp + 112] + db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8 + movdqu [rsp + 112], xmm3 + movdqu xmm8, [rsp + 128] + db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8 + movdqu [rsp + 128], xmm4 + movdqu xmm8, [rsp + 144] + db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8 + movdqu [rsp + 144], xmm5 + movdqu xmm8, [rsp + 160] + db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8 + movdqu [rsp + 160], xmm6 + movdqu xmm8, [rsp + 176] + db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8 + movdqu [rsp + 176], xmm7 + movdqu xmm8, [rsp + 256] + db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 + movdqu [rsp + 256], xmm12 + movdqu xmm8, [rsp + 272] + db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8 + movdqu [rsp + 272], xmm13 + movdqu xmm8, [rsp + 288] + db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8 + movdqu [rsp + 288], xmm14 + movdqu xmm8, [rsp + 304] + db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8 + movdqu [rsp + 304], xmm15 + movdqu xmm8, [rsp] + movdqu xmm9, [rsp + 192] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 192], xmm8 + movdqu xmm8, [rsp + 16] + movdqu xmm9, [rsp + 208] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 208], xmm8 + movdqu xmm8, [rsp + 32] + movdqu xmm9, [rsp + 224] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 224], xmm8 + movdqu xmm8, [rsp + 48] + movdqu xmm9, [rsp + 240] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 240], xmm8 + add dword ptr [rdx + 48], 4 +{$IFDEF CHACHA_CTR64} + adc dword ptr [rdx + 52], 0 +{$ENDIF} + movdqu xmm0, [rsp + 64] + movdqu xmm1, [rsp + 80] + movdqu xmm2, [rsp + 96] + movdqu xmm3, [rsp + 112] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8] + pxor xmm0, xmm6 + movdqu [r9], xmm0 + movdqu xmm6, [r8 + 64] + pxor xmm1, xmm6 + movdqu [r9 + 64], xmm1 + movdqu xmm6, [r8 + 128] + pxor xmm4, xmm6 + movdqu [r9 + 128], xmm4 + movdqu xmm6, [r8 + 192] + pxor xmm3, xmm6 + movdqu [r9 + 192], xmm3 + movdqu xmm0, [rsp + 128] + movdqu xmm1, [rsp + 144] + movdqu xmm2, [rsp + 160] + movdqu xmm3, [rsp + 176] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 16] + pxor xmm0, xmm6 + movdqu [r9 + 16], xmm0 + movdqu xmm6, [r8 + 80] + pxor xmm1, xmm6 + movdqu [r9 + 80], xmm1 + movdqu xmm6, [r8 + 144] + pxor xmm4, xmm6 + movdqu [r9 + 144], xmm4 + movdqu xmm6, [r8 + 208] + pxor xmm3, xmm6 + movdqu [r9 + 208], xmm3 + movdqu xmm0, [rsp + 192] + movdqu xmm1, [rsp + 208] + movdqu xmm2, [rsp + 224] + movdqu xmm3, [rsp + 240] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 32] + pxor xmm0, xmm6 + movdqu [r9 + 32], xmm0 + movdqu xmm6, [r8 + 96] + pxor xmm1, xmm6 + movdqu [r9 + 96], xmm1 + movdqu xmm6, [r8 + 160] + pxor xmm4, xmm6 + movdqu [r9 + 160], xmm4 + movdqu xmm6, [r8 + 224] + pxor xmm3, xmm6 + movdqu [r9 + 224], xmm3 + movdqu xmm0, [rsp + 256] + movdqu xmm1, [rsp + 272] + movdqu xmm2, [rsp + 288] + movdqu xmm3, [rsp + 304] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 48] + pxor xmm0, xmm6 + movdqu [r9 + 48], xmm0 + movdqu xmm6, [r8 + 112] + pxor xmm1, xmm6 + movdqu [r9 + 112], xmm1 + movdqu xmm6, [r8 + 176] + pxor xmm4, xmm6 + movdqu [r9 + 176], xmm4 + movdqu xmm6, [r8 + 240] + pxor xmm3, xmm6 + movdqu [r9 + 240], xmm3 + add rsp, 336 +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc new file mode 100644 index 00000000..7af122fb --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_i386.inc @@ -0,0 +1,493 @@ +// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (i386). +// ------------------------------------------------------------------- +// +// Four 64-byte ChaCha20 blocks, state words held vertically (one block per +// xmm lane). i386 has only 8 xmm, so the 16 state words live in a stack frame; +// each quarter-round loads its 4 words into xmm0-3 (temp xmm4) and the rot16/rot8 masks in xmm6-7. +// rot16/rot8 use pshufb; rot12/rot7 use shift+or. +// +// On entry (after ClpSimdProc4Begin_i386.inc): +// ebx = ARounds, esi = AState (16 UInt32), edi = AIn, eax = AOut. +// +// On exit: +// AState word-12 counter (byte 48) advanced by 4; the keystream is xored +// with AIn into AOut (256 bytes). +// +// punpck* / pshufb are db-encoded for broad assembler compatibility. + sub esp, 560 + mov dword ptr [esp + 528], $01000302 + mov dword ptr [esp + 532], $05040706 + mov dword ptr [esp + 536], $09080B0A + mov dword ptr [esp + 540], $0D0C0F0E + mov dword ptr [esp + 544], $02010003 + mov dword ptr [esp + 548], $06050407 + mov dword ptr [esp + 552], $0A09080B + mov dword ptr [esp + 556], $0E0D0C0F + movdqu xmm6, [esp + 528] + movdqu xmm7, [esp + 544] + mov dword ptr [esp + 512], 0 + mov dword ptr [esp + 516], 1 + mov dword ptr [esp + 520], 2 + mov dword ptr [esp + 524], 3 + movd xmm0, [esi] + pshufd xmm0, xmm0, $00 + movdqu [esp], xmm0 + movdqu [esp + 256], xmm0 + movd xmm0, [esi + 4] + pshufd xmm0, xmm0, $00 + movdqu [esp + 16], xmm0 + movdqu [esp + 272], xmm0 + movd xmm0, [esi + 8] + pshufd xmm0, xmm0, $00 + movdqu [esp + 32], xmm0 + movdqu [esp + 288], xmm0 + movd xmm0, [esi + 12] + pshufd xmm0, xmm0, $00 + movdqu [esp + 48], xmm0 + movdqu [esp + 304], xmm0 + movd xmm0, [esi + 16] + pshufd xmm0, xmm0, $00 + movdqu [esp + 64], xmm0 + movdqu [esp + 320], xmm0 + movd xmm0, [esi + 20] + pshufd xmm0, xmm0, $00 + movdqu [esp + 80], xmm0 + movdqu [esp + 336], xmm0 + movd xmm0, [esi + 24] + pshufd xmm0, xmm0, $00 + movdqu [esp + 96], xmm0 + movdqu [esp + 352], xmm0 + movd xmm0, [esi + 28] + pshufd xmm0, xmm0, $00 + movdqu [esp + 112], xmm0 + movdqu [esp + 368], xmm0 + movd xmm0, [esi + 32] + pshufd xmm0, xmm0, $00 + movdqu [esp + 128], xmm0 + movdqu [esp + 384], xmm0 + movd xmm0, [esi + 36] + pshufd xmm0, xmm0, $00 + movdqu [esp + 144], xmm0 + movdqu [esp + 400], xmm0 + movd xmm0, [esi + 40] + pshufd xmm0, xmm0, $00 + movdqu [esp + 160], xmm0 + movdqu [esp + 416], xmm0 + movd xmm0, [esi + 44] + pshufd xmm0, xmm0, $00 + movdqu [esp + 176], xmm0 + movdqu [esp + 432], xmm0 + movd xmm0, [esi + 48] + pshufd xmm0, xmm0, $00 + movdqu xmm1, [esp + 512] + paddd xmm0, xmm1 + movdqu [esp + 192], xmm0 + movdqu [esp + 448], xmm0 + movd xmm0, [esi + 52] + pshufd xmm0, xmm0, $00 + movdqu [esp + 208], xmm0 + movdqu [esp + 464], xmm0 + movd xmm0, [esi + 56] + pshufd xmm0, xmm0, $00 + movdqu [esp + 224], xmm0 + movdqu [esp + 480], xmm0 + movd xmm0, [esi + 60] + pshufd xmm0, xmm0, $00 + movdqu [esp + 240], xmm0 + movdqu [esp + 496], xmm0 +@@c7539_ssse3_round: + movdqu xmm0, [esp] + movdqu xmm1, [esp + 64] + movdqu xmm2, [esp + 128] + movdqu xmm3, [esp + 192] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp], xmm0 + movdqu [esp + 64], xmm1 + movdqu [esp + 128], xmm2 + movdqu [esp + 192], xmm3 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 144] + movdqu xmm3, [esp + 208] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 16], xmm0 + movdqu [esp + 80], xmm1 + movdqu [esp + 144], xmm2 + movdqu [esp + 208], xmm3 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 96] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 224] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 32], xmm0 + movdqu [esp + 96], xmm1 + movdqu [esp + 160], xmm2 + movdqu [esp + 224], xmm3 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 112] + movdqu xmm2, [esp + 176] + movdqu xmm3, [esp + 240] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 48], xmm0 + movdqu [esp + 112], xmm1 + movdqu [esp + 176], xmm2 + movdqu [esp + 240], xmm3 + movdqu xmm0, [esp] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 240] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp], xmm0 + movdqu [esp + 80], xmm1 + movdqu [esp + 160], xmm2 + movdqu [esp + 240], xmm3 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 96] + movdqu xmm2, [esp + 176] + movdqu xmm3, [esp + 192] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 16], xmm0 + movdqu [esp + 96], xmm1 + movdqu [esp + 176], xmm2 + movdqu [esp + 192], xmm3 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 112] + movdqu xmm2, [esp + 128] + movdqu xmm3, [esp + 208] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 32], xmm0 + movdqu [esp + 112], xmm1 + movdqu [esp + 128], xmm2 + movdqu [esp + 208], xmm3 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 64] + movdqu xmm2, [esp + 144] + movdqu xmm3, [esp + 224] + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 12 + psrld xmm4, 20 + por xmm1, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm4, xmm1 + pslld xmm1, 7 + psrld xmm4, 25 + por xmm1, xmm4 + movdqu [esp + 48], xmm0 + movdqu [esp + 64], xmm1 + movdqu [esp + 144], xmm2 + movdqu [esp + 224], xmm3 + sub ebx, 2 + jg @@c7539_ssse3_round + movdqu xmm0, [esp] + movdqu xmm1, [esp + 256] + paddd xmm0, xmm1 + movdqu [esp], xmm0 + movdqu xmm0, [esp + 16] + movdqu xmm1, [esp + 272] + paddd xmm0, xmm1 + movdqu [esp + 16], xmm0 + movdqu xmm0, [esp + 32] + movdqu xmm1, [esp + 288] + paddd xmm0, xmm1 + movdqu [esp + 32], xmm0 + movdqu xmm0, [esp + 48] + movdqu xmm1, [esp + 304] + paddd xmm0, xmm1 + movdqu [esp + 48], xmm0 + movdqu xmm0, [esp + 64] + movdqu xmm1, [esp + 320] + paddd xmm0, xmm1 + movdqu [esp + 64], xmm0 + movdqu xmm0, [esp + 80] + movdqu xmm1, [esp + 336] + paddd xmm0, xmm1 + movdqu [esp + 80], xmm0 + movdqu xmm0, [esp + 96] + movdqu xmm1, [esp + 352] + paddd xmm0, xmm1 + movdqu [esp + 96], xmm0 + movdqu xmm0, [esp + 112] + movdqu xmm1, [esp + 368] + paddd xmm0, xmm1 + movdqu [esp + 112], xmm0 + movdqu xmm0, [esp + 128] + movdqu xmm1, [esp + 384] + paddd xmm0, xmm1 + movdqu [esp + 128], xmm0 + movdqu xmm0, [esp + 144] + movdqu xmm1, [esp + 400] + paddd xmm0, xmm1 + movdqu [esp + 144], xmm0 + movdqu xmm0, [esp + 160] + movdqu xmm1, [esp + 416] + paddd xmm0, xmm1 + movdqu [esp + 160], xmm0 + movdqu xmm0, [esp + 176] + movdqu xmm1, [esp + 432] + paddd xmm0, xmm1 + movdqu [esp + 176], xmm0 + movdqu xmm0, [esp + 192] + movdqu xmm1, [esp + 448] + paddd xmm0, xmm1 + movdqu [esp + 192], xmm0 + movdqu xmm0, [esp + 208] + movdqu xmm1, [esp + 464] + paddd xmm0, xmm1 + movdqu [esp + 208], xmm0 + movdqu xmm0, [esp + 224] + movdqu xmm1, [esp + 480] + paddd xmm0, xmm1 + movdqu [esp + 224], xmm0 + movdqu xmm0, [esp + 240] + movdqu xmm1, [esp + 496] + paddd xmm0, xmm1 + movdqu [esp + 240], xmm0 + add dword ptr [esi + 48], 4 +{$IFDEF CHACHA_CTR64} + adc dword ptr [esi + 52], 0 +{$ENDIF} + movdqu xmm0, [esp] + movdqu xmm1, [esp + 16] + movdqu xmm2, [esp + 32] + movdqu xmm3, [esp + 48] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi] + pxor xmm0, xmm5 + movdqu [eax], xmm0 + movdqu xmm5, [edi + 64] + pxor xmm1, xmm5 + movdqu [eax + 64], xmm1 + movdqu xmm5, [edi + 128] + pxor xmm4, xmm5 + movdqu [eax + 128], xmm4 + movdqu xmm5, [edi + 192] + pxor xmm3, xmm5 + movdqu [eax + 192], xmm3 + movdqu xmm0, [esp + 64] + movdqu xmm1, [esp + 80] + movdqu xmm2, [esp + 96] + movdqu xmm3, [esp + 112] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 16] + pxor xmm0, xmm5 + movdqu [eax + 16], xmm0 + movdqu xmm5, [edi + 80] + pxor xmm1, xmm5 + movdqu [eax + 80], xmm1 + movdqu xmm5, [edi + 144] + pxor xmm4, xmm5 + movdqu [eax + 144], xmm4 + movdqu xmm5, [edi + 208] + pxor xmm3, xmm5 + movdqu [eax + 208], xmm3 + movdqu xmm0, [esp + 128] + movdqu xmm1, [esp + 144] + movdqu xmm2, [esp + 160] + movdqu xmm3, [esp + 176] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 32] + pxor xmm0, xmm5 + movdqu [eax + 32], xmm0 + movdqu xmm5, [edi + 96] + pxor xmm1, xmm5 + movdqu [eax + 96], xmm1 + movdqu xmm5, [edi + 160] + pxor xmm4, xmm5 + movdqu [eax + 160], xmm4 + movdqu xmm5, [edi + 224] + pxor xmm3, xmm5 + movdqu [eax + 224], xmm3 + movdqu xmm0, [esp + 192] + movdqu xmm1, [esp + 208] + movdqu xmm2, [esp + 224] + movdqu xmm3, [esp + 240] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm5, [edi + 48] + pxor xmm0, xmm5 + movdqu [eax + 48], xmm0 + movdqu xmm5, [edi + 112] + pxor xmm1, xmm5 + movdqu [eax + 112], xmm1 + movdqu xmm5, [edi + 176] + pxor xmm4, xmm5 + movdqu [eax + 176], xmm4 + movdqu xmm5, [edi + 240] + pxor xmm3, xmm5 + movdqu [eax + 240], xmm3 + add esp, 560 + pop edi + pop esi + pop ebx + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc new file mode 100644 index 00000000..ac08b118 --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks4Ssse3_x86_64.inc @@ -0,0 +1,422 @@ +// ChaCha7539 (IETF) 4-way (256B) SSSE3 keystream kernel, vertical layout (x86-64). +// ------------------------------------------------------------------- +// +// Four 64-byte ChaCha20 blocks, state words held vertically (one block per +// xmm lane): xmm0-3 = words a (0-3), xmm4-7 = words b (4-7), xmm12-15 = words +// d (12-15). Words c (8-11) live in a stack slot; xmm8-9 are scratch, xmm10-11 the rot16/rot8 pshufb masks. +// rot16/rot8 use pshufb; rot12/rot7 use shift+or. +// +// On entry (after ClpSimdProc4Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// +// On exit: +// AState word-12 counter (byte 48) advanced by 4; the keystream is xored +// with AIn into AOut (256 bytes). +// +// Ops touching the extended xmm8-15 registers are db-encoded for broad +// assembler compatibility. + mov r10, rcx +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + sub rsp, 368 + mov dword ptr [rsp + 336], $01000302 + mov dword ptr [rsp + 340], $05040706 + mov dword ptr [rsp + 344], $09080B0A + mov dword ptr [rsp + 348], $0D0C0F0E + mov dword ptr [rsp + 352], $02010003 + mov dword ptr [rsp + 356], $06050407 + mov dword ptr [rsp + 360], $0A09080B + mov dword ptr [rsp + 364], $0E0D0C0F + movdqu xmm10, [rsp + 336] + movdqu xmm11, [rsp + 352] + mov dword ptr [rsp + 320], 0 + mov dword ptr [rsp + 324], 1 + mov dword ptr [rsp + 328], 2 + mov dword ptr [rsp + 332], 3 + movd xmm0, [rdx] + pshufd xmm0, xmm0, $00 + movdqu [rsp + 64], xmm0 + movd xmm1, [rdx + 4] + pshufd xmm1, xmm1, $00 + movdqu [rsp + 80], xmm1 + movd xmm2, [rdx + 8] + pshufd xmm2, xmm2, $00 + movdqu [rsp + 96], xmm2 + movd xmm3, [rdx + 12] + pshufd xmm3, xmm3, $00 + movdqu [rsp + 112], xmm3 + movd xmm4, [rdx + 16] + pshufd xmm4, xmm4, $00 + movdqu [rsp + 128], xmm4 + movd xmm5, [rdx + 20] + pshufd xmm5, xmm5, $00 + movdqu [rsp + 144], xmm5 + movd xmm6, [rdx + 24] + pshufd xmm6, xmm6, $00 + movdqu [rsp + 160], xmm6 + movd xmm7, [rdx + 28] + pshufd xmm7, xmm7, $00 + movdqu [rsp + 176], xmm7 + movd xmm8, [rdx + 32] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp], xmm8 + movdqu [rsp + 192], xmm8 + movd xmm8, [rdx + 36] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 16], xmm8 + movdqu [rsp + 208], xmm8 + movd xmm8, [rdx + 40] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 32], xmm8 + movdqu [rsp + 224], xmm8 + movd xmm8, [rdx + 44] + db $66, $45, $0F, $70, $C0, $00 // pshufd xmm8, xmm8, $00 + movdqu [rsp + 48], xmm8 + movdqu [rsp + 240], xmm8 + movd xmm12, [rdx + 48] + db $66, $45, $0F, $70, $E4, $00 // pshufd xmm12, xmm12, $00 + movdqu xmm8, [rsp + 320] + db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 + movdqu [rsp + 256], xmm12 + movd xmm13, [rdx + 52] + db $66, $45, $0F, $70, $ED, $00 // pshufd xmm13, xmm13, $00 + movdqu [rsp + 272], xmm13 + movd xmm14, [rdx + 56] + db $66, $45, $0F, $70, $F6, $00 // pshufd xmm14, xmm14, $00 + movdqu [rsp + 288], xmm14 + movd xmm15, [rdx + 60] + db $66, $45, $0F, $70, $FF, $00 // pshufd xmm15, xmm15, $00 + movdqu [rsp + 304], xmm15 +@@c7539_ssse3_round: + movdqu xmm8, [rsp] + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + paddd xmm0, xmm4 + db $66, $44, $0F, $EF, $E0 // pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + movdqu [rsp], xmm8 + movdqu xmm8, [rsp + 16] + paddd xmm1, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + paddd xmm1, xmm5 + db $66, $44, $0F, $EF, $E9 // pxor xmm13, xmm1 + db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + movdqu [rsp + 16], xmm8 + movdqu xmm8, [rsp + 32] + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + paddd xmm2, xmm6 + db $66, $44, $0F, $EF, $F2 // pxor xmm14, xmm2 + db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + movdqu [rsp + 32], xmm8 + movdqu xmm8, [rsp + 48] + paddd xmm3, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + paddd xmm3, xmm7 + db $66, $44, $0F, $EF, $FB // pxor xmm15, xmm3 + db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + movdqu [rsp + 48], xmm8 + movdqu xmm8, [rsp + 32] + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $38, $00, $FA // pshufb xmm15, xmm10 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + paddd xmm0, xmm5 + db $66, $44, $0F, $EF, $F8 // pxor xmm15, xmm0 + db $66, $45, $0F, $38, $00, $FB // pshufb xmm15, xmm11 + db $66, $45, $0F, $FE, $C7 // paddd xmm8, xmm15 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $44, $0F, $6F, $CD // movdqa xmm9, xmm5 + pslld xmm5, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E9 // por xmm5, xmm9 + movdqu [rsp + 32], xmm8 + movdqu xmm8, [rsp + 48] + paddd xmm1, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E2 // pshufb xmm12, xmm10 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + paddd xmm1, xmm6 + db $66, $44, $0F, $EF, $E1 // pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E3 // pshufb xmm12, xmm11 + db $66, $45, $0F, $FE, $C4 // paddd xmm8, xmm12 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $44, $0F, $6F, $CE // movdqa xmm9, xmm6 + pslld xmm6, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F1 // por xmm6, xmm9 + movdqu [rsp + 48], xmm8 + movdqu xmm8, [rsp] + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $38, $00, $EA // pshufb xmm13, xmm10 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + paddd xmm2, xmm7 + db $66, $44, $0F, $EF, $EA // pxor xmm13, xmm2 + db $66, $45, $0F, $38, $00, $EB // pshufb xmm13, xmm11 + db $66, $45, $0F, $FE, $C5 // paddd xmm8, xmm13 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + db $66, $44, $0F, $6F, $CF // movdqa xmm9, xmm7 + pslld xmm7, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $F9 // por xmm7, xmm9 + movdqu [rsp], xmm8 + movdqu xmm8, [rsp + 16] + paddd xmm3, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $38, $00, $F2 // pshufb xmm14, xmm10 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 12 + db $66, $41, $0F, $72, $D1, $14 // psrld xmm9, 20 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + paddd xmm3, xmm4 + db $66, $44, $0F, $EF, $F3 // pxor xmm14, xmm3 + db $66, $45, $0F, $38, $00, $F3 // pshufb xmm14, xmm11 + db $66, $45, $0F, $FE, $C6 // paddd xmm8, xmm14 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $44, $0F, $6F, $CC // movdqa xmm9, xmm4 + pslld xmm4, 7 + db $66, $41, $0F, $72, $D1, $19 // psrld xmm9, 25 + db $66, $41, $0F, $EB, $E1 // por xmm4, xmm9 + movdqu [rsp + 16], xmm8 + sub r10, 2 + jg @@c7539_ssse3_round + movdqu xmm8, [rsp + 64] + db $66, $41, $0F, $FE, $C0 // paddd xmm0, xmm8 + movdqu [rsp + 64], xmm0 + movdqu xmm8, [rsp + 80] + db $66, $41, $0F, $FE, $C8 // paddd xmm1, xmm8 + movdqu [rsp + 80], xmm1 + movdqu xmm8, [rsp + 96] + db $66, $41, $0F, $FE, $D0 // paddd xmm2, xmm8 + movdqu [rsp + 96], xmm2 + movdqu xmm8, [rsp + 112] + db $66, $41, $0F, $FE, $D8 // paddd xmm3, xmm8 + movdqu [rsp + 112], xmm3 + movdqu xmm8, [rsp + 128] + db $66, $41, $0F, $FE, $E0 // paddd xmm4, xmm8 + movdqu [rsp + 128], xmm4 + movdqu xmm8, [rsp + 144] + db $66, $41, $0F, $FE, $E8 // paddd xmm5, xmm8 + movdqu [rsp + 144], xmm5 + movdqu xmm8, [rsp + 160] + db $66, $41, $0F, $FE, $F0 // paddd xmm6, xmm8 + movdqu [rsp + 160], xmm6 + movdqu xmm8, [rsp + 176] + db $66, $41, $0F, $FE, $F8 // paddd xmm7, xmm8 + movdqu [rsp + 176], xmm7 + movdqu xmm8, [rsp + 256] + db $66, $45, $0F, $FE, $E0 // paddd xmm12, xmm8 + movdqu [rsp + 256], xmm12 + movdqu xmm8, [rsp + 272] + db $66, $45, $0F, $FE, $E8 // paddd xmm13, xmm8 + movdqu [rsp + 272], xmm13 + movdqu xmm8, [rsp + 288] + db $66, $45, $0F, $FE, $F0 // paddd xmm14, xmm8 + movdqu [rsp + 288], xmm14 + movdqu xmm8, [rsp + 304] + db $66, $45, $0F, $FE, $F8 // paddd xmm15, xmm8 + movdqu [rsp + 304], xmm15 + movdqu xmm8, [rsp] + movdqu xmm9, [rsp + 192] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 192], xmm8 + movdqu xmm8, [rsp + 16] + movdqu xmm9, [rsp + 208] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 208], xmm8 + movdqu xmm8, [rsp + 32] + movdqu xmm9, [rsp + 224] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 224], xmm8 + movdqu xmm8, [rsp + 48] + movdqu xmm9, [rsp + 240] + db $66, $45, $0F, $FE, $C1 // paddd xmm8, xmm9 + movdqu [rsp + 240], xmm8 + add dword ptr [rdx + 48], 4 +{$IFDEF CHACHA_CTR64} + adc dword ptr [rdx + 52], 0 +{$ENDIF} + movdqu xmm0, [rsp + 64] + movdqu xmm1, [rsp + 80] + movdqu xmm2, [rsp + 96] + movdqu xmm3, [rsp + 112] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8] + pxor xmm0, xmm6 + movdqu [r9], xmm0 + movdqu xmm6, [r8 + 64] + pxor xmm1, xmm6 + movdqu [r9 + 64], xmm1 + movdqu xmm6, [r8 + 128] + pxor xmm4, xmm6 + movdqu [r9 + 128], xmm4 + movdqu xmm6, [r8 + 192] + pxor xmm3, xmm6 + movdqu [r9 + 192], xmm3 + movdqu xmm0, [rsp + 128] + movdqu xmm1, [rsp + 144] + movdqu xmm2, [rsp + 160] + movdqu xmm3, [rsp + 176] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 16] + pxor xmm0, xmm6 + movdqu [r9 + 16], xmm0 + movdqu xmm6, [r8 + 80] + pxor xmm1, xmm6 + movdqu [r9 + 80], xmm1 + movdqu xmm6, [r8 + 144] + pxor xmm4, xmm6 + movdqu [r9 + 144], xmm4 + movdqu xmm6, [r8 + 208] + pxor xmm3, xmm6 + movdqu [r9 + 208], xmm3 + movdqu xmm0, [rsp + 192] + movdqu xmm1, [rsp + 208] + movdqu xmm2, [rsp + 224] + movdqu xmm3, [rsp + 240] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 32] + pxor xmm0, xmm6 + movdqu [r9 + 32], xmm0 + movdqu xmm6, [r8 + 96] + pxor xmm1, xmm6 + movdqu [r9 + 96], xmm1 + movdqu xmm6, [r8 + 160] + pxor xmm4, xmm6 + movdqu [r9 + 160], xmm4 + movdqu xmm6, [r8 + 224] + pxor xmm3, xmm6 + movdqu [r9 + 224], xmm3 + movdqu xmm0, [rsp + 256] + movdqu xmm1, [rsp + 272] + movdqu xmm2, [rsp + 288] + movdqu xmm3, [rsp + 304] + movdqa xmm4, xmm0 + db $66, $0F, $62, $C1 // punpckldq xmm0, xmm1 + db $66, $0F, $6A, $E1 // punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + db $66, $0F, $62, $D3 // punpckldq xmm2, xmm3 + db $66, $0F, $6A, $EB // punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + db $66, $0F, $6C, $C2 // punpcklqdq xmm0, xmm2 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + db $66, $0F, $6C, $E5 // punpcklqdq xmm4, xmm5 + db $66, $0F, $6D, $DD // punpckhqdq xmm3, xmm5 + movdqu xmm6, [r8 + 48] + pxor xmm0, xmm6 + movdqu [r9 + 48], xmm0 + movdqu xmm6, [r8 + 112] + pxor xmm1, xmm6 + movdqu [r9 + 112], xmm1 + movdqu xmm6, [r8 + 176] + pxor xmm4, xmm6 + movdqu [r9 + 176], xmm4 + movdqu xmm6, [r8 + 240] + pxor xmm3, xmm6 + movdqu [r9 + 240], xmm3 + add rsp, 368 +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc new file mode 100644 index 00000000..0fd47d0c --- /dev/null +++ b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539Blocks8Avx2_x86_64.inc @@ -0,0 +1,452 @@ +// ChaCha7539 (IETF) 8-way (512B) AVX2 keystream kernel, vertical layout (x86-64). +// ------------------------------------------------------------------- +// +// Eight 64-byte ChaCha20 blocks, state words held vertically (one block per +// ymm lane). ymm0-3 = words a (0-3), ymm4-7 = words b (4-7), ymm12-15 = words +// d (12-15); words c (8-11) live in a stack slot; ymm8-11 are scratch. The four +// quarter-rounds of each round are step-interleaved; rot16/rot8 use vpshufb with +// the mask read from memory; rot12/rot7 use vpslld/vpsrld/vpor. +// +// On entry (after ClpSimdProc4Begin_x86_64.inc): +// rcx = ARounds, rdx = AState (16 UInt32), r8 = AIn, r9 = AOut. +// +// On exit: +// AState word-12 counter (byte 48) advanced by 8; the keystream is xored +// with AIn into AOut (512 bytes). +// +// VEX-encoded instructions are db-encoded for broad assembler compatibility. + mov r10, rcx +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + sub rsp, 1248 + mov dword ptr [rsp + 640], 0 + mov dword ptr [rsp + 644], 1 + mov dword ptr [rsp + 648], 2 + mov dword ptr [rsp + 652], 3 + mov dword ptr [rsp + 656], 4 + mov dword ptr [rsp + 660], 5 + mov dword ptr [rsp + 664], 6 + mov dword ptr [rsp + 668], 7 + mov dword ptr [rsp + 672], $01000302 + mov dword ptr [rsp + 676], $05040706 + mov dword ptr [rsp + 680], $09080B0A + mov dword ptr [rsp + 684], $0D0C0F0E + mov dword ptr [rsp + 688], $01000302 + mov dword ptr [rsp + 692], $05040706 + mov dword ptr [rsp + 696], $09080B0A + mov dword ptr [rsp + 700], $0D0C0F0E + mov dword ptr [rsp + 704], $02010003 + mov dword ptr [rsp + 708], $06050407 + mov dword ptr [rsp + 712], $0A09080B + mov dword ptr [rsp + 716], $0E0D0C0F + mov dword ptr [rsp + 720], $02010003 + mov dword ptr [rsp + 724], $06050407 + mov dword ptr [rsp + 728], $0A09080B + mov dword ptr [rsp + 732], $0E0D0C0F + db $C4, $E2, $7D, $58, $02 // vpbroadcastd ymm0, [rdx] + db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0 + db $C4, $E2, $7D, $58, $4A, $04 // vpbroadcastd ymm1, [rdx + 4] + db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1 + db $C4, $E2, $7D, $58, $52, $08 // vpbroadcastd ymm2, [rdx + 8] + db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2 + db $C4, $E2, $7D, $58, $5A, $0C // vpbroadcastd ymm3, [rdx + 12] + db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3 + db $C4, $E2, $7D, $58, $62, $10 // vpbroadcastd ymm4, [rdx + 16] + db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4 + db $C4, $E2, $7D, $58, $6A, $14 // vpbroadcastd ymm5, [rdx + 20] + db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5 + db $C4, $E2, $7D, $58, $72, $18 // vpbroadcastd ymm6, [rdx + 24] + db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6 + db $C4, $E2, $7D, $58, $7A, $1C // vpbroadcastd ymm7, [rdx + 28] + db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7 + db $C4, $62, $7D, $58, $42, $20 // vpbroadcastd ymm8, [rdx + 32] + db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8 + db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 + db $C4, $62, $7D, $58, $42, $24 // vpbroadcastd ymm8, [rdx + 36] + db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8 + db $C5, $7E, $7F, $44, $24, $20 // vmovdqu [rsp + 32], ymm8 + db $C4, $62, $7D, $58, $42, $28 // vpbroadcastd ymm8, [rdx + 40] + db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8 + db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 + db $C4, $62, $7D, $58, $42, $2C // vpbroadcastd ymm8, [rdx + 44] + db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8 + db $C5, $7E, $7F, $44, $24, $60 // vmovdqu [rsp + 96], ymm8 + db $C4, $62, $7D, $58, $62, $30 // vpbroadcastd ymm12, [rdx + 48] + db $C5, $1D, $FE, $A4, $24, $80, $02, $00, $00 // vpaddd ymm12, ymm12, [rsp + 640] + db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12 + db $C4, $62, $7D, $58, $6A, $34 // vpbroadcastd ymm13, [rdx + 52] + db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13 + db $C4, $62, $7D, $58, $72, $38 // vpbroadcastd ymm14, [rdx + 56] + db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14 + db $C4, $62, $7D, $58, $7A, $3C // vpbroadcastd ymm15, [rdx + 60] + db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15 +@@c7539_avx2_8x_round: + db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 + db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 + db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 + db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 + db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 + db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 + db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 + db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 + db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672] + db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 + db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 + db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 + db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 + db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] + db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32] + db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64] + db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96] + db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 + db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 + db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14 + db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15 + db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 + db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 + db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 + db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 + db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 + db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9 + db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10 + db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11 + db $C5, $BD, $72, $F4, $0C // vpslld ymm8, ymm4, 12 + db $C5, $B5, $72, $F5, $0C // vpslld ymm9, ymm5, 12 + db $C5, $AD, $72, $F6, $0C // vpslld ymm10, ymm6, 12 + db $C5, $A5, $72, $F7, $0C // vpslld ymm11, ymm7, 12 + db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 + db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 + db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 + db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 + db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4 + db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5 + db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 + db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 + db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 + db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 + db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 + db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 + db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 + db $C5, $15, $EF, $E9 // vpxor ymm13, ymm13, ymm1 + db $C5, $0D, $EF, $F2 // vpxor ymm14, ymm14, ymm2 + db $C5, $05, $EF, $FB // vpxor ymm15, ymm15, ymm3 + db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704] + db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 + db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 + db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 + db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 + db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] + db $C5, $7E, $6F, $4C, $24, $20 // vmovdqu ymm9, [rsp + 32] + db $C5, $7E, $6F, $54, $24, $40 // vmovdqu ymm10, [rsp + 64] + db $C5, $7E, $6F, $5C, $24, $60 // vmovdqu ymm11, [rsp + 96] + db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 + db $C4, $41, $35, $FE, $CD // vpaddd ymm9, ymm9, ymm13 + db $C4, $41, $2D, $FE, $D6 // vpaddd ymm10, ymm10, ymm14 + db $C4, $41, $25, $FE, $DF // vpaddd ymm11, ymm11, ymm15 + db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 + db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 + db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 + db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 + db $C5, $7E, $7F, $04, $24 // vmovdqu [rsp], ymm8 + db $C5, $7E, $7F, $4C, $24, $20 // vmovdqu [rsp + 32], ymm9 + db $C5, $7E, $7F, $54, $24, $40 // vmovdqu [rsp + 64], ymm10 + db $C5, $7E, $7F, $5C, $24, $60 // vmovdqu [rsp + 96], ymm11 + db $C5, $BD, $72, $F4, $07 // vpslld ymm8, ymm4, 7 + db $C5, $B5, $72, $F5, $07 // vpslld ymm9, ymm5, 7 + db $C5, $AD, $72, $F6, $07 // vpslld ymm10, ymm6, 7 + db $C5, $A5, $72, $F7, $07 // vpslld ymm11, ymm7, 7 + db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 + db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 + db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 + db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 + db $C5, $BD, $EB, $E4 // vpor ymm4, ymm8, ymm4 + db $C5, $B5, $EB, $ED // vpor ymm5, ymm9, ymm5 + db $C5, $AD, $EB, $F6 // vpor ymm6, ymm10, ymm6 + db $C5, $A5, $EB, $FF // vpor ymm7, ymm11, ymm7 + db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 + db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 + db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 + db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 + db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 + db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 + db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 + db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 + db $C5, $7E, $6F, $84, $24, $A0, $02, $00, $00 // vmovdqu ymm8, [rsp + 672] + db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 + db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 + db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 + db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 + db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] + db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96] + db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp] + db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32] + db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 + db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 + db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13 + db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14 + db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 + db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 + db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10 + db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11 + db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 + db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9 + db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10 + db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11 + db $C5, $BD, $72, $F5, $0C // vpslld ymm8, ymm5, 12 + db $C5, $B5, $72, $F6, $0C // vpslld ymm9, ymm6, 12 + db $C5, $AD, $72, $F7, $0C // vpslld ymm10, ymm7, 12 + db $C5, $A5, $72, $F4, $0C // vpslld ymm11, ymm4, 12 + db $C5, $D5, $72, $D5, $14 // vpsrld ymm5, ymm5, 20 + db $C5, $CD, $72, $D6, $14 // vpsrld ymm6, ymm6, 20 + db $C5, $C5, $72, $D7, $14 // vpsrld ymm7, ymm7, 20 + db $C5, $DD, $72, $D4, $14 // vpsrld ymm4, ymm4, 20 + db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5 + db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6 + db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 + db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 + db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 + db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 + db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 + db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 + db $C5, $05, $EF, $F8 // vpxor ymm15, ymm15, ymm0 + db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 + db $C5, $15, $EF, $EA // vpxor ymm13, ymm13, ymm2 + db $C5, $0D, $EF, $F3 // vpxor ymm14, ymm14, ymm3 + db $C5, $7E, $6F, $84, $24, $C0, $02, $00, $00 // vmovdqu ymm8, [rsp + 704] + db $C4, $42, $05, $00, $F8 // vpshufb ymm15, ymm15, ymm8 + db $C4, $42, $1D, $00, $E0 // vpshufb ymm12, ymm12, ymm8 + db $C4, $42, $15, $00, $E8 // vpshufb ymm13, ymm13, ymm8 + db $C4, $42, $0D, $00, $F0 // vpshufb ymm14, ymm14, ymm8 + db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] + db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, [rsp + 96] + db $C5, $7E, $6F, $14, $24 // vmovdqu ymm10, [rsp] + db $C5, $7E, $6F, $5C, $24, $20 // vmovdqu ymm11, [rsp + 32] + db $C4, $41, $3D, $FE, $C7 // vpaddd ymm8, ymm8, ymm15 + db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 + db $C4, $41, $2D, $FE, $D5 // vpaddd ymm10, ymm10, ymm13 + db $C4, $41, $25, $FE, $DE // vpaddd ymm11, ymm11, ymm14 + db $C4, $C1, $55, $EF, $E8 // vpxor ymm5, ymm5, ymm8 + db $C4, $C1, $4D, $EF, $F1 // vpxor ymm6, ymm6, ymm9 + db $C4, $C1, $45, $EF, $FA // vpxor ymm7, ymm7, ymm10 + db $C4, $C1, $5D, $EF, $E3 // vpxor ymm4, ymm4, ymm11 + db $C5, $7E, $7F, $44, $24, $40 // vmovdqu [rsp + 64], ymm8 + db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu [rsp + 96], ymm9 + db $C5, $7E, $7F, $14, $24 // vmovdqu [rsp], ymm10 + db $C5, $7E, $7F, $5C, $24, $20 // vmovdqu [rsp + 32], ymm11 + db $C5, $BD, $72, $F5, $07 // vpslld ymm8, ymm5, 7 + db $C5, $B5, $72, $F6, $07 // vpslld ymm9, ymm6, 7 + db $C5, $AD, $72, $F7, $07 // vpslld ymm10, ymm7, 7 + db $C5, $A5, $72, $F4, $07 // vpslld ymm11, ymm4, 7 + db $C5, $D5, $72, $D5, $19 // vpsrld ymm5, ymm5, 25 + db $C5, $CD, $72, $D6, $19 // vpsrld ymm6, ymm6, 25 + db $C5, $C5, $72, $D7, $19 // vpsrld ymm7, ymm7, 25 + db $C5, $DD, $72, $D4, $19 // vpsrld ymm4, ymm4, 25 + db $C5, $BD, $EB, $ED // vpor ymm5, ymm8, ymm5 + db $C5, $B5, $EB, $F6 // vpor ymm6, ymm9, ymm6 + db $C5, $AD, $EB, $FF // vpor ymm7, ymm10, ymm7 + db $C5, $A5, $EB, $E4 // vpor ymm4, ymm11, ymm4 + sub r10, 2 + jg @@c7539_avx2_8x_round + db $C5, $7E, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm8, [rsp + 128] + db $C4, $C1, $7D, $FE, $C0 // vpaddd ymm0, ymm0, ymm8 + db $C5, $FE, $7F, $84, $24, $80, $00, $00, $00 // vmovdqu [rsp + 128], ymm0 + db $C5, $7E, $6F, $84, $24, $A0, $00, $00, $00 // vmovdqu ymm8, [rsp + 160] + db $C4, $C1, $75, $FE, $C8 // vpaddd ymm1, ymm1, ymm8 + db $C5, $FE, $7F, $8C, $24, $A0, $00, $00, $00 // vmovdqu [rsp + 160], ymm1 + db $C5, $7E, $6F, $84, $24, $C0, $00, $00, $00 // vmovdqu ymm8, [rsp + 192] + db $C4, $C1, $6D, $FE, $D0 // vpaddd ymm2, ymm2, ymm8 + db $C5, $FE, $7F, $94, $24, $C0, $00, $00, $00 // vmovdqu [rsp + 192], ymm2 + db $C5, $7E, $6F, $84, $24, $E0, $00, $00, $00 // vmovdqu ymm8, [rsp + 224] + db $C4, $C1, $65, $FE, $D8 // vpaddd ymm3, ymm3, ymm8 + db $C5, $FE, $7F, $9C, $24, $E0, $00, $00, $00 // vmovdqu [rsp + 224], ymm3 + db $C5, $7E, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm8, [rsp + 256] + db $C4, $C1, $5D, $FE, $E0 // vpaddd ymm4, ymm4, ymm8 + db $C5, $FE, $7F, $A4, $24, $00, $01, $00, $00 // vmovdqu [rsp + 256], ymm4 + db $C5, $7E, $6F, $84, $24, $20, $01, $00, $00 // vmovdqu ymm8, [rsp + 288] + db $C4, $C1, $55, $FE, $E8 // vpaddd ymm5, ymm5, ymm8 + db $C5, $FE, $7F, $AC, $24, $20, $01, $00, $00 // vmovdqu [rsp + 288], ymm5 + db $C5, $7E, $6F, $84, $24, $40, $01, $00, $00 // vmovdqu ymm8, [rsp + 320] + db $C4, $C1, $4D, $FE, $F0 // vpaddd ymm6, ymm6, ymm8 + db $C5, $FE, $7F, $B4, $24, $40, $01, $00, $00 // vmovdqu [rsp + 320], ymm6 + db $C5, $7E, $6F, $84, $24, $60, $01, $00, $00 // vmovdqu ymm8, [rsp + 352] + db $C4, $C1, $45, $FE, $F8 // vpaddd ymm7, ymm7, ymm8 + db $C5, $FE, $7F, $BC, $24, $60, $01, $00, $00 // vmovdqu [rsp + 352], ymm7 + db $C5, $7E, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm8, [rsp + 512] + db $C4, $41, $1D, $FE, $E0 // vpaddd ymm12, ymm12, ymm8 + db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp + 512], ymm12 + db $C5, $7E, $6F, $84, $24, $20, $02, $00, $00 // vmovdqu ymm8, [rsp + 544] + db $C4, $41, $15, $FE, $E8 // vpaddd ymm13, ymm13, ymm8 + db $C5, $7E, $7F, $AC, $24, $20, $02, $00, $00 // vmovdqu [rsp + 544], ymm13 + db $C5, $7E, $6F, $84, $24, $40, $02, $00, $00 // vmovdqu ymm8, [rsp + 576] + db $C4, $41, $0D, $FE, $F0 // vpaddd ymm14, ymm14, ymm8 + db $C5, $7E, $7F, $B4, $24, $40, $02, $00, $00 // vmovdqu [rsp + 576], ymm14 + db $C5, $7E, $6F, $84, $24, $60, $02, $00, $00 // vmovdqu ymm8, [rsp + 608] + db $C4, $41, $05, $FE, $F8 // vpaddd ymm15, ymm15, ymm8 + db $C5, $7E, $7F, $BC, $24, $60, $02, $00, $00 // vmovdqu [rsp + 608], ymm15 + db $C5, $7E, $6F, $04, $24 // vmovdqu ymm8, [rsp] + db $C5, $7E, $6F, $8C, $24, $80, $01, $00, $00 // vmovdqu ymm9, [rsp + 384] + db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 + db $C5, $7E, $7F, $84, $24, $80, $01, $00, $00 // vmovdqu [rsp + 384], ymm8 + db $C5, $7E, $6F, $44, $24, $20 // vmovdqu ymm8, [rsp + 32] + db $C5, $7E, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm9, [rsp + 416] + db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 + db $C5, $7E, $7F, $84, $24, $A0, $01, $00, $00 // vmovdqu [rsp + 416], ymm8 + db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, [rsp + 64] + db $C5, $7E, $6F, $8C, $24, $C0, $01, $00, $00 // vmovdqu ymm9, [rsp + 448] + db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 + db $C5, $7E, $7F, $84, $24, $C0, $01, $00, $00 // vmovdqu [rsp + 448], ymm8 + db $C5, $7E, $6F, $44, $24, $60 // vmovdqu ymm8, [rsp + 96] + db $C5, $7E, $6F, $8C, $24, $E0, $01, $00, $00 // vmovdqu ymm9, [rsp + 480] + db $C4, $41, $3D, $FE, $C1 // vpaddd ymm8, ymm8, ymm9 + db $C5, $7E, $7F, $84, $24, $E0, $01, $00, $00 // vmovdqu [rsp + 480], ymm8 + add dword ptr [rdx + 48], 8 +{$IFDEF CHACHA_CTR64} + adc dword ptr [rdx + 52], 0 +{$ENDIF} + db $C5, $FE, $6F, $84, $24, $80, $00, $00, $00 // vmovdqu ymm0, [rsp + 128] + db $C5, $FE, $6F, $8C, $24, $A0, $00, $00, $00 // vmovdqu ymm1, [rsp + 160] + db $C5, $FE, $6F, $94, $24, $C0, $00, $00, $00 // vmovdqu ymm2, [rsp + 192] + db $C5, $FE, $6F, $9C, $24, $E0, $00, $00, $00 // vmovdqu ymm3, [rsp + 224] + db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 + db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 + db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 + db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 + db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 + db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 + db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 + db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 + db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 + db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 + db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 + db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 + db $C5, $FA, $7F, $84, $24, $E0, $02, $00, $00 // vmovdqu [rsp + 736], xmm0 + db $C4, $E3, $7D, $39, $84, $24, $E0, $03, $00, $00, $01 // vextracti128 [rsp + 992], ymm0, 1 + db $C5, $FA, $7F, $8C, $24, $20, $03, $00, $00 // vmovdqu [rsp + 800], xmm1 + db $C4, $E3, $7D, $39, $8C, $24, $20, $04, $00, $00, $01 // vextracti128 [rsp + 1056], ymm1, 1 + db $C5, $FA, $7F, $A4, $24, $60, $03, $00, $00 // vmovdqu [rsp + 864], xmm4 + db $C4, $E3, $7D, $39, $A4, $24, $60, $04, $00, $00, $01 // vextracti128 [rsp + 1120], ymm4, 1 + db $C5, $FA, $7F, $9C, $24, $A0, $03, $00, $00 // vmovdqu [rsp + 928], xmm3 + db $C4, $E3, $7D, $39, $9C, $24, $A0, $04, $00, $00, $01 // vextracti128 [rsp + 1184], ymm3, 1 + db $C5, $FE, $6F, $84, $24, $00, $01, $00, $00 // vmovdqu ymm0, [rsp + 256] + db $C5, $FE, $6F, $8C, $24, $20, $01, $00, $00 // vmovdqu ymm1, [rsp + 288] + db $C5, $FE, $6F, $94, $24, $40, $01, $00, $00 // vmovdqu ymm2, [rsp + 320] + db $C5, $FE, $6F, $9C, $24, $60, $01, $00, $00 // vmovdqu ymm3, [rsp + 352] + db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 + db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 + db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 + db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 + db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 + db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 + db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 + db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 + db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 + db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 + db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 + db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 + db $C5, $FA, $7F, $84, $24, $F0, $02, $00, $00 // vmovdqu [rsp + 752], xmm0 + db $C4, $E3, $7D, $39, $84, $24, $F0, $03, $00, $00, $01 // vextracti128 [rsp + 1008], ymm0, 1 + db $C5, $FA, $7F, $8C, $24, $30, $03, $00, $00 // vmovdqu [rsp + 816], xmm1 + db $C4, $E3, $7D, $39, $8C, $24, $30, $04, $00, $00, $01 // vextracti128 [rsp + 1072], ymm1, 1 + db $C5, $FA, $7F, $A4, $24, $70, $03, $00, $00 // vmovdqu [rsp + 880], xmm4 + db $C4, $E3, $7D, $39, $A4, $24, $70, $04, $00, $00, $01 // vextracti128 [rsp + 1136], ymm4, 1 + db $C5, $FA, $7F, $9C, $24, $B0, $03, $00, $00 // vmovdqu [rsp + 944], xmm3 + db $C4, $E3, $7D, $39, $9C, $24, $B0, $04, $00, $00, $01 // vextracti128 [rsp + 1200], ymm3, 1 + db $C5, $FE, $6F, $84, $24, $80, $01, $00, $00 // vmovdqu ymm0, [rsp + 384] + db $C5, $FE, $6F, $8C, $24, $A0, $01, $00, $00 // vmovdqu ymm1, [rsp + 416] + db $C5, $FE, $6F, $94, $24, $C0, $01, $00, $00 // vmovdqu ymm2, [rsp + 448] + db $C5, $FE, $6F, $9C, $24, $E0, $01, $00, $00 // vmovdqu ymm3, [rsp + 480] + db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 + db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 + db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 + db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 + db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 + db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 + db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 + db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 + db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 + db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 + db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 + db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 + db $C5, $FA, $7F, $84, $24, $00, $03, $00, $00 // vmovdqu [rsp + 768], xmm0 + db $C4, $E3, $7D, $39, $84, $24, $00, $04, $00, $00, $01 // vextracti128 [rsp + 1024], ymm0, 1 + db $C5, $FA, $7F, $8C, $24, $40, $03, $00, $00 // vmovdqu [rsp + 832], xmm1 + db $C4, $E3, $7D, $39, $8C, $24, $40, $04, $00, $00, $01 // vextracti128 [rsp + 1088], ymm1, 1 + db $C5, $FA, $7F, $A4, $24, $80, $03, $00, $00 // vmovdqu [rsp + 896], xmm4 + db $C4, $E3, $7D, $39, $A4, $24, $80, $04, $00, $00, $01 // vextracti128 [rsp + 1152], ymm4, 1 + db $C5, $FA, $7F, $9C, $24, $C0, $03, $00, $00 // vmovdqu [rsp + 960], xmm3 + db $C4, $E3, $7D, $39, $9C, $24, $C0, $04, $00, $00, $01 // vextracti128 [rsp + 1216], ymm3, 1 + db $C5, $FE, $6F, $84, $24, $00, $02, $00, $00 // vmovdqu ymm0, [rsp + 512] + db $C5, $FE, $6F, $8C, $24, $20, $02, $00, $00 // vmovdqu ymm1, [rsp + 544] + db $C5, $FE, $6F, $94, $24, $40, $02, $00, $00 // vmovdqu ymm2, [rsp + 576] + db $C5, $FE, $6F, $9C, $24, $60, $02, $00, $00 // vmovdqu ymm3, [rsp + 608] + db $C5, $FD, $6F, $E0 // vmovdqa ymm4, ymm0 + db $C5, $FD, $62, $C1 // vpunpckldq ymm0, ymm0, ymm1 + db $C5, $DD, $6A, $E1 // vpunpckhdq ymm4, ymm4, ymm1 + db $C5, $FD, $6F, $EA // vmovdqa ymm5, ymm2 + db $C5, $ED, $62, $D3 // vpunpckldq ymm2, ymm2, ymm3 + db $C5, $D5, $6A, $EB // vpunpckhdq ymm5, ymm5, ymm3 + db $C5, $FD, $6F, $C8 // vmovdqa ymm1, ymm0 + db $C5, $FD, $6C, $C2 // vpunpcklqdq ymm0, ymm0, ymm2 + db $C5, $F5, $6D, $CA // vpunpckhqdq ymm1, ymm1, ymm2 + db $C5, $FD, $6F, $DC // vmovdqa ymm3, ymm4 + db $C5, $DD, $6C, $E5 // vpunpcklqdq ymm4, ymm4, ymm5 + db $C5, $E5, $6D, $DD // vpunpckhqdq ymm3, ymm3, ymm5 + db $C5, $FA, $7F, $84, $24, $10, $03, $00, $00 // vmovdqu [rsp + 784], xmm0 + db $C4, $E3, $7D, $39, $84, $24, $10, $04, $00, $00, $01 // vextracti128 [rsp + 1040], ymm0, 1 + db $C5, $FA, $7F, $8C, $24, $50, $03, $00, $00 // vmovdqu [rsp + 848], xmm1 + db $C4, $E3, $7D, $39, $8C, $24, $50, $04, $00, $00, $01 // vextracti128 [rsp + 1104], ymm1, 1 + db $C5, $FA, $7F, $A4, $24, $90, $03, $00, $00 // vmovdqu [rsp + 912], xmm4 + db $C4, $E3, $7D, $39, $A4, $24, $90, $04, $00, $00, $01 // vextracti128 [rsp + 1168], ymm4, 1 + db $C5, $FA, $7F, $9C, $24, $D0, $03, $00, $00 // vmovdqu [rsp + 976], xmm3 + db $C4, $E3, $7D, $39, $9C, $24, $D0, $04, $00, $00, $01 // vextracti128 [rsp + 1232], ymm3, 1 + db $C5, $FE, $6F, $84, $24, $E0, $02, $00, $00 // vmovdqu ymm0, [rsp + 736] + db $C4, $C1, $7D, $EF, $00 // vpxor ymm0, ymm0, [r8 + 0] + db $C4, $C1, $7E, $7F, $01 // vmovdqu [r9], ymm0 + db $C5, $FE, $6F, $84, $24, $00, $03, $00, $00 // vmovdqu ymm0, [rsp + 768] + db $C4, $C1, $7D, $EF, $40, $20 // vpxor ymm0, ymm0, [r8 + 32] + db $C4, $C1, $7E, $7F, $41, $20 // vmovdqu [r9 + 32], ymm0 + db $C5, $FE, $6F, $84, $24, $20, $03, $00, $00 // vmovdqu ymm0, [rsp + 800] + db $C4, $C1, $7D, $EF, $40, $40 // vpxor ymm0, ymm0, [r8 + 64] + db $C4, $C1, $7E, $7F, $41, $40 // vmovdqu [r9 + 64], ymm0 + db $C5, $FE, $6F, $84, $24, $40, $03, $00, $00 // vmovdqu ymm0, [rsp + 832] + db $C4, $C1, $7D, $EF, $40, $60 // vpxor ymm0, ymm0, [r8 + 96] + db $C4, $C1, $7E, $7F, $41, $60 // vmovdqu [r9 + 96], ymm0 + db $C5, $FE, $6F, $84, $24, $60, $03, $00, $00 // vmovdqu ymm0, [rsp + 864] + db $C4, $C1, $7D, $EF, $80, $80, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 128] + db $C4, $C1, $7E, $7F, $81, $80, $00, $00, $00 // vmovdqu [r9 + 128], ymm0 + db $C5, $FE, $6F, $84, $24, $80, $03, $00, $00 // vmovdqu ymm0, [rsp + 896] + db $C4, $C1, $7D, $EF, $80, $A0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 160] + db $C4, $C1, $7E, $7F, $81, $A0, $00, $00, $00 // vmovdqu [r9 + 160], ymm0 + db $C5, $FE, $6F, $84, $24, $A0, $03, $00, $00 // vmovdqu ymm0, [rsp + 928] + db $C4, $C1, $7D, $EF, $80, $C0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 192] + db $C4, $C1, $7E, $7F, $81, $C0, $00, $00, $00 // vmovdqu [r9 + 192], ymm0 + db $C5, $FE, $6F, $84, $24, $C0, $03, $00, $00 // vmovdqu ymm0, [rsp + 960] + db $C4, $C1, $7D, $EF, $80, $E0, $00, $00, $00 // vpxor ymm0, ymm0, [r8 + 224] + db $C4, $C1, $7E, $7F, $81, $E0, $00, $00, $00 // vmovdqu [r9 + 224], ymm0 + db $C5, $FE, $6F, $84, $24, $E0, $03, $00, $00 // vmovdqu ymm0, [rsp + 992] + db $C4, $C1, $7D, $EF, $80, $00, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 256] + db $C4, $C1, $7E, $7F, $81, $00, $01, $00, $00 // vmovdqu [r9 + 256], ymm0 + db $C5, $FE, $6F, $84, $24, $00, $04, $00, $00 // vmovdqu ymm0, [rsp + 1024] + db $C4, $C1, $7D, $EF, $80, $20, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 288] + db $C4, $C1, $7E, $7F, $81, $20, $01, $00, $00 // vmovdqu [r9 + 288], ymm0 + db $C5, $FE, $6F, $84, $24, $20, $04, $00, $00 // vmovdqu ymm0, [rsp + 1056] + db $C4, $C1, $7D, $EF, $80, $40, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 320] + db $C4, $C1, $7E, $7F, $81, $40, $01, $00, $00 // vmovdqu [r9 + 320], ymm0 + db $C5, $FE, $6F, $84, $24, $40, $04, $00, $00 // vmovdqu ymm0, [rsp + 1088] + db $C4, $C1, $7D, $EF, $80, $60, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 352] + db $C4, $C1, $7E, $7F, $81, $60, $01, $00, $00 // vmovdqu [r9 + 352], ymm0 + db $C5, $FE, $6F, $84, $24, $60, $04, $00, $00 // vmovdqu ymm0, [rsp + 1120] + db $C4, $C1, $7D, $EF, $80, $80, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 384] + db $C4, $C1, $7E, $7F, $81, $80, $01, $00, $00 // vmovdqu [r9 + 384], ymm0 + db $C5, $FE, $6F, $84, $24, $80, $04, $00, $00 // vmovdqu ymm0, [rsp + 1152] + db $C4, $C1, $7D, $EF, $80, $A0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 416] + db $C4, $C1, $7E, $7F, $81, $A0, $01, $00, $00 // vmovdqu [r9 + 416], ymm0 + db $C5, $FE, $6F, $84, $24, $A0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1184] + db $C4, $C1, $7D, $EF, $80, $C0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 448] + db $C4, $C1, $7E, $7F, $81, $C0, $01, $00, $00 // vmovdqu [r9 + 448], ymm0 + db $C5, $FE, $6F, $84, $24, $C0, $04, $00, $00 // vmovdqu ymm0, [rsp + 1216] + db $C4, $C1, $7D, $EF, $80, $E0, $01, $00, $00 // vpxor ymm0, ymm0, [r8 + 480] + db $C4, $C1, $7E, $7F, $81, $E0, $01, $00, $00 // vmovdqu [r9 + 480], ymm0 + db $C5, $F8, $77 // vzeroupper + add rsp, 1248 +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} + diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_i386.inc deleted file mode 100644 index 647c3305..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_i386.inc +++ /dev/null @@ -1,16 +0,0 @@ -// ChaCha7539 (IETF) 128B, AVX2, i386. Little-endian state in [esi]; two 32-bit block counters in YMM lanes. -// Fused I/O xor: read plaintext at AIn (edi), xor with ChaCha output, write to AOut (eax). -// Fragments: ChaCha7539_Avx2_128B_LoadState_i386, ChaCha7539_Avx2_DoubleRound.inc, ChaCha7539_Avx2_128B_XorStore_i386 -// After ClpSimdProc4Begin_i386.inc: ebx=ARounds, esi=AState, edi=AIn, eax=AOut. - mov edx, ebx -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_i386.inc} - @@chacha7539_avx2_i386_pb2: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub edx, 2 - jg @@chacha7539_avx2_i386_pb2 -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_i386.inc} - db $81, $C4, $C8, $00, $00, $00 // add esp, 0xc8 - db $C5, $F8, $77 // vzeroupper - pop edi - pop esi - pop ebx diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_x86_64.inc deleted file mode 100644 index 1fbebd20..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks2Avx2_x86_64.inc +++ /dev/null @@ -1,13 +0,0 @@ -// ChaCha7539 (IETF) 128B, AVX2. Little-endian state in [rdx]; two 32-bit block counters in YMM lanes. -// Fused I/O xor: read plaintext at AIn (r8), xor with ChaCha output, write to AOut (r9) — no 128B keystream temp in Pascal; xor in asm (XorStore) after rounds. -// Fragments: ChaCha7539_Avx2_128B_LoadState, ChaCha7539_Avx2_DoubleRound.inc (VEX db), ChaCha7539_Avx2_128B_XorStore .inc -// After ClpSimdProc4Begin_x86_64.inc: rcx=ARounds, rdx=AState, r8=AIn, r9=AOut. - mov r10, rcx -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_x86_64.inc} - @@chacha7539_avx2_dround: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub r10, 2 - jg @@chacha7539_avx2_dround -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_x86_64.inc} - db $48, $81, $C4, $C8, $00, $00, $00 // add rsp, 0xc8 - db $C5, $F8, $77 // vzeroupper diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_i386.inc deleted file mode 100644 index 8eea7cf1..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_i386.inc +++ /dev/null @@ -1,27 +0,0 @@ -// ChaCha7539 (IETF) 256B, AVX2, i386. Two 128B blocks: same fused I/O xor as ProcessBlocks2 (AIn=edi, AOut=eax). -// One vzeroupper; add edi/eax,+128 between blocks. LoadState .inc reloads IETF state at [esi] (word 12 already advanced after block 1). -// After ClpSimdProc4Begin_i386.inc: ebx=ARounds, esi=AState, edi=AIn, eax=AOut. - mov edx, ebx - // --- first 128B --- -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_i386.inc} - @@c7539_avx2_i386_pb4_b1: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub edx, 2 - jg @@c7539_avx2_i386_pb4_b1 -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_i386.inc} - db $81, $C4, $C8, $00, $00, $00 // add esp, 0xc8 - add edi, 128 - add eax, 128 - mov edx, ebx - // --- second 128B (IETF state at [esi+48] is already advanced) --- -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_i386.inc} - @@c7539_avx2_i386_pb4_b2: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub edx, 2 - jg @@c7539_avx2_i386_pb4_b2 -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_i386.inc} - db $81, $C4, $C8, $00, $00, $00 // add esp, 0xc8 - db $C5, $F8, $77 // vzeroupper - pop edi - pop esi - pop ebx diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_x86_64.inc deleted file mode 100644 index e2a9a728..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539ProcessBlocks4Avx2_x86_64.inc +++ /dev/null @@ -1,24 +0,0 @@ -// ChaCha7539 (IETF) 256B, AVX2. Two 128B blocks: same fused I/O xor as ProcessBlocks2 (AIn=r8, AOut=r9; no large keystream temp in Pascal). -// One vzeroupper; add r8/r9,+128 between blocks. LoadState .inc reloads IETF state at [rdx] (word 12 already advanced after block 1). -// After ClpSimdProc4Begin_x86_64.inc: rcx=ARounds, rdx=AState, r8=AIn, r9=AOut. - mov r10, rcx - // --- first 128B --- -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_x86_64.inc} - @@c7539_avx2_pb4_b1: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub r10, 2 - jg @@c7539_avx2_pb4_b1 -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_x86_64.inc} - db $48, $81, $C4, $C8, $00, $00, $00 // add rsp, 0xc8 - add r8, 128 - add r9, 128 - mov r10, rcx - // --- second 128B (IETF state at [rdx+48] is already advanced) --- -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_LoadState_x86_64.inc} - @@c7539_avx2_pb4_b2: -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_DoubleRound.inc} - sub r10, 2 - jg @@c7539_avx2_pb4_b2 -{$I ..\..\Include\Simd\ChaCha\ChaCha7539_Avx2_128B_XorStore_x86_64.inc} - db $48, $81, $C4, $C8, $00, $00, $00 // add rsp, 0xc8 - db $C5, $F8, $77 // vzeroupper diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_i386.inc deleted file mode 100644 index 98f4ad4b..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_i386.inc +++ /dev/null @@ -1,27 +0,0 @@ -// ChaCha7539 (IETF) AVX2: 128B block prologue, i386. Fused I/O xor path: full keystream is not written to a Pascal temp; XorStore .inc xors in asm. -// sub esp,0xC8; state from ESI; bump IETF word-12 counter twice; save YMM scratch to [esp]. -// Parent must set edx := ebx (ARounds) before this include, once per 128B block (ProcessBlocks2 and each half of ProcessBlocks4). - db $81, $EC, $C8, $00, $00, $00 // sub esp, 0xc8 - db $C5, $FA, $6F, $26 // vmovdqu xmm4, xmmword ptr [esi] - db $C5, $FC, $57, $C0 // vxorps ymm0, ymm0, ymm0 - db $C4, $E3, $7D, $18, $C4, $00 // vinsertf128 ymm0, ymm0, xmm4, 0 - db $C4, $E3, $7D, $18, $C4, $01 // vinsertf128 ymm0, ymm0, xmm4, 1 - db $C5, $FA, $6F, $66, $10 // vmovdqu xmm4, xmmword ptr [esi + 0x10] - db $C5, $F4, $57, $C9 // vxorps ymm1, ymm1, ymm1 - db $C4, $E3, $75, $18, $CC, $00 // vinsertf128 ymm1, ymm1, xmm4, 0 - db $C4, $E3, $75, $18, $CC, $01 // vinsertf128 ymm1, ymm1, xmm4, 1 - db $C5, $FA, $6F, $66, $20 // vmovdqu xmm4, xmmword ptr [esi + 0x20] - db $C5, $EC, $57, $D2 // vxorps ymm2, ymm2, ymm2 - db $C4, $E3, $6D, $18, $D4, $00 // vinsertf128 ymm2, ymm2, xmm4, 0 - db $C4, $E3, $6D, $18, $D4, $01 // vinsertf128 ymm2, ymm2, xmm4, 1 - db $C5, $FA, $6F, $66, $30 // vmovdqu xmm4, xmmword ptr [esi + 0x30] - db $83, $46, $30, $01 // add dword ptr [esi + 0x30], 1 - db $C5, $FA, $6F, $6E, $30 // vmovdqu xmm5, xmmword ptr [esi + 0x30] - db $83, $46, $30, $01 // add dword ptr [esi + 0x30], 1 - db $C5, $E4, $57, $DB // vxorps ymm3, ymm3, ymm3 - db $C4, $E3, $65, $18, $DC, $00 // vinsertf128 ymm3, ymm3, xmm4, 0 - db $C4, $E3, $65, $18, $DD, $01 // vinsertf128 ymm3, ymm3, xmm5, 1 - db $C5, $FE, $7F, $04, $24 // vmovdqu ymmword ptr [esp], ymm0 - db $C5, $FE, $7F, $4C, $24, $20 // vmovdqu ymmword ptr [esp + 0x20], ymm1 - db $C5, $FE, $7F, $54, $24, $40 // vmovdqu ymmword ptr [esp + 0x40], ymm2 - db $C5, $FE, $7F, $5C, $24, $60 // vmovdqu ymmword ptr [esp + 0x60], ymm3 diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_x86_64.inc deleted file mode 100644 index 9cb47994..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_LoadState_x86_64.inc +++ /dev/null @@ -1,27 +0,0 @@ -// ChaCha7539 (IETF) AVX2: 128B block prologue. Fused I/O xor path: full keystream is not written to a Pascal temp; XorStore .inc xors in asm. -// sub rsp,0xC8; state from RDX; bump IETF word-12 counter twice; save YMM scratch to [rsp]. -// Parent must set r10 := rcx (rounds) before this include, once per 128B block (ProcessBlocks2 and each half of ProcessBlocks4). - db $48, $81, $EC, $C8, $00, $00, $00 // sub rsp, 0xc8 - db $C5, $FA, $6F, $22 // vmovdqu xmm4, xmmword ptr [rdx] - db $C5, $FC, $57, $C0 // vxorps ymm0, ymm0, ymm0 - db $C4, $E3, $7D, $18, $C4, $00 // vinsertf128 ymm0, ymm0, xmm4, 0 - db $C4, $E3, $7D, $18, $C4, $01 // vinsertf128 ymm0, ymm0, xmm4, 1 - db $C5, $FA, $6F, $62, $10 // vmovdqu xmm4, xmmword ptr [rdx + 0x10] - db $C5, $F4, $57, $C9 // vxorps ymm1, ymm1, ymm1 - db $C4, $E3, $75, $18, $CC, $00 // vinsertf128 ymm1, ymm1, xmm4, 0 - db $C4, $E3, $75, $18, $CC, $01 // vinsertf128 ymm1, ymm1, xmm4, 1 - db $C5, $FA, $6F, $62, $20 // vmovdqu xmm4, xmmword ptr [rdx + 0x20] - db $C5, $EC, $57, $D2 // vxorps ymm2, ymm2, ymm2 - db $C4, $E3, $6D, $18, $D4, $00 // vinsertf128 ymm2, ymm2, xmm4, 0 - db $C4, $E3, $6D, $18, $D4, $01 // vinsertf128 ymm2, ymm2, xmm4, 1 - db $C5, $FA, $6F, $62, $30 // vmovdqu xmm4, xmmword ptr [rdx + 0x30] - db $83, $42, $30, $01 // add dword ptr [rdx + 0x30], 1 - db $C5, $FA, $6F, $6A, $30 // vmovdqu xmm5, xmmword ptr [rdx + 0x30] - db $83, $42, $30, $01 // add dword ptr [rdx + 0x30], 1 - db $C5, $E4, $57, $DB // vxorps ymm3, ymm3, ymm3 - db $C4, $E3, $65, $18, $DC, $00 // vinsertf128 ymm3, ymm3, xmm4, 0 - db $C4, $E3, $65, $18, $DD, $01 // vinsertf128 ymm3, ymm3, xmm5, 1 - db $C5, $FE, $7F, $04, $24 // vmovdqu ymmword ptr [rsp], ymm0 - db $C5, $FE, $7F, $4C, $24, $20 // vmovdqu ymmword ptr [rsp + 0x20], ymm1 - db $C5, $FE, $7F, $54, $24, $40 // vmovdqu ymmword ptr [rsp + 0x40], ymm2 - db $C5, $FE, $7F, $5C, $24, $60 // vmovdqu ymmword ptr [rsp + 0x60], ymm3 diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_i386.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_i386.inc deleted file mode 100644 index 67a6ba7d..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_i386.inc +++ /dev/null @@ -1,26 +0,0 @@ -// ChaCha7539 (IETF) AVX2: after rounds, add back initial state from [esp] scratch, vperm2f128, then fused I/O xor, i386. -// Fused I/O xor: plaintext at EDI, ciphertext at EAX (encrypt; in-place if AIn=AOut). Offsets 0..0x7F. - db $C5, $FE, $6F, $24, $24 // vmovdqu ymm4, ymmword ptr [esp] - db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 - db $C5, $FE, $6F, $64, $24, $20 // vmovdqu ymm4, ymmword ptr [esp + 0x20] - db $C5, $F5, $FE, $CC // vpaddd ymm1, ymm1, ymm4 - db $C5, $FE, $6F, $64, $24, $40 // vmovdqu ymm4, ymmword ptr [esp + 0x40] - db $C5, $ED, $FE, $D4 // vpaddd ymm2, ymm2, ymm4 - db $C5, $FE, $6F, $64, $24, $60 // vmovdqu ymm4, ymmword ptr [esp + 0x60] - db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 - db $C4, $E3, $7D, $06, $E1, $20 // vperm2f128 ymm4, ymm0, ymm1, 0x20 - db $C5, $FE, $6F, $2F // vmovdqu ymm5, ymmword ptr [edi] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $20 // vmovdqu ymmword ptr [eax], ymm4 - db $C4, $E3, $6D, $06, $E3, $20 // vperm2f128 ymm4, ymm2, ymm3, 0x20 - db $C5, $FE, $6F, $6F, $20 // vmovdqu ymm5, ymmword ptr [edi + 0x20] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $60, $20 // vmovdqu ymmword ptr [eax + 0x20], ymm4 - db $C4, $E3, $7D, $06, $E1, $31 // vperm2f128 ymm4, ymm0, ymm1, 0x31 - db $C5, $FE, $6F, $6F, $40 // vmovdqu ymm5, ymmword ptr [edi + 0x40] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $60, $40 // vmovdqu ymmword ptr [eax + 0x40], ymm4 - db $C4, $E3, $6D, $06, $E3, $31 // vperm2f128 ymm4, ymm2, ymm3, 0x31 - db $C5, $FE, $6F, $6F, $60 // vmovdqu ymm5, ymmword ptr [edi + 0x60] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $60, $60 // vmovdqu ymmword ptr [eax + 0x60], ymm4 diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_x86_64.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_x86_64.inc deleted file mode 100644 index 010dd6c8..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_128B_XorStore_x86_64.inc +++ /dev/null @@ -1,26 +0,0 @@ -// ChaCha7539 (IETF) AVX2: after rounds, add back initial state from [rsp] scratch, vperm2f128, then fused I/O xor. -// Fused I/O xor: produce running output, xor with plaintext at r8, store ciphertext at r9 (encrypt; in-place if AIn=AOut). Offsets 0..0x7F. - db $C5, $FE, $6F, $24, $24 // vmovdqu ymm4, ymmword ptr [rsp] - db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 - db $C5, $FE, $6F, $64, $24, $20 // vmovdqu ymm4, ymmword ptr [rsp + 0x20] - db $C5, $F5, $FE, $CC // vpaddd ymm1, ymm1, ymm4 - db $C5, $FE, $6F, $64, $24, $40 // vmovdqu ymm4, ymmword ptr [rsp + 0x40] - db $C5, $ED, $FE, $D4 // vpaddd ymm2, ymm2, ymm4 - db $C5, $FE, $6F, $64, $24, $60 // vmovdqu ymm4, ymmword ptr [rsp + 0x60] - db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 - db $C4, $E3, $7D, $06, $E1, $20 // vperm2f128 ymm4, ymm0, ymm1, 0x20 - db $C4, $C1, $7E, $6F, $28 // vmovdqu ymm5, ymmword ptr [r8] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C4, $C1, $7E, $7F, $21 // vmovdqu ymmword ptr [r9], ymm4 - db $C4, $E3, $6D, $06, $E3, $20 // vperm2f128 ymm4, ymm2, ymm3, 0x20 - db $C4, $C1, $7E, $6F, $68, $20 // vmovdqu ymm5, ymmword ptr [r8 + 0x20] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C4, $C1, $7E, $7F, $61, $20 // vmovdqu ymmword ptr [r9 + 0x20], ymm4 - db $C4, $E3, $7D, $06, $E1, $31 // vperm2f128 ymm4, ymm0, ymm1, 0x31 - db $C4, $C1, $7E, $6F, $68, $40 // vmovdqu ymm5, ymmword ptr [r8 + 0x40] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C4, $C1, $7E, $7F, $61, $40 // vmovdqu ymmword ptr [r9 + 0x40], ymm4 - db $C4, $E3, $6D, $06, $E3, $31 // vperm2f128 ymm4, ymm2, ymm3, 0x31 - db $C4, $C1, $7E, $6F, $68, $60 // vmovdqu ymm5, ymmword ptr [r8 + 0x60] - db $C5, $DD, $EF, $E5 // vpxor ymm4, ymm4, ymm5 - db $C4, $C1, $7E, $7F, $61, $60 // vmovdqu ymmword ptr [r9 + 0x60], ymm4 diff --git a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_DoubleRound.inc b/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_DoubleRound.inc deleted file mode 100644 index f7b1db25..00000000 --- a/CryptoLib/src/Include/Simd/ChaCha/ChaCha7539_Avx2_DoubleRound.inc +++ /dev/null @@ -1,57 +0,0 @@ -// ChaCha7539 (IETF) AVX2: one "double round" in two 32-lane YMMs (4 quarter-rounds). VEX db-encoded; ymm-only — same bytes in IA-32 and x86-64. -// ARX only — no fused I/O xor here (that is XorStore .inc after the round loop). Parent supplies: sub , 2 / jg . -// ChaCha20 in this tree uses SSE2, not this VEX include. - db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 - db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $FD, $6F, $E3 // vmovdqa ymm4, ymm3 - db $C5, $E5, $72, $D3, $10 // vpsrld ymm3, ymm3, 0x10 - db $C5, $DD, $72, $F4, $10 // vpslld ymm4, ymm4, 0x10 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 - db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 - db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $E1 // vmovdqa ymm4, ymm1 - db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 0x14 - db $C5, $DD, $72, $F4, $0C // vpslld ymm4, ymm4, 0xc - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 - db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 - db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $FD, $6F, $E3 // vmovdqa ymm4, ymm3 - db $C5, $E5, $72, $D3, $18 // vpsrld ymm3, ymm3, 0x18 - db $C5, $DD, $72, $F4, $08 // vpslld ymm4, ymm4, 8 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 - db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 - db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $E1 // vmovdqa ymm4, ymm1 - db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 0x19 - db $C5, $DD, $72, $F4, $07 // vpslld ymm4, ymm4, 7 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 - db $C5, $FD, $70, $C9, $39 // vpshufd ymm1, ymm1, 0x39 - db $C5, $FD, $70, $D2, $4E // vpshufd ymm2, ymm2, 0x4e - db $C5, $FD, $70, $DB, $93 // vpshufd ymm3, ymm3, 0x93 - db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 - db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $FD, $6F, $E3 // vmovdqa ymm4, ymm3 - db $C5, $E5, $72, $D3, $10 // vpsrld ymm3, ymm3, 0x10 - db $C5, $DD, $72, $F4, $10 // vpslld ymm4, ymm4, 0x10 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 - db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 - db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $E1 // vmovdqa ymm4, ymm1 - db $C5, $F5, $72, $D1, $14 // vpsrld ymm1, ymm1, 0x14 - db $C5, $DD, $72, $F4, $0C // vpslld ymm4, ymm4, 0xc - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 - db $C5, $FD, $FE, $C1 // vpaddd ymm0, ymm0, ymm1 - db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $FD, $6F, $E3 // vmovdqa ymm4, ymm3 - db $C5, $E5, $72, $D3, $18 // vpsrld ymm3, ymm3, 0x18 - db $C5, $DD, $72, $F4, $08 // vpslld ymm4, ymm4, 8 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 - db $C5, $ED, $FE, $D3 // vpaddd ymm2, ymm2, ymm3 - db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $FD, $6F, $E1 // vmovdqa ymm4, ymm1 - db $C5, $F5, $72, $D1, $19 // vpsrld ymm1, ymm1, 0x19 - db $C5, $DD, $72, $F4, $07 // vpslld ymm4, ymm4, 7 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 - db $C5, $FD, $70, $C9, $93 // vpshufd ymm1, ymm1, 0x93 - db $C5, $FD, $70, $D2, $4E // vpshufd ymm2, ymm2, 0x4e - db $C5, $FD, $70, $DB, $39 // vpshufd ymm3, ymm3, 0x39 diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc index 851e1b50..89324967 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_i386.inc @@ -1,417 +1,351 @@ // Poly1305 4-way AVX2 bulk MAC kernel (i386). // ------------------------------------------------------------------- -// Algorithm: Goll-Gueron 2015 / Polyakov OpenSSL `poly1305_blocks_avx2`. -// Sibling of the x86-64 kernel; same 4-way design, same precomputed -// r^1..r^4 power table consumed at row offset (broadcast r^4 in the -// bulk loop) and row offset+4 (shifted [r^1,r^2,r^3,r^4] in the tail -// iteration). The Pascal dispatcher rounds the SIMD count down to a -// multiple of 4 and forwards 0..3 leftover blocks to the scalar path. -// -// VEX-encoded AVX2 instructions are db-encoded for broad assembler compatibility. -// Each db line carries a mnemonic comment. The encoding uses ymm0..ymm7 only -// (i386 has no ymm8..ymm15) so all VEX prefixes are 2-byte / 3-byte -// without REX.{R,X,B}; the bytes of the inner kernel are therefore -// identical to the equivalent ymm0..ymm7 sequences in the x86-64 -// sibling - only the base register names change to match the i386 -// argument map. +// Goll-Gueron / Polyakov: 4 blocks per iteration in 4 ymm lanes against an +// r^1..r^4 power table (320 bytes, 32-byte rows). radix-2^26 schoolbook +// multiply mod 2^130-5 (5*r wraparound); VEX tolerates unaligned memory so +// the table is read straight into vpmuludq. The bulk-loop carry-reduce uses +// the carry-reduce stays serial: i386 has only ymm0..7, and the two-chain +// schedule the x86-64 kernel uses needs a second carry temp that would +// clobber padbit (rebuild each iter) - measured slower on this +// throughput-bound, spill-heavy loop. D0..D4 and an H backup live in a +// 320-byte esp frame; the multiply spills each product accumulator per term. // // On entry (after ClpSimdProc5Begin_i386.inc): -// ebx = ACtx (PTPoly1305State; 72-byte R/S/H/K record) -// esi = APowTable (PByte; 320-byte 10x8 power table; layout below) -// edi = AInp (PByte) -// eax = ALen (NativeUInt; multiple of 64; >= 64) -// ecx = APad (always 1; ignored - padbit unconditional in bulk) -// -// TPoly1305State byte offsets (no SIMD member): -// R0..R4 : 0..16 (5 dwords) -// S1..S4 : 20..32 (4 dwords) -// H0..H4 : 36..52 (5 dwords) <- loaded into ymm0..ymm4 lane 0 -// K0..K3 : 56..68 (4 dwords; not read by this kernel) -// -// Power-table byte offsets (separate 320-byte buffer; esi-based loads): -// row j at [esi + 32*j]; bulk loop reads at row offset (broadcast -// r^4), tail iteration reads at row offset+4 (shifted [r^4,r^3,r^2,r^1]). -// Row 0..4 = r-limbs 0..4; row 5..8 = s-limbs 1..4 (s = 5*r); row 9 pad. +// ebx = ACtx (72-byte R/S/H/K state); esi = APowTable (320-byte r^1..r^4) +// edi = AInp; eax = ALen (multiple of 64, >= 64); ecx = APad (ignored) // -// Register allocation (uses ymm0..ymm7 only): -// ymm0..ymm4 : H0..H4 (4-lane radix-2^26 accumulator) -// ymm5 : mask26 (0x3FFFFFF per 64-bit lane low half) -// ymm6 : padbit (0x1000000 per 64-bit lane low half) -// ymm7 : scratch (D0..D4 / M0..M4 transient via stack spills) -// -// Stack frame (320 bytes): -// [esp + 0..159] = D0..D4 (5 x 32 bytes; product accumulators) -// [esp + 160..319] = H backup (5 x 32 bytes; spilled while loading M) +// Power table: row j at [esi + 32*j]; rows 0..4 r-limbs, rows 5..8 s-limbs +// (5*r), row 9 pad. Bulk reads [+32*j] (broadcast r^4); tail reads +// [+32*j+4] ([r^4,r^3,r^2,r^1]). sub esp, 320 - - // ---- Prologue: build mask26 (ymm5) and padbit (ymm6) constants ---- - db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 - db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 ; mask26 - db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 - db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 - db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 ; padbit = 1<<24 - - // ---- Load h0..h4 from ctx into lane 0 of ymm0..ymm4 ---- - db $C5, $F9, $6E, $43, $24 // vmovd xmm0, dword ptr [ebx + 36] ; H0 - db $C5, $F9, $6E, $4B, $28 // vmovd xmm1, dword ptr [ebx + 40] ; H1 - db $C5, $F9, $6E, $53, $2C // vmovd xmm2, dword ptr [ebx + 44] ; H2 - db $C5, $F9, $6E, $5B, $30 // vmovd xmm3, dword ptr [ebx + 48] ; H3 - db $C5, $F9, $6E, $63, $34 // vmovd xmm4, dword ptr [ebx + 52] ; H4 - - // ---- Pre-loop: spill H, load+decompress 4 blocks, absorb H back ---- - db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 160], ymm0 - db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 192], ymm1 - db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 224], ymm2 - db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu ymmword ptr [esp + 256], ymm3 - db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu ymmword ptr [esp + 288], ymm4 - - db $C5, $FA, $6F, $07 // vmovdqu xmm0, xmmword ptr [edi] - db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, xmmword ptr [edi + 16] - db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, xmmword ptr [edi + 32], 1 - db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, xmmword ptr [edi + 48], 1 - add edi, 64 - - // Decompress 4 blocks into limbs M0..M4 (ymm0, ymm1, ymm2, ymm3, ymm4) - db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 - db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 - db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 ; T4 (high 64) - db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 ; T0 (low 64) - db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 ; T2 (limbs 2/3) - db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 ; limb3 raw - db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 ; limb2 raw - db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 ; limb1 raw - db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 ; limb4 raw - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 ; M0 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 ; M1 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 ; M2 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 ; M3 - db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 ; M4 |= padbit - - // Absorb: H += M (H reload from stack since it was spilled) - db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, ymmword ptr [esp + 160] - db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, ymmword ptr [esp + 192] - db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, ymmword ptr [esp + 224] - db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, ymmword ptr [esp + 256] - db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, ymmword ptr [esp + 288] - - sub eax, 64 - jz @@poly1305_avx2_i386_tail + // ---- Prologue: mask26 (ymm5), padbit (ymm6) ---- + db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 + db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 + db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 + db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 + db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 + // ---- Load H0..H4 into lane 0 ---- + db $C5, $F9, $6E, $43, $24 // vmovd xmm0, dword ptr [ebx + 36] + db $C5, $F9, $6E, $4B, $28 // vmovd xmm1, dword ptr [ebx + 40] + db $C5, $F9, $6E, $53, $2C // vmovd xmm2, dword ptr [ebx + 44] + db $C5, $F9, $6E, $5B, $30 // vmovd xmm3, dword ptr [ebx + 48] + db $C5, $F9, $6E, $63, $34 // vmovd xmm4, dword ptr [ebx + 52] + // ---- Pre-loop: spill H, absorb first 4 blocks ---- + db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0 + db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1 + db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2 + db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3 + db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4 + db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi] + db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16] + db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1 + db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1 + add edi, 64 + db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 + db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 + db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 + db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 + db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 + db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 + db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 + db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 + db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 + db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192] + db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224] + db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256] + db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288] + sub eax, 64 + jz @@poly1305_avx2_i386_tail @@poly1305_avx2_i386_loop: - // ---- Bulk multiply: H *= r^4 (broadcast row form, base offset +0) ---- - // D4 spilled at [esp + 128] - db $C5, $FD, $F4, $BE, $80, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 128] ; H0 * R4 - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $F5, $F4, $BE, $60, $00, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 96] ; H1 * R3 - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $ED, $F4, $BE, $40, $00, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 64] ; H2 * R2 - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $E5, $F4, $7E, $20 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 32] ; H3 * R1 - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $DD, $F4, $7E, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi] ; H4 * R0 - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - - // D3 spilled at [esp + 96] - db $C5, $FD, $F4, $BE, $60, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $F5, $F4, $BE, $40, $00, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 64] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $ED, $F4, $7E, $20 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 32] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $E5, $F4, $7E, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $DD, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 256] ; S4 - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - - // D2 spilled at [esp + 64] - db $C5, $FD, $F4, $BE, $40, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $F5, $F4, $7E, $20 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 32] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $ED, $F4, $7E, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $E5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 256] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $DD, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 224] ; S3 - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - - // D1 spilled at [esp + 32] - db $C5, $FD, $F4, $7E, $20 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $F5, $F4, $7E, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $ED, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 256] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $E5, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 224] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $DD, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 192] ; S2 - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - - // D0 spilled at [esp + 0] - db $C5, $FD, $F4, $7E, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $F5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 256] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $ED, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 224] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $E5, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 192] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $DD, $F4, $BE, $A0, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 160] ; S1 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - - // ---- Lazy reduce: D0..D4 (stack) -> H0..H4 (ymm0..ymm4) ---- - db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, ymmword ptr [esp] - db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, ymmword ptr [esp + 32] - db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, ymmword ptr [esp + 64] - db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, ymmword ptr [esp + 96] - db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, ymmword ptr [esp + 128] - - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 ; (D4>>26) backup - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] ; *5 - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - - // ---- Spill H, load + decompress + absorb next 4 blocks ---- - db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 160], ymm0 - db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 192], ymm1 - db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu ymmword ptr [esp + 224], ymm2 - db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu ymmword ptr [esp + 256], ymm3 - db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu ymmword ptr [esp + 288], ymm4 - - db $C5, $FA, $6F, $07 // vmovdqu xmm0, xmmword ptr [edi] - db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, xmmword ptr [edi + 16] - db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, xmmword ptr [edi + 32], 1 - db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, xmmword ptr [edi + 48], 1 - add edi, 64 - - db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 - db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 - db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 - db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 - db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 - db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 - db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 - db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 - db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 - - db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, ymmword ptr [esp + 160] - db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, ymmword ptr [esp + 192] - db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, ymmword ptr [esp + 224] - db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, ymmword ptr [esp + 256] - db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, ymmword ptr [esp + 288] - - sub eax, 64 - jnz @@poly1305_avx2_i386_loop + // ---- Bulk: H *= r^4 (broadcast), D0..D4 spilled ---- + db $C5, $FD, $F4, $BE, $80, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $F5, $F4, $7E, $60 // vpmuludq ymm7, ymm1, [esi + 96] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $ED, $F4, $7E, $40 // vpmuludq ymm7, ymm2, [esi + 64] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $E5, $F4, $7E, $20 // vpmuludq ymm7, ymm3, [esi + 32] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $DD, $F4, $3E // vpmuludq ymm7, ymm4, [esi] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $FD, $F4, $7E, $60 // vpmuludq ymm7, ymm0, [esi + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $F5, $F4, $7E, $40 // vpmuludq ymm7, ymm1, [esi + 64] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $ED, $F4, $7E, $20 // vpmuludq ymm7, ymm2, [esi + 32] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $E5, $F4, $3E // vpmuludq ymm7, ymm3, [esi] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $DD, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 256] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $FD, $F4, $7E, $40 // vpmuludq ymm7, ymm0, [esi + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $F5, $F4, $7E, $20 // vpmuludq ymm7, ymm1, [esi + 32] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $ED, $F4, $3E // vpmuludq ymm7, ymm2, [esi] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $E5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 256] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $DD, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 224] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $FD, $F4, $7E, $20 // vpmuludq ymm7, ymm0, [esi + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $F5, $F4, $3E // vpmuludq ymm7, ymm1, [esi] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $ED, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 256] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $E5, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 224] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $DD, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 192] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $FD, $F4, $3E // vpmuludq ymm7, ymm0, [esi] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $F5, $F4, $BE, $00, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 256] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $ED, $F4, $BE, $E0, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 224] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $E5, $F4, $BE, $C0, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 192] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $DD, $F4, $BE, $A0, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 160] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + // ---- Lazy reduce ---- + db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp] + db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32] + db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64] + db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96] + db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128] + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 + db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + // ---- Absorb next 4 blocks ---- + db $C5, $FE, $7F, $84, $24, $A0, $00, $00, $00 // vmovdqu [esp + 160], ymm0 + db $C5, $FE, $7F, $8C, $24, $C0, $00, $00, $00 // vmovdqu [esp + 192], ymm1 + db $C5, $FE, $7F, $94, $24, $E0, $00, $00, $00 // vmovdqu [esp + 224], ymm2 + db $C5, $FE, $7F, $9C, $24, $00, $01, $00, $00 // vmovdqu [esp + 256], ymm3 + db $C5, $FE, $7F, $A4, $24, $20, $01, $00, $00 // vmovdqu [esp + 288], ymm4 + db $C5, $FA, $6F, $07 // vmovdqu xmm0, [edi] + db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, [edi + 16] + db $C4, $E3, $7D, $38, $47, $20, $01 // vinserti128 ymm0, ymm0, [edi + 32], 1 + db $C4, $E3, $75, $38, $4F, $30, $01 // vinserti128 ymm1, ymm1, [edi + 48], 1 + add edi, 64 + db $C5, $ED, $73, $D8, $06 // vpsrldq ymm2, ymm0, 6 + db $C5, $E5, $73, $D9, $06 // vpsrldq ymm3, ymm1, 6 + db $C5, $FD, $6D, $E1 // vpunpckhqdq ymm4, ymm0, ymm1 + db $C5, $FD, $6C, $C1 // vpunpcklqdq ymm0, ymm0, ymm1 + db $C5, $ED, $6C, $D3 // vpunpcklqdq ymm2, ymm2, ymm3 + db $C5, $E5, $73, $D2, $1E // vpsrlq ymm3, ymm2, 30 + db $C5, $ED, $73, $D2, $04 // vpsrlq ymm2, ymm2, 4 + db $C5, $F5, $73, $D0, $1A // vpsrlq ymm1, ymm0, 26 + db $C5, $DD, $73, $D4, $28 // vpsrlq ymm4, ymm4, 40 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C5, $DD, $EB, $E6 // vpor ymm4, ymm4, ymm6 + db $C5, $FD, $D4, $84, $24, $A0, $00, $00, $00 // vpaddq ymm0, ymm0, [esp + 160] + db $C5, $F5, $D4, $8C, $24, $C0, $00, $00, $00 // vpaddq ymm1, ymm1, [esp + 192] + db $C5, $ED, $D4, $94, $24, $E0, $00, $00, $00 // vpaddq ymm2, ymm2, [esp + 224] + db $C5, $E5, $D4, $9C, $24, $00, $01, $00, $00 // vpaddq ymm3, ymm3, [esp + 256] + db $C5, $DD, $D4, $A4, $24, $20, $01, $00, $00 // vpaddq ymm4, ymm4, [esp + 288] + sub eax, 64 + jnz @@poly1305_avx2_i386_loop @@poly1305_avx2_i386_tail: - // ---- Tail: H *= [r^1, r^2, r^3, r^4] (offset+4 = shifted form) ---- - // D4 spilled at [esp + 128] - table loads at +4 over the bulk offsets - db $C5, $FD, $F4, $BE, $84, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 132] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $F5, $F4, $BE, $64, $00, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 100] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $ED, $F4, $BE, $44, $00, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 68] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $E5, $F4, $7E, $24 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 36] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - db $C5, $DD, $F4, $7E, $04 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 4] - db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, ymmword ptr [esp + 128] - db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu ymmword ptr [esp + 128], ymm7 - - // D3 spilled at [esp + 96] - db $C5, $FD, $F4, $BE, $64, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 100] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $F5, $F4, $BE, $44, $00, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 68] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $ED, $F4, $7E, $24 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 36] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $E5, $F4, $7E, $04 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 4] - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - db $C5, $DD, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 260] ; S4 shifted - db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, ymmword ptr [esp + 96] - db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu ymmword ptr [esp + 96], ymm7 - - // D2 spilled at [esp + 64] - db $C5, $FD, $F4, $BE, $44, $00, $00, $00 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 68] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $F5, $F4, $7E, $24 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 36] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $ED, $F4, $7E, $04 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 4] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $E5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 260] - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - db $C5, $DD, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 228] ; S3 shifted - db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, ymmword ptr [esp + 64] - db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu ymmword ptr [esp + 64], ymm7 - - // D1 spilled at [esp + 32] - db $C5, $FD, $F4, $7E, $24 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 36] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $F5, $F4, $7E, $04 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 4] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $ED, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 260] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $E5, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 228] - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - db $C5, $DD, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 196] ; S2 shifted - db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, ymmword ptr [esp + 32] - db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu ymmword ptr [esp + 32], ymm7 - - // D0 spilled at [esp + 0] - db $C5, $FD, $F4, $7E, $04 // vpmuludq ymm7, ymm0, ymmword ptr [esi + 4] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $F5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm1, ymmword ptr [esi + 260] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $ED, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm2, ymmword ptr [esi + 228] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $E5, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm3, ymmword ptr [esi + 196] - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $DD, $F4, $BE, $A4, $00, $00, $00 // vpmuludq ymm7, ymm4, ymmword ptr [esi + 164] ; S1 shifted - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - + // ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ---- + db $C5, $FD, $F4, $BE, $84, $00, $00, $00 // vpmuludq ymm7, ymm0, [esi + 132] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $F5, $F4, $7E, $64 // vpmuludq ymm7, ymm1, [esi + 100] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $ED, $F4, $7E, $44 // vpmuludq ymm7, ymm2, [esi + 68] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $E5, $F4, $7E, $24 // vpmuludq ymm7, ymm3, [esi + 36] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $DD, $F4, $7E, $04 // vpmuludq ymm7, ymm4, [esi + 4] + db $C5, $C5, $D4, $BC, $24, $80, $00, $00, $00 // vpaddq ymm7, ymm7, [esp + 128] + db $C5, $FE, $7F, $BC, $24, $80, $00, $00, $00 // vmovdqu [esp + 128], ymm7 + db $C5, $FD, $F4, $7E, $64 // vpmuludq ymm7, ymm0, [esi + 100] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $F5, $F4, $7E, $44 // vpmuludq ymm7, ymm1, [esi + 68] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $ED, $F4, $7E, $24 // vpmuludq ymm7, ymm2, [esi + 36] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $E5, $F4, $7E, $04 // vpmuludq ymm7, ymm3, [esi + 4] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $DD, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm4, [esi + 260] + db $C5, $C5, $D4, $7C, $24, $60 // vpaddq ymm7, ymm7, [esp + 96] + db $C5, $FE, $7F, $7C, $24, $60 // vmovdqu [esp + 96], ymm7 + db $C5, $FD, $F4, $7E, $44 // vpmuludq ymm7, ymm0, [esi + 68] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $F5, $F4, $7E, $24 // vpmuludq ymm7, ymm1, [esi + 36] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $ED, $F4, $7E, $04 // vpmuludq ymm7, ymm2, [esi + 4] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $E5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm3, [esi + 260] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $DD, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 228] + db $C5, $C5, $D4, $7C, $24, $40 // vpaddq ymm7, ymm7, [esp + 64] + db $C5, $FE, $7F, $7C, $24, $40 // vmovdqu [esp + 64], ymm7 + db $C5, $FD, $F4, $7E, $24 // vpmuludq ymm7, ymm0, [esi + 36] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $F5, $F4, $7E, $04 // vpmuludq ymm7, ymm1, [esi + 4] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $ED, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm2, [esi + 260] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $E5, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 228] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $DD, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 196] + db $C5, $C5, $D4, $7C, $24, $20 // vpaddq ymm7, ymm7, [esp + 32] + db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu [esp + 32], ymm7 + db $C5, $FD, $F4, $7E, $04 // vpmuludq ymm7, ymm0, [esi + 4] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $F5, $F4, $BE, $04, $01, $00, $00 // vpmuludq ymm7, ymm1, [esi + 260] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $ED, $F4, $BE, $E4, $00, $00, $00 // vpmuludq ymm7, ymm2, [esi + 228] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $E5, $F4, $BE, $C4, $00, $00, $00 // vpmuludq ymm7, ymm3, [esi + 196] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $DD, $F4, $BE, $A4, $00, $00, $00 // vpmuludq ymm7, ymm4, [esi + 164] + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 // ---- Tail lazy reduce ---- - db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, ymmword ptr [esp] - db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, ymmword ptr [esp + 32] - db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, ymmword ptr [esp + 64] - db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, ymmword ptr [esp + 96] - db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, ymmword ptr [esp + 128] - - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - - // ---- Horizontal sum across 4 lanes for ymm0..ymm4 ---- - db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8 - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C4, $E3, $FD, $00, $F8, $4E // vpermq ymm7, ymm0, 0x4E - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D9, $08 // vpsrldq ymm7, ymm1, 8 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C4, $E3, $FD, $00, $F9, $4E // vpermq ymm7, ymm1, 0x4E - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $DA, $08 // vpsrldq ymm7, ymm2, 8 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C4, $E3, $FD, $00, $FA, $4E // vpermq ymm7, ymm2, 0x4E - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $DB, $08 // vpsrldq ymm7, ymm3, 8 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C4, $E3, $FD, $00, $FB, $4E // vpermq ymm7, ymm3, 0x4E - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $DC, $08 // vpsrldq ymm7, ymm4, 8 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - - // ---- Final lazy reduce on the now-scalar limbs (lane 0) ---- - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C5, $FE, $7F, $3C, $24 // vmovdqu ymmword ptr [esp], ymm7 - db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 - db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, ymmword ptr [esp] - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - - // ---- Store h0..h4 back to ctx ---- - db $C5, $F9, $7E, $43, $24 // vmovd dword ptr [ebx + 36], xmm0 - db $C5, $F9, $7E, $4B, $28 // vmovd dword ptr [ebx + 40], xmm1 - db $C5, $F9, $7E, $53, $2C // vmovd dword ptr [ebx + 44], xmm2 - db $C5, $F9, $7E, $5B, $30 // vmovd dword ptr [ebx + 48], xmm3 - db $C5, $F9, $7E, $63, $34 // vmovd dword ptr [ebx + 52], xmm4 - - db $C5, $F8, $77 // vzeroupper - add esp, 320 - - pop edi - pop esi - pop ebx + db $C5, $FE, $6F, $04, $24 // vmovdqu ymm0, [esp] + db $C5, $FE, $6F, $4C, $24, $20 // vmovdqu ymm1, [esp + 32] + db $C5, $FE, $6F, $54, $24, $40 // vmovdqu ymm2, [esp + 64] + db $C5, $FE, $6F, $5C, $24, $60 // vmovdqu ymm3, [esp + 96] + db $C5, $FE, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu ymm4, [esp + 128] + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 + db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + // ---- Horizontal sum 4 lanes -> lane 0 ---- + db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8 + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C4, $E3, $FD, $00, $F8, $4E // vpermq ymm7, ymm0, 0x4E + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C5, $C5, $73, $D9, $08 // vpsrldq ymm7, ymm1, 8 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C4, $E3, $FD, $00, $F9, $4E // vpermq ymm7, ymm1, 0x4E + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $C5, $73, $DA, $08 // vpsrldq ymm7, ymm2, 8 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C4, $E3, $FD, $00, $FA, $4E // vpermq ymm7, ymm2, 0x4E + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $C5, $73, $DB, $08 // vpsrldq ymm7, ymm3, 8 + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C4, $E3, $FD, $00, $FB, $4E // vpermq ymm7, ymm3, 0x4E + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C5, $C5, $73, $DC, $08 // vpsrldq ymm7, ymm4, 8 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + // ---- Final lazy reduce (lane 0 scalars) ---- + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $C5, $73, $D1, $1A // vpsrlq ymm7, ymm1, 26 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $C5, $73, $D2, $1A // vpsrlq ymm7, ymm2, 26 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C5, $C5, $73, $D3, $1A // vpsrlq ymm7, ymm3, 26 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C5, $C5, $73, $D4, $1A // vpsrlq ymm7, ymm4, 26 + db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 + db $C5, $FE, $7F, $3C, $24 // vmovdqu [esp], ymm7 + db $C5, $C5, $73, $F7, $02 // vpsllq ymm7, ymm7, 2 + db $C5, $C5, $D4, $3C, $24 // vpaddq ymm7, ymm7, [esp] + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C5, $C5, $73, $D0, $1A // vpsrlq ymm7, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + // ---- Store H0..H4 back to ctx ---- + db $C5, $F9, $7E, $43, $24 // vmovd dword ptr [ebx + 36], xmm0 + db $C5, $F9, $7E, $4B, $28 // vmovd dword ptr [ebx + 40], xmm1 + db $C5, $F9, $7E, $53, $2C // vmovd dword ptr [ebx + 44], xmm2 + db $C5, $F9, $7E, $5B, $30 // vmovd dword ptr [ebx + 48], xmm3 + db $C5, $F9, $7E, $63, $34 // vmovd dword ptr [ebx + 52], xmm4 + db $C5, $F8, $77 // vzeroupper + add esp, 320 + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc index dd2429b6..50df2c65 100644 --- a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkAvx2Core_x86_64.inc @@ -1,332 +1,276 @@ // Poly1305 4-way AVX2 bulk MAC kernel (x86-64). // ------------------------------------------------------------------- -// Algorithm: Goll-Gueron 2015 / Polyakov OpenSSL `poly1305_blocks_avx2`. -// Processes 4 16-byte blocks per iteration in 4-way SIMD with a precomputed -// r^1..r^4 power table (rebuilt at SetKey when AVX2 is the chosen path). -// The bulk loop multiplies the 4-lane accumulator H by r^4 broadcast in all -// lanes; the final tail iteration uses the same table read at +4 byte -// offset, giving lanes [r^1, r^2, r^3, r^4] for a branch-free fold-back via -// a horizontal sum. No scalar-tail block from the SIMD kernel; the Pascal -// dispatcher rounds the SIMD count down to a multiple of 4 and forwards -// 0..3 leftover blocks to the scalar code path. -// -// VEX-encoded AVX2 instructions are db-encoded for broad assembler compatibility. -// Each db line carries a mnemonic comment. Most VEX bytes are shared across IA-32 -// and x86-64 since the inner kernel uses ymm0..ymm15 / r-class registers -// addressable in both modes via 2-/3-byte VEX prefixes; the i386 sibling -// include shares the same algorithm but uses a different register mapping -// (ebx/esi/edi/eax instead of rcx/rdx/r8/r9) and stays inside ymm0..ymm7. +// Goll-Gueron / Polyakov: 4 blocks per iteration in 4 ymm lanes against an +// r^1..r^4 power table (320 bytes, 32-byte rows). radix-2^26 schoolbook +// multiply mod 2^130-5 (5*r wraparound); VEX tolerates unaligned memory so +// the table is read straight into vpmuludq. The bulk-loop carry-reduce uses +// two interleaved chains (low D0->D1->D2->D3->H4, high D3->D4->H0->H1) to +// shorten the serial critical path; the run-once tail/final reduces stay +// serial. A vpermq-based horizontal sum folds the 4 lanes at the end. // // On entry (after ClpSimdProc5Begin_x86_64.inc): -// rcx = ACtx (PTPoly1305State; 72-byte R/S/H/K record) -// rdx = APowTable (PByte; 320-byte 10x8 power table; layout below) -// r8 = AInp (PByte; 16-byte aligned-or-not) -// r9 = ALen (NativeUInt; multiple of 64; >= 64) -// r10 = APad (always 1; ignored - padbit unconditional in bulk) -// -// TPoly1305State byte offsets (no SIMD member): -// R0..R4 : 0..16 (5 dwords) -// S1..S4 : 20..32 (4 dwords) -// H0..H4 : 36..52 (5 dwords) <- loaded into ymm0..ymm4 lane 0 -// K0..K3 : 56..68 (4 dwords; not read by this kernel) -// -// Power-table byte offsets (separate 320-byte buffer; rdx-based loads): -// row j at [rdx + 32*j]; bulk loop reads at row offset (broadcast r^4), -// tail iteration reads at row offset+4 (shifted [r^4,r^3,r^2,r^1]). -// Row 0..4 = r-limbs 0..4; row 5..8 = s-limbs 1..4 (s = 5*r); row 9 pad. -// -// Register allocation (uses ymm0..ymm15): -// ymm0..ymm4 : H0..H4 (4-lane radix-2^26 accumulator, persistent) -// ymm5 : mask26 (0x3FFFFFF per 64-bit lane low half) -// ymm6 : padbit (0x1000000 per 64-bit lane low half) -// ymm7..ymm15: D0..D4 / M0..M4 / scratch (transient) +// rcx = ACtx (72-byte R/S/H/K state); rdx = APowTable (320-byte r^1..r^4) +// r8 = AInp; r9 = ALen (multiple of 64, >= 64); r10 = APad (ignored) // -{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} - // ---- Prologue: build mask26 (ymm5) and padbit (ymm6) constants ---- - db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 - db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 ; mask26 = 0x3FFFFFF per lane - db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 - db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 - db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 ; padbit = 1<<24 per lane - - // ---- Load h0..h4 from ctx into lane 0 of ymm0..ymm4 ---- - db $C5, $F9, $6E, $41, $24 // vmovd xmm0, dword ptr [rcx + 36] ; H0 - db $C5, $F9, $6E, $49, $28 // vmovd xmm1, dword ptr [rcx + 40] ; H1 - db $C5, $F9, $6E, $51, $2C // vmovd xmm2, dword ptr [rcx + 44] ; H2 - db $C5, $F9, $6E, $59, $30 // vmovd xmm3, dword ptr [rcx + 48] ; H3 - db $C5, $F9, $6E, $61, $34 // vmovd xmm4, dword ptr [rcx + 52] ; H4 +// Power table: row j at [rdx + 32*j]; rows 0..4 r-limbs, rows 5..8 s-limbs +// (5*r), row 9 pad. Bulk reads [+32*j] (broadcast r^4); tail reads +// [+32*j+4] ([r^4,r^3,r^2,r^1]). - // ---- Pre-loop: load + decompress + absorb first 4 blocks ---- - db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, xmmword ptr [r8] - db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, xmmword ptr [r8 + 16] - db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, xmmword ptr [r8 + 32], 1 - db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, xmmword ptr [r8 + 48], 1 +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + // ---- Prologue: mask26 (ymm5), padbit (ymm6) ---- + db $C5, $D5, $76, $ED // vpcmpeqd ymm5, ymm5, ymm5 + db $C5, $D5, $73, $D5, $26 // vpsrlq ymm5, ymm5, 38 + db $C5, $CD, $76, $F6 // vpcmpeqd ymm6, ymm6, ymm6 + db $C5, $CD, $73, $D6, $3F // vpsrlq ymm6, ymm6, 63 + db $C5, $CD, $73, $F6, $18 // vpsllq ymm6, ymm6, 24 + // ---- Load H0..H4 into lane 0 ---- + db $C5, $F9, $6E, $41, $24 // vmovd xmm0, dword ptr [rcx + 36] + db $C5, $F9, $6E, $49, $28 // vmovd xmm1, dword ptr [rcx + 40] + db $C5, $F9, $6E, $51, $2C // vmovd xmm2, dword ptr [rcx + 44] + db $C5, $F9, $6E, $59, $30 // vmovd xmm3, dword ptr [rcx + 48] + db $C5, $F9, $6E, $61, $34 // vmovd xmm4, dword ptr [rcx + 52] + // ---- Pre-loop: absorb first 4 blocks ---- + db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8] + db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16] + db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1 + db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1 add r8, 64 - - // Decompress 4 blocks into limbs M0..M4 in (ymm12, ymm13, ymm14, ymm15, ymm7) - db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 - db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 - db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 ; T4 (high 64) - db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 ; T0 (low 64) - db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 ; T2 (limbs 2/3) - db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 ; limb3 raw - db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 ; limb2 raw - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 ; limb1 raw - db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 ; limb4 raw - db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 ; M0 = limb0 - db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 ; M1 = limb1 - db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 ; M2 = limb2 - db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 ; M3 = limb3 - db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 ; M4 = limb4 | padbit - - // Absorb: H += M - db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 ; H0 += M0 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 ; H1 += M1 - db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 ; H2 += M2 - db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 ; H3 += M3 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 ; H4 += M4 - + db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 + db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 + db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 + db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 + db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 + db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 + db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 + db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 + db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 + db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 + db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 + db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 + db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 + db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 + db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 + db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 + db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 + db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 sub r9, 64 jz @@poly1305_avx2_tail @@poly1305_avx2_loop: - // ---- Bulk multiply: H *= r^4 (broadcast row form, base offset +0) ---- - // D4 (ymm12) = H0*R4 + H1*R3 + H2*R2 + H3*R1 + H4*R0 - db $C5, $7D, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm12, ymm0, ymmword ptr [rdx + 128] ; H0 * R4 - db $C5, $75, $F4, $AA, $60, $00, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 96] ; H1 * R3 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $6D, $F4, $AA, $40, $00, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 64] ; H2 * R2 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $65, $F4, $6A, $20 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 32] ; H3 * R1 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $5D, $F4, $6A, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx] ; H4 * R0 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - - // D3 (ymm11) = H0*R3 + H1*R2 + H2*R1 + H3*R0 + H4*S4 - db $C5, $7D, $F4, $9A, $60, $00, $00, $00 // vpmuludq ymm11, ymm0, ymmword ptr [rdx + 96] - db $C5, $75, $F4, $AA, $40, $00, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 64] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $6D, $F4, $6A, $20 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 32] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $65, $F4, $6A, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $5D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 256] ; S4 - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - - // D2 (ymm10) = H0*R2 + H1*R1 + H2*R0 + H3*S4 + H4*S3 - db $C5, $7D, $F4, $92, $40, $00, $00, $00 // vpmuludq ymm10, ymm0, ymmword ptr [rdx + 64] - db $C5, $75, $F4, $6A, $20 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 32] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $6D, $F4, $6A, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $65, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 256] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $5D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 224] ; S3 - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - - // D1 (ymm9) = H0*R1 + H1*R0 + H2*S4 + H3*S3 + H4*S2 - db $C5, $7D, $F4, $4A, $20 // vpmuludq ymm9, ymm0, ymmword ptr [rdx + 32] - db $C5, $75, $F4, $6A, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $6D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 256] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $65, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 224] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $5D, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 192] ; S2 - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - - // D0 (ymm8) = H0*R0 + H1*S4 + H2*S3 + H3*S2 + H4*S1 - db $C5, $7D, $F4, $42, $00 // vpmuludq ymm8, ymm0, ymmword ptr [rdx] - db $C5, $75, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 256] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $6D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 224] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $65, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 192] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $5D, $F4, $AA, $A0, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 160] ; S1 - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - - // ---- Lazy reduce D0..D4 (ymm8..ymm12) -> H0..H4 (ymm0..ymm4) ---- - db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26 - db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5 - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 ; D1 += D0>>26 - db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26 - db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5 - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 ; D2 += D1>>26 - db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26 - db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5 - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 ; D3 += D2>>26 - db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26 - db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 ; D4 += D3>>26 - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 - db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5 - db $C4, $C1, $05, $73, $F5, $02 // vpsllq ymm15, ymm13, 2 - db $C4, $41, $15, $D4, $EF // vpaddq ymm13, ymm13, ymm15 ; (D4>>26) *= 5 - db $C4, $C1, $7D, $D4, $C5 // vpaddq ymm0, ymm0, ymm13 ; H0 += (D4>>26)*5 - db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 ; H1 += H0>>26 - - // ---- Load + decompress + absorb next 4 blocks ---- - db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, xmmword ptr [r8] - db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, xmmword ptr [r8 + 16] - db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, xmmword ptr [r8 + 32], 1 - db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, xmmword ptr [r8 + 48], 1 + // ---- Bulk: H *= r^4 (broadcast) ---- + db $C5, $7D, $F4, $A2, $80, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 128] + db $C5, $75, $F4, $6A, $60 // vpmuludq ymm13, ymm1, [rdx + 96] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $6D, $F4, $6A, $40 // vpmuludq ymm13, ymm2, [rdx + 64] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $65, $F4, $6A, $20 // vpmuludq ymm13, ymm3, [rdx + 32] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $5D, $F4, $2A // vpmuludq ymm13, ymm4, [rdx] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $7D, $F4, $5A, $60 // vpmuludq ymm11, ymm0, [rdx + 96] + db $C5, $75, $F4, $6A, $40 // vpmuludq ymm13, ymm1, [rdx + 64] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $6D, $F4, $6A, $20 // vpmuludq ymm13, ymm2, [rdx + 32] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $65, $F4, $2A // vpmuludq ymm13, ymm3, [rdx] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $5D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 256] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $7D, $F4, $52, $40 // vpmuludq ymm10, ymm0, [rdx + 64] + db $C5, $75, $F4, $6A, $20 // vpmuludq ymm13, ymm1, [rdx + 32] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $6D, $F4, $2A // vpmuludq ymm13, ymm2, [rdx] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $65, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 256] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $5D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 224] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $7D, $F4, $4A, $20 // vpmuludq ymm9, ymm0, [rdx + 32] + db $C5, $75, $F4, $2A // vpmuludq ymm13, ymm1, [rdx] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $6D, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 256] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $65, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 224] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $5D, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 192] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $7D, $F4, $02 // vpmuludq ymm8, ymm0, [rdx] + db $C5, $75, $F4, $AA, $00, $01, $00, $00 // vpmuludq ymm13, ymm1, [rdx + 256] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $6D, $F4, $AA, $E0, $00, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 224] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $65, $F4, $AA, $C0, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 192] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $5D, $F4, $AA, $A0, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 160] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + // ---- Lazy reduce (two interleaved carry chains) ---- + db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26 + db $C4, $C1, $0D, $73, $D3, $1A // vpsrlq ymm14, ymm11, 26 + db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5 + db $C5, $25, $DB, $DD // vpand ymm11, ymm11, ymm5 + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C4, $41, $1D, $D4, $E6 // vpaddq ymm12, ymm12, ymm14 + db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26 + db $C4, $C1, $0D, $73, $D4, $1A // vpsrlq ymm14, ymm12, 26 + db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5 + db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5 + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C4, $C1, $05, $73, $F6, $02 // vpsllq ymm15, ymm14, 2 + db $C4, $41, $0D, $D4, $F7 // vpaddq ymm14, ymm14, ymm15 + db $C4, $C1, $7D, $D4, $C6 // vpaddq ymm0, ymm0, ymm14 + db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26 + db $C5, $8D, $73, $D0, $1A // vpsrlq ymm14, ymm0, 26 + db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C4, $C1, $75, $D4, $CE // vpaddq ymm1, ymm1, ymm14 + db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26 + db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5 + db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13 + // ---- Absorb next 4 blocks ---- + db $C4, $41, $7A, $6F, $20 // vmovdqu xmm12, [r8] + db $C4, $41, $7A, $6F, $68, $10 // vmovdqu xmm13, [r8 + 16] + db $C4, $43, $1D, $38, $60, $20, $01 // vinserti128 ymm12, ymm12, [r8 + 32], 1 + db $C4, $43, $15, $38, $68, $30, $01 // vinserti128 ymm13, ymm13, [r8 + 48], 1 add r8, 64 - db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 - db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 - db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 - db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 - db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 - db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 - db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 - db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 - db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 - db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 - db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 - db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 - db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 - db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 - db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - + db $C4, $C1, $0D, $73, $DC, $06 // vpsrldq ymm14, ymm12, 6 + db $C4, $C1, $05, $73, $DD, $06 // vpsrldq ymm15, ymm13, 6 + db $C4, $C1, $1D, $6D, $FD // vpunpckhqdq ymm7, ymm12, ymm13 + db $C4, $41, $1D, $6C, $E5 // vpunpcklqdq ymm12, ymm12, ymm13 + db $C4, $41, $0D, $6C, $F7 // vpunpcklqdq ymm14, ymm14, ymm15 + db $C4, $C1, $05, $73, $D6, $1E // vpsrlq ymm15, ymm14, 30 + db $C4, $C1, $0D, $73, $D6, $04 // vpsrlq ymm14, ymm14, 4 + db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 + db $C5, $C5, $73, $D7, $28 // vpsrlq ymm7, ymm7, 40 + db $C5, $1D, $DB, $E5 // vpand ymm12, ymm12, ymm5 + db $C5, $15, $DB, $ED // vpand ymm13, ymm13, ymm5 + db $C5, $0D, $DB, $F5 // vpand ymm14, ymm14, ymm5 + db $C5, $05, $DB, $FD // vpand ymm15, ymm15, ymm5 + db $C5, $C5, $EB, $FE // vpor ymm7, ymm7, ymm6 + db $C4, $C1, $7D, $D4, $C4 // vpaddq ymm0, ymm0, ymm12 + db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 + db $C4, $C1, $6D, $D4, $D6 // vpaddq ymm2, ymm2, ymm14 + db $C4, $C1, $65, $D4, $DF // vpaddq ymm3, ymm3, ymm15 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 sub r9, 64 jnz @@poly1305_avx2_loop @@poly1305_avx2_tail: - // ---- Tail: H *= [r^1, r^2, r^3, r^4] (offset+4 = shifted form) ---- - // D4 (ymm12) - table loads at +4 over the bulk offsets - db $C5, $7D, $F4, $A2, $84, $00, $00, $00 // vpmuludq ymm12, ymm0, ymmword ptr [rdx + 132] - db $C5, $75, $F4, $AA, $64, $00, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 100] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $6D, $F4, $AA, $44, $00, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 68] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $65, $F4, $6A, $24 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 36] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C5, $5D, $F4, $6A, $04 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 4] - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - - // D3 (ymm11) - db $C5, $7D, $F4, $9A, $64, $00, $00, $00 // vpmuludq ymm11, ymm0, ymmword ptr [rdx + 100] - db $C5, $75, $F4, $AA, $44, $00, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 68] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $6D, $F4, $6A, $24 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 36] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $65, $F4, $6A, $04 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 4] - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C5, $5D, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 260] ; S4 shifted - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - - // D2 (ymm10) - db $C5, $7D, $F4, $92, $44, $00, $00, $00 // vpmuludq ymm10, ymm0, ymmword ptr [rdx + 68] - db $C5, $75, $F4, $6A, $24 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 36] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $6D, $F4, $6A, $04 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 4] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $65, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 260] - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C5, $5D, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 228] ; S3 shifted - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - - // D1 (ymm9) - db $C5, $7D, $F4, $4A, $24 // vpmuludq ymm9, ymm0, ymmword ptr [rdx + 36] - db $C5, $75, $F4, $6A, $04 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 4] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $6D, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 260] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $65, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 228] - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C5, $5D, $F4, $AA, $C4, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 196] ; S2 shifted - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - - // D0 (ymm8) - db $C5, $7D, $F4, $42, $04 // vpmuludq ymm8, ymm0, ymmword ptr [rdx + 4] - db $C5, $75, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm1, ymmword ptr [rdx + 260] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $6D, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm2, ymmword ptr [rdx + 228] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $65, $F4, $AA, $C4, $00, $00, $00 // vpmuludq ymm13, ymm3, ymmword ptr [rdx + 196] - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - db $C5, $5D, $F4, $AA, $A4, $00, $00, $00 // vpmuludq ymm13, ymm4, ymmword ptr [rdx + 164] ; S1 shifted - db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 - + // ---- Tail: H *= [r^1, r^2, r^3, r^4] (shifted table read) ---- + db $C5, $7D, $F4, $A2, $84, $00, $00, $00 // vpmuludq ymm12, ymm0, [rdx + 132] + db $C5, $75, $F4, $6A, $64 // vpmuludq ymm13, ymm1, [rdx + 100] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $6D, $F4, $6A, $44 // vpmuludq ymm13, ymm2, [rdx + 68] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $65, $F4, $6A, $24 // vpmuludq ymm13, ymm3, [rdx + 36] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $5D, $F4, $6A, $04 // vpmuludq ymm13, ymm4, [rdx + 4] + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C5, $7D, $F4, $5A, $64 // vpmuludq ymm11, ymm0, [rdx + 100] + db $C5, $75, $F4, $6A, $44 // vpmuludq ymm13, ymm1, [rdx + 68] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $6D, $F4, $6A, $24 // vpmuludq ymm13, ymm2, [rdx + 36] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $65, $F4, $6A, $04 // vpmuludq ymm13, ymm3, [rdx + 4] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $5D, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 260] + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C5, $7D, $F4, $52, $44 // vpmuludq ymm10, ymm0, [rdx + 68] + db $C5, $75, $F4, $6A, $24 // vpmuludq ymm13, ymm1, [rdx + 36] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $6D, $F4, $6A, $04 // vpmuludq ymm13, ymm2, [rdx + 4] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $65, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 260] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $5D, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 228] + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C5, $7D, $F4, $4A, $24 // vpmuludq ymm9, ymm0, [rdx + 36] + db $C5, $75, $F4, $6A, $04 // vpmuludq ymm13, ymm1, [rdx + 4] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $6D, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 260] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $65, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 228] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $5D, $F4, $AA, $C4, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 196] + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C5, $7D, $F4, $42, $04 // vpmuludq ymm8, ymm0, [rdx + 4] + db $C5, $75, $F4, $AA, $04, $01, $00, $00 // vpmuludq ymm13, ymm1, [rdx + 260] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $6D, $F4, $AA, $E4, $00, $00, $00 // vpmuludq ymm13, ymm2, [rdx + 228] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $65, $F4, $AA, $C4, $00, $00, $00 // vpmuludq ymm13, ymm3, [rdx + 196] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 + db $C5, $5D, $F4, $AA, $A4, $00, $00, $00 // vpmuludq ymm13, ymm4, [rdx + 164] + db $C4, $41, $3D, $D4, $C5 // vpaddq ymm8, ymm8, ymm13 // ---- Tail lazy reduce ---- - db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26 - db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5 - db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 - db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26 - db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5 - db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 - db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26 - db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5 - db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 - db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26 - db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5 - db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 - db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 - db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5 - db $C4, $C1, $05, $73, $F5, $02 // vpsllq ymm15, ymm13, 2 - db $C4, $41, $15, $D4, $EF // vpaddq ymm13, ymm13, ymm15 - db $C4, $C1, $7D, $D4, $C5 // vpaddq ymm0, ymm0, ymm13 - db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - - // ---- Horizontal sum across 4 lanes for ymm0..ymm4 ---- - db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8 - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C4, $E3, $FD, $00, $F8, $4E // vpermq ymm7, ymm0, 0x4E - db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 - db $C5, $C5, $73, $D9, $08 // vpsrldq ymm7, ymm1, 8 - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C4, $E3, $FD, $00, $F9, $4E // vpermq ymm7, ymm1, 0x4E - db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 - db $C5, $C5, $73, $DA, $08 // vpsrldq ymm7, ymm2, 8 - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C4, $E3, $FD, $00, $FA, $4E // vpermq ymm7, ymm2, 0x4E - db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 - db $C5, $C5, $73, $DB, $08 // vpsrldq ymm7, ymm3, 8 - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C4, $E3, $FD, $00, $FB, $4E // vpermq ymm7, ymm3, 0x4E - db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 - db $C5, $C5, $73, $DC, $08 // vpsrldq ymm7, ymm4, 8 - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E - db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 - - // ---- Final lazy reduce on the now-scalar limbs (lane 0) ---- - db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - db $C5, $95, $73, $D1, $1A // vpsrlq ymm13, ymm1, 26 - db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 - db $C4, $C1, $6D, $D4, $D5 // vpaddq ymm2, ymm2, ymm13 - db $C5, $95, $73, $D2, $1A // vpsrlq ymm13, ymm2, 26 - db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 - db $C4, $C1, $65, $D4, $DD // vpaddq ymm3, ymm3, ymm13 - db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26 - db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 - db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13 - db $C5, $95, $73, $D4, $1A // vpsrlq ymm13, ymm4, 26 - db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 - db $C4, $C1, $05, $73, $F5, $02 // vpsllq ymm15, ymm13, 2 - db $C4, $41, $15, $D4, $EF // vpaddq ymm13, ymm13, ymm15 - db $C4, $C1, $7D, $D4, $C5 // vpaddq ymm0, ymm0, ymm13 - db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 - db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 - db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 - - // ---- Store h0..h4 back to ctx ---- - db $C5, $F9, $7E, $41, $24 // vmovd dword ptr [rcx + 36], xmm0 - db $C5, $F9, $7E, $49, $28 // vmovd dword ptr [rcx + 40], xmm1 - db $C5, $F9, $7E, $51, $2C // vmovd dword ptr [rcx + 44], xmm2 - db $C5, $F9, $7E, $59, $30 // vmovd dword ptr [rcx + 48], xmm3 - db $C5, $F9, $7E, $61, $34 // vmovd dword ptr [rcx + 52], xmm4 - - db $C5, $F8, $77 // vzeroupper - // restore caller xmm6..15 (legacy SSE moves are safe after vzeroupper) + db $C4, $C1, $15, $73, $D0, $1A // vpsrlq ymm13, ymm8, 26 + db $C5, $BD, $DB, $C5 // vpand ymm0, ymm8, ymm5 + db $C4, $41, $35, $D4, $CD // vpaddq ymm9, ymm9, ymm13 + db $C4, $C1, $15, $73, $D1, $1A // vpsrlq ymm13, ymm9, 26 + db $C5, $B5, $DB, $CD // vpand ymm1, ymm9, ymm5 + db $C4, $41, $2D, $D4, $D5 // vpaddq ymm10, ymm10, ymm13 + db $C4, $C1, $15, $73, $D2, $1A // vpsrlq ymm13, ymm10, 26 + db $C5, $AD, $DB, $D5 // vpand ymm2, ymm10, ymm5 + db $C4, $41, $25, $D4, $DD // vpaddq ymm11, ymm11, ymm13 + db $C4, $C1, $15, $73, $D3, $1A // vpsrlq ymm13, ymm11, 26 + db $C5, $A5, $DB, $DD // vpand ymm3, ymm11, ymm5 + db $C4, $41, $1D, $D4, $E5 // vpaddq ymm12, ymm12, ymm13 + db $C4, $C1, $15, $73, $D4, $1A // vpsrlq ymm13, ymm12, 26 + db $C5, $9D, $DB, $E5 // vpand ymm4, ymm12, ymm5 + db $C4, $C1, $05, $73, $F5, $02 // vpsllq ymm15, ymm13, 2 + db $C4, $41, $15, $D4, $EF // vpaddq ymm13, ymm13, ymm15 + db $C4, $C1, $7D, $D4, $C5 // vpaddq ymm0, ymm0, ymm13 + db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 + // ---- Horizontal sum 4 lanes -> lane 0 ---- + db $C5, $C5, $73, $D8, $08 // vpsrldq ymm7, ymm0, 8 + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C4, $E3, $FD, $00, $F8, $4E // vpermq ymm7, ymm0, 0x4E + db $C5, $FD, $D4, $C7 // vpaddq ymm0, ymm0, ymm7 + db $C5, $C5, $73, $D9, $08 // vpsrldq ymm7, ymm1, 8 + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C4, $E3, $FD, $00, $F9, $4E // vpermq ymm7, ymm1, 0x4E + db $C5, $F5, $D4, $CF // vpaddq ymm1, ymm1, ymm7 + db $C5, $C5, $73, $DA, $08 // vpsrldq ymm7, ymm2, 8 + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C4, $E3, $FD, $00, $FA, $4E // vpermq ymm7, ymm2, 0x4E + db $C5, $ED, $D4, $D7 // vpaddq ymm2, ymm2, ymm7 + db $C5, $C5, $73, $DB, $08 // vpsrldq ymm7, ymm3, 8 + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C4, $E3, $FD, $00, $FB, $4E // vpermq ymm7, ymm3, 0x4E + db $C5, $E5, $D4, $DF // vpaddq ymm3, ymm3, ymm7 + db $C5, $C5, $73, $DC, $08 // vpsrldq ymm7, ymm4, 8 + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + db $C4, $E3, $FD, $00, $FC, $4E // vpermq ymm7, ymm4, 0x4E + db $C5, $DD, $D4, $E7 // vpaddq ymm4, ymm4, ymm7 + // ---- Final lazy reduce (lane 0 scalars) ---- + db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 + db $C5, $95, $73, $D1, $1A // vpsrlq ymm13, ymm1, 26 + db $C5, $F5, $DB, $CD // vpand ymm1, ymm1, ymm5 + db $C4, $C1, $6D, $D4, $D5 // vpaddq ymm2, ymm2, ymm13 + db $C5, $95, $73, $D2, $1A // vpsrlq ymm13, ymm2, 26 + db $C5, $ED, $DB, $D5 // vpand ymm2, ymm2, ymm5 + db $C4, $C1, $65, $D4, $DD // vpaddq ymm3, ymm3, ymm13 + db $C5, $95, $73, $D3, $1A // vpsrlq ymm13, ymm3, 26 + db $C5, $E5, $DB, $DD // vpand ymm3, ymm3, ymm5 + db $C4, $C1, $5D, $D4, $E5 // vpaddq ymm4, ymm4, ymm13 + db $C5, $95, $73, $D4, $1A // vpsrlq ymm13, ymm4, 26 + db $C5, $DD, $DB, $E5 // vpand ymm4, ymm4, ymm5 + db $C4, $C1, $05, $73, $F5, $02 // vpsllq ymm15, ymm13, 2 + db $C4, $41, $15, $D4, $EF // vpaddq ymm13, ymm13, ymm15 + db $C4, $C1, $7D, $D4, $C5 // vpaddq ymm0, ymm0, ymm13 + db $C5, $95, $73, $D0, $1A // vpsrlq ymm13, ymm0, 26 + db $C5, $FD, $DB, $C5 // vpand ymm0, ymm0, ymm5 + db $C4, $C1, $75, $D4, $CD // vpaddq ymm1, ymm1, ymm13 + // ---- Store H0..H4 back to ctx ---- + db $C5, $F9, $7E, $41, $24 // vmovd dword ptr [rcx + 36], xmm0 + db $C5, $F9, $7E, $49, $28 // vmovd dword ptr [rcx + 40], xmm1 + db $C5, $F9, $7E, $51, $2C // vmovd dword ptr [rcx + 44], xmm2 + db $C5, $F9, $7E, $59, $30 // vmovd dword ptr [rcx + 48], xmm3 + db $C5, $F9, $7E, $61, $34 // vmovd dword ptr [rcx + 52], xmm4 + db $C5, $F8, $77 // vzeroupper {$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc new file mode 100644 index 00000000..7c288564 --- /dev/null +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_i386.inc @@ -0,0 +1,508 @@ +// Poly1305 2-way SSE2 bulk MAC kernel (i386). +// ------------------------------------------------------------------- +// 2-lane (128-bit xmm) radix-2^26 narrowing of the 4-way AVX2 kernel: +// same 5x5 schoolbook multiply mod 2^130-5 (with the 5*r wraparound), +// 2 blocks per iteration against an r/r^2 power table, then a single +// pshufd-based horizontal fold of the 2 lanes. +// +// i386 has only xmm0..7 (no callee-saved xmm), so the accumulator +// H0..H4 and the transient products D0..D4 live in a 160-byte esp +// stack frame; xmm7 = mask26 and xmm6 = padbit stay resident. Every +// table/state access is an unaligned movdqu load/store (SSE memory- +// operand arithmetic would #GP on the non-16-aligned power table). +// +// On entry (after ClpSimdProc5Begin_i386.inc): +// ebx = ACtx (72-byte R/S/H/K state); esi = APowTable (160-byte r/r^2) +// edi = AInp; eax = ALen (multiple of 32, >= 32); ecx = APad (ignored) +// +// Power table: row j at [esi + 16*j]; rows 0..4 r-limbs [r2,r2,r2,r1], +// rows 5..8 s-limbs (5*r), row 9 pad. Bulk reads [+16*j] (broadcast r^2); +// tail reads [+16*j+4] ([r^2,r^1]). + + sub esp, 160 + // ---- Prologue: mask26 (xmm7), padbit (xmm6) ---- + db $66, $0F, $76, $FF // pcmpeqd xmm7, xmm7 + db $66, $0F, $73, $D7, $26 // psrlq xmm7, 38 + db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6 + db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63 + db $66, $0F, $73, $F6, $18 // psllq xmm6, 24 + // ---- Load H0..H4 (lane 0) into the stack frame ---- + db $66, $0F, $6E, $43, $24 // movd xmm0, dword ptr [ebx + 36] + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $66, $0F, $6E, $43, $28 // movd xmm0, dword ptr [ebx + 40] + db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 + db $66, $0F, $6E, $43, $2C // movd xmm0, dword ptr [ebx + 44] + db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 + db $66, $0F, $6E, $43, $30 // movd xmm0, dword ptr [ebx + 48] + db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 + db $66, $0F, $6E, $43, $34 // movd xmm0, dword ptr [ebx + 52] + db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + // ---- Pre-loop: absorb first 2 blocks ---- + db $F3, $0F, $6F, $07 // movdqu xmm0, [edi] + db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16] + add edi, 32 + db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0 + db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6 + db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1 + db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6 + db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0 + db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1 + db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1 + db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 + db $66, $0F, $6F, $DA // movdqa xmm3, xmm2 + db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30 + db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4 + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $66, $0F, $DB, $CF // pand xmm1, xmm7 + db $66, $0F, $DB, $D7 // pand xmm2, xmm7 + db $66, $0F, $DB, $DF // pand xmm3, xmm7 + db $66, $0F, $EB, $E6 // por xmm4, xmm6 + db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp] + db $66, $0F, $D4, $E8 // paddq xmm5, xmm0 + db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5 + db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16] + db $66, $0F, $D4, $E9 // paddq xmm5, xmm1 + db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5 + db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32] + db $66, $0F, $D4, $EA // paddq xmm5, xmm2 + db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5 + db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48] + db $66, $0F, $D4, $EB // paddq xmm5, xmm3 + db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5 + db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64] + db $66, $0F, $D4, $EC // paddq xmm5, xmm4 + db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5 + sub eax, 32 + jz @@poly1305_sse2_tail + +@@poly1305_sse2_loop: + // ---- Bulk: H *= r^2 (broadcast) ---- + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $40 // movdqu xmm1, [esi + 64] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $30 // movdqu xmm2, [esi + 48] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $30 // movdqu xmm1, [esi + 48] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $20 // movdqu xmm2, [esi + 32] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $20 // movdqu xmm1, [esi + 32] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $10 // movdqu xmm2, [esi + 16] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $10 // movdqu xmm1, [esi + 16] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $16 // movdqu xmm2, [esi] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $0E // movdqu xmm1, [esi] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $96, $80, $00, $00, $00 // movdqu xmm2, [esi + 128] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $70 // movdqu xmm2, [esi + 112] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $60 // movdqu xmm2, [esi + 96] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $50 // movdqu xmm2, [esi + 80] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0 + // ---- Lazy reduce D -> H ---- + db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2 + db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 + db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2 + db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 + db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2 + db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 + db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2 + db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 + db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 + db $66, $0F, $D4, $CA // paddq xmm1, xmm2 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + // ---- Absorb next 2 blocks ---- + db $F3, $0F, $6F, $07 // movdqu xmm0, [edi] + db $F3, $0F, $6F, $4F, $10 // movdqu xmm1, [edi + 16] + add edi, 32 + db $66, $0F, $6F, $D0 // movdqa xmm2, xmm0 + db $66, $0F, $73, $DA, $06 // psrldq xmm2, 6 + db $66, $0F, $6F, $D9 // movdqa xmm3, xmm1 + db $66, $0F, $73, $DB, $06 // psrldq xmm3, 6 + db $66, $0F, $6F, $E0 // movdqa xmm4, xmm0 + db $66, $0F, $6D, $E1 // punpckhqdq xmm4, xmm1 + db $66, $0F, $6C, $C1 // punpcklqdq xmm0, xmm1 + db $66, $0F, $6C, $D3 // punpcklqdq xmm2, xmm3 + db $66, $0F, $6F, $DA // movdqa xmm3, xmm2 + db $66, $0F, $73, $D3, $1E // psrlq xmm3, 30 + db $66, $0F, $73, $D2, $04 // psrlq xmm2, 4 + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $73, $D4, $28 // psrlq xmm4, 40 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $66, $0F, $DB, $CF // pand xmm1, xmm7 + db $66, $0F, $DB, $D7 // pand xmm2, xmm7 + db $66, $0F, $DB, $DF // pand xmm3, xmm7 + db $66, $0F, $EB, $E6 // por xmm4, xmm6 + db $F3, $0F, $6F, $2C, $24 // movdqu xmm5, [esp] + db $66, $0F, $D4, $E8 // paddq xmm5, xmm0 + db $F3, $0F, $7F, $2C, $24 // movdqu [esp], xmm5 + db $F3, $0F, $6F, $6C, $24, $10 // movdqu xmm5, [esp + 16] + db $66, $0F, $D4, $E9 // paddq xmm5, xmm1 + db $F3, $0F, $7F, $6C, $24, $10 // movdqu [esp + 16], xmm5 + db $F3, $0F, $6F, $6C, $24, $20 // movdqu xmm5, [esp + 32] + db $66, $0F, $D4, $EA // paddq xmm5, xmm2 + db $F3, $0F, $7F, $6C, $24, $20 // movdqu [esp + 32], xmm5 + db $F3, $0F, $6F, $6C, $24, $30 // movdqu xmm5, [esp + 48] + db $66, $0F, $D4, $EB // paddq xmm5, xmm3 + db $F3, $0F, $7F, $6C, $24, $30 // movdqu [esp + 48], xmm5 + db $F3, $0F, $6F, $6C, $24, $40 // movdqu xmm5, [esp + 64] + db $66, $0F, $D4, $EC // paddq xmm5, xmm4 + db $F3, $0F, $7F, $6C, $24, $40 // movdqu [esp + 64], xmm5 + sub eax, 32 + jnz @@poly1305_sse2_loop + +@@poly1305_sse2_tail: + // ---- Tail: H *= [r^2, r^1] (shifted table read) ---- + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $44 // movdqu xmm1, [esi + 68] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $34 // movdqu xmm2, [esi + 52] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $84, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $34 // movdqu xmm1, [esi + 52] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $24 // movdqu xmm2, [esi + 36] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $84, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $24 // movdqu xmm1, [esi + 36] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $14 // movdqu xmm2, [esi + 20] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $70 // movdqu [esp + 112], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $14 // movdqu xmm1, [esi + 20] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $56, $04 // movdqu xmm2, [esi + 4] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $60 // movdqu [esp + 96], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $F3, $0F, $6F, $4E, $04 // movdqu xmm1, [esi + 4] + db $66, $0F, $F4, $C1 // pmuludq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $10 // movdqu xmm1, [esp + 16] + db $F3, $0F, $6F, $96, $84, $00, $00, $00 // movdqu xmm2, [esi + 132] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $20 // movdqu xmm1, [esp + 32] + db $F3, $0F, $6F, $56, $74 // movdqu xmm2, [esi + 116] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $30 // movdqu xmm1, [esp + 48] + db $F3, $0F, $6F, $56, $64 // movdqu xmm2, [esi + 100] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $6F, $4C, $24, $40 // movdqu xmm1, [esp + 64] + db $F3, $0F, $6F, $56, $54 // movdqu xmm2, [esi + 84] + db $66, $0F, $F4, $CA // pmuludq xmm1, xmm2 + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $50 // movdqu [esp + 80], xmm0 + db $F3, $0F, $6F, $44, $24, $50 // movdqu xmm0, [esp + 80] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $60 // movdqu xmm2, [esp + 96] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $60 // movdqu [esp + 96], xmm2 + db $F3, $0F, $6F, $44, $24, $60 // movdqu xmm0, [esp + 96] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 + db $F3, $0F, $6F, $54, $24, $70 // movdqu xmm2, [esp + 112] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $70 // movdqu [esp + 112], xmm2 + db $F3, $0F, $6F, $44, $24, $70 // movdqu xmm0, [esp + 112] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 + db $F3, $0F, $6F, $94, $24, $80, $00, $00, $00 // movdqu xmm2, [esp + 128] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $94, $24, $80, $00, $00, $00 // movdqu [esp + 128], xmm2 + db $F3, $0F, $6F, $84, $24, $80, $00, $00, $00 // movdqu xmm0, [esp + 128] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 + db $F3, $0F, $6F, $94, $24, $90, $00, $00, $00 // movdqu xmm2, [esp + 144] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $94, $24, $90, $00, $00, $00 // movdqu [esp + 144], xmm2 + db $F3, $0F, $6F, $84, $24, $90, $00, $00, $00 // movdqu xmm0, [esp + 144] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 + db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 + db $66, $0F, $D4, $CA // paddq xmm1, xmm2 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + // ---- Horizontal fold 2 lanes -> 1, final reduce, store ---- + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] + db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 + db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] + db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 + db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] + db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 + db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] + db $66, $0F, $70, $C8, $4E // pshufd xmm1, xmm0, 0x4E + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $10 // movdqu [esp + 16], xmm0 + db $F3, $0F, $6F, $54, $24, $20 // movdqu xmm2, [esp + 32] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $20 // movdqu [esp + 32], xmm2 + db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $20 // movdqu [esp + 32], xmm0 + db $F3, $0F, $6F, $54, $24, $30 // movdqu xmm2, [esp + 48] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $30 // movdqu [esp + 48], xmm2 + db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $30 // movdqu [esp + 48], xmm0 + db $F3, $0F, $6F, $54, $24, $40 // movdqu xmm2, [esp + 64] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $40 // movdqu [esp + 64], xmm2 + db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $44, $24, $40 // movdqu [esp + 64], xmm0 + db $66, $0F, $6F, $D1 // movdqa xmm2, xmm1 + db $66, $0F, $73, $F2, $02 // psllq xmm2, 2 + db $66, $0F, $D4, $CA // paddq xmm1, xmm2 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $D4, $C1 // paddq xmm0, xmm1 + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 + db $66, $0F, $73, $D1, $1A // psrlq xmm1, 26 + db $66, $0F, $DB, $C7 // pand xmm0, xmm7 + db $F3, $0F, $7F, $04, $24 // movdqu [esp], xmm0 + db $F3, $0F, $6F, $54, $24, $10 // movdqu xmm2, [esp + 16] + db $66, $0F, $D4, $D1 // paddq xmm2, xmm1 + db $F3, $0F, $7F, $54, $24, $10 // movdqu [esp + 16], xmm2 + db $F3, $0F, $6F, $04, $24 // movdqu xmm0, [esp] + db $66, $0F, $7E, $43, $24 // movd dword ptr [ebx + 36], xmm0 + db $F3, $0F, $6F, $44, $24, $10 // movdqu xmm0, [esp + 16] + db $66, $0F, $7E, $43, $28 // movd dword ptr [ebx + 40], xmm0 + db $F3, $0F, $6F, $44, $24, $20 // movdqu xmm0, [esp + 32] + db $66, $0F, $7E, $43, $2C // movd dword ptr [ebx + 44], xmm0 + db $F3, $0F, $6F, $44, $24, $30 // movdqu xmm0, [esp + 48] + db $66, $0F, $7E, $43, $30 // movd dword ptr [ebx + 48], xmm0 + db $F3, $0F, $6F, $44, $24, $40 // movdqu xmm0, [esp + 64] + db $66, $0F, $7E, $43, $34 // movd dword ptr [ebx + 52], xmm0 + add esp, 160 + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc new file mode 100644 index 00000000..e8172d0b --- /dev/null +++ b/CryptoLib/src/Include/Simd/Poly1305/Poly1305BlocksBulkSse2Core_x86_64.inc @@ -0,0 +1,394 @@ +// Poly1305 2-way SSE2 bulk MAC kernel (x86-64). +// ------------------------------------------------------------------- +// 2-lane (128-bit xmm) radix-2^26 narrowing of the 4-way AVX2 kernel: +// same 5x5 schoolbook multiply mod 2^130-5 (with the 5*r wraparound), +// 2 blocks per iteration against an r/r^2 power table, then a single +// pshufd-based horizontal fold of the 2 lanes. Every instruction is +// db-encoded with its mnemonic beside it for broad assembler compatibility. +// +// On entry (after ClpSimdProc5Begin_x86_64.inc): +// rcx = ACtx (72-byte R/S/H/K state); rdx = APowTable (160-byte r/r^2) +// r8 = AInp; r9 = ALen (multiple of 32, >= 32); r10 = APad (ignored) +// +// Power table: row j at [rdx + 16*j]; rows 0..4 r-limbs [r2,r2,r2,r1], +// rows 5..8 s-limbs (5*r), row 9 pad. Bulk reads [+16*j] (broadcast r^2); +// tail reads [+16*j+4] ([r^2,r^1]). + +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileSave_x86_64.inc} + // ---- Prologue: mask26 (xmm5), padbit (xmm6) ---- + db $66, $0F, $76, $ED // pcmpeqd xmm5, xmm5 + db $66, $0F, $73, $D5, $26 // psrlq xmm5, 38 + db $66, $0F, $76, $F6 // pcmpeqd xmm6, xmm6 + db $66, $0F, $73, $D6, $3F // psrlq xmm6, 63 + db $66, $0F, $73, $F6, $18 // psllq xmm6, 24 + // ---- Load H0..H4 into lane 0 ---- + db $66, $0F, $6E, $41, $24 // movd xmm0, dword ptr [rcx + 36] + db $66, $0F, $6E, $49, $28 // movd xmm1, dword ptr [rcx + 40] + db $66, $0F, $6E, $51, $2C // movd xmm2, dword ptr [rcx + 44] + db $66, $0F, $6E, $59, $30 // movd xmm3, dword ptr [rcx + 48] + db $66, $0F, $6E, $61, $34 // movd xmm4, dword ptr [rcx + 52] + // ---- Pre-loop: absorb first 2 blocks ---- + db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] + db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] + add r8, 32 + db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 + db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 + db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 + db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 + db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 + db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 + db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 + db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 + db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 + db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 + db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 + db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 + db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 + db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 + db $66, $0F, $EB, $FE // por xmm7, xmm6 + db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 + db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + sub r9, 32 + jz @@poly1305_sse2_tail + +@@poly1305_sse2_loop: + // ---- Bulk: H *= r^2 (broadcast) ---- + db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 + db $F3, $44, $0F, $6F, $6A, $40 // movdqu xmm13, [rdx + 64] + db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $F3, $44, $0F, $6F, $6A, $30 // movdqu xmm13, [rdx + 48] + db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 + db $F3, $44, $0F, $6F, $6A, $20 // movdqu xmm13, [rdx + 32] + db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 + db $F3, $44, $0F, $6F, $6A, $10 // movdqu xmm13, [rdx + 16] + db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $F3, $44, $0F, $6F, $2A // movdqu xmm13, [rdx] + db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $AA, $80, $00, $00, $00 // movdqu xmm13, [rdx + 128] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $70 // movdqu xmm13, [rdx + 112] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $60 // movdqu xmm13, [rdx + 96] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $50 // movdqu xmm13, [rdx + 80] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + // ---- Lazy reduce D -> H ---- + db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $FD // pand xmm7, xmm5 + db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 + db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 + db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 + db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 + db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 + db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 + db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 + db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 + db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 + db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 + db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 + db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 + db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 + db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + // ---- Absorb next 2 blocks ---- + db $F3, $45, $0F, $6F, $20 // movdqu xmm12, [r8] + db $F3, $45, $0F, $6F, $68, $10 // movdqu xmm13, [r8 + 16] + add r8, 32 + db $66, $45, $0F, $6F, $F4 // movdqa xmm14, xmm12 + db $66, $41, $0F, $73, $DE, $06 // psrldq xmm14, 6 + db $66, $45, $0F, $6F, $FD // movdqa xmm15, xmm13 + db $66, $41, $0F, $73, $DF, $06 // psrldq xmm15, 6 + db $66, $41, $0F, $6F, $FC // movdqa xmm7, xmm12 + db $66, $41, $0F, $6D, $FD // punpckhqdq xmm7, xmm13 + db $66, $45, $0F, $6C, $E5 // punpcklqdq xmm12, xmm13 + db $66, $45, $0F, $6C, $F7 // punpcklqdq xmm14, xmm15 + db $66, $45, $0F, $6F, $FE // movdqa xmm15, xmm14 + db $66, $41, $0F, $73, $D7, $1E // psrlq xmm15, 30 + db $66, $41, $0F, $73, $D6, $04 // psrlq xmm14, 4 + db $66, $45, $0F, $6F, $EC // movdqa xmm13, xmm12 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $73, $D7, $28 // psrlq xmm7, 40 + db $66, $44, $0F, $DB, $E5 // pand xmm12, xmm5 + db $66, $44, $0F, $DB, $ED // pand xmm13, xmm5 + db $66, $44, $0F, $DB, $F5 // pand xmm14, xmm5 + db $66, $44, $0F, $DB, $FD // pand xmm15, xmm5 + db $66, $0F, $EB, $FE // por xmm7, xmm6 + db $66, $41, $0F, $D4, $C4 // paddq xmm0, xmm12 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + db $66, $41, $0F, $D4, $D6 // paddq xmm2, xmm14 + db $66, $41, $0F, $D4, $DF // paddq xmm3, xmm15 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + sub r9, 32 + jnz @@poly1305_sse2_loop + +@@poly1305_sse2_tail: + // ---- Tail: H *= [r^2, r^1] (shifted table read) ---- + db $66, $44, $0F, $6F, $D8 // movdqa xmm11, xmm0 + db $F3, $44, $0F, $6F, $6A, $44 // movdqu xmm13, [rdx + 68] + db $66, $45, $0F, $F4, $DD // pmuludq xmm11, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $DC // paddq xmm11, xmm12 + db $66, $44, $0F, $6F, $D0 // movdqa xmm10, xmm0 + db $F3, $44, $0F, $6F, $6A, $34 // movdqu xmm13, [rdx + 52] + db $66, $45, $0F, $F4, $D5 // pmuludq xmm10, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $D4 // paddq xmm10, xmm12 + db $66, $44, $0F, $6F, $C8 // movdqa xmm9, xmm0 + db $F3, $44, $0F, $6F, $6A, $24 // movdqu xmm13, [rdx + 36] + db $66, $45, $0F, $F4, $CD // pmuludq xmm9, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $CC // paddq xmm9, xmm12 + db $66, $44, $0F, $6F, $C0 // movdqa xmm8, xmm0 + db $F3, $44, $0F, $6F, $6A, $14 // movdqu xmm13, [rdx + 20] + db $66, $45, $0F, $F4, $C5 // pmuludq xmm8, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $45, $0F, $D4, $C4 // paddq xmm8, xmm12 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $F3, $44, $0F, $6F, $6A, $04 // movdqu xmm13, [rdx + 4] + db $66, $41, $0F, $F4, $FD // pmuludq xmm7, xmm13 + db $66, $44, $0F, $6F, $E1 // movdqa xmm12, xmm1 + db $F3, $44, $0F, $6F, $AA, $84, $00, $00, $00 // movdqu xmm13, [rdx + 132] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E2 // movdqa xmm12, xmm2 + db $F3, $44, $0F, $6F, $6A, $74 // movdqu xmm13, [rdx + 116] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E3 // movdqa xmm12, xmm3 + db $F3, $44, $0F, $6F, $6A, $64 // movdqu xmm13, [rdx + 100] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $E4 // movdqa xmm12, xmm4 + db $F3, $44, $0F, $6F, $6A, $54 // movdqu xmm13, [rdx + 84] + db $66, $45, $0F, $F4, $E5 // pmuludq xmm12, xmm13 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $44, $0F, $6F, $EF // movdqa xmm13, xmm7 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $FD // pand xmm7, xmm5 + db $66, $0F, $6F, $C7 // movdqa xmm0, xmm7 + db $66, $45, $0F, $D4, $C5 // paddq xmm8, xmm13 + db $66, $45, $0F, $6F, $E8 // movdqa xmm13, xmm8 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $C5 // pand xmm8, xmm5 + db $66, $41, $0F, $6F, $C8 // movdqa xmm1, xmm8 + db $66, $45, $0F, $D4, $CD // paddq xmm9, xmm13 + db $66, $45, $0F, $6F, $E9 // movdqa xmm13, xmm9 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $CD // pand xmm9, xmm5 + db $66, $41, $0F, $6F, $D1 // movdqa xmm2, xmm9 + db $66, $45, $0F, $D4, $D5 // paddq xmm10, xmm13 + db $66, $45, $0F, $6F, $EA // movdqa xmm13, xmm10 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $D5 // pand xmm10, xmm5 + db $66, $41, $0F, $6F, $DA // movdqa xmm3, xmm10 + db $66, $45, $0F, $D4, $DD // paddq xmm11, xmm13 + db $66, $45, $0F, $6F, $EB // movdqa xmm13, xmm11 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $44, $0F, $DB, $DD // pand xmm11, xmm5 + db $66, $41, $0F, $6F, $E3 // movdqa xmm4, xmm11 + db $66, $45, $0F, $6F, $E5 // movdqa xmm12, xmm13 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $45, $0F, $D4, $EC // paddq xmm13, xmm12 + db $66, $41, $0F, $D4, $C5 // paddq xmm0, xmm13 + db $66, $44, $0F, $6F, $E8 // movdqa xmm13, xmm0 + db $66, $41, $0F, $73, $D5, $1A // psrlq xmm13, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $41, $0F, $D4, $CD // paddq xmm1, xmm13 + // ---- Horizontal fold 2 lanes -> 1, final reduce, store ---- + db $66, $0F, $70, $F8, $4E // pshufd xmm7, xmm0, 0x4E + db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 + db $66, $0F, $70, $F9, $4E // pshufd xmm7, xmm1, 0x4E + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $70, $FA, $4E // pshufd xmm7, xmm2, 0x4E + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $70, $FB, $4E // pshufd xmm7, xmm3, 0x4E + db $66, $0F, $D4, $DF // paddq xmm3, xmm7 + db $66, $0F, $70, $FC, $4E // pshufd xmm7, xmm4, 0x4E + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $6F, $F9 // movdqa xmm7, xmm1 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $CD // pand xmm1, xmm5 + db $66, $0F, $D4, $D7 // paddq xmm2, xmm7 + db $66, $0F, $6F, $FA // movdqa xmm7, xmm2 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $D5 // pand xmm2, xmm5 + db $66, $0F, $D4, $DF // paddq xmm3, xmm7 + db $66, $0F, $6F, $FB // movdqa xmm7, xmm3 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $DD // pand xmm3, xmm5 + db $66, $0F, $D4, $E7 // paddq xmm4, xmm7 + db $66, $0F, $6F, $FC // movdqa xmm7, xmm4 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $E5 // pand xmm4, xmm5 + db $66, $44, $0F, $6F, $E7 // movdqa xmm12, xmm7 + db $66, $41, $0F, $73, $F4, $02 // psllq xmm12, 2 + db $66, $41, $0F, $D4, $FC // paddq xmm7, xmm12 + db $66, $0F, $D4, $C7 // paddq xmm0, xmm7 + db $66, $0F, $6F, $F8 // movdqa xmm7, xmm0 + db $66, $0F, $73, $D7, $1A // psrlq xmm7, 26 + db $66, $0F, $DB, $C5 // pand xmm0, xmm5 + db $66, $0F, $D4, $CF // paddq xmm1, xmm7 + db $66, $0F, $7E, $41, $24 // movd dword ptr [rcx + 36], xmm0 + db $66, $0F, $7E, $49, $28 // movd dword ptr [rcx + 40], xmm1 + db $66, $0F, $7E, $51, $2C // movd dword ptr [rcx + 44], xmm2 + db $66, $0F, $7E, $59, $30 // movd dword ptr [rcx + 48], xmm3 + db $66, $0F, $7E, $61, $34 // movd dword ptr [rcx + 52], xmm4 +{$I ..\..\Include\Simd\Common\ClpSimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Interfaces/Crypto/ClpIBulkStreamCipher.pas b/CryptoLib/src/Interfaces/Crypto/ClpIBulkStreamCipher.pas new file mode 100644 index 00000000..5973718e --- /dev/null +++ b/CryptoLib/src/Interfaces/Crypto/ClpIBulkStreamCipher.pas @@ -0,0 +1,53 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpIBulkStreamCipher; + +{$I ..\..\Include\CryptoLib.inc} + +interface + +uses + ClpIStreamCipher, + ClpCryptoLibTypes; + +type + /// + /// Capability interface for stream ciphers that emit many consecutive 64-byte + /// keystream blocks per call via a SIMD kernel. Resolved with Supports() and + /// driven through ProcessBlocks; its absence means "no fast path". + /// + IBulkStreamCipher = interface(IStreamCipher) + ['{7F1C4E62-9A3D-4B18-8E5A-2D6F1B0C7A94}'] + + /// + /// Transform ABlockCount 64-byte blocks; byte-identical to ABlockCount + /// single-block transforms and advances the counter by ABlockCount. Requires + /// block-aligned state; input and output must be identical or fully disjoint. + /// + /// Bytes produced (ABlockCount * 64). + function ProcessBlocks(const AInBuf: TCryptoLibByteArray; + AInOff, ABlockCount: Int32; const AOutBuf: TCryptoLibByteArray; + AOutOff: Int32): Int32; overload; + + /// Pointer overload of ProcessBlocks; same semantics. + function ProcessBlocks(AInput, AOutput: PByte; + ABlockCount: Int32): Int32; overload; + end; + +implementation + +end. diff --git a/CryptoLib/src/Interfaces/Crypto/Engines/ClpIAesHardwareEngine.pas b/CryptoLib/src/Interfaces/Crypto/Engines/ClpIAesHardwareEngine.pas index 691036ec..76722b00 100644 --- a/CryptoLib/src/Interfaces/Crypto/Engines/ClpIAesHardwareEngine.pas +++ b/CryptoLib/src/Interfaces/Crypto/Engines/ClpIAesHardwareEngine.pas @@ -28,12 +28,12 @@ interface /// /// Architecture-neutral capability interface for hardware-accelerated AES /// engines (e.g. AES-NI on x86 via TAesEngineX86; a NEON/Crypto-Ext - /// ARMv8 engine could implement the same surface). It exposes the - /// fixed-width 4-/8-block batch entry points and the raw-pointer single-block - /// overload that sit beneath the generic - /// ladder. The aliasing contract for every overload here is identical to - /// IBulkBlockCipher: AInput / AOutput MUST be either identical pointers - /// (in-place) or reference fully disjoint ranges of the stated size. + /// ARMv8 engine could implement the same surface). Bulk work goes through the + /// width-agnostic ladder, which each engine + /// drives at its own internal batch widths; this interface adds only a + /// raw-pointer single-block overload. Aliasing for the pointer overload is + /// identical to IBulkBlockCipher: AInput / AOutput MUST be identical pointers + /// (in-place) or reference fully disjoint ranges. /// /// /// This interface is the shared surface that engine-agnostic test suites and @@ -49,30 +49,6 @@ interface /// behind each pointer and the standard identical-or-disjoint aliasing. /// function ProcessBlock(AInput, AOutput: PByte): Int32; overload; - - /// - /// Four consecutive 16-byte blocks (64 bytes) from AInput[AInOff] into - /// AOutput[AOutOff]. - /// - function ProcessFourBlocks(const AInput: TCryptoLibByteArray; AInOff: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; overload; - - /// - /// Four consecutive 16-byte blocks (64 bytes) via raw pointers. - /// - function ProcessFourBlocks(AInput, AOutput: PByte): Int32; overload; - - /// - /// Eight consecutive 16-byte blocks (128 bytes) from AInput[AInOff] into - /// AOutput[AOutOff]. - /// - function ProcessEightBlocks(const AInput: TCryptoLibByteArray; AInOff: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; overload; - - /// - /// Eight consecutive 16-byte blocks (128 bytes) via raw pointers. - /// - function ProcessEightBlocks(AInput, AOutput: PByte): Int32; overload; end; implementation diff --git a/CryptoLib/src/Interfaces/Crypto/Engines/ClpIChaCha7539Engine.pas b/CryptoLib/src/Interfaces/Crypto/Engines/ClpIChaCha7539Engine.pas index 7a8d7061..81972072 100644 --- a/CryptoLib/src/Interfaces/Crypto/Engines/ClpIChaCha7539Engine.pas +++ b/CryptoLib/src/Interfaces/Crypto/Engines/ClpIChaCha7539Engine.pas @@ -35,12 +35,6 @@ interface procedure ProcessBlock(const AInBytes: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - procedure ProcessBlocks2(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - - procedure ProcessBlocks4(const AInBytes: TCryptoLibByteArray; AInOff: Int32; - const AOutBytes: TCryptoLibByteArray; AOutOff: Int32); - end; implementation diff --git a/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk b/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk index e4a1fc52..ddbdc48c 100644 --- a/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk +++ b/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk @@ -657,6 +657,7 @@ contains ClpIBlockCipherMode in '..\..\Interfaces\Crypto\Modes\ClpIBlockCipherMode.pas', ClpIBulkBlockCipherMode in '..\..\Interfaces\Crypto\Modes\ClpIBulkBlockCipherMode.pas', ClpIBulkBlockCipher in '..\..\Interfaces\Crypto\ClpIBulkBlockCipher.pas', + ClpIBulkStreamCipher in '..\..\Interfaces\Crypto\ClpIBulkStreamCipher.pas', ClpIEcbBlockCipher in '..\..\Interfaces\Crypto\Modes\ClpIEcbBlockCipher.pas', ClpEcbBlockCipher in '..\..\Crypto\Modes\ClpEcbBlockCipher.pas', ClpGnuObjectIdentifiers in '..\..\Asn1\Gnu\ClpGnuObjectIdentifiers.pas', diff --git a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk index c5296e79..37b2c89c 100644 --- a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk +++ b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk @@ -31,7 +31,7 @@ Acknowledgements: Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring the development of this library "/> - + @@ -3213,6 +3213,10 @@ Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring the devel + + + + diff --git a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas index 4e35c4b3..9ce3f30f 100644 --- a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas +++ b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas @@ -257,7 +257,7 @@ interface ClpPoly1305X86Backend, ClpGhashSimd, ClpGhashX86Backend, ClpGcmSivSimd, ClpGcmSivX86Backend, ClpAesNiFusedX86Backend, ClpBinPolySimd, ClpAesSimd, ClpIFusedCtrKernel, ClpAesNiCtrKernel, ClpByteXorSimd, ClpByteXorX86Backend, - ClpIFusedCbcKernel, ClpAesNiCbcKernel; + ClpIFusedCbcKernel, ClpAesNiCbcKernel, ClpIBulkStreamCipher; implementation