From 16ca5f1c95ca359a82f2ef6653d74b96d43bb3fb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 23:29:01 +0000 Subject: [PATCH] ML-KEM AArch64: decapsulation's decryption with 16-bit coefficients Optimized.decryptM decodes and decompresses each row of u into PP with 32 bits, narrows it to 16 bits at yOff i (Narrow16.narrow, uzp1) and transforms it with vg_mlkem16_ntt; the secret vector is decoded with 16 bits (Codec12W), its product with u' is the 16-bit fused dot product (DotW) and the inverse transform invW, widened into PP for the subtraction from v. The small 16-bit helpers move from Kem16 to Ops16 so that Optimized can use them. Deletes what decaps's u32 decryption alone used: ReadonlyDecrypt, the inline u32 readonly transforms (Readonly.fwd, inv, nttAt, nttInvAt, nttAtWith, nttInvAtWith) and their proofs (Readonly Fwd/Inv/Contract, nttAtWith_ok, NttR fwd_run/inv_run and their checks and literals, KemReadonlyTransform), the u32 dot product's proofs (Dot Core, Loop, Setup, LoopInput, KemDot) and the u32 decode (KemDecode, decoded_ready, DecryptU, DecryptReadonlyU/Tail). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FoYWDDEDFq1DFev5X7n2aq --- .../Impl/MlKem/AArch64/Kem16.lean | 27 +- .../Impl/MlKem/AArch64/Narrow16.lean | 26 + .../Impl/MlKem/AArch64/Ntt16/Readonly.lean | 13 +- .../MlKem/AArch64/Ntt16/ReadonlyCalls.lean | 15 +- .../Impl/MlKem/AArch64/Ops16.lean | 44 + .../Impl/MlKem/AArch64/Optimized.lean | 20 +- .../Impl/MlKem/AArch64/ReadonlyDecrypt.lean | 9 - .../Proof/MlKem/AArch64/Dot/Core.lean | 24 - .../Proof/MlKem/AArch64/Dot/Loop.lean | 43 - .../Proof/MlKem/AArch64/Dot/LoopInput.lean | 50 - .../Proof/MlKem/AArch64/Dot/Setup.lean | 43 - .../Proof/MlKem/AArch64/Kem16/Decaps.lean | 1 - .../Proof/MlKem/AArch64/Kem16/Ops.lean | 53 +- .../Proof/MlKem/AArch64/Kem16P/Decaps.lean | 1 - .../Proof/MlKem/AArch64/Narrow16.lean | 150 + .../MlKem/AArch64/Ntt16/Readonly/At.lean | 42 - .../MlKem/AArch64/Ntt16/Readonly/Call.lean | 50 +- .../AArch64/Ntt16/Readonly/Contract.lean | 81 - .../MlKem/AArch64/Ntt16/Readonly/Fwd.lean | 17 - .../MlKem/AArch64/Ntt16/Readonly/Inv.lean | 17 - .../Proof/MlKem/AArch64/Ntt16/W/Call.lean | 7 +- .../Proof/MlKem/AArch64/NttR/FwdU.lean | 20 +- .../Proof/MlKem/AArch64/NttR/Inv.lean | 26 +- .../Proof/MlKem/AArch64/NttR/Lit.lean | 6 - .../Proof/MlKem/AArch64/NttR/LitI.lean | 3 +- .../Proof/MlKem/AArch64/NttR/RunI.lean | 18 +- .../Proof/MlKem/AArch64/NttR/RunU.lean | 17 +- .../MlKem/AArch64/Optimized/DecryptM.lean | 24 +- .../Optimized/DecryptReadonlyTail.lean | 51 - .../AArch64/Optimized/DecryptReadonlyU.lean | 65 - .../MlKem/AArch64/Optimized/DecryptU.lean | 56 - .../MlKem/AArch64/Optimized/DecryptW.lean | 127 + .../MlKem/AArch64/Optimized/KemDecode.lean | 65 - .../Proof/MlKem/AArch64/Optimized/KemDot.lean | 29 - .../Optimized/KemReadonlyTransform.lean | 40 - .../MlKem/AArch64/Optimized/KemReady.lean | 61 +- .../Proof/MlKem/AArch64/Optimized/Taints.lean | 4 +- src/asm/aarch64/mlkem1024.rs | 13676 +++++++--------- src/asm/aarch64/mlkem768.rs | 7510 ++++----- 39 files changed, 9507 insertions(+), 13024 deletions(-) create mode 100644 lean/VerifiedGarbage/Impl/MlKem/AArch64/Narrow16.lean create mode 100644 lean/VerifiedGarbage/Impl/MlKem/AArch64/Ops16.lean delete mode 100644 lean/VerifiedGarbage/Impl/MlKem/AArch64/ReadonlyDecrypt.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Core.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Loop.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/LoopInput.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Setup.lean create mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Narrow16.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Contract.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Fwd.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Inv.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyTail.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyU.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptU.lean create mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptW.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDecode.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDot.lean delete mode 100644 lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReadonlyTransform.lean diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Kem16.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Kem16.lean index 295e76b6c2..06d692d295 100644 --- a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Kem16.lean +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Kem16.lean @@ -2,10 +2,7 @@ module public import VerifiedGarbage.Impl.MlKem.AArch64.Hybrid public import VerifiedGarbage.Impl.MlKem.AArch64.FixedHashDecaps -public import VerifiedGarbage.Impl.MlKem.AArch64.AddSubW -public import VerifiedGarbage.Impl.MlKem.AArch64.Codec12W -public import VerifiedGarbage.Impl.MlKem.AArch64.DotW -public import VerifiedGarbage.Impl.MlKem.AArch64.Widen16 +public import VerifiedGarbage.Impl.MlKem.AArch64.Ops16 public import VerifiedGarbage.Impl.MlKem.AArch64.Decaps public import VerifiedGarbage.Impl.MlKem.AArch64.NoiseWith import VerifiedGarbage.Impl.MlKem.AArch64.Encaps @@ -21,7 +18,8 @@ the sampler and the CBD write halfwords (`w16`), the transforms are and the sums `AddSubW.addW`, and keys are encoded and decoded by `Codec12W`. Compression still reads 32-bit coefficients: each row of `u` and `v` is widened (`Widen16.widen`) into `PP` first, and `v` adds the decoded -message there with `vg_mlkem_add`. Decapsulation's decryption stays 32-bit. +message there with `vg_mlkem_add`. Decapsulation's decryption (`Optimized.decryptM`) is +16-bit too. The pieces of key generation (`Kem16P.keyGen`): each row of `t̂` (`kgRow`) as soon as the matrix has sampled it, the pairs absorbing `ek` into the third state of the matrix's `vg_keccak_f1600_x3_sha3` calls as the rows encode it @@ -35,25 +33,6 @@ open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM open VG.Impl.MlKem.AArch64.Noise (noiseOff noiseAllWith) open VG.Impl.MlKem.AArch64.Ntt16.Readonly (nttAtWithW nttInvAtWithW) -/-- `f ← f + g`, both stored with 16 bits in `scratch`. -/ -def addWAt (f g : Nat) : Prog isa := - .seq (.block (ptrTo .x0 .x28 f ++ ptrTo .x1 .x28 g)) AddSubW.addW - -/-- The polynomial at `f`, stored with 16 bits, at `g` with 32. -/ -def widenAt (f g : Nat) : Prog isa := - .seq (.block (ptrTo .x0 .x28 f ++ ptrTo .x1 .x28 g)) Widen16.widen - -/-- `ByteDecode₁₂` of the 384 bytes at `b + o` into `off`, with 16 bits. -/ -def dec12WAt (b : Reg) (o off : Nat) : Prog isa := - .seq (.block (ptrTo .x0 b o ++ ptrTo .x1 .x28 off)) Codec12W.decW - -/-- `ByteEncode₁₂` of the polynomial at `off`, stored with 16 bits, into `b + o`. -/ -def enc12WAt (off : Nat) (b : Reg) (o : Nat) : Prog isa := - .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 b o)) Codec12W.encW - -/-- The fused dot product into `out`, of terms in `scratch`. -/ -def dotAt (out : Nat) (terms : List (Nat × Nat)) : Prog isa := DotW.dotW .x28 .x28 .x28 out terms - /-- Row `i` of `u`: `NTT⁻¹(Σⱼ Âⱼᵢ ∘ ŷⱼ) + e₁[i]`, widened, compressed and encoded. -/ def uRow (L : KemLay) (ct : Reg) (co i : Nat) : Prog isa := seqs [dotAt (TP L) ((List.range L.k).map fun j => (aOff L j i, yOff L j)), diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Narrow16.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Narrow16.lean new file mode 100644 index 0000000000..d90138f057 --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Narrow16.lean @@ -0,0 +1,26 @@ +module +public import VerifiedGarbage.TCB.AArch64.Isa + + +/-! +# ML-KEM on AArch64: a polynomial from 32-bit to 16-bit coefficients + +`narrow` copies the polynomial stored with 32-bit coefficients at `x0`, each less than `2¹⁶`, +to `x1` with 16-bit ones (each eight coefficients loaded at once, their low halfwords taken by +`uzp1`). +-/ + +@[expose] public section + +namespace VG.Impl.MlKem.AArch64.Narrow16 + +open VG.AArch64 + +def narrowBody : List Instr := + [.ldrq .v0 .x0 0, .ldrq .v1 .x0 16, .vop (.perm .uzp1 .h8 .v2 .v0 .v1), .strq .v2 .x1 0, + .addImm .x .x0 .x0 32, .addImm .x .x1 .x1 16, .subImm .x .x10 .x10 1] + +def narrow : Prog isa := + .seq (.block [.movz .x .x10 32 0]) (.loop (.block narrowBody) (.nonzero .x .x10)) + +end VG.Impl.MlKem.AArch64.Narrow16 diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/Readonly.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/Readonly.lean index d002464d2b..b83869d86a 100644 --- a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/Readonly.lean +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/Readonly.lean @@ -9,17 +9,8 @@ namespace VG.Impl.MlKem.AArch64.Ntt16.Readonly open VG.AArch64 open VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.Ntt16 -def fwd : Prog isa := - .seq (.block (consts ++ [mov .x2 .x0,mov .x3 .x1,.movz .x .x5 4 0])) - (.seq (.loop (.block pass1Body) (.nonzero .x .x5)) - (.seq (.block [mov .x2 .x1,.adrSym .x6 "VG_MLKEM_NTT16_ZETAS",mov .x4 .x0,.movz .x .x5 8 0]) - (.loop (.block pass2Body) (.nonzero .x .x5)))) -def inv : Prog isa := - .seq (.block (qConsts ++ [mov .x2 .x0,mov .x3 .x1,.adrSym .x6 "VG_MLKEM_INTT16_ZETAS",.movz .x .x5 8 0])) - (.seq (.loop (.block invPass1Body) (.nonzero .x .x5)) - (.seq (.block (invConsts ++ [mov .x2 .x1,mov .x3 .x0,.movz .x .x5 4 0])) - (.loop (.block invPass2Body) (.nonzero .x .x5)))) -/-- `inv` of a polynomial stored with 16-bit coefficients (`invPass1BodyW`, `invPass2BodyW`). -/ +/-- The inverse transform of a polynomial stored with 16-bit coefficients (`invPass1BodyW`, +`invPass2BodyW`). -/ def invW : Prog isa := .seq (.block (qConsts ++ [mov .x2 .x0,mov .x3 .x1,.adrSym .x6 "VG_MLKEM_INTT16_ZETAS",.movz .x .x5 8 0])) (.seq (.loop (.block invPass1BodyW) (.nonzero .x .x5)) diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/ReadonlyCalls.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/ReadonlyCalls.lean index 847441436f..09ba58a8cf 100644 --- a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/ReadonlyCalls.lean +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ntt16/ReadonlyCalls.lean @@ -8,22 +8,11 @@ public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.Fast namespace VG.Impl.MlKem.AArch64.Ntt16.Readonly open VG.AArch64 VG.Impl.MlKem.AArch64 -def fwdName : String := "vg_mlkem_ntt_readonly" -def invName : String := "vg_mlkem_inv_ntt_readonly" -def fwdConsts := [("VG_MLKEM_NTT16_ZETAS",words 16 tableHalves)] -def invConstsTable := [("VG_MLKEM_INTT16_ZETAS",words 16 invTableHalves)] - -def nttAtWith (off ns : Nat) : Prog isa := - .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 .x28 ns)) fwd -def nttInvAtWith (off ns : Nat) : Prog isa := - .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 .x28 ns)) inv -/-- `nttAtWith` and `nttInvAtWith` of a polynomial stored with 16-bit coefficients: the -forward transform a call of `vg_mlkem16_ntt`. -/ +/-- The transforms of the polynomial stored with 16-bit coefficients at `off` from `x28`, with +working space at `ns`: the forward transform a call of `vg_mlkem16_ntt`. -/ def nttAtWithW (off ns : Nat) : Prog isa := .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 .x28 ns)) (.call "vg_mlkem16_ntt" Fast.fwdFast) def nttInvAtWithW (off ns : Nat) : Prog isa := .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 .x28 ns)) invW -def nttAt (off : Nat) : Prog isa := nttAtWith off KEM.NS -def nttInvAt (off : Nat) : Prog isa := nttInvAtWith off KEM.NS end VG.Impl.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ops16.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ops16.lean new file mode 100644 index 0000000000..35af4183e7 --- /dev/null +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Ops16.lean @@ -0,0 +1,44 @@ +module + +import VerifiedGarbage.Impl.MlKem.AArch64.Kem +public import VerifiedGarbage.Impl.MlKem.AArch64.AddSubW +public import VerifiedGarbage.Impl.MlKem.AArch64.Codec12W +public import VerifiedGarbage.Impl.MlKem.AArch64.DotW +public import VerifiedGarbage.Impl.MlKem.AArch64.Widen16 +public import VerifiedGarbage.Impl.MlKem.AArch64.Narrow16 +public import VerifiedGarbage.Impl.MlKem.AArch64.Top + +/-! # AArch64 ML-KEM with 16-bit coefficients: the operations on `scratch` + +The sums, conversions, codecs and dot products of polynomials stored with 16 bits in the first +half of each 1024-byte slot of `scratch`, which `Kem16` and the decryption of +`Optimized.decryptM` use. -/ + +@[expose] public section +namespace VG.Impl.MlKem.AArch64.Kem16 +open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM + +/-- `f ← f + g`, both stored with 16 bits in `scratch`. -/ +def addWAt (f g : Nat) : Prog isa := + .seq (.block (ptrTo .x0 .x28 f ++ ptrTo .x1 .x28 g)) AddSubW.addW + +/-- The polynomial at `f`, stored with 16 bits, at `g` with 32. -/ +def widenAt (f g : Nat) : Prog isa := + .seq (.block (ptrTo .x0 .x28 f ++ ptrTo .x1 .x28 g)) Widen16.widen + +/-- `ByteDecode₁₂` of the 384 bytes at `b + o` into `off`, with 16 bits. -/ +def dec12WAt (b : Reg) (o off : Nat) : Prog isa := + .seq (.block (ptrTo .x0 b o ++ ptrTo .x1 .x28 off)) Codec12W.decW + +/-- `ByteEncode₁₂` of the polynomial at `off`, stored with 16 bits, into `b + o`. -/ +def enc12WAt (off : Nat) (b : Reg) (o : Nat) : Prog isa := + .seq (.block (ptrTo .x0 .x28 off ++ ptrTo .x1 b o)) Codec12W.encW + +/-- The polynomial at `f`, stored with 32 bits, at `g` with 16. -/ +def narrowAt (f g : Nat) : Prog isa := + .seq (.block (ptrTo .x0 .x28 f ++ ptrTo .x1 .x28 g)) Narrow16.narrow + +/-- The fused dot product into `out`, of terms in `scratch`. -/ +def dotAt (out : Nat) (terms : List (Nat × Nat)) : Prog isa := DotW.dotW .x28 .x28 .x28 out terms + +end VG.Impl.MlKem.AArch64.Kem16 diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Optimized.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Optimized.lean index eefcc4df59..3f2907d99a 100644 --- a/lean/VerifiedGarbage/Impl/MlKem/AArch64/Optimized.lean +++ b/lean/VerifiedGarbage/Impl/MlKem/AArch64/Optimized.lean @@ -1,7 +1,7 @@ module -public import VerifiedGarbage.Impl.MlKem.AArch64.ReadonlyDecrypt -public import VerifiedGarbage.Impl.MlKem.AArch64.Dot +public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.ReadonlyCalls +public import VerifiedGarbage.Impl.MlKem.AArch64.Ops16 import VerifiedGarbage.Impl.MlKem.AArch64.Noise /-! # Paired sampling and fused polynomial products for AArch64 ML-KEM @@ -16,12 +16,20 @@ before re-encryption. @[expose] public section namespace VG.Impl.MlKem.AArch64.Optimized open VG VG.AArch64 VG.Impl.MlKem.AArch64 -open VG.Impl.MlKem.AArch64.Dot VG.Impl.MlKem.AArch64.Noise +open VG.Impl.MlKem.AArch64.Noise +/-- Row `i` of `u`: decoded and decompressed with 32 bits into `PP`, narrowed to 16 bits at +`yOff i` and transformed by `vg_mlkem16_ntt`. -/ +def deU (L : KemLay) (i : Nat) : Prog isa := + seqs [L.ddLAt .x26 (32*L.du*i) L.du (KEM.PP L),Kem16.narrowAt (KEM.PP L) (KEM.yOff L i), + Ntt16.Readonly.nttAtWithW (KEM.yOff L i) KEM.NS] +/-- `m = Compress₁(v - NTT⁻¹(ŝᵀ ∘ û))`, with `ŝ` and `û` stored with 16 bits: the dot product +and the inverse transform in 16 bits, widened into `PP` for the subtraction from `v`. -/ def decryptM (L : KemLay) : Prog isa := - seqs (((List.range L.k).map (readonlyDeU L)) ++ ((List.range L.k).map fun j => dec12At .x25 (384*j) (KEM.aOff L 0 j)) ++ - [dot (KEM.TP L) ((List.range L.k).map fun j => (KEM.aOff L 0 j,KEM.yOff L j)), - Ntt16.Readonly.nttInvAt (KEM.TP L),L.ddLAt .x26 (32*L.du*L.k) L.dv (KEM.EP L),subAt (KEM.EP L) (KEM.TP L),ceAt (KEM.EP L) 1 .x28 KEM.MB]) + seqs (((List.range L.k).map (deU L)) ++ ((List.range L.k).map fun j => Kem16.dec12WAt .x25 (384*j) (KEM.aOff L 0 j)) ++ + [Kem16.dotAt (KEM.TP L) ((List.range L.k).map fun j => (KEM.aOff L 0 j,KEM.yOff L j)), + Ntt16.Readonly.nttInvAtWithW (KEM.TP L) KEM.NS,Kem16.widenAt (KEM.TP L) (KEM.PP L), + L.ddLAt .x26 (32*L.du*L.k) L.dv (KEM.EP L),subAt (KEM.EP L) (KEM.PP L),ceAt (KEM.EP L) 1 .x28 KEM.MB]) def deStart (callee : Impl.Sha3.AArch64.Callee) (L : KemLay) : Prog isa := seqs [.block (L.kemPrologue 3 [0,1,2,3]),decryptM L, hashWith callee .x28 KEM.ST KEM.WK 72 6 [⟨.x28,KEM.MB,32⟩,⟨.x25,768*L.k+32,32⟩] diff --git a/lean/VerifiedGarbage/Impl/MlKem/AArch64/ReadonlyDecrypt.lean b/lean/VerifiedGarbage/Impl/MlKem/AArch64/ReadonlyDecrypt.lean deleted file mode 100644 index 558516ec75..0000000000 --- a/lean/VerifiedGarbage/Impl/MlKem/AArch64/ReadonlyDecrypt.lean +++ /dev/null @@ -1,9 +0,0 @@ -module -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.ReadonlyCalls -@[expose] public section -namespace VG.Impl.MlKem.AArch64.Optimized -open VG VG.AArch64 VG.Impl.MlKem.AArch64 - -def readonlyDeU (L : KemLay) (i : Nat) : Prog isa := - .seq (L.ddLAt .x26 (32*L.du*i) L.du (KEM.yOff L i)) (Ntt16.Readonly.nttAt (KEM.yOff L i)) -end VG.Impl.MlKem.AArch64.Optimized diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Core.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Core.lean deleted file mode 100644 index dc3e7cbb62..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Core.lean +++ /dev/null @@ -1,24 +0,0 @@ -module - -import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Setup -public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Result -import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Loop - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Dot -open VG.AArch64 VG.Spec.MlKem - -theorem dot_ok {s : State} {out : Nat} (ho : out < 65536) (terms : List (Nat × Nat)) - {F G : (Nat × Nat) → Poly} - (hd : LoopData s (s.gpr .x28 + BitVec.ofNat 64 out) (s.syms "VG_MLKEM_GAMMA") terms - (polyValues F) (polyValues G) gammaValues) : - WP isa (Impl.MlKem.AArch64.Dot.dot out terms) s fun s' => - Keep clob s s' ∧ Frame [polyRegion (s.gpr .x28 + BitVec.ofNat 64 out)] s.mem s'.mem ∧ - PolyIs s'.mem (s.gpr .x28 + BitVec.ofNat 64 out) (polySum terms F G) := by - rw [dot_partition] - refine WP.seq (WP.mono (setup_ok ho (result terms (polyValues F) (polyValues G) gammaValues)) - fun s₁ h₁ => ?_) - refine WP.mono (loop_ok hd h₁) fun s₂ h₂ => ?_ - exact ⟨h₂.keep,h₂.frame,h₂.out.polyIs (fun i hi => result_field terms F G hi)⟩ - -end VG.Proof.MlKem.AArch64.Dot diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Loop.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Loop.lean deleted file mode 100644 index 040f85d257..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Loop.lean +++ /dev/null @@ -1,43 +0,0 @@ -module -public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.LoopState - -import VerifiedGarbage.Proof.MlKem.AArch64.Dot.LoopInput -import VerifiedGarbage.Proof.Sha3.AArch64.Sums - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Dot -open VG.AArch64 VG.Spec.MlKem - -theorem loop_step {s₀ s : State} {p t : Addr} {terms : List (Nat × Nat)} - {F G : (Nat × Nat) → Nat → Nat} {Γ : Nat → Nat} {c : Nat} - (hd : LoopData s₀ p t terms F G Γ) (hc : c < 32) - (h : LoopInv s₀ p t (result terms F G Γ) c s) : - WP isa (.block (body terms)) s fun s' => - LoopInv s₀ p t (result terms F G Γ) (c+1) s' ∧ - ((s'.gpr .x11).toNat ≠ 0 ↔ c+1 ≠ 32) := by - refine WP.mono (step_ok terms h.constants hd.count (fun _ hf => loop_input hd hc h hf) - (by rw [h.keep.rd,h.keep.wr,h.x3,BitVec.add_zero]; exact hd.tableRead c hc) - (loop_gamma hd hc h) (fun e he => hd.tableBound _ (by omega)) h.x0 (by omega) - (fun i hi => by rw [h.keep.wr]; exact hd.write _ (by omega)) h.out - (fun e he => result_block terms F G Γ c he)) fun s' hs => ?_ - have counter : (s'.gpr .x11).toNat = 32-(c+1) := by - rw [hs.x11,toNat_sub_n (by rw [h.x11]; simp only [BitVec.toNat_ofNat]; omega),h.x11] - simp only [BitVec.toNat_ofNat] - omega - refine ⟨⟨(h.keep.trans hs.keep).mono, h.frame.trans hs.frame, hs.constants, ?_, ?_, ?_, - counter, ?_⟩, by rw [counter]; omega⟩ - · rw [hs.x0,h.x0,coeffAddr,ptr_add,show 4*(8*c)+32 = 4*(8*(c+1)) by omega] - · rw [hs.x3,h.x3,ptr_add,show 16*c+16 = 16*(c+1) by omega] - · rw [hs.x12,h.x12,←BitVec.ofNat_add,show 32*c+32 = 32*(c+1) by omega] - · simpa only [show 8*c+8 = 8*(c+1) by omega] using hs.out - -theorem loop_ok {s₀ s : State} {p t : Addr} {terms : List (Nat × Nat)} - {F G : (Nat × Nat) → Nat → Nat} {Γ : Nat → Nat} - (hd : LoopData s₀ p t terms F G Γ) - (h : LoopInv s₀ p t (result terms F G Γ) 0 s) : - WP isa (.loop (.block (body terms)) (.nonzero .x .x11)) s - (LoopInv s₀ p t (result terms F G Γ) 32) := - count_loop (by decide) (LoopInv s₀ p t (result terms F G Γ)) - (fun _ hc _ hi => loop_step hd hc hi) h - -end VG.Proof.MlKem.AArch64.Dot diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/LoopInput.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/LoopInput.lean deleted file mode 100644 index a05a740fd8..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/LoopInput.lean +++ /dev/null @@ -1,50 +0,0 @@ -module - -public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.LoopState -import VerifiedGarbage.Proof.MlKem.AArch64.NttVec -import VerifiedGarbage.Proof.Sha3.AArch64.Sums - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Dot -open VG.AArch64 VG.Spec.MlKem - -theorem coeffAt_shift (m : Mem) (p : Addr) (j e : Nat) : - coeffAt m (coeffAddr p j) e = coeffAt m p (j+e) := by - change m.readW (coeffAddr p j + BitVec.ofNat 64 (4*e)) 32 = _ - rw [coeffAddr_off]; rfl - -theorem loop_input {s₀ s : State} {p t : Addr} {terms : List (Nat × Nat)} - {F G : (Nat × Nat) → Nat → Nat} {Γ : Nat → Nat} {O : Nat → BitVec 32} {c : Nat} - (hd : LoopData s₀ p t terms F G Γ) (hc : c < 32) (h : LoopInv s₀ p t O c s) - {fg : Nat × Nat} (hf : fg ∈ terms) : - TermInput s fg (blockValues F c fg) (blockValues G c fg) := by - have addr (off : Nat) : s.gpr .x28 + BitVec.ofNat 64 off + s.gpr .x12 = - coeffAddr (s₀.gpr .x28 + BitVec.ofNat 64 off) (8*c) := by - rw [h.keep.get .x28,h.x12,coeffAddr, show 4*(8*c) = 32*c by omega] - refine ⟨hd.offF fg hf, hd.offG fg hf, ?_, ?_, ?_, ?_, ?_, ?_⟩ - · intro e he; exact hd.boundF fg hf _ (by omega) - · intro e he; exact hd.boundG fg hf _ (by omega) - · intro e he - rw [addr, coeffAt_shift, coeffAt_frame h.frame (fun r hr => by - rw [List.mem_singleton.mp hr]; exact (hd.apartF fg hf).symm) (by omega)] - exact hd.valueF fg hf _ (by omega) - · intro e he - rw [addr, coeffAt_shift, coeffAt_frame h.frame (fun r hr => by - rw [List.mem_singleton.mp hr]; exact (hd.apartG fg hf).symm) (by omega)] - exact hd.valueG fg hf _ (by omega) - · intro i hi - rw [h.keep.rd,h.keep.wr,addr,show 16*i = 4*(4*i) by omega,coeffAddr_off] - exact hd.readF fg hf _ (by omega) - · intro i hi - rw [h.keep.rd,h.keep.wr,addr,show 16*i = 4*(4*i) by omega,coeffAddr_off] - exact hd.readG fg hf _ (by omega) - -theorem loop_gamma {s₀ s : State} {p t : Addr} {terms : List (Nat × Nat)} - {F G : (Nat × Nat) → Nat → Nat} {Γ : Nat → Nat} {O : Nat → BitVec 32} {c : Nat} - (hd : LoopData s₀ p t terms F G Γ) (hc : c < 32) (h : LoopInv s₀ p t O c s) : - Lanes (s.mem.read (s.gpr .x3 + BitVec.ofNat 64 0) 16) (blockGamma Γ c) := by - rw [h.x3,BitVec.add_zero,h.frame.read (r := ⟨t,512⟩) (contains_off (by omega) (by decide)) - (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.tableApart.symm) (by decide)] - exact hd.tableValue c hc - -end VG.Proof.MlKem.AArch64.Dot diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Setup.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Setup.lean deleted file mode 100644 index e1e3bd278b..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Dot/Setup.lean +++ /dev/null @@ -1,43 +0,0 @@ -module -public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.LoopState - -import VerifiedGarbage.Proof.Framework.AArch64.Syms -import VerifiedGarbage.Proof.MlKem.AArch64.HashProof -import VerifiedGarbage.Proof.MlKem.AArch64.NttCommon -import VerifiedGarbage.Proof.Framework.Block - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Dot -open VG.AArch64 VG.Impl.MlKem.AArch64 VG.Spec.MlKem - -def setup (out : Nat) : List Instr := ptrTo .x0 .x28 out ++ vconsts ++ - [.adrSym .x3 "VG_MLKEM_GAMMA", .movz .x .x11 32 0, .movz .x .x12 0 0] - -theorem setup_ok {s : State} {out : Nat} (ho : out < 65536) (O : Nat → BitVec 32) : - WP isa (.block (setup out)) s - (LoopInv s (s.gpr .x28 + BitVec.ofNat 64 out) (s.syms "VG_MLKEM_GAMMA") O 0) := by - unfold setup - rw [List.append_assoc,WP.block_append_iff] - refine WP.mono_syms (wp_ptrTo' (d := .x0) (b := .x28) (s := s) - (Q := fun s' => Only [.x0] s s' ∧ s'.gpr .x0 = s.gpr .x28 + BitVec.ofNat 64 out) - (by decide) ho (fun _ hk hx => ⟨hk,hx⟩)) ?_ - intro s₁ h₁ sy₁ - rw [WP.block_append_iff] - refine WP.mono_syms (Ntt.vconsts_ok s₁) ?_ - intro s₂ ⟨k₂,m₂,c₂,_⟩ sy₂ - refine WP.mono (WP.keepV (Q := fun s₃ => Only [.x3,.x11,.x12] s₂ s₃ ∧ - s₃.gpr .x3 = s₂.syms "VG_MLKEM_GAMMA" ∧ s₃.gpr .x11 = BitVec.ofNat 64 32 ∧ s₃.gpr .x12 = 0) - (by rfl) (wp_x (by rfl) fun s₃ k₃ e₃ => wp_movz fun s₄ k₄ e₄ => - wp_movz fun s₅ k₅ e₅ => wp_nil ⟨((k₃.trans k₄).trans k₅).mono, - by rw [k₅.get .x3,k₄.get .x3,e₃], by rw [k₅.get .x11,e₄]; rfl, by rw [e₅]; rfl⟩)) - fun s₃ ⟨⟨k₃,e₃,e11,e12⟩,v₃⟩ => ?_ - have hm : s₃.mem = s.mem := k₃.mem.trans (m₂.trans h₁.1.mem) - refine ⟨((h₁.1.keep.trans k₂).trans k₃.keep).mono, - by rw [hm]; exact Frame.refl _ _, ⟨by rw [v₃]; exact c₂.q,by rw [v₃]; exact c₂.m⟩, - ?_, ?_, e12, ?_, ?_⟩ - · rw [k₃.get .x0,k₂.get .x0,h₁.2]; exact (BitVec.add_zero _).symm - · rw [e₃,sy₂,sy₁]; exact (BitVec.add_zero _).symm - · rw [e11]; rfl - · rw [hm]; exact CoeffsUpTo.zero _ - -end VG.Proof.MlKem.AArch64.Dot diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Decaps.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Decaps.lean index a1984906ce..c4cff824c9 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Decaps.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Decaps.lean @@ -8,7 +8,6 @@ public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.Tables import VerifiedGarbage.Proof.MlKem.AArch64.Kem16.Encrypt import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemVDecode -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemReadonlyTransform import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemURow import VerifiedGarbage.Proof.MlKem.AArch64.Noise.ReadonlyLayoutTables import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemNoise diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Ops.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Ops.lean index d29dbd2cc4..24a7a6b8cf 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Ops.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16/Ops.lean @@ -5,6 +5,7 @@ import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.W.Call import VerifiedGarbage.Proof.MlKem.AArch64.AddSubW import VerifiedGarbage.Proof.MlKem.AArch64.Codec12W.Dec import VerifiedGarbage.Proof.MlKem.AArch64.Widen16 +import VerifiedGarbage.Proof.MlKem.AArch64.Narrow16 public import VerifiedGarbage.Impl.MlKem.AArch64.Kem16 public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables public import VerifiedGarbage.Proof.MlKem.AArch64.KemOps @@ -17,8 +18,8 @@ import VerifiedGarbage.Proof.MlKem.AArch64.DotW As `KemOps.lean`, for the blocks of `Impl.MlKem.AArch64.Kem16` on polynomials stored with 16 bits in the first half of a buffer of `scratch`: the fused dot -product (`fusedW_ok`), the inverse transform (`invW_ok`), the sum (`addW_read_ok`), -the widening (`widenW_ok`) and `ByteDecode₁₂` (`dec12W_ok`). Each keeps `KB` and +product (`fusedW_ok`), the transforms (`nttW_ok`, `invW_ok`), the sum (`addW_read_ok`), +the widening (`widenW_ok`), the narrowing (`narrowW_ok`) and `ByteDecode₁₂` (`dec12W_ok`). Each keeps `KB` and `x24` and changes only its whole 1024-byte buffers. -/ @[expose] public section @@ -48,6 +49,28 @@ theorem fusedW_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P of exact ⟨hk.frame kt ft' (by decide) (fun r hr => by rw [List.mem_singleton.mp hr]; exact ho.safe hp),ft',pt,kt.get .x24 (by decide)⟩ +theorem nttW_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P off) + (hk : KB P L s₀ s) {F : Poly} (pf : PolyIsW 16 s.mem (sA L s₀ off) F) (ht : Optimized.TransformTables s) : + WP isa (Ntt16.Readonly.nttAtWithW off NS) s fun t => + KB P L s₀ t ∧ Frame [R (kA s₀) L.sc off 1024,R (kA s₀) L.sc NS 1024] s.mem t.mem ∧ + PolyIsW 16 t.mem (sA L s₀ off) (ntt F) ∧ t.gpr .x24 = s.gpr .x24 := by + have hw := hp.wf + have hc := covers_cons (cov_s hp hk ho.le) (cov_s hp hk (o:=NS) (l:=1024) (by kom)) + have hc' := covers_cons (cov_s hp hk (o:=off) (l:=512) (by have := ho.le; omega)) + (cov_s hp hk (o:=NS) (l:=512) (by kom)) + refine WP.mono (Ntt16.W.nttAtWithW_ok (by obtain ⟨_,hh⟩:=ho; kom) (by decide) + (by rw [e28 hk]) (by rw [e28 hk]) (((ho.ns hp).sub_left (half_sub _)).sub_right (half_sub _)) + pf hc' (ht.fast.ready hc)) fun t ⟨kt,pt⟩ => ?_ + have kt' : Kept [R (kA s₀) L.sc off 1024,R (kA s₀) L.sc NS 1024] s t := kt.sub fun r hh => by + rcases mem2' hh with rfl|rfl + · exact ⟨_, List.mem_cons_self .., half_sub _⟩ + · exact ⟨_, List.mem_cons_of_mem _ (List.mem_singleton_self _), half_sub _⟩ + refine ⟨hk.call kt' ?_,kt'.frame,pt,kt.cs _ (by decide) (by decide)⟩ + intro r hh + rcases mem2' hh with rfl|rfl + · exact ho.safe hp + · exact safe_scr hp (by kom) + theorem invW_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P off) (hk : KB P L s₀ s) (hr : ReducedW 16 s.mem (sA L s₀ off)) (ht : Optimized.TransformTables s) : WP isa (Ntt16.Readonly.nttInvAtWithW off NS) s fun t => @@ -126,6 +149,32 @@ theorem widenW_ok {s₀ : State} (hp : Pre P L s₀) {f g : Nat} (hf : PO P f) ( refine ⟨kb₂.frame k₃ f₃ (by decide) fun r hr => by rw [List.mem_singleton.mp hr]; exact hg.safe hp, by rw [← m₂]; exact f₃, p₃, by rw [k₃.get .x24 (by decide), h₂.get .x24, h₁.get .x24]⟩ +theorem narrowW_ok {s₀ : State} (hp : Pre P L s₀) {f g : Nat} (hf : PO P f) (hg : PO P g) + (d : f + 1024 ≤ g ∨ g + 1024 ≤ f) {s : State} (hk : KB P L s₀ s) {F : Poly} + (pf : PolyIs s.mem (sA L s₀ f) F) : + WP isa (Kem16.narrowAt f g) s fun s' => KB P L s₀ s' ∧ Frame [R (kA s₀) L.sc g 1024] s.mem s'.mem ∧ + PolyIsW 16 s'.mem (sA L s₀ g) F ∧ s'.gpr .x24 = s.gpr .x24 := by + have ff := hf.le + have fg := hg.le + refine WP.seq (wp_ptrTo (by decide) (by kom) fun s₁ h₁ e₁ => wp_ptrTo' (by decide) (by kom) + fun s₂ h₂ e₂ => ?_) + have kb₂ := hk.block (h₁.trans h₂).keep (by rw [h₂.mem, h₁.mem]) (by decide) + have m₂ : s₂.mem = s.mem := by rw [h₂.mem, h₁.mem] + have x0 : s₂.gpr .x0 = sA L s₀ f := by rw [h₂.get .x0, e₁, e28 hk] + have x1 : s₂.gpr .x1 = sA L s₀ g := by rw [e₂, h₁.get .x28, e28 hk] + have cw := cov_s hp kb₂ fg + have cr := cov_sr hp kb₂ ff + refine WP.mono (Narrow16.narrow_ok (F := F) ⟨?_, fun j hj => ?_, fun j hj => ?_, ?_⟩) fun s₃ ⟨k₃,f₃,p₃⟩ => ?_ + · rw [x0, m₂]; exact pf + · rw [x0]; exact cr _ _ (in_regions (List.mem_singleton_self _) (contains_off (by omega) (by decide))) + · rw [x1]; exact cw _ _ (in_regions (List.mem_singleton_self _) (contains_off (by omega) (by decide))) + · rw [x0, x1]; exact (papart hp hf hg d).sub_right (half_sub _) + rw [x1] at f₃ p₃ + have f₃' : Frame [R (kA s₀) L.sc g 1024] s₂.mem s₃.mem := + f₃.sub fun r hr => ⟨_, List.mem_singleton_self _, by rw [List.mem_singleton.mp hr]; exact half_sub _⟩ + refine ⟨kb₂.frame k₃ f₃' (by decide) fun r hr => by rw [List.mem_singleton.mp hr]; exact hg.safe hp, + by rw [← m₂]; exact f₃', p₃, by rw [k₃.get .x24 (by decide), h₂.get .x24, h₁.get .x24]⟩ + theorem dec12W_ok {s₀ : State} (hp : Pre P L s₀) {k o off : Nat} (hk4 : k < 4) (f : o + 384 ≤ L.len (L.slot k)) (ho : PO P off) (hs : L.slot k ≠ L.sc ∨ o + 384 ≤ off ∨ off + 1024 ≤ o) {s : State} (hk : KB P L s₀ s) : diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16P/Decaps.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16P/Decaps.lean index 8f88864ccb..6a1df150e2 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16P/Decaps.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Kem16P/Decaps.lean @@ -4,7 +4,6 @@ import VerifiedGarbage.Proof.MlKem.AArch64.Kem16P.Encrypt public import VerifiedGarbage.Proof.MlKem.AArch64.Kem16.Decaps public import VerifiedGarbage.Proof.MlKem.AArch64.Hybrid.Correct import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemVDecode -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemReadonlyTransform import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemURow import VerifiedGarbage.Proof.MlKem.AArch64.Noise.ReadonlyLayoutTables import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemNoise diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Narrow16.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Narrow16.lean new file mode 100644 index 0000000000..44a8cc4219 --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Narrow16.lean @@ -0,0 +1,150 @@ +module + +public import VerifiedGarbage.Impl.MlKem.AArch64.Narrow16 +import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Pass1 +import VerifiedGarbage.Proof.MlKem.AArch64.NttVec +public import VerifiedGarbage.Spec.MlKem.Kpke16 +import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Vec +public import VerifiedGarbage.Proof.MlKem.AArch64.Coeff16 +import VerifiedGarbage.Proof.MlKem.PolyW +import VerifiedGarbage.Proof.Framework.Offset +import VerifiedGarbage.Proof.MlKem.Arith +import VerifiedGarbage.Proof.Rc4.AArch64.Vec +import VerifiedGarbage.Proof.Sha3.AArch64.Sums +public import VerifiedGarbage.Proof.MlKem.AArch64.Wp +public import VerifiedGarbage.Proof.MlKem.Mem + +/-! +# ML-KEM on AArch64: a polynomial from 32-bit to 16-bit coefficients + +`narrow_ok`: `Impl.MlKem.AArch64.Narrow16.narrow` stores the polynomial at `x0`, with 32-bit +coefficients, at `x1` with 16-bit ones. Each iteration loads eight coefficients in two vectors, +whose low halfwords `uzp1` takes (`vhalf_perm_h8`), and stores them as one. +-/ + +@[expose] public section + +namespace VG.Proof.MlKem.AArch64.Narrow16 + +open VG VG.AArch64 VG.Proof.MlKem VG.Proof.MlKem.AArch64 VG.Spec.MlKem +open VG.Proof.MlKem.AArch64.Ntt16 (vhalf_perm_h8 vword_lo) +open VG.Impl.MlKem.AArch64.Narrow16 (narrowBody narrow) + +/-- The registers `narrow` writes. -/ +abbrev clob : List Reg := [.x0, .x1, .x10] + +/-- What `narrow` needs: the polynomial `F` with 32-bit coefficients at `x0`, readable, and 512 +writable bytes at `x1` apart from it. -/ +structure NarrowData (s₀ : State) (F : Poly) : Prop where + src : PolyIs s₀.mem (s₀.gpr .x0) F + read : ∀ j, j + 16 ≤ 1024 → InRegions (s₀.rd ++ s₀.wr) (s₀.gpr .x0 + BitVec.ofNat 64 j) 16 + write : ∀ j, j + 16 ≤ 512 → InRegions s₀.wr (s₀.gpr .x1 + BitVec.ofNat 64 j) 16 + apart : (polyRegion (s₀.gpr .x0)).Disjoint ⟨s₀.gpr .x1, 512⟩ + +/-- Coefficient `i` of the output. -/ +def G (F : Poly) (i : Nat) : BitVec 16 := BitVec.ofNat 16 (F[i]!).val + +theorem G_toNat (F : Poly) (i : Nat) : (G F i).toNat = (F[i]!).val := by + rw [G, BitVec.toNat_ofNat, Nat.mod_eq_of_lt (by have := val_lt F[i]!; omega)] + +/-- After `k` iterations. -/ +structure Inv (s₀ : State) (F : Poly) (k : Nat) (s : State) : Prop where + gpr : ∀ r, r ∉ clob → s.gpr r = s₀.gpr r + rd : s.rd = s₀.rd + wr : s.wr = s₀.wr + sp : s.sp = s₀.sp + x0 : s.gpr .x0 = s₀.gpr .x0 + BitVec.ofNat 64 (32 * k) + x1 : s.gpr .x1 = s₀.gpr .x1 + BitVec.ofNat 64 (16 * k) + count : (s.gpr .x10).toNat = 32 - k + vcs : ∀ r ∈ VG.AArch64.preservedV, (s.v r).extractLsb' 0 64 = (s₀.v r).extractLsb' 0 64 + out : CoeffsInW s.mem (s₀.gpr .x1) (· < 8 * k) (G F) fun i => coeffAtW 16 s₀.mem (s₀.gpr .x1) i + frame : Frame [⟨s₀.gpr .x1, 512⟩] s₀.mem s.mem + +theorem step {s₀ : State} {F : Poly} (hd : NarrowData s₀ F) {k : Nat} (hk : k < 32) {s : State} + (h : Inv s₀ F k s) : + WP isa (.block narrowBody) s fun t => Inv s₀ F (k + 1) t ∧ ((t.gpr .x10).toNat ≠ 0 ↔ k + 1 ≠ 32) := by + have src := polyIs_frame h.frame (fun r hr => by rw [List.mem_singleton.mp hr]; exact hd.apart) hd.src + have a0 : s.gpr .x0 + BitVec.ofNat 64 0 = coeffAddr (s₀.gpr .x0) (8 * k) := by + rw [h.x0, BitVec.add_zero, coeffAddr]; congr 2; omega + refine wp_ldrq (a := coeffAddr (s₀.gpr .x0) (8 * k)) (by decide) a0 ?_ fun s1 h1 => ?_ + · rw [h.rd, h.wr, coeffAddr, show 4 * (8 * k) = 32 * k by omega]; exact hd.read _ (by omega) + have a1 : s1.gpr .x0 + BitVec.ofNat 64 16 = coeffAddr (s₀.gpr .x0) (8 * k + 4) := by + rw [h1.gpr, h.x0, coeffAddr, ptr_add]; congr 2; omega + refine wp_ldrq (a := coeffAddr (s₀.gpr .x0) (8 * k + 4)) (by decide) a1 ?_ fun s2 h2 => ?_ + · rw [h1.rd, h1.wr, h.rd, h.wr, coeffAddr, show 4 * (8 * k + 4) = 32 * k + 16 by omega] + exact hd.read _ (by omega) + refine wp_vop (d := .v2) rfl fun s3 h3 => ?_ + have key : ∀ (x : BitVec 128) (c e : Nat), (vword x c).toNat = (F[8 * k + e]!).val → + vhalf x (2 * c) = G F (8 * k + e) := fun x c e hx => BitVec.eq_of_toNat_eq (by + rw [← vword_lo, BitVec.extractLsb'_toNat, Nat.shiftRight_zero, hx, G_toNat, + Nat.mod_eq_of_lt (by have := val_lt F[8 * k + e]!; omega)]) + have hv : ∀ e < 8, vhalf (s3.v .v2) e = G F (8 * k + e) := fun e he => by + rw [h3.v, h2.other .v0 (by decide), h2.v, h1.v, h1.mem, vhalf_perm_h8 _ _ _ he] + by_cases c : e < 4 + · rw [ite_eq_left c]; exact key _ e e (lanes_load src (j := 8 * k) (by omega) e c) + · rw [ite_eq_right c, show 2 * e - 8 = 2 * (e - 4) by omega] + have := lanes_load src (j := 8 * k + 4) (by omega) (e - 4) (by omega) + simp only [show 8 * k + 4 + (e - 4) = 8 * k + e by omega] at this + exact key _ _ e this + have g3 : s3.gpr = s.gpr := by rw [h3.gpr, h2.gpr, h1.gpr] + have dst : s3.gpr .x1 + BitVec.ofNat 64 0 = s₀.gpr .x1 + BitVec.ofNat 64 (2 * (8 * k)) := by + rw [g3, h.x1, BitVec.add_zero]; congr 2; omega + refine wp_strq (a := s₀.gpr .x1 + BitVec.ofNat 64 (2 * (8 * k))) (by decide) dst ?_ fun s4 h4 => ?_ + · rw [h3.wr, h2.wr, h1.wr, h.wr, show 2 * (8 * k) = 16 * k by omega] + exact hd.write _ (by omega) + have scalar : WP isa (.block [.addImm .x .x0 .x0 32, .addImm .x .x1 .x1 16, .subImm .x .x10 .x10 1]) s4 + fun t => Only [.x0, .x1, .x10] s4 t ∧ t.gpr .x0 = s4.gpr .x0 + 32 ∧ + t.gpr .x1 = s4.gpr .x1 + 16 ∧ (t.gpr .x10).toNat = 32 - (k + 1) := by + refine wp_addImm (by decide) fun s6 h6 e6 => wp_addImm (by decide) fun s7 h7 e7 => + wp_subImm (by decide) fun s8 h8 e8 => wp_nil ?_ + refine ⟨((h6.trans h7).trans h8).mono (by decide), ?_, ?_, ?_⟩ + · rw [h8.get .x0, h7.get .x0, e6]; rfl + · rw [h8.get .x1, e7, h6.get .x1]; rfl + · have cnt : (s7.gpr .x10).toNat = 32 - k := by + rw [h7.get .x10, h6.get .x10, h4.gpr, g3, h.count] + rw [e8, toNat_sub_n (by rw [cnt]; simp; omega), cnt]; simp; omega + refine WP.mono (WP.keepV (by decide) scalar) ?_ + intro t ⟨⟨ht, p0, p1, cnt⟩, vt⟩ + have g4 : s4.gpr = s.gpr := by rw [h4.gpr, g3] + have vv (r : VReg) (r0 : r ≠ .v0) (r1 : r ≠ .v1) (r2 : r ≠ .v2) : t.v r = s.v r := by + rw [vt, h4.v, h3.other _ r2, h2.other _ r1, h1.other _ r0] + have mt : t.mem = s.mem.write (s₀.gpr .x1 + BitVec.ofNat 64 (2 * (8 * k))) 16 (s3.v .v2) := by + rw [ht.mem, h4.mem, h3.mem, h2.mem, h1.mem] + refine ⟨⟨fun r hr => ?_, ?_, ?_, ?_, ?_, ?_, cnt, fun r hr => ?_, ?_, ?_⟩, by rw [cnt]; omega⟩ + · simp only [List.mem_cons, List.not_mem_nil, or_false, not_or] at hr + rw [ht.gpr r (by simp [hr.1, hr.2.1, hr.2.2]), g4, h.gpr r (by simp [hr.1, hr.2.1, hr.2.2])] + · rw [ht.rd, h4.rd, h3.rd, h2.rd, h1.rd, h.rd] + · rw [ht.wr, h4.wr, h3.wr, h2.wr, h1.wr, h.wr] + · rw [ht.sp, h4.sp, h3.sp, h2.sp, h1.sp, h.sp] + · rw [p0, g4, h.x0, show (32 : BitVec 64) = BitVec.ofNat 64 32 from rfl, ptr_add]; congr 2 + · rw [p1, g4, h.x1, show (16 : BitVec 64) = BitVec.ofNat 64 16 from rfl, ptr_add]; congr 2 + · have hr0 : r ≠ .v0 := by revert r hr; decide + have hr1 : r ≠ .v1 := by revert r hr; decide + have hr2 : r ≠ .v2 := by revert r hr; decide + rw [vv r hr0 hr1 hr2]; exact h.vcs r hr + · rw [mt] + exact h.out.write16 (by omega) hv (fun i _ => by omega) + · rw [mt] + refine h.frame.write (List.mem_singleton_self _) _ ?_ + rw [show 2 * (8 * k) = 16 * k by omega] + exact Offset.contains_base (s₀.gpr .x1) (d := 16 * k) (n := 16) (k := 512) (by omega) (by omega) + +theorem init (s : State) : WP isa (.block [.movz .x .x10 32 0]) s fun t => ∀ F, Inv s F 0 t := by + refine wp_movz fun s1 h1 e1 => wp_nil fun F => ?_ + refine ⟨fun r hr => ?_, h1.rd, h1.wr, h1.sp, ?_, ?_, by rw [e1]; rfl, fun r hr => h1.vcs r hr, + fun i _ => ?_, ?_⟩ + · rw [h1.get r (by simp at hr ⊢; exact hr.2.2)] + · rw [h1.get .x0 (by decide)]; exact (BitVec.add_zero _).symm + · rw [h1.get .x1 (by decide)]; exact (BitVec.add_zero _).symm + · rw [ite_eq_right (by omega), h1.mem] + · rw [h1.mem]; exact Frame.refl _ _ + +theorem narrow_ok {s : State} {F : Poly} (hd : NarrowData s F) : + WP isa narrow s fun s' => Keep clob s s' ∧ Frame [⟨s.gpr .x1, 512⟩] s.mem s'.mem ∧ + PolyIsW 16 s'.mem (s.gpr .x1) F := by + refine WP.seq (WP.mono (init s) fun s₁ h₁ => + WP.mono (count_loop (by decide) (Inv s F) (fun k hk t ht => step hd hk ht) (h₁ F)) fun s' h' => ?_) + exact ⟨⟨h'.gpr, h'.rd, h'.wr, h'.sp, h'.vcs⟩, h'.frame, + h'.out.polyIs (fun i hi => by omega) fun i _ => G_toNat F i⟩ + +end VG.Proof.MlKem.AArch64.Narrow16 diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/At.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/At.lean index 9259f9707e..9c3f204337 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/At.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/At.lean @@ -20,46 +20,4 @@ theorem TableReady.frame {s t : State} {symbol : String} {H : List Nat} {f w : A rw [hs,hf.readW (Offset.contains_base _ (by omega) (by omega)) hd (by decide)] exact ht.table i hi -theorem nttAtWith_ok {s : State} {off ns : Nat} {f w : Addr} {F : Poly} - (hoff : off<65536) (hns : ns<65536) - (hf : s.gpr .x28+BitVec.ofNat 64 off=f) (hw : s.gpr .x28+BitVec.ofNat 64 ns=w) - (hd : (polyRegion f).Disjoint (polyRegion w)) (hp : PolyIs s.mem f F) - (hc : Covers [polyRegion f,polyRegion w] s.wr) - (ht : TableReady s "VG_MLKEM_NTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.tableHalves f w) : - WP isa (Impl.MlKem.AArch64.Ntt16.Readonly.nttAtWith off ns) s fun t=> - Kept [polyRegion f,polyRegion w] s t ∧ PolyIs t.mem f (ntt F) := by - have prep : WP isa (.block (Impl.MlKem.AArch64.ptrTo .x0 .x28 off ++ Impl.MlKem.AArch64.ptrTo .x1 .x28 ns)) s - fun z=>Only [.x0,.x1] s z ∧ z.gpr .x0=f ∧ z.gpr .x1=w := by - refine wp_ptrTo (by decide) hoff fun a ha ea=>wp_ptrTo' (by decide) hns fun z hz ez=>?_ - exact ⟨(ha.trans hz).mono (by simp),by rw [hz.get .x0 (by decide),ea]; exact hf, - by rw [ez,ha.get .x28 (by decide)]; exact hw⟩ - refine WP.seq (WP.mono_syms prep fun z ⟨oz,z0,z1⟩ zs=>?_) - have table : TableReady z "VG_MLKEM_NTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.tableHalves f w := by - exact ⟨by simpa only [zs,oz.mem] using ht.table, - by simpa only [zs,oz.rd,oz.wr] using ht.readable, - by simpa only [zs] using ht.output,by simpa only [zs] using ht.scratch⟩ - exact fwd_call z0 z1 hd (by rw [oz.mem]; exact hp.1) - (by rw [oz.wr]; exact hc) table fun t kt pt=> - ⟨(Kept.of_keep oz.keep oz.mem (by decide)).trans kt,by simpa only [oz.mem,hp.2] using pt⟩ -theorem nttInvAtWith_ok {s : State} {off ns : Nat} {f w : Addr} {F : Poly} - (hoff : off<65536) (hns : ns<65536) - (hf : s.gpr .x28+BitVec.ofNat 64 off=f) (hw : s.gpr .x28+BitVec.ofNat 64 ns=w) - (hd : (polyRegion f).Disjoint (polyRegion w)) (hp : PolyIs s.mem f F) - (hc : Covers [polyRegion f,polyRegion w] s.wr) - (ht : TableReady s "VG_MLKEM_INTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.invTableHalves f w) : - WP isa (Impl.MlKem.AArch64.Ntt16.Readonly.nttInvAtWith off ns) s fun t=> - Kept [polyRegion f,polyRegion w] s t ∧ PolyIs t.mem f (nttInv F) := by - have prep : WP isa (.block (Impl.MlKem.AArch64.ptrTo .x0 .x28 off ++ Impl.MlKem.AArch64.ptrTo .x1 .x28 ns)) s - fun z=>Only [.x0,.x1] s z ∧ z.gpr .x0=f ∧ z.gpr .x1=w := by - refine wp_ptrTo (by decide) hoff fun a ha ea=>wp_ptrTo' (by decide) hns fun z hz ez=>?_ - exact ⟨(ha.trans hz).mono (by simp),by rw [hz.get .x0 (by decide),ea]; exact hf, - by rw [ez,ha.get .x28 (by decide)]; exact hw⟩ - refine WP.seq (WP.mono_syms prep fun z ⟨oz,z0,z1⟩ zs=>?_) - have table : TableReady z "VG_MLKEM_INTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.invTableHalves f w := by - exact ⟨by simpa only [zs,oz.mem] using ht.table, - by simpa only [zs,oz.rd,oz.wr] using ht.readable, - by simpa only [zs] using ht.output,by simpa only [zs] using ht.scratch⟩ - exact inv_call z0 z1 hd (by rw [oz.mem]; exact hp.1) - (by rw [oz.wr]; exact hc) table fun t kt pt=> - ⟨(Kept.of_keep oz.keep oz.mem (by decide)).trans kt,by simpa only [oz.mem,hp.2] using pt⟩ end VG.Proof.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Call.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Call.lean index c7942cc3ef..6fff3834bc 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Call.lean @@ -1,10 +1,13 @@ module -public import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Contract +import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Pre public import VerifiedGarbage.Proof.MlKem.AArch64.KeccakCall import VerifiedGarbage.Proof.Framework.Contract import VerifiedGarbage.Proof.Framework.TacticTemplate import VerifiedGarbage.Proof.Rc4.AArch64.Vec +public import VerifiedGarbage.Proof.Framework.Covers +public import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Table +public import VerifiedGarbage.Proof.MlKem.Mem @[expose] public section namespace VG.Proof.MlKem.AArch64.Ntt16.Readonly @@ -17,49 +20,4 @@ structure TableReady (s : State) (symbol : String) (H : List Nat) (f w : Addr) : output : (polyRegion f).Disjoint ⟨s.syms symbol,512⟩ scratch : (polyRegion w).Disjoint ⟨s.syms symbol,512⟩ -theorem transform_call {op : Poly → Poly} {symbol : String} {H : List Nat} {c : Prog isa} - (hv : ∀ s, (contract op symbol H).pre s → ∃ tr t, Exec isa c s tr t ∧ - abiPreserved s t ∧ (contract op symbol H).post s t) - (hn : c.noCalls=true) {s : State} {f w : Addr} (h0 : s.gpr .x0=f) (h1 : s.gpr .x1=w) - (hd : (polyRegion f).Disjoint (polyRegion w)) (hr : Reduced s.mem f) - (hw : Covers [polyRegion f,polyRegion w] s.wr) (ht : TableReady s symbol H f w) {Q : State → Prop} - (hQ : ∀ s', Kept [polyRegion f,polyRegion w] s s' → PolyIs s'.mem f (op (polyAt s.mem f)) → Q s') : - WP isa c s Q := by - refine WP.inline (k:=contract op symbol H) hv (rd:=[⟨s.syms symbol,512⟩]) - (wr:=[polyRegion f,polyRegion w]) ?_ (ht.readable.append_left hw.right) hw ?_ hn - · refine ⟨⟨?_,?_,?_⟩,?_,?_,?_,?_⟩ - · simp only [fP,sP,State.withRegions_wr,State.withRegions_gpr,h0,h1] - · simpa only [fP,sP,State.withRegions_gpr,h0,h1] using hd - · simpa only [fP,State.withRegions_mem,State.callEntry_mem,State.withRegions_gpr,h0] using hr - · exact ht.table - · exact Covers.of_mem (fun r h => List.mem_append_left _ h) - · simp only [fP,State.withRegions_gpr,h0] - change (polyRegion f).Disjoint ⟨s.syms symbol,512⟩ - exact ht.output - · simp only [sP,State.withRegions_gpr,h1] - change (polyRegion w).Disjoint ⟨s.syms symbol,512⟩ - exact ht.scratch - · intro s' hrd hwr habi hf _ hpost - change PolyIs s'.mem (s.gpr .x0) (op (polyAt s.mem (s.gpr .x0))) at hpost - rw [h0] at hpost - exact hQ s' ⟨fun r hr _ => habi.1 r hr,habi.2.1,hrd,hwr,hf,habi.2.2⟩ hpost - -theorem fwd_call {s : State} {f w : Addr} (h0 : s.gpr .x0=f) (h1 : s.gpr .x1=w) - (hd : (polyRegion f).Disjoint (polyRegion w)) (hr : Reduced s.mem f) - (hw : Covers [polyRegion f,polyRegion w] s.wr) - (ht : TableReady s "VG_MLKEM_NTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.tableHalves f w) - {Q : State → Prop} (hQ : ∀ s', Kept [polyRegion f,polyRegion w] s s' → - PolyIs s'.mem f (ntt (polyAt s.mem f)) → Q s') : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.fwd s Q := - transform_call fwd_correct (by decide +kernel) h0 h1 hd hr hw ht hQ - -theorem inv_call {s : State} {f w : Addr} (h0 : s.gpr .x0=f) (h1 : s.gpr .x1=w) - (hd : (polyRegion f).Disjoint (polyRegion w)) (hr : Reduced s.mem f) - (hw : Covers [polyRegion f,polyRegion w] s.wr) - (ht : TableReady s "VG_MLKEM_INTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.invTableHalves f w) - {Q : State → Prop} (hQ : ∀ s', Kept [polyRegion f,polyRegion w] s s' → - PolyIs s'.mem f (nttInv (polyAt s.mem f)) → Q s') : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.inv s Q := - transform_call inv_correct (by decide +kernel) h0 h1 hd hr hw ht hQ - end VG.Proof.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Contract.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Contract.lean deleted file mode 100644 index 8b246803a2..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Contract.lean +++ /dev/null @@ -1,81 +0,0 @@ -module -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.Readonly -public import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Pre - -import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Fwd -import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Inv -import VerifiedGarbage.Proof.Framework.AArch64.TaintSym -import VerifiedGarbage.Proof.MlKem.AArch64.Common -import VerifiedGarbage.Proof.Framework.Offset -import VerifiedGarbage.Proof.Framework.RegionsSub -import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Lit -meta import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.Readonly -meta import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Pre -meta import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Fwd -meta import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Inv -meta import VerifiedGarbage.Proof.Framework.AArch64.TaintSym -meta import VerifiedGarbage.Proof.MlKem.AArch64.Common -meta import VerifiedGarbage.Proof.Framework.Offset -meta import VerifiedGarbage.Proof.Framework.RegionsSub -meta import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Lit - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Ntt16.Readonly -open VG VG.AArch64 VG.Spec.MlKem VG.Proof.MlKem.AArch64 -open VG.Impl.MlKem.AArch64.Ntt16 - -def contract (op : Poly → Poly) (symbol : String) (H : List Nat) : Contract isa where - pre s := Input s symbol H - post s t := PolyIs t.mem (s.gpr .x0) (op (polyAt s.mem (s.gpr .x0))) - pub s t := s.sp=t.sp ∧ s.gpr .x0=t.gpr .x0 ∧ s.gpr .x1=t.gpr .x1 ∧ s.syms symbol=t.syms symbol - -def fwdContract : Contract isa := contract ntt "VG_MLKEM_NTT16_ZETAS" tableHalves -def invContract : Contract isa := contract nttInv "VG_MLKEM_INTT16_ZETAS" invTableHalves - -theorem fwd_correct (s : State) (h : fwdContract.pre s) : - ∃ tr t, Exec isa Impl.MlKem.AArch64.Ntt16.Readonly.fwd s tr t ∧ - abiPreserved s t ∧ fwdContract.post s t := by - obtain ⟨tr,t,he,_,hf⟩ := fwd_ok s h - exact ⟨tr,t,he,abi_of rfl (by lit_decide) he (by lit_decide),hf⟩ - -theorem inv_correct (s : State) (h : invContract.pre s) : - ∃ tr t, Exec isa Impl.MlKem.AArch64.Ntt16.Readonly.inv s tr t ∧ - abiPreserved s t ∧ invContract.post s t := by - obtain ⟨tr,t,he,_,hf⟩ := inv_ok s h - exact ⟨tr,t,he,abi_of rfl (by lit_decide) he (by lit_decide),hf⟩ - -theorem fwd_ct : ConstantTime isa fwdContract.pre fwdContract.pub - Impl.MlKem.AArch64.Ntt16.Readonly.fwd := by - have hp : ∀ s t, fwdContract.pre s → fwdContract.pre t → fwdContract.pub s t → - VG.AArch64.Taint.AgreeS ["VG_MLKEM_NTT16_ZETAS"] (VG.AArch64.Taint.ofRegs [.x0,.x1]) s t := by - intro s t _ _ h - refine ⟨agree_of h.1 ?_,?_⟩ - · intro r hr - simp only [List.mem_cons,List.not_mem_nil,or_false] at hr - rcases hr with rfl | rfl - · exact h.2.1 - · exact h.2.2.1 - · intro name hn - rw [List.mem_singleton.mp hn] - exact h.2.2.2 - exact VG.Taint.constantTime (A:=taintS ["VG_MLKEM_NTT16_ZETAS"]) - (VG.AArch64.Taint.ofRegs [.x0,.x1]) hp (by taint_decide) - -theorem inv_ct : ConstantTime isa invContract.pre invContract.pub - Impl.MlKem.AArch64.Ntt16.Readonly.inv := by - have hp : ∀ s t, invContract.pre s → invContract.pre t → invContract.pub s t → - VG.AArch64.Taint.AgreeS ["VG_MLKEM_INTT16_ZETAS"] (VG.AArch64.Taint.ofRegs [.x0,.x1]) s t := by - intro s t _ _ h - refine ⟨agree_of h.1 ?_,?_⟩ - · intro r hr - simp only [List.mem_cons,List.not_mem_nil,or_false] at hr - rcases hr with rfl | rfl - · exact h.2.1 - · exact h.2.2.1 - · intro name hn - rw [List.mem_singleton.mp hn] - exact h.2.2.2 - exact VG.Taint.constantTime (A:=taintS ["VG_MLKEM_INTT16_ZETAS"]) - (VG.AArch64.Taint.ofRegs [.x0,.x1]) hp (by taint_decide) - -end VG.Proof.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Fwd.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Fwd.lean deleted file mode 100644 index 57c16c82de..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Fwd.lean +++ /dev/null @@ -1,17 +0,0 @@ -module - -public import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Pre -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.Readonly -import VerifiedGarbage.Proof.MlKem.AArch64.NttR.FwdU - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Ntt16.Readonly -open VG VG.AArch64 VG.Proof.MlKem.AArch64 VG.Proof.MlKem VG.Spec.MlKem -open VG.Impl.MlKem.AArch64.Ntt16 - -theorem fwd_ok (s₀ : State) (hi : Input s₀ "VG_MLKEM_NTT16_ZETAS" tableHalves) : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.fwd s₀ fun s' => - s'.sp=s₀.sp ∧ PolyIs s'.mem (fP s₀) (Spec.MlKem.ntt (P₀ s₀)) := by - exact NttR.fwd_run s₀ ⟨hi.wr, hi.disj, hi.red, hi.table, hi.readable, hi.apartOutput, hi.apartScratch⟩ - -end VG.Proof.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Inv.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Inv.lean deleted file mode 100644 index 04df6e9307..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/Readonly/Inv.lean +++ /dev/null @@ -1,17 +0,0 @@ -module - -public import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.Pre -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.Readonly -import VerifiedGarbage.Proof.MlKem.AArch64.NttR.Inv - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Ntt16.Readonly -open VG VG.AArch64 VG.Proof.MlKem.AArch64 VG.Proof.MlKem VG.Spec.MlKem -open VG.Impl.MlKem.AArch64.Ntt16 - -theorem inv_ok (s₀ : State) (hi : Input s₀ "VG_MLKEM_INTT16_ZETAS" invTableHalves) : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.inv s₀ fun s' => - s'.sp=s₀.sp ∧ PolyIs s'.mem (fP s₀) (Spec.MlKem.nttInv (P₀ s₀)) := by - exact NttR.inv_run s₀ ⟨hi.wr, hi.disj, hi.red, hi.table, hi.readable, hi.apartOutput, hi.apartScratch⟩ - -end VG.Proof.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/W/Call.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/W/Call.lean index 3d151eac04..f7f1395a21 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/W/Call.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Ntt16/W/Call.lean @@ -14,10 +14,9 @@ import VerifiedGarbage.Proof.Framework.Offset /-! # ML-KEM on AArch64: the 16-bit NTT and its inverse of polynomials stored with 16 bits, at offsets -`invW` (`Impl.MlKem.AArch64.Ntt16.Readonly`) is used as `Readonly.inv` is: a -contract (`contractW`), the code inlined where its caller's pointers are -(`transformW_call`), and from offsets of `x28` (`nttInvAtWithW_ok`), as -`vg_mlkem16_ntt` is called (`nttAtWithW_ok`). The polynomial and the working +`invW` (`Impl.MlKem.AArch64.Ntt16.Readonly`) has a contract (`contractW`), its code is inlined +where its caller's pointers are (`transformW_call`), and from offsets of `x28` +(`nttInvAtWithW_ok`), as `vg_mlkem16_ntt` is called (`nttAtWithW_ok`). The polynomial and the working space are 512 bytes each. -/ diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/FwdU.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/FwdU.lean index 2728c28730..c5e0e27b54 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/FwdU.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/FwdU.lean @@ -20,9 +20,8 @@ import VerifiedGarbage.Proof.Framework.Block Coefficient `p` of the polynomial at `x0` is the word at `4p`: the machine follows its low halfword, `Wu p` (`initU_rel`), and finds the reduced coefficients of `NTT` in the low halfwords and zero in the high ones -(`polyIs_of_outU`). So `fwd` (`fwd_run`), with the table at its symbol, and -`ntt` (`ntt_run`), which first stores the table in the working space -(`halves_ok`), transform the polynomial in place. +(`polyIs_of_outU`). So `ntt` (`ntt_run`), which first stores the table in the working space +(`halves_ok`), transforms the polynomial in place. -/ @[expose] public section @@ -178,21 +177,6 @@ theorem ctxU {sym : String} {H : List Nat} {s₀ : State} (hi : InU sym H s₀) | exact d01 | exact d01.symm | exact hi.apartOutput | exact hi.apartOutput.symm | exact d12 | exact d12.symm -theorem fwd_run (s₀ : State) (hi : InU "VG_MLKEM_NTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.tableHalves s₀) : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.fwd s₀ fun s' => - s'.sp = s₀.sp ∧ PolyIs s'.mem (s₀.gpr .x0) (ntt (polyAt s₀.mem (s₀.gpr .x0))) := by - have hc := checkU_ok - generalize hx : initU.runP fwdRules rsU 16 Impl.MlKem.AArch64.Ntt16.Readonly.fwd = x at hc - cases x with - | none => cases hc - | some σ => - have hrel := initU_rel (rs := rsU) (s₀ := s₀) (tb := s₀.syms "VG_MLKEM_NTT16_ZETAS") (L := Lf (Wu s₀)) - (O := Of (Wu s₀)) (fun _ => rfl) rfl rfl rfl rfl rfl - (by show s₀.syms _ + BitVec.ofNat 64 0 = _; rw [ptr_zero]) tabT_get - (fun j hj => by rw [Nat.zero_add]; exact hi.table j hj) - refine WP.mono (runP_ok (model (Wu s₀)) (ctxU hi) 16 _ hrel hx) fun s' h' => ⟨h'.sp, ?_⟩ - exact polyIs_of_outU rfl rfl rfl h' hc fun p hp => Of_ntt (Wu s₀) _ (fun p hp => lrep_redU hi.red hp) hp - /-- What `ntt` needs: the polynomial at `x0` and the working space at `x1`, writable and apart, and the polynomial reduced. -/ structure InN (s : State) : Prop where diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Inv.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Inv.lean index 9c3353bb82..63fc1b82c7 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Inv.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Inv.lean @@ -17,12 +17,11 @@ public import VerifiedGarbage.Proof.MlKem.AArch64.NttR.LitI /-! # ML-KEM on AArch64: the 16-bit inverse NTTs, by the machine -`inv` (`inv_run`) and `invW` (`invW_run`), with the table at its symbol, and -`nttInv` (`nttInv_run`), which first stores the table in the working space -(`halves_ok`), transform the polynomial at `x0` in place: the machine's state -at the start holds of the caller's (`initU_rel`, `initW_rel`), with the -inverse's model (`modelI`), and its runs (`checkIU_ok`, `checkIW_ok`, -`checkIN_ok`) leave the reduced coefficients of `NTT⁻¹` in the polynomial. +`invW` (`invW_run`), with the table at its symbol, and `nttInv` (`nttInv_run`), which first +stores the table in the working space (`halves_ok`), transform the polynomial at `x0` in place: +the machine's state at the start holds of the caller's (`initU_rel`, `initW_rel`), with the +inverse's model (`modelI`), and its runs (`checkIW_ok`, `checkIN_ok`) leave the reduced +coefficients of `NTT⁻¹` in the polynomial. -/ @[expose] public section @@ -32,21 +31,6 @@ namespace VG.Proof.MlKem.AArch64.NttR open VG VG.AArch64 VG.Spec.MlKem VG.Proof.MlKem VG.Proof.MlKem.AArch64 open VG.Proof.MlKem.AArch64.Ntt16 (halves_ok) -theorem inv_run (s₀ : State) (hi : InU "VG_MLKEM_INTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.invTableHalves s₀) : - WP isa Impl.MlKem.AArch64.Ntt16.Readonly.inv s₀ fun s' => - s'.sp = s₀.sp ∧ PolyIs s'.mem (s₀.gpr .x0) (nttInv (polyAt s₀.mem (s₀.gpr .x0))) := by - have hc := checkIU_ok - generalize hx : initIU.runP invRules rsIU 16 Impl.MlKem.AArch64.Ntt16.Readonly.inv = x at hc - cases x with - | none => cases hc - | some σ => - have hrel := initU_rel (rs := rsIU) (s₀ := s₀) (tb := s₀.syms "VG_MLKEM_INTT16_ZETAS") - (L := Li (Wu s₀)) (O := Oi (Wu s₀)) (fun _ => rfl) rfl rfl rfl rfl rfl - (by show s₀.syms _ + BitVec.ofNat 64 0 = _; rw [ptr_zero]) itabT_get - (fun j hj => by rw [Nat.zero_add]; exact hi.table j hj) - refine WP.mono (runP_ok (modelI (Wu s₀)) (ctxU hi) 16 _ hrel hx) fun s' h' => ⟨h'.sp, ?_⟩ - exact polyIs_of_outU rfl rfl rfl h' hc fun p hp => Oi_nttInv (Wu s₀) _ (fun p hp => lrep_redU hi.red hp) hp - theorem invW_run (s₀ : State) (hi : InW "VG_MLKEM_INTT16_ZETAS" Impl.MlKem.AArch64.Ntt16.invTableHalves s₀) : WP isa Impl.MlKem.AArch64.Ntt16.Readonly.invW s₀ fun s' => s'.sp = s₀.sp ∧ PolyIsW 16 s'.mem (s₀.gpr .x0) (nttInv (polyAtW 16 s₀.mem (s₀.gpr .x0))) := by diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Lit.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Lit.lean index 262ca97cb1..d3f51a046d 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Lit.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/Lit.lean @@ -15,12 +15,6 @@ meta import VerifiedGarbage.Proof.Framework.AArch64.Lit @[expose] public section -namespace VG.Impl.MlKem.AArch64.Ntt16.Readonly - -materialize_code fwd - -end VG.Impl.MlKem.AArch64.Ntt16.Readonly - namespace VG.Impl.MlKem.AArch64.Ntt16 materialize_value tableHalves diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/LitI.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/LitI.lean index a35b642774..e2076a1767 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/LitI.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/LitI.lean @@ -11,13 +11,12 @@ meta import VerifiedGarbage.Proof.Framework.Taint import VerifiedGarbage.Proof.Framework.AArch64.Lit meta import VerifiedGarbage.Proof.Framework.AArch64.Lit -/-! # Literals of the 16-bit inverse NTTs, for the runs the kernel evaluates -/ +/-! # Literals of the 16-bit inverse NTT of a polynomial stored with 16 bits, for the runs the kernel evaluates -/ @[expose] public section namespace VG.Impl.MlKem.AArch64.Ntt16.Readonly -materialize_code inv materialize_code invW end VG.Impl.MlKem.AArch64.Ntt16.Readonly diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunI.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunI.lean index bbc4a3dffd..1f8c752674 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunI.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunI.lean @@ -13,13 +13,11 @@ meta import VerifiedGarbage.Proof.Framework.Lit /-! # ML-KEM on AArch64: the runs of the 16-bit inverse NTTs -The regions of `inv` (`rsIU`: the polynomial stored with 32 bits at `x0`, the -working space at `x1`, the table of zetas) and `invW` (`rsIW`: stored with 16 -bits), the terms they start from (`initIU`, `initIW`: the input's coefficients, -the table's halfwords as constants), and the checks, by the kernel, that the -runs leave the reduced coefficients of `NTT⁻¹` in the polynomial (`checkIU_ok`, -`checkIW_ok`, and `checkIN_ok` for the end of `nttInv`, with the table in the -working space). +The regions of `invW` (`rsIW`: the polynomial stored with 16 bits at `x0`, the working space at +`x1`, the table of zetas), the terms it and the end of `nttInv` start from (`initIW`, `initIU`: +the input's coefficients, the table's halfwords as constants), and the checks, by the kernel, +that the runs leave the reduced coefficients of `NTT⁻¹` in the polynomial (`checkIW_ok`, and +`checkIN_ok` for the end of `nttInv`, with the table in the working space). -/ @[expose] public section @@ -28,18 +26,12 @@ namespace VG.Proof.MlKem.AArch64.NttR open VG VG.AArch64 -def rsIU : List Rg := rsUs "VG_MLKEM_INTT16_ZETAS" - def rsIW : List Rg := rsWs "VG_MLKEM_INTT16_ZETAS" def initIU : St := initUs itabT def initIW : St := initWs itabT -theorem checkIU_ok : - (initIU.runP invRules rsIU 16 Impl.MlKem.AArch64.Ntt16.Readonly.inv).elim false outU = true := by - lit_decide - theorem checkIN_ok : (initIU.runP invRules rsN 16 nttInvTail).elim false outU = true := by lit_decide diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunU.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunU.lean index 10ea7d75c6..c202b1d492 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunU.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/NttR/RunU.lean @@ -8,12 +8,11 @@ meta import VerifiedGarbage.Proof.Framework.Lit /-! # ML-KEM on AArch64: the runs of the 16-bit NTT of a polynomial stored with 32 bits -`fwd` and the end of `ntt` (`nttTail`) read and write coefficient `p` as -the word at `4p`, halfwords `2p` (the coefficient) and `2p + 1` (zero). Their -regions (`rsU`, `rsN`: the table in the working space, as `ntt` stores it), -the terms they start from (`initU`), and the checks, by the kernel, that the -runs leave the reduced coefficients of `NTT` in the low halfwords and zero -in the high ones (`checkU_ok`, `checkN_ok`). +The end of `ntt` (`nttTail`) reads and writes coefficient `p` as the word at `4p`, halfwords +`2p` (the coefficient) and `2p + 1` (zero). Its regions (`rsN`: the table in the working space, +as `ntt` stores it), the terms it starts from (`initU`), and the check, by the kernel, that the +run leaves the reduced coefficients of `NTT` in the low halfwords and zero in the high ones +(`checkN_ok`). -/ @[expose] public section @@ -25,8 +24,6 @@ open VG VG.AArch64 /-- The polynomial at `x0`, the working space at `x1`, the table at `sym`. -/ def rsUs (sym : String) : List Rg := [⟨.reg .x0, 0, 64, true⟩, ⟨.reg .x1, 0, 32, true⟩, ⟨.sym sym, 0, 32, false⟩] -def rsU : List Rg := rsUs "VG_MLKEM_NTT16_ZETAS" - def rsN : List Rg := [⟨.reg .x0, 0, 64, true⟩, ⟨.reg .x1, 0, 32, true⟩, ⟨.reg .x1, 512, 32, false⟩] @@ -46,10 +43,6 @@ def outU (σ : St) : Bool := (List.range 64).all fun j => (List.range 8).all fun e => Nat.beq ((σ.chunk 0 j).get e) (if e % 2 = 0 then outT (4 * j + e / 2) else cT 0) -theorem checkU_ok : - (initU.runP fwdRules rsU 16 Impl.MlKem.AArch64.Ntt16.Readonly.fwd).elim false outU = true := by - lit_decide - theorem checkN_ok : (initU.runP fwdRules rsN 16 nttTail).elim false outU = true := by lit_decide diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptM.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptM.lean index 5d063bf5eb..ddf49ca26b 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptM.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptM.lean @@ -5,15 +5,15 @@ public import VerifiedGarbage.Proof.MlKem.AArch64.DecapsA public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables public import VerifiedGarbage.Proof.Sha3.AArch64.Neon.X2Lit -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.DecryptReadonlyTail -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.DecryptReadonlyU +import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.DecryptW +import VerifiedGarbage.Proof.MlKem.AArch64.Kem16.VRest +import VerifiedGarbage.Proof.MlKem.AArch64.Kem16.Ops import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.Lists import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemUContext import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemReady -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemDot import VerifiedGarbage.Proof.Framework.AArch64.Syms -/-! # Complete decryption using a single fused secret-vector dot product -/ +/-! # Complete decryption with 16-bit coefficients and a single fused secret-vector dot product -/ @[expose] public section namespace VG.Proof.MlKem.AArch64.Decaps @@ -39,22 +39,24 @@ theorem fused_m_ok {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) (ht apply WPs.seqs (by simp) rw [List.append_assoc] apply WPs.append - refine WPs.mono_syms (readonly_u_all_frame hp hc htr h) fun t ⟨dt,ft,trt⟩ hs => ?_ + refine WPs.mono_syms (uW_all hp hc htr ⟨h.kb,h.x24,fun _ hj => by omega⟩) fun t ⟨dt,ft,trt⟩ hs => ?_ have gt : Dot.GammaArtifact t.mem (t.syms "VG_MLKEM_GAMMA") := by rw [hs]; apply gamma_frame ht ft intro r hh - rcases mem2' hh with rfl | rfl <;> exact far _ _ (by lom) + simp only [uWritesW, List.mem_cons, List.not_mem_nil, or_false] at hh + rcases hh with rfl | rfl | rfl <;> exact far _ _ (by lom) have hT : ∀j ntt (KPke.dcU P.params (cD P s₀) j)) 1 0 t := ⟨dt.kb,dt.x24,fun _ hj => by omega,dt.u,Frame.refl _ _⟩ apply WPs.append - have dec := decode_all hp (kE := 0) (eo := 0) (by decide) (by change 0<2; decide) (by dek) hT d0 + have dec := WPs.range (fun _ hj _ h => decode_stepW hp (kE := 0) (eo := 0) (by decide) (by change 0<2; decide) + (by dek) hj hT h) d0 simp only [Nat.zero_add] at dec refine WPs.mono_syms dec fun u du hsu => ?_ have gu : Dot.GammaArtifact u.mem (u.syms "VG_MLKEM_GAMMA") := by @@ -71,11 +73,11 @@ theorem fused_m_ok {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) (ht rw [hsu,hs]; exact (far _ _ (by lom)).symm have tru := trt.frame hsu (du.kb.rd.trans dt.kb.rd.symm) (du.kb.wr.trans dt.kb.wr.symm) (du.kb.sp.trans dt.kb.sp.symm) du.frame (cov_w hp dt.kb ⟨hp.scw,hp.scb⟩ (hp.fs (by lom))) - refine WPs.cons (WP.mono_syms (fused_ok hp (po_TP hw) du.kb (decodedTerms P) - (decoded_ready hp du gu cu au).data) fun w ⟨kw,fw,pw,xw⟩ hus => ?_) + refine WPs.cons (WP.mono_syms (fusedW_ok hp (po_TP hw) du.kb (decodedTerms P) + (decoded_readyW hp du gu cu au).data) fun w ⟨kw,fw,pw,xw⟩ hus => ?_) rw [decoded_sum] at pw have trw := tru.frame hus (kw.rd.trans du.kb.rd.symm) (kw.wr.trans du.kb.wr.symm) (kw.sp.trans du.kb.sp.symm) fw (cov_w hp du.kb ⟨hp.scw,hp.scb⟩ (hp.fs (by lom))) - exact readonly_m_tail hp hc trw kw (xw.trans du.x24) pw + exact m_tailW hp hc trw kw (xw.trans du.x24) pw end VG.Proof.MlKem.AArch64.Decaps diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyTail.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyTail.lean deleted file mode 100644 index 15e05bbce3..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyTail.lean +++ /dev/null @@ -1,51 +0,0 @@ -module -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.ReadonlyCalls -public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables -public import VerifiedGarbage.Proof.MlKem.AArch64.DecapsA -public import VerifiedGarbage.Proof.Sha3.AArch64.Neon.X2Lit - -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemReadonlyTransform -import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc - -/-! # Finish decryption after the fused secret-vector product -/ - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Decaps -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 -open VG.Proof.MlKem.AArch64.Kem -open VG.Spec.MlKem -open VG.Spec.Sha3 (bytesAt) -variable {P : KemLay} - -theorem readonly_m_tail {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) - (ht : Optimized.TransformTables s) - (hk : KB P (deL P) s₀ s) (hx : s.gpr .x24=1) - (t₄ : PolyIs s.mem (sA (deL P) s₀ (TP P)) - (KPke.dotK (dcS (KPke.dkPke P.params (dkD P s₀))) - (fun j => ntt (KPke.dcU P.params (cD P s₀) j)) P.k)) : - WPs [Ntt16.Readonly.nttInvAt (TP P),P.ddLAt .x26 (32*P.du*P.k) P.dv (EP P),subAt (EP P) (TP P), - ceAt (EP P) 1 .x28 MB] s fun t => KB P (deL P) s₀ t ∧ t.gpr .x24=1 ∧ - bytesAt t.mem (sA (deL P) s₀ MB) 32=mD P s₀ := by - have hw := hp.wf - refine WPs.cons (WP.mono (readonly_nttInv_ok hp (po_TP hw) hk t₄.1 ht) fun s₅ ⟨kb₅, f₅, t₅, x₅⟩ => ?_) - rw [t₄.2] at t₅ - have fv : 32 * P.du * P.k + 32 * P.dv ≤ (deL P).len 1 := by - simp only [deL, List.getD_cons_zero, List.getD_cons_succ, KemLay.ctLen] - rw [Nat.mul_add, Nat.mul_assoc] - refine WPs.cons (WP.mono (ddL_ok hp hc (k := 1) (o := 32 * P.du * P.k) (d := P.dv) (off := EP P) (by decide) - fv (.inr rfl) (po_EP hw) (.inl (by dek)) kb₅) fun s₆ ⟨kb₆, f₆, p₆, x₆⟩ => ?_) - rw [slice_eq kb₅ (b := 1) (by decide) fv] at p₆ - have t₆ := polyIs_frame f₆ (fun r hr => by - rw [List.mem_singleton.mp hr]; exact papart hp (po_TP hw) (po_EP hw) (.inr (by kom))) t₅ - refine WPs.cons (WP.mono (sub_ok hp (po_EP hw) (po_TP hw) (.inl (by kom)) kb₆ p₆.1 t₆.1) - fun s₇ ⟨kb₇, f₇, p₇, x₇⟩ => ?_) - rw [p₆.2, t₆.2] at p₇ - refine WPs.single (WP.mono (ce_ok hp (off := EP P) (d := 1) (k := 3) (o := MB) (po_EP hw) (by decide) - (by decide) (by dek) (by dek) (.inr ⟨by kom, .inl (by kom)⟩) kb₇ p₇.1) fun s₈ ⟨kb₈, f₈, b₈, x₈⟩ => ?_) - refine ⟨kb₈, by rw [x₈, x₇, x₆, x₅, hx], ?_⟩ - rw [show sA (deL P) s₀ MB = kA s₀ ((deL P).slot 3) + BitVec.ofNat 64 MB from rfl, b₈, p₇.2, mD, KPke.decM, - KPke.kpkeDecrypt_eq] - rfl - - -end VG.Proof.MlKem.AArch64.Decaps diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyU.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyU.lean deleted file mode 100644 index f9f807989c..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptReadonlyU.lean +++ /dev/null @@ -1,65 +0,0 @@ -module - -import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemReadonlyTransform -public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.DecryptU -public import VerifiedGarbage.Impl.MlKem.AArch64.ReadonlyDecrypt -public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables -import VerifiedGarbage.Proof.Framework.AArch64.Syms -import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc - -/-! # Ciphertext vector transformation with an explicit memory footprint -/ - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Decaps -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 -open VG.Proof.MlKem.AArch64.Kem -open VG.Spec.MlKem -open VG.Spec.Sha3 (bytesAt) -variable {P : KemLay} - -theorem readonly_u_step_frame {s₀ : State} (hp : Pre P (deL P) s₀) (hc : Calls P) {i : Nat} (hi : i < P.k) {s : State} - (ht : Optimized.TransformTables s) - (h : DInv P s₀ i s) : WP isa (Impl.MlKem.AArch64.Optimized.readonlyDeU P i) s (fun t => DInv P s₀ (i + 1) t ∧ Frame [R (kA s₀) (deL P).sc (yOff P i) 1024,R (kA s₀) (deL P).sc NS 1024] s.mem t.mem) := by - have hw := hp.wf - have hpo := po_y hw hi - have hu := du_le hi - have fu : 32 * P.du * i + 32 * P.du ≤ (deL P).len 1 := by dek - refine WP.seq (WP.mono_syms (ddL_ok hp hc (k := 1) (o := 32 * P.du * i) (d := P.du) (off := yOff P i) (by decide) - fu (.inl rfl) hpo (.inl (by dek)) h.kb) - fun s₁ ⟨kb₁, f₁, p₁, x₁⟩ sy₁ => ?_) - have e₁ := h.frame kb₁ x₁ f₁ fun r hr => by - rw [List.mem_singleton.mp hr]; exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) - rw [slice_eq h.kb (b := 1) (by decide) fu] at p₁ - have ht₁ := ht.frame sy₁ (kb₁.rd.trans h.kb.rd.symm) (kb₁.wr.trans h.kb.wr.symm) (kb₁.sp.trans h.kb.sp.symm) - f₁ (cov_w hp h.kb ⟨hp.scw,hp.scb⟩ (o:=yOff P i) (l:=1024) (hp.fs hpo.le)) - refine WP.mono (readonly_ntt_ok hp hpo kb₁ p₁.1 ht₁) fun s₂ ⟨kb₂, f₂, p₂, x₂⟩ => ?_ - have e₂ := e₁.frame kb₂ x₂ f₂ fun r hr => by - rcases mem2' hr with rfl | rfl - · exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) - · exact dfar hp (by omega) (by kom) (.inl (by kom)) - rw [p₁.2] at p₂ - refine ⟨⟨e₂.kb,e₂.x24,fun i' hi' => ?_⟩,(f₁.mono (by simp)).trans f₂⟩ - rcases (by omega : i' < i ∨ i' = i) with hi' | rfl - · exact e₂.u i' hi' - · refine (congrArg (fun x => PolyIs s₂.mem (sA (deL P) s₀ (yOff P i')) (ntt x)) ?_).mp p₂ - simp only [KPke.dcU, cD, deL, List.getD_cons_zero, List.getD_cons_succ, KemLay.params] - - -theorem readonly_u_all_frame {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) - (ht : Optimized.TransformTables s) (h : DInv P s₀ 0 s) : - WPs ((List.range P.k).map (Impl.MlKem.AArch64.Optimized.readonlyDeU P)) s fun t => - DInv P s₀ P.k t ∧ Frame (uWrites P s₀) s.mem t.mem ∧ Optimized.TransformTables t := by - apply WPs.range (I := fun i t => DInv P s₀ i t ∧ Frame (uWrites P s₀) s.mem t.mem ∧ Optimized.TransformTables t) - · intro i hi t ⟨di,fi,ti⟩ - refine WP.mono_syms (readonly_u_step_frame hp hc hi ti di) fun u ⟨du,fu⟩ us => ?_ - have hw := hp.wf - have cc := covers_cons (cov_s hp di.kb (po_y hw hi).le) (cov_s hp di.kb (o:=NS) (l:=1024) (by kom)) - exact ⟨du,fi.trans (fu.sub (fun r hh => by - rcases mem2' hh with rfl | rfl - · exact ⟨R (kA s₀) (deL P).sc (YH P) (1024*P.k),by simp [uWrites], - R.sub2 (by simp [yOff]) (by simp only [yOff]; omega)⟩ - · exact ⟨_,by simp [uWrites],fun _ hh => hh⟩)), - ti.frame us (du.kb.rd.trans di.kb.rd.symm) (du.kb.wr.trans di.kb.wr.symm) (du.kb.sp.trans di.kb.sp.symm) fu cc⟩ - · exact ⟨h,Frame.refl _ _,ht⟩ - -end VG.Proof.MlKem.AArch64.Decaps diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptU.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptU.lean deleted file mode 100644 index 6ba1b05d02..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptU.lean +++ /dev/null @@ -1,56 +0,0 @@ -module - -public import VerifiedGarbage.Proof.MlKem.AArch64.DecapsA -public import VerifiedGarbage.Proof.Sha3.AArch64.Neon.X2Lit -import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc - -/-! # Ciphertext vector transformation with an explicit memory footprint -/ - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Decaps -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 -open VG.Proof.MlKem.AArch64.Kem -open VG.Spec.MlKem -open VG.Spec.Sha3 (bytesAt) -variable {P : KemLay} - -theorem u_step_frame {s₀ : State} (hp : Pre P (deL P) s₀) (hc : Calls P) {i : Nat} (hi : i < P.k) {s : State} - (h : DInv P s₀ i s) : WP isa (P.deU i) s (fun t => DInv P s₀ (i + 1) t ∧ Frame [R (kA s₀) (deL P).sc (yOff P i) 1024,R (kA s₀) (deL P).sc NS 1024] s.mem t.mem) := by - have hw := hp.wf - have hpo := po_y hw hi - have hu := du_le hi - have fu : 32 * P.du * i + 32 * P.du ≤ (deL P).len 1 := by dek - refine WP.seq (WP.mono (ddL_ok hp hc (k := 1) (o := 32 * P.du * i) (d := P.du) (off := yOff P i) (by decide) - fu (.inl rfl) hpo (.inl (by dek)) h.kb) - fun s₁ ⟨kb₁, f₁, p₁, x₁⟩ => ?_) - have e₁ := h.frame kb₁ x₁ f₁ fun r hr => by - rw [List.mem_singleton.mp hr]; exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) - rw [slice_eq h.kb (b := 1) (by decide) fu] at p₁ - refine WP.mono (ntt_ok hp hpo kb₁ p₁.1) fun s₂ ⟨kb₂, f₂, p₂, x₂⟩ => ?_ - have e₂ := e₁.frame kb₂ x₂ f₂ fun r hr => by - rcases mem2' hr with rfl | rfl - · exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) - · exact dfar hp (by omega) (by kom) (.inl (by kom)) - rw [p₁.2] at p₂ - refine ⟨⟨e₂.kb,e₂.x24,fun i' hi' => ?_⟩,(f₁.mono (by simp)).trans f₂⟩ - rcases (by omega : i' < i ∨ i' = i) with hi' | rfl - · exact e₂.u i' hi' - · refine (congrArg (fun x => PolyIs s₂.mem (sA (deL P) s₀ (yOff P i')) (ntt x)) ?_).mp p₂ - simp only [KPke.dcU, cD, deL, List.getD_cons_zero, List.getD_cons_succ, KemLay.params] - - -def uWrites (P : KemLay) (s₀ : State) : List Region := - [R (kA s₀) (deL P).sc (YH P) (1024*P.k),R (kA s₀) (deL P).sc NS 1024] - -theorem u_all_frame {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) (h : DInv P s₀ 0 s) : - WPs ((List.range P.k).map P.deU) s fun t => DInv P s₀ P.k t ∧ Frame (uWrites P s₀) s.mem t.mem := by - apply WPs.range (I := fun i t => DInv P s₀ i t ∧ Frame (uWrites P s₀) s.mem t.mem) - · intro i hi t ⟨di,fi⟩ - exact WP.mono (u_step_frame hp hc hi di) fun u ⟨du,fu⟩ => ⟨du,fi.trans (fu.sub (fun r hh => by - rcases mem2' hh with rfl | rfl - · exact ⟨R (kA s₀) (deL P).sc (YH P) (1024*P.k),by simp [uWrites], - R.sub2 (by simp [yOff]) (by simp only [yOff]; omega)⟩ - · exact ⟨_,by simp [uWrites],fun _ hh => hh⟩))⟩ - · exact ⟨h,Frame.refl _ _⟩ - -end VG.Proof.MlKem.AArch64.Decaps diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptW.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptW.lean new file mode 100644 index 0000000000..ed6f9becbd --- /dev/null +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/DecryptW.lean @@ -0,0 +1,127 @@ +module + +public import VerifiedGarbage.Proof.MlKem.AArch64.DecapsA +public import VerifiedGarbage.Impl.MlKem.AArch64.Optimized +public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables +public import VerifiedGarbage.Proof.Sha3.AArch64.Neon.X2Lit +import VerifiedGarbage.Proof.MlKem.AArch64.Kem16.Ops +import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc +import VerifiedGarbage.Proof.Framework.AArch64.Syms + +/-! # Decryption with 16-bit coefficients: `û'` and the tail + +`uW_all`: each row of `u` decoded and decompressed into `PP`, narrowed to 16 bits at `yOff i` +and transformed by `vg_mlkem16_ntt`. `m_tailW`: after the 16-bit dot product `ŝᵀ ∘ û'`, its +inverse transform, widened into `PP`, subtracted from the decompressed `v` and compressed into +`m'`. -/ + +@[expose] public section +namespace VG.Proof.MlKem.AArch64.Decaps +open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 +open VG.Proof.MlKem.AArch64.Kem +open VG.Spec.MlKem +open VG.Spec.Sha3 (bytesAt) +variable {P : KemLay} + +/-- After `û'[i]` for `i < n`, stored with 16 bits. -/ +structure DInvW (P : KemLay) (s₀ : State) (n : Nat) (s : State) : Prop where + kb : KB P (deL P) s₀ s + x24 : s.gpr .x24 = 1 + u : ∀ i < n, PolyIsW 16 s.mem (sA (deL P) s₀ (yOff P i)) (ntt (KPke.dcU P.params (cD P s₀) i)) + +theorem DInvW.frame {s₀ : State} {n : Nat} {s s' : State} (h : DInvW P s₀ n s) (hk : KB P (deL P) s₀ s') + (hx : s'.gpr .x24 = s.gpr .x24) {W : List Region} (hf : Frame W s.mem s'.mem) + (hW : ∀ r ∈ W, (R (kA s₀) (deL P).sc (YH P) (1024 * n)).Disjoint r) : DInvW P s₀ n s' := + ⟨hk, by rw [hx, h.x24], fun i hi => polyIsW_frame_poly (w16 := true) hf (fun r hr => (hW r hr).sub_left + (R.sub2 (by simp only [yOff]; omega) (by simp only [yOff]; omega))) (h.u i hi)⟩ + +theorem uW_step {s₀ : State} (hp : Pre P (deL P) s₀) (hc : Calls P) {i : Nat} (hi : i < P.k) {s : State} + (ht : Optimized.TransformTables s) (h : DInvW P s₀ i s) : + WP isa (Impl.MlKem.AArch64.Optimized.deU P i) s fun t => DInvW P s₀ (i + 1) t ∧ + Frame [R (kA s₀) (deL P).sc (PP P) 1024,R (kA s₀) (deL P).sc (yOff P i) 1024, + R (kA s₀) (deL P).sc NS 1024] s.mem t.mem ∧ Optimized.TransformTables t := by + have hw := hp.wf + have hpo := po_y hw hi + have hu := du_le hi + have fu : 32 * P.du * i + 32 * P.du ≤ (deL P).len 1 := by dek + apply WPs.seqs (by simp) + refine WPs.cons (WP.mono_syms (ddL_ok hp hc (k := 1) (o := 32 * P.du * i) (d := P.du) (off := PP P) + (by decide) fu (.inl rfl) (po_PP hw) (.inl (by dek)) h.kb) fun s₁ ⟨kb₁, f₁, p₁, x₁⟩ sy₁ => ?_) + have e₁ := h.frame kb₁ x₁ f₁ fun r hr => by + rw [List.mem_singleton.mp hr]; exact dfar hp (by omega) (po_PP hw).le (by lom) + rw [slice_eq h.kb (b := 1) (by decide) fu] at p₁ + have ht₁ := ht.frame sy₁ (kb₁.rd.trans h.kb.rd.symm) (kb₁.wr.trans h.kb.wr.symm) (kb₁.sp.trans h.kb.sp.symm) + f₁ (cov_w hp h.kb ⟨hp.scw,hp.scb⟩ (o:=PP P) (l:=1024) (hp.fs (po_PP hw).le)) + refine WPs.cons (WP.mono_syms (narrowW_ok hp (po_PP hw) hpo (by lom) kb₁ p₁) + fun s₂ ⟨kb₂, f₂, p₂, x₂⟩ sy₂ => ?_) + have e₂ := e₁.frame kb₂ x₂ f₂ fun r hr => by + rw [List.mem_singleton.mp hr]; exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) + have ht₂ := ht₁.frame sy₂ (kb₂.rd.trans kb₁.rd.symm) (kb₂.wr.trans kb₁.wr.symm) (kb₂.sp.trans kb₁.sp.symm) + f₂ (cov_w hp kb₁ ⟨hp.scw,hp.scb⟩ (o:=yOff P i) (l:=1024) (hp.fs hpo.le)) + refine WPs.single (WP.mono_syms (nttW_ok hp hpo kb₂ p₂ ht₂) fun s₃ ⟨kb₃, f₃, p₃, x₃⟩ sy₃ => ?_) + have e₃ := e₂.frame kb₃ x₃ f₃ fun r hr => by + rcases mem2' hr with rfl | rfl + · exact dfar hp (by omega) hpo.le (.inr (by simp only [yOff]; omega)) + · exact dfar hp (by omega) (by kom) (.inl (by kom)) + have ht₃ := ht₂.frame sy₃ (kb₃.rd.trans kb₂.rd.symm) (kb₃.wr.trans kb₂.wr.symm) (kb₃.sp.trans kb₂.sp.symm) + f₃ (covers_cons (cov_s hp kb₂ hpo.le) (cov_s hp kb₂ (o:=NS) (l:=1024) (by kom))) + refine ⟨⟨e₃.kb,e₃.x24,fun i' hi' => ?_⟩, + ((f₁.mono (by simp)).trans (f₂.mono (by simp))).trans (f₃.mono (by simp)),ht₃⟩ + rcases (by omega : i' < i ∨ i' = i) with hi' | rfl + · exact e₃.u i' hi' + · refine (congrArg (fun x => PolyIsW 16 s₃.mem (sA (deL P) s₀ (yOff P i')) (ntt x)) ?_).mp p₃ + simp only [KPke.dcU, cD, deL, List.getD_cons_zero, List.getD_cons_succ, KemLay.params] + +/-- What decoding `û'` writes. -/ +def uWritesW (P : KemLay) (s₀ : State) : List Region := + [R (kA s₀) (deL P).sc (YH P) (1024*P.k),R (kA s₀) (deL P).sc NS 1024,R (kA s₀) (deL P).sc (PP P) 1024] + +theorem uW_all {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) + (ht : Optimized.TransformTables s) (h : DInvW P s₀ 0 s) : + WPs ((List.range P.k).map (Impl.MlKem.AArch64.Optimized.deU P)) s fun t => + DInvW P s₀ P.k t ∧ Frame (uWritesW P s₀) s.mem t.mem ∧ Optimized.TransformTables t := by + apply WPs.range (I := fun i t => DInvW P s₀ i t ∧ Frame (uWritesW P s₀) s.mem t.mem ∧ + Optimized.TransformTables t) + · intro i hi t ⟨di,fi,ti⟩ + refine WP.mono (uW_step hp hc hi ti di) fun u ⟨du,fu,tu⟩ => ⟨du,fi.trans (fu.sub (fun r hh => ?_)),tu⟩ + simp only [List.mem_cons, List.not_mem_nil, or_false] at hh + rcases hh with rfl | rfl | rfl + · exact ⟨_,by simp [uWritesW],fun _ hh => hh⟩ + · exact ⟨R (kA s₀) (deL P).sc (YH P) (1024*P.k),by simp [uWritesW], + R.sub2 (by simp [yOff]) (by simp only [yOff]; omega)⟩ + · exact ⟨_,by simp [uWritesW],fun _ hh => hh⟩ + · exact ⟨h,Frame.refl _ _,ht⟩ + +theorem m_tailW {s₀ s : State} (hp : Pre P (deL P) s₀) (hc : Calls P) + (ht : Optimized.TransformTables s) + (hk : KB P (deL P) s₀ s) (hx : s.gpr .x24=1) + (t₄ : PolyIsW 16 s.mem (sA (deL P) s₀ (TP P)) + (KPke.dotK (dcS (KPke.dkPke P.params (dkD P s₀))) + (fun j => ntt (KPke.dcU P.params (cD P s₀) j)) P.k)) : + WPs [Ntt16.Readonly.nttInvAtWithW (TP P) NS,Kem16.widenAt (TP P) (PP P), + P.ddLAt .x26 (32*P.du*P.k) P.dv (EP P),subAt (EP P) (PP P),ceAt (EP P) 1 .x28 MB] s + fun t => KB P (deL P) s₀ t ∧ t.gpr .x24=1 ∧ bytesAt t.mem (sA (deL P) s₀ MB) 32=mD P s₀ := by + have hw := hp.wf + refine WPs.cons (WP.mono (invW_ok hp (po_TP hw) hk t₄.1 ht) fun s₅ ⟨kb₅, f₅, t₅, x₅⟩ => ?_) + rw [t₄.2] at t₅ + refine WPs.cons (WP.mono (widenW_ok hp (po_TP hw) (po_PP hw) (by lom) kb₅ t₅) + fun s₆ ⟨kb₆, f₆, t₆, x₆⟩ => ?_) + have fv : 32 * P.du * P.k + 32 * P.dv ≤ (deL P).len 1 := by + simp only [deL, List.getD_cons_zero, List.getD_cons_succ, KemLay.ctLen] + rw [Nat.mul_add, Nat.mul_assoc] + refine WPs.cons (WP.mono (ddL_ok hp hc (k := 1) (o := 32 * P.du * P.k) (d := P.dv) (off := EP P) (by decide) + fv (.inr rfl) (po_EP hw) (.inl (by dek)) kb₆) fun s₇ ⟨kb₇, f₇, p₇, x₇⟩ => ?_) + rw [slice_eq kb₆ (b := 1) (by decide) fv] at p₇ + have t₇ := polyIs_frame f₇ (fun r hr => by + rw [List.mem_singleton.mp hr]; exact papart hp (po_PP hw) (po_EP hw) (by lom)) t₆ + refine WPs.cons (WP.mono (sub_ok hp (po_EP hw) (po_PP hw) (by lom) kb₇ p₇.1 t₇.1) + fun s₈ ⟨kb₈, f₈, p₈, x₈⟩ => ?_) + rw [p₇.2, t₇.2] at p₈ + refine WPs.single (WP.mono (ce_ok hp (off := EP P) (d := 1) (k := 3) (o := MB) (po_EP hw) (by decide) + (by decide) (by dek) (by dek) (.inr ⟨by kom, .inl (by kom)⟩) kb₈ p₈.1) fun s₉ ⟨kb₉, f₉, b₉, x₉⟩ => ?_) + refine ⟨kb₉, by rw [x₉, x₈, x₇, x₆, x₅, hx], ?_⟩ + rw [show sA (deL P) s₀ MB = kA s₀ ((deL P).slot 3) + BitVec.ofNat 64 MB from rfl, b₉, p₈.2, mD, KPke.decM, + KPke.kpkeDecrypt_eq] + rfl + +end VG.Proof.MlKem.AArch64.Decaps diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDecode.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDecode.lean deleted file mode 100644 index 7a4508df9c..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDecode.lean +++ /dev/null @@ -1,65 +0,0 @@ -module -public import VerifiedGarbage.Proof.MlKem.AArch64.KemOps - -import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc - -/-! # Decode a key vector into the consumed first matrix row -/ - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Kem -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 -open VG.Spec.MlKem -open VG.Spec.Sha3 (bytesAt) -variable {P : KemLay} {L : Layout} - -structure DecodeInv (P : KemLay) (L : Layout) (s₀ : State) (mD : Mem) - (T Y : Nat → Poly) (v : BitVec 64) (n : Nat) (s : State) : Prop where - kb : KB P L s₀ s - x24 : s.gpr .x24=v - decoded : ∀ j ?_ - rw [h.kb.ro_bytes hread (by omega),ht j hj] at pt - refine ⟨kt,xt.trans h.x24,?_,?_,?_⟩ - · intro i hi - rcases (by omega : i by - rw [List.mem_singleton.mp hh] - exact papart hp (po_a hw h0 (by omega)) (po_a hw h0 hj) (.inl (by lom))) (h.decoded i hi) - · exact pt - · intro i hi - exact polyIs_frame ft (fun r hh => by - rw [List.mem_singleton.mp hh] - exact papart hp (po_y hw hi) (po_a hw h0 hj) (.inr (by - have := Nat.mul_le_mul_right P.k hw.facts.1 - lom))) (h.y i hi) - · exact (h.frame.sub (fun r hh => by - rw [List.mem_singleton.mp hh] - exact ⟨_,List.mem_singleton_self _,R.sub2 (by omega) (by omega)⟩)).trans - (ft.sub (fun r hh => by - rw [List.mem_singleton.mp hh] - exact ⟨_,List.mem_singleton_self _,R.sub2 (by simp [aOff]) (by simp [aOff]; omega)⟩)) - -theorem decode_all {s₀ s : State} (hp : Pre P L s₀) {kE eo : Nat} - (hkE : kE<4) (hread : L.slot kE dec12At (slotReg kE) (eo+384*j) (aOff P 0 j)) s - (DecodeInv P L s₀ mD T Y v P.k) := - WPs.range (fun _ hj _ h => decode_step hp hkE hread hfit hj ht h) h - -end VG.Proof.MlKem.AArch64.Kem diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDot.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDot.lean deleted file mode 100644 index 25bafddbcd..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemDot.lean +++ /dev/null @@ -1,29 +0,0 @@ -module -public import VerifiedGarbage.Proof.MlKem.AArch64.KemOps -public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Result - -import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Core - -/-! # Fused polynomial products preserve the KEM call invariant -/ - -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Kem -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 -open VG.Spec.MlKem -variable {P : KemLay} {L : Layout} - -theorem fused_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P off) - (hk : KB P L s₀ s) (terms : List (Nat × Nat)) {F G : (Nat × Nat) → Poly} - (hd : Dot.LoopData s (s.gpr .x28+BitVec.ofNat 64 off) (s.syms "VG_MLKEM_GAMMA") terms - (Dot.polyValues F) (Dot.polyValues G) Dot.gammaValues) : - WP isa (Impl.MlKem.AArch64.Dot.dot off terms) s fun t => - KB P L s₀ t ∧ Frame [R (kA s₀) L.sc off 1024] s.mem t.mem ∧ - PolyIs t.mem (sA L s₀ off) (Dot.polySum terms F G) ∧ t.gpr .x24 = s.gpr .x24 := by - have hw := hp.wf - have hoff := ho.le - refine WP.mono (Dot.dot_ok (by kom) terms hd) fun t ⟨kt,ft,pt⟩ => ?_ - rw [e28 hk] at ft pt - exact ⟨hk.frame kt ft (by decide) (fun r hr => by - rw [List.mem_singleton.mp hr]; exact ho.safe hp),ft,pt,kt.get .x24 (by decide)⟩ - -end VG.Proof.MlKem.AArch64.Kem diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReadonlyTransform.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReadonlyTransform.lean deleted file mode 100644 index edd27777cd..0000000000 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReadonlyTransform.lean +++ /dev/null @@ -1,40 +0,0 @@ -module -public import VerifiedGarbage.Proof.MlKem.AArch64.KemOps -public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.TransformTables -public import VerifiedGarbage.Impl.MlKem.AArch64.Ntt16.ReadonlyCalls -import VerifiedGarbage.Proof.MlKem.AArch64.Ntt16.Readonly.At -@[expose] public section -namespace VG.Proof.MlKem.AArch64.Kem -open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKem.AArch64 VG.Spec.MlKem -variable {P : KemLay} {L : Layout} - -theorem readonly_ntt_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P off) - (hk : KB P L s₀ s) (hr : Reduced s.mem (sA L s₀ off)) (ht : Optimized.TransformTables s) : - WP isa (Ntt16.Readonly.nttAt off) s fun t=> - KB P L s₀ t ∧ Frame [R (kA s₀) L.sc off 1024,R (kA s₀) L.sc NS 1024] s.mem t.mem ∧ - PolyIs t.mem (sA L s₀ off) (ntt (polyAt s.mem (sA L s₀ off))) ∧ t.gpr .x24=s.gpr .x24 := by - have hw := hp.wf - have hc := covers_cons (cov_s hp hk ho.le) (cov_s hp hk (o:=NS) (l:=1024) (by kom)) - refine WP.mono (Ntt16.Readonly.nttAtWith_ok (by obtain ⟨_,hh⟩:=ho; kom) (by decide) - (by rw [e28 hk]) (by rw [e28 hk]) (ho.ns hp) ⟨hr,rfl⟩ hc (ht.forward.ready hc)) fun t ⟨kt,pt⟩=>?_ - refine ⟨hk.call kt ?_,kt.frame,pt,kt.cs _ (by decide) (by decide)⟩ - intro r hh - rcases mem2' hh with rfl|rfl - · exact ho.safe hp - · exact safe_scr hp (by kom) - -theorem readonly_nttInv_ok {s₀ s : State} (hp : Pre P L s₀) {off : Nat} (ho : PO P off) - (hk : KB P L s₀ s) (hr : Reduced s.mem (sA L s₀ off)) (ht : Optimized.TransformTables s) : - WP isa (Ntt16.Readonly.nttInvAt off) s fun t=> - KB P L s₀ t ∧ Frame [R (kA s₀) L.sc off 1024,R (kA s₀) L.sc NS 1024] s.mem t.mem ∧ - PolyIs t.mem (sA L s₀ off) (nttInv (polyAt s.mem (sA L s₀ off))) ∧ t.gpr .x24=s.gpr .x24 := by - have hw := hp.wf - have hc := covers_cons (cov_s hp hk ho.le) (cov_s hp hk (o:=NS) (l:=1024) (by kom)) - refine WP.mono (Ntt16.Readonly.nttInvAtWith_ok (by obtain ⟨_,hh⟩:=ho; kom) (by decide) - (by rw [e28 hk]) (by rw [e28 hk]) (ho.ns hp) ⟨hr,rfl⟩ hc (ht.inverse.ready hc)) fun t ⟨kt,pt⟩=>?_ - refine ⟨hk.call kt ?_,kt.frame,pt,kt.cs _ (by decide) (by decide)⟩ - intro r hh - rcases mem2' hh with rfl|rfl - · exact ho.safe hp - · exact safe_scr hp (by kom) -end VG.Proof.MlKem.AArch64.Kem diff --git a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReady.lean b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReady.lean index f03f8a9501..7b0f3f3398 100644 --- a/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReady.lean +++ b/lean/VerifiedGarbage/Proof/MlKem/AArch64/Optimized/KemReady.lean @@ -1,11 +1,13 @@ module -public import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.KemDecode public import VerifiedGarbage.Proof.MlKem.AArch64.Dot.Ready import VerifiedGarbage.Proof.MlKem.AArch64.Optimized.DotSpec import VerifiedGarbage.Proof.MlKem.AArch64.KemEnc +public import Init.Data.List.Basic +public import VerifiedGarbage.Impl.MlKem.AArch64.Kem +public import VerifiedGarbage.Proof.MlKem.KPke -/-! # Establish the fused arithmetic preconditions from KEM buffer facts -/ +/-! # The terms of the fused dot product of `ŝ` and `û'` and their sum -/ @[expose] public section namespace VG.Proof.MlKem.AArch64.Kem @@ -13,30 +15,6 @@ open VG VG.AArch64 VG.Impl.MlKem.AArch64 VG.Impl.MlKem.AArch64.KEM VG.Proof.MlKe open VG.Spec.MlKem variable {P : KemLay} {L : Layout} -theorem fused_ready {s₀ s : State} (hp : Pre P L s₀) (hk : KB P L s₀ s) - {off : Nat} (ho : PO P off) {terms : List (Nat × Nat)} {F G : (Nat × Nat) → Poly} - (hn : terms.length≤4) - (hf : ∀ fg∈terms, PO P fg.1) (hg : ∀ fg∈terms, PO P fg.2) - (hF : ∀ fg∈terms, PolyIs s.mem (sA L s₀ fg.1) (F fg)) - (hG : ∀ fg∈terms, PolyIs s.mem (sA L s₀ fg.2) (G fg)) - (df : ∀ fg∈terms, off+1024≤fg.1 ∨ fg.1+1024≤off) - (dg : ∀ fg∈terms, off+1024≤fg.2 ∨ fg.2+1024≤off) - (ht : Dot.GammaArtifact s.mem (s.syms "VG_MLKEM_GAMMA")) - (hc : Covers [⟨s.syms "VG_MLKEM_GAMMA",512⟩] (s.rd++s.wr)) - (ha : (polyRegion (sA L s₀ off)).Disjoint ⟨s.syms "VG_MLKEM_GAMMA",512⟩) : - Dot.Ready s (s.gpr .x28+BitVec.ofNat 64 off) (s.syms "VG_MLKEM_GAMMA") terms F G := by - refine ⟨hn,?_,?_,?_,?_,?_,?_,?_,?_,?_,?_,hc,ht⟩ - · intro fg hh; have := (hf fg hh).le; kom - · intro fg hh; have := (hg fg hh).le; kom - · intro fg hh; rw [e28 hk]; exact hF fg hh - · intro fg hh; rw [e28 hk]; exact hG fg hh - · intro fg hh; rw [e28 hk,e28 hk]; exact papart hp ho (hf fg hh) (df fg hh) - · intro fg hh; rw [e28 hk,e28 hk]; exact papart hp ho (hg fg hh) (dg fg hh) - · intro fg hh; rw [e28 hk]; exact cov_sr hp hk (hf fg hh).le - · intro fg hh; rw [e28 hk]; exact cov_sr hp hk (hg fg hh).le - · rw [e28 hk]; exact cov_s hp hk ho.le - · rwa [e28 hk] - def decodedTerms (P : KemLay) : List (Nat × Nat) := (List.range P.k).map fun j => (aOff P 0 j,yOff P j) @@ -49,35 +27,4 @@ theorem decoded_sum (T Y : Nat → Poly) : · intro j hj; simp [decodedF,aOff] · intro j hj; simp [decodedG,yOff] - -theorem decoded_ready {s₀ s : State} (hp : Pre P L s₀) - {mD : Mem} {T Y : Nat → Poly} {v : BitVec 64} - (h : DecodeInv P L s₀ mD T Y v P.k s) - (ht : Dot.GammaArtifact s.mem (s.syms "VG_MLKEM_GAMMA")) - (hc : Covers [⟨s.syms "VG_MLKEM_GAMMA",512⟩] (s.rd++s.wr)) - (ha : (polyRegion (sA L s₀ (TP P))).Disjoint ⟨s.syms "VG_MLKEM_GAMMA",512⟩) : - Dot.Ready s (s.gpr .x28+BitVec.ofNat 64 (TP P)) (s.syms "VG_MLKEM_GAMMA") - (decodedTerms P) (decodedF T) (decodedG P Y) := by - have hw := hp.wf - have h0 : 0 u32 { - core::arch::naked_asm!( - r".arch_extension sha3 -str x24, [x3, #30392] -str x30, [x3, #30400] -str x25, [x3, #30408] -str x26, [x3, #30416] -str x27, [x3, #30424] -str x28, [x3, #30432] -add x25, x0, #0 -add x26, x1, #0 -add x27, x2, #0 -add x28, x3, #0 -movz x24, #1, lsl #0 -add x0, x26, #0 -movz x1, #352, lsl #0 -movz x2, #11, lsl #0 -movz x3, #20632, lsl #0 -add x3, x28, x3 -bl {vg_mlkem1024_decode_decompress} -movz x0, #20632, lsl #0 -add x0, x28, x0 -add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -20: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 20b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -21: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 21b -add x0, x26, #352 -movz x1, #352, lsl #0 -movz x2, #11, lsl #0 -movz x3, #21656, lsl #0 -add x3, x28, x3 -bl {vg_mlkem1024_decode_decompress} -movz x0, #21656, lsl #0 -add x0, x28, x0 -add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -22: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 22b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -23: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 23b -add x0, x26, #704 -movz x1, #352, lsl #0 -movz x2, #11, lsl #0 -movz x3, #22680, lsl #0 -add x3, x28, x3 -bl {vg_mlkem1024_decode_decompress} -movz x0, #22680, lsl #0 -add x0, x28, x0 -add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -24: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 24b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -25: +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h +ldr q18, [x3, #0] +ldr q28, [x3, #16] +sqdmulh v24.4s, v22.4s, v17.4s +mls v22.4s, v24.4s, v16.4s +mla v21.4s, v22.4s, v18.4s +sqdmulh v24.4s, v21.4s, v17.4s +mls v21.4s, v24.4s, v16.4s +sub v24.4s, v21.4s, v16.4s +umin v21.4s, v21.4s, v24.4s +sqdmulh v24.4s, v23.4s, v17.4s +mls v23.4s, v24.4s, v16.4s +sub v24.4s, v23.4s, v16.4s +umin v23.4s, v23.4s, v24.4s +sqdmulh v24.4s, v26.4s, v17.4s +mls v26.4s, v24.4s, v16.4s +mla v25.4s, v26.4s, v28.4s +sqdmulh v24.4s, v25.4s, v17.4s +mls v25.4s, v24.4s, v16.4s +sub v24.4s, v25.4s, v16.4s +umin v25.4s, v25.4s, v24.4s +sqdmulh v24.4s, v27.4s, v17.4s +mls v27.4s, v24.4s, v16.4s +sub v24.4s, v27.4s, v16.4s +umin v27.4s, v27.4s, v24.4s +uzp1 v0.8h, v21.8h, v25.8h +uzp1 v1.8h, v23.8h, v27.8h +zip1 v2.8h, v0.8h, v1.8h +zip2 v3.8h, v0.8h, v1.8h +str q2, [x0, #0] +str q3, [x0, #16] +add x0, x0, #32 +add x3, x3, #32 +add x12, x12, #32 +sub x11, x11, #1 +cbnz x11, 21273b +movz x0, #25752, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +movz x9, #3329, lsl #0 +dup v31.8h, w9 +movz x9, #10, lsl #0 +dup v30.8h, w9 +movi v29.2d, #0 +add x2, x0, #0 +add x3, x1, #0", + concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), + concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), + r"movz x5, #8, lsl #0 +21274: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -95585,418 +93566,652 @@ ldr q20, [x6, #0] ldr q21, [x6, #16] ldr q22, [x6, #32] ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h zip2 v24.8h, v20.8h, v20.8h zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s zip1 v28.8h, v22.8h, v22.8h zip1 v19.8h, v23.8h, v23.8h +zip1 v26.4s, v24.4s, v24.4s +zip1 v27.4s, v25.4s, v25.4s trn1 v4.2d, v0.2d, v1.2d trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h trn1 v0.4s, v4.4s, v5.4s trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h +sub v17.8h, v1.8h, v0.8h +add v0.8h, v0.8h, v1.8h +mul v1.8h, v17.8h, v28.8h +sqrdmulh v17.8h, v17.8h, v19.8h +mls v1.8h, v17.8h, v31.8h trn1 v4.4s, v0.4s, v1.4s trn2 v5.4s, v0.4s, v1.4s +sub v17.8h, v5.8h, v4.8h +add v4.8h, v4.8h, v5.8h +mul v5.8h, v17.8h, v26.8h +sqrdmulh v17.8h, v17.8h, v27.8h +mls v5.8h, v17.8h, v31.8h trn1 v0.2d, v4.2d, v5.2d trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s zip2 v28.8h, v22.8h, v22.8h zip2 v19.8h, v23.8h, v23.8h +zip2 v26.4s, v24.4s, v24.4s +zip2 v27.4s, v25.4s, v25.4s trn1 v4.2d, v2.2d, v3.2d trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h trn1 v2.4s, v4.4s, v5.4s trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h +sub v17.8h, v3.8h, v2.8h +add v2.8h, v2.8h, v3.8h +mul v3.8h, v17.8h, v28.8h +sqrdmulh v17.8h, v17.8h, v19.8h +mls v3.8h, v17.8h, v31.8h trn1 v4.4s, v2.4s, v3.4s trn2 v5.4s, v2.4s, v3.4s +sub v17.8h, v5.8h, v4.8h +add v4.8h, v4.8h, v5.8h +mul v5.8h, v17.8h, v26.8h +sqrdmulh v17.8h, v17.8h, v27.8h +mls v5.8h, v17.8h, v31.8h trn1 v2.2d, v4.2d, v5.2d trn2 v3.2d, v4.2d, v5.2d +dup v24.8h, v20.h[1] +dup v25.8h, v21.h[1] +sub v17.8h, v1.8h, v0.8h +add v0.8h, v0.8h, v1.8h +mul v1.8h, v17.8h, v24.8h +sqrdmulh v17.8h, v17.8h, v25.8h +mls v1.8h, v17.8h, v31.8h +dup v24.8h, v20.h[2] +dup v25.8h, v21.h[2] +sub v17.8h, v3.8h, v2.8h +add v2.8h, v2.8h, v3.8h +mul v3.8h, v17.8h, v24.8h +sqrdmulh v17.8h, v17.8h, v25.8h +mls v3.8h, v17.8h, v31.8h sqrdmulh v16.8h, v0.8h, v30.8h mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h sqrdmulh v16.8h, v1.8h, v30.8h mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h sqrdmulh v16.8h, v2.8h, v30.8h mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h sqrdmulh v16.8h, v3.8h, v30.8h mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] +dup v24.8h, v20.h[0] +dup v25.8h, v21.h[0] +sub v17.8h, v2.8h, v0.8h +add v0.8h, v0.8h, v2.8h +mul v2.8h, v17.8h, v24.8h +sqrdmulh v17.8h, v17.8h, v25.8h +mls v2.8h, v17.8h, v31.8h +sub v17.8h, v3.8h, v1.8h +add v1.8h, v1.8h, v3.8h +mul v3.8h, v17.8h, v24.8h +sqrdmulh v17.8h, v17.8h, v25.8h +mls v3.8h, v17.8h, v31.8h +str q0, [x3, #0] +str q1, [x3, #16] +str q2, [x3, #32] +str q3, [x3, #48] add x2, x2, #64 +add x3, x3, #64 add x6, x6, #64 -add x4, x4, #128 sub x5, x5, #1 -cbnz x5, 25b -add x0, x26, #1056 -movz x1, #352, lsl #0 -movz x2, #11, lsl #0 -movz x3, #23704, lsl #0 -add x3, x28, x3 -bl {vg_mlkem1024_decode_decompress} -movz x0, #23704, lsl #0 -add x0, x28, x0 -add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 +cbnz x5, 21274b +movz x9, #848, lsl #0 mov v18.h[0], w9 -movz x9, #17019, lsl #0 +movz x9, #8347, lsl #0 mov v19.h[0], w9 -movz x9, #2580, lsl #0 +movz x9, #1897, lsl #0 mov v18.h[1], w9 -movz x9, #25395, lsl #0 +movz x9, #18673, lsl #0 mov v19.h[1], w9 -movz x9, #3289, lsl #0 +movz x9, #630, lsl #0 mov v18.h[2], w9 -movz x9, #32374, lsl #0 +movz x9, #6201, lsl #0 mov v19.h[2], w9 movz x9, #2642, lsl #0 mov v18.h[3], w9 movz x9, #26006, lsl #0 mov v19.h[3], w9 -movz x9, #630, lsl #0 +movz x9, #3289, lsl #0 mov v18.h[4], w9 -movz x9, #6201, lsl #0 +movz x9, #32374, lsl #0 mov v19.h[4], w9 -movz x9, #1897, lsl #0 +movz x9, #2580, lsl #0 mov v18.h[5], w9 -movz x9, #18673, lsl #0 +movz x9, #25395, lsl #0 mov v19.h[5], w9 -movz x9, #848, lsl #0 +movz x9, #1729, lsl #0 mov v18.h[6], w9 -movz x9, #8347, lsl #0 +movz x9, #17019, lsl #0 mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 +movz x9, #3303, lsl #0 +mov v18.h[7], w9 +movz x9, #32512, lsl #0 +mov v19.h[7], w9 +add x2, x1, #0 +add x3, x0, #0 movz x5, #4, lsl #0 -26: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h +21275: +ldr q0, [x2, #0] +ldr q1, [x2, #64] +ldr q2, [x2, #128] +ldr q3, [x2, #192] +ldr q4, [x2, #256] +ldr q5, [x2, #320] +ldr q6, [x2, #384] +ldr q7, [x2, #448] dup v22.8h, v18.h[0] dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h +sub v17.8h, v1.8h, v0.8h +add v0.8h, v0.8h, v1.8h +mul v1.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v1.8h, v17.8h, v31.8h dup v22.8h, v18.h[1] dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h +sub v17.8h, v3.8h, v2.8h +add v2.8h, v2.8h, v3.8h +mul v3.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v3.8h, v17.8h, v31.8h dup v22.8h, v18.h[2] dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h +sub v17.8h, v5.8h, v4.8h +add v4.8h, v4.8h, v5.8h +mul v5.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v5.8h, v17.8h, v31.8h dup v22.8h, v18.h[3] dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h +sub v17.8h, v7.8h, v6.8h +add v6.8h, v6.8h, v7.8h +mul v7.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v7.8h, v17.8h, v31.8h dup v22.8h, v18.h[4] dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h +sub v17.8h, v2.8h, v0.8h +add v0.8h, v0.8h, v2.8h +mul v2.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v2.8h, v17.8h, v31.8h +sub v17.8h, v3.8h, v1.8h +add v1.8h, v1.8h, v3.8h +mul v3.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v3.8h, v17.8h, v31.8h dup v22.8h, v18.h[5] dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h +sub v17.8h, v6.8h, v4.8h +add v4.8h, v4.8h, v6.8h +mul v6.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v6.8h, v17.8h, v31.8h +sub v17.8h, v7.8h, v5.8h +add v5.8h, v5.8h, v7.8h +mul v7.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v7.8h, v17.8h, v31.8h +sqrdmulh v16.8h, v0.8h, v30.8h +mls v0.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v1.8h, v30.8h +mls v1.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v2.8h, v30.8h +mls v2.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v3.8h, v30.8h +mls v3.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v4.8h, v30.8h +mls v4.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v5.8h, v30.8h +mls v5.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v6.8h, v30.8h +mls v6.8h, v16.8h, v31.8h +sqrdmulh v16.8h, v7.8h, v30.8h +mls v7.8h, v16.8h, v31.8h dup v22.8h, v18.h[6] dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 26b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -27: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h +sub v17.8h, v4.8h, v0.8h +add v0.8h, v0.8h, v4.8h +mul v4.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v4.8h, v17.8h, v31.8h +sub v17.8h, v5.8h, v1.8h +add v1.8h, v1.8h, v5.8h +mul v5.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v5.8h, v17.8h, v31.8h +sub v17.8h, v6.8h, v2.8h +add v2.8h, v2.8h, v6.8h +mul v6.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v6.8h, v17.8h, v31.8h +sub v17.8h, v7.8h, v3.8h +add v3.8h, v3.8h, v7.8h +mul v7.8h, v17.8h, v22.8h +sqrdmulh v17.8h, v17.8h, v23.8h +mls v7.8h, v17.8h, v31.8h +dup v22.8h, v18.h[7] +dup v23.8h, v19.h[7] +mul v16.8h, v0.8h, v22.8h +sqrdmulh v17.8h, v0.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #0] +mul v16.8h, v1.8h, v22.8h +sqrdmulh v17.8h, v1.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #64] +mul v16.8h, v2.8h, v22.8h +sqrdmulh v17.8h, v2.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #128] +mul v16.8h, v3.8h, v22.8h +sqrdmulh v17.8h, v3.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #192] +mul v16.8h, v4.8h, v22.8h +sqrdmulh v17.8h, v4.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #256] +mul v16.8h, v5.8h, v22.8h +sqrdmulh v17.8h, v5.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #320] +mul v16.8h, v6.8h, v22.8h +sqrdmulh v17.8h, v6.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #384] +mul v16.8h, v7.8h, v22.8h +sqrdmulh v17.8h, v7.8h, v23.8h mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 +sshr v17.8h, v16.8h, #15 +and v17.16b, v17.16b, v31.16b +add v16.8h, v16.8h, v17.8h +str q16, [x3, #448] +add x2, x2, #16 +add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 27b +cbnz x5, 21275b +movz x0, #25752, lsl #0 +add x0, x28, x0 +movz x1, #24728, lsl #0 +add x1, x28, x1 +movz x9, #3329, lsl #0 +dup v16.8h, w9 +movz x11, #32, lsl #0 +21276: +ldr q0, [x0, #0] +ldr q1, [x1, #0] +add v2.8h, v0.8h, v1.8h +cmhs v3.8h, v2.8h, v16.8h +and v3.16b, v3.16b, v16.16b +sub v2.8h, v2.8h, v3.8h +str q2, [x0, #0] +add x0, x0, #16 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 21276b +movz x0, #25752, lsl #0 +add x0, x28, x0 +movz x1, #26776, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +movi v29.2d, #0 +21277: +ldr q0, [x0, #0] +zip1 v1.8h, v0.8h, v29.8h +zip2 v2.8h, v0.8h, v29.8h +str q1, [x1, #0] +str q2, [x1, #16] +add x0, x0, #16 +add x1, x1, #32 +sub x10, x10, #1 +cbnz x10, 21277b +add x0, x28, #912 +movz x1, #32, lsl #0 +movz x2, #1, lsl #0 +movz x3, #24728, lsl #0 +add x3, x28, x3 +bl {vg_mlkem_decode_decompress} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #24728, lsl #0 +add x1, x28, x1 +bl {vg_mlkem_add} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #5, lsl #0 +add x2, x27, #1408 +movz x3, #160, lsl #0 +bl {vg_mlkem1024_compress_encode} +add x0, x24, #0 +ldr x24, [x28, #30392] +ldr x30, [x28, #30400] +ldr x25, [x28, #30408] +ldr x26, [x28, #30416] +ldr x27, [x28, #30424] +ldr x28, [x28, #30432] +ret +.arch_extension nosha3 +.p2align 6", + vg_keccak_f1600_x3_sha3 = sym super::sha3::vg_keccak_f1600_x3_sha3, + vg_keccak_f1600_x2_sha3 = sym super::sha3::vg_keccak_f1600_x2_sha3, + vg_keccak_f1600_sha3 = sym super::sha3::vg_keccak_f1600_sha3, + vg_mlkem16_cbd2 = sym super::mlkem16::vg_mlkem16_cbd2, + vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, + vg_mlkem1024_compress_encode = sym super::mlkem1024::vg_mlkem1024_compress_encode, + vg_mlkem_decode_decompress = sym super::mlkem::vg_mlkem_decode_decompress, + vg_mlkem_add = sym super::mlkem::vg_mlkem_add, + VG_MLKEM_REJ_SHUFFLE = sym super::consts::VG_MLKEM_REJ_SHUFFLE, + VG_MLKEM_GAMMA = sym super::consts::VG_MLKEM_GAMMA, + VG_MLKEM_INTT16_ZETAS = sym super::consts::VG_MLKEM_INTT16_ZETAS, + ) +} + +/// The CPU features `vg_mlkem1024_decaps_x2_sha3_keccak_sha3` requires (`Artifact.features`). +pub(crate) const VG_MLKEM1024_DECAPS_X2_SHA3_KECCAK_SHA3_FEATURES: crate::cpu::Features = crate::cpu::Features::of(&["sha3"]); + +/// ML-KEM-1024 decapsulation, `ML-KEM.Decaps_internal(dk, c)` (FIPS 203 Algorithm 18): with the decapsulation key `*dk` and the ciphertext `*ct`, writes the shared secret key to `*key`, which is the implicit rejection key `J(z ‖ c)` if the ciphertext does not re-encrypt to itself. Returns 1 on success. Returns 0 if a `SampleNTT` (FIPS 203 Algorithm 7) reaches the bound on its loop's iterations, which is at least 280 (FIPS 203 Appendix B; this happens with probability less than 2^-261): the outputs are then unspecified, and the caller must destroy them and treat the operation as failed. +/// +/// Contract: `VG.Spec.MlKem1024.decapsContract`. Constant time but for `ρ`: timing may depend on the pointers and on `ρ` (bytes 3072–3103 of `*dk`), but not on anything else of the key, on the ciphertext, or on whether it was rejected. +/// +/// Computes `J(z ‖ c)` in the third state of `vg_keccak_f1600_x3_sha3`, one block of a copy of `z ‖ c` per main squeeze of the matrix's first pairs, while the vector unit runs their SHAKE128 states. +/// +/// Computes `PRF₂(r', N)` for the first noise polynomials of the re-encryption in the third state of the pairs after those hashing `z ‖ c` and the second state of the singleton, one per main squeeze, while the vector unit runs their SHAKE128 states. +/// +/// # Safety +/// +/// * `dk` must be valid for reads of 3168 bytes. +/// * `ct` must be valid for reads of 1568 bytes. +/// * `key` must be valid for reads and writes of 32 bytes. +/// * `scratch` must be valid for reads and writes of 49152 bytes. +/// * `dk` must have been written by `vg_mlkem1024_keygen` (so that it passes the checks of FIPS 203 §7.3). +/// * `scratch` is working space: on return it holds intermediate values, which the caller must destroy (FIPS 203 §3.3). +/// * `key` and `scratch` must not overlap each other, `dk` or `ct` (distinct Rust objects never do). +/// * None of `dk`, `ct`, `key` and `scratch` may overlap the 128 bytes of stack below the stack pointer, or wrap around the end of the address space (no Rust object does). +/// * The CPU must support the `sha3` target feature. +#[unsafe(naked)] +pub(crate) unsafe extern "C" fn vg_mlkem1024_decaps_x2_sha3_keccak_sha3(dk: *const [u8; 3168], ct: *const [u8; 1568], key: *mut [u8; 32], scratch: *mut [u64; 6144]) -> u32 { + core::arch::naked_asm!( + r".arch_extension sha3 +str x24, [x3, #30392] +str x30, [x3, #30400] +str x25, [x3, #30408] +str x26, [x3, #30416] +str x27, [x3, #30424] +str x28, [x3, #30432] +add x25, x0, #0 +add x26, x1, #0 +add x27, x2, #0 +add x28, x3, #0 +movz x24, #1, lsl #0 +add x0, x26, #0 +movz x1, #352, lsl #0 +movz x2, #11, lsl #0 +movz x3, #26776, lsl #0 +add x3, x28, x3 +bl {vg_mlkem1024_decode_decompress} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #20632, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +20: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 20b +movz x0, #20632, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +bl {vg_mlkem16_ntt} +add x0, x26, #352 +movz x1, #352, lsl #0 +movz x2, #11, lsl #0 +movz x3, #26776, lsl #0 +add x3, x28, x3 +bl {vg_mlkem1024_decode_decompress} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #21656, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +21: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 21b +movz x0, #21656, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +bl {vg_mlkem16_ntt} +add x0, x26, #704 +movz x1, #352, lsl #0 +movz x2, #11, lsl #0 +movz x3, #26776, lsl #0 +add x3, x28, x3 +bl {vg_mlkem1024_decode_decompress} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #22680, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +22: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 22b +movz x0, #22680, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +bl {vg_mlkem16_ntt} +add x0, x26, #1056 +movz x1, #352, lsl #0 +movz x2, #11, lsl #0 +movz x3, #26776, lsl #0 +add x3, x28, x3 +bl {vg_mlkem1024_decode_decompress} +movz x0, #26776, lsl #0 +add x0, x28, x0 +movz x1, #23704, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +23: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 23b +movz x0, #23704, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +bl {vg_mlkem16_ntt} add x0, x25, #0 movz x1, #4248, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +24: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 24b add x0, x25, #384 movz x1, #5272, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +25: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 25b add x0, x25, #768 movz x1, #6296, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +26: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 26b add x0, x25, #1152 movz x1, #7320, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +27: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 27b movz x0, #25752, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -96008,12 +94223,15 @@ dup v16.4s, w9 dup v17.4s, w10", concat!("adrp x3, ", vg_sym_page!("{VG_MLKEM_GAMMA}")), concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), - r"movz x11, #32, lsl #0 + r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 28: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 +movi v25.2d, #0 +movi v26.2d, #0 +movi v27.2d, #0 movz x1, #4248, lsl #0 add x1, x28, x1 movz x2, #20632, lsl #0 @@ -96024,14 +94242,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #5272, lsl #0 add x1, x28, x1 movz x2, #21656, lsl #0 @@ -96042,14 +94264,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #6296, lsl #0 add x1, x28, x1 movz x2, #22680, lsl #0 @@ -96060,14 +94286,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #7320, lsl #0 add x1, x28, x1 movz x2, #23704, lsl #0 @@ -96078,15 +94308,20 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h ldr q18, [x3, #0] +ldr q28, [x3, #16] sqdmulh v24.4s, v22.4s, v17.4s mls v22.4s, v24.4s, v16.4s mla v21.4s, v22.4s, v18.4s @@ -96098,12 +94333,25 @@ sqdmulh v24.4s, v23.4s, v17.4s mls v23.4s, v24.4s, v16.4s sub v24.4s, v23.4s, v16.4s umin v23.4s, v23.4s, v24.4s -zip1 v0.4s, v21.4s, v23.4s -zip2 v1.4s, v21.4s, v23.4s -str q0, [x0, #0] -str q1, [x0, #16] +sqdmulh v24.4s, v26.4s, v17.4s +mls v26.4s, v24.4s, v16.4s +mla v25.4s, v26.4s, v28.4s +sqdmulh v24.4s, v25.4s, v17.4s +mls v25.4s, v24.4s, v16.4s +sub v24.4s, v25.4s, v16.4s +umin v25.4s, v25.4s, v24.4s +sqdmulh v24.4s, v27.4s, v17.4s +mls v27.4s, v24.4s, v16.4s +sub v24.4s, v27.4s, v16.4s +umin v27.4s, v27.4s, v24.4s +uzp1 v0.8h, v21.8h, v25.8h +uzp1 v1.8h, v23.8h, v27.8h +zip1 v2.8h, v0.8h, v1.8h +zip2 v3.8h, v0.8h, v1.8h +str q2, [x0, #0] +str q3, [x0, #16] add x0, x0, #32 -add x3, x3, #16 +add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 cbnz x11, 28b @@ -96121,18 +94369,10 @@ add x3, x1, #0", concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 29: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #32] -ldr q21, [x2, #48] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #64] -ldr q21, [x2, #80] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #96] -ldr q21, [x2, #112] -uzp1 v3.8h, v20.8h, v21.8h +ldr q0, [x2, #0] +ldr q1, [x2, #16] +ldr q2, [x2, #32] +ldr q3, [x2, #48] ldr q20, [x6, #0] ldr q21, [x6, #16] ldr q22, [x6, #32] @@ -96221,7 +94461,7 @@ str q0, [x3, #0] str q1, [x3, #16] str q2, [x3, #32] str q3, [x3, #48] -add x2, x2, #128 +add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 @@ -96368,84 +94608,76 @@ mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #0] -str q25, [x3, #16] +str q16, [x3, #0] mul v16.8h, v1.8h, v22.8h sqrdmulh v17.8h, v1.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #128] -str q25, [x3, #144] +str q16, [x3, #64] mul v16.8h, v2.8h, v22.8h sqrdmulh v17.8h, v2.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #256] -str q25, [x3, #272] +str q16, [x3, #128] mul v16.8h, v3.8h, v22.8h sqrdmulh v17.8h, v3.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #384] -str q25, [x3, #400] +str q16, [x3, #192] mul v16.8h, v4.8h, v22.8h sqrdmulh v17.8h, v4.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #512] -str q25, [x3, #528] +str q16, [x3, #256] mul v16.8h, v5.8h, v22.8h sqrdmulh v17.8h, v5.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #640] -str q25, [x3, #656] +str q16, [x3, #320] mul v16.8h, v6.8h, v22.8h sqrdmulh v17.8h, v6.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #768] -str q25, [x3, #784] +str q16, [x3, #384] mul v16.8h, v7.8h, v22.8h sqrdmulh v17.8h, v7.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #896] -str q25, [x3, #912] +str q16, [x3, #448] add x2, x2, #16 -add x3, x3, #32 +add x3, x3, #16 sub x5, x5, #1 cbnz x5, 210b +movz x0, #25752, lsl #0 +add x0, x28, x0 +movz x1, #26776, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +movi v29.2d, #0 +211: +ldr q0, [x0, #0] +zip1 v1.8h, v0.8h, v29.8h +zip2 v2.8h, v0.8h, v29.8h +str q1, [x1, #0] +str q2, [x1, #16] +add x0, x0, #16 +add x1, x1, #32 +sub x10, x10, #1 +cbnz x10, 211b add x0, x26, #1408 movz x1, #160, lsl #0 movz x2, #5, lsl #0 @@ -96454,7 +94686,7 @@ add x3, x28, x3 bl {vg_mlkem1024_decode_decompress} movz x0, #24728, lsl #0 add x0, x28, x0 -movz x1, #25752, lsl #0 +movz x1, #26776, lsl #0 add x1, x28, x1 bl {vg_mlkem_sub} movz x0, #24728, lsl #0 @@ -97585,10 +95817,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -211: +212: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 212f +cbz x6, 213f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97600,26 +95832,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 214f +cbz x4, 215f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 216f +cbz x4, 217f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 217f -216: +b 218f 217: -b 215f -214: +218: +b 216f 215: +216: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97631,26 +95863,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 218f +cbz x4, 219f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 220f +cbz x4, 221f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 221f -220: +b 222f 221: -b 219f -218: +222: +b 220f 219: +220: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97662,26 +95894,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 222f +cbz x4, 223f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 224f +cbz x4, 225f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 225f -224: +b 226f 225: -b 223f -222: +226: +b 224f 223: +224: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97693,33 +95925,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 226f +cbz x4, 227f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 228f +cbz x4, 229f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 229f -228: +b 230f 229: -b 227f -226: -227: -cbz x4, 230f -b 231f 230: -movz x5, #0, lsl #0 +b 228f +227: +228: +cbz x4, 231f +b 232f 231: -b 213f -212: +movz x5, #0, lsl #0 +232: +b 214f +213: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -97743,14 +95975,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -213: -cbnz x5, 211b -cbz x4, 232f +214: +cbnz x5, 212b +cbz x4, 233f movz x13, #0, lsl #0 str w13, [x3, #0] -b 233f -232: +b 234f 233: +234: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -97763,10 +95995,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -234: +235: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 235f +cbz x6, 236f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97778,26 +96010,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 237f +cbz x4, 238f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 239f +cbz x4, 240f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 240f -239: +b 241f 240: -b 238f -237: +241: +b 239f 238: +239: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97809,26 +96041,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 241f +cbz x4, 242f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 243f +cbz x4, 244f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 244f -243: +b 245f 244: -b 242f -241: +245: +b 243f 242: +243: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97840,26 +96072,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 245f +cbz x4, 246f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 247f +cbz x4, 248f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 248f -247: +b 249f 248: -b 246f -245: +249: +b 247f 246: +247: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -97871,33 +96103,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 249f +cbz x4, 250f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 251f +cbz x4, 252f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 252f -251: +b 253f 252: -b 250f -249: -250: -cbz x4, 253f -b 254f 253: -movz x5, #0, lsl #0 +b 251f +250: +251: +cbz x4, 254f +b 255f 254: -b 236f -235: +movz x5, #0, lsl #0 +255: +b 237f +236: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -97921,23 +96153,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -236: -cbnz x5, 234b -cbz x4, 255f +237: +cbnz x5, 235b +cbz x4, 256f movz x13, #0, lsl #0 str w13, [x3, #0] -b 256f -255: +b 257f 256: +257: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 257f -b 258f -257: +cbz x6, 258f +b 259f +258: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -98069,15 +96301,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 259f +cbz x4, 260f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -261: +262: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 262f +cbz x6, 263f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98089,26 +96321,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 264f +cbz x4, 265f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 266f +cbz x4, 267f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 267f -266: +b 268f 267: -b 265f -264: +268: +b 266f 265: +266: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98120,26 +96352,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 268f +cbz x4, 269f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 270f +cbz x4, 271f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 271f -270: +b 272f 271: -b 269f -268: +272: +b 270f 269: +270: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98151,26 +96383,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 272f +cbz x4, 273f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 274f +cbz x4, 275f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 275f -274: +b 276f 275: -b 273f -272: +276: +b 274f 273: +274: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98182,33 +96414,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 276f +cbz x4, 277f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 278f +cbz x4, 279f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 279f -278: +b 280f 279: -b 277f -276: -277: -cbz x4, 280f -b 281f 280: -movz x5, #0, lsl #0 +b 278f +277: +278: +cbz x4, 281f +b 282f 281: -b 263f -262: +movz x5, #0, lsl #0 +282: +b 264f +263: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -98232,34 +96464,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -263: -cbnz x5, 261b -cbz x4, 282f +264: +cbnz x5, 262b +cbz x4, 283f movz x13, #0, lsl #0 str w13, [x3, #0] -b 283f -282: +b 284f 283: +284: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 260f -259: -movz x22, #1, lsl #0 +b 261f 260: +movz x22, #1, lsl #0 +261: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 284f +cbz x4, 285f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -286: +287: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 287f +cbz x6, 288f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98271,26 +96503,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 289f +cbz x4, 290f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 291f +cbz x4, 292f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 292f -291: +b 293f 292: -b 290f -289: +293: +b 291f 290: +291: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98302,26 +96534,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 293f +cbz x4, 294f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 295f +cbz x4, 296f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 296f -295: +b 297f 296: -b 294f -293: +297: +b 295f 294: +295: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98333,26 +96565,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 297f +cbz x4, 298f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 299f +cbz x4, 2100f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2100f -299: +b 2101f 2100: -b 298f -297: +2101: +b 299f 298: +299: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98364,33 +96596,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2101f +cbz x4, 2102f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2103f +cbz x4, 2104f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2104f -2103: +b 2105f 2104: -b 2102f -2101: -2102: -cbz x4, 2105f -b 2106f 2105: -movz x5, #0, lsl #0 +b 2103f +2102: +2103: +cbz x4, 2106f +b 2107f 2106: -b 288f -287: +movz x5, #0, lsl #0 +2107: +b 289f +288: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -98414,27 +96646,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -288: -cbnz x5, 286b -cbz x4, 2107f +289: +cbnz x5, 287b +cbz x4, 2108f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2108f -2107: +b 2109f 2108: +2109: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 285f -284: -movz x23, #1, lsl #0 +b 286f 285: +movz x23, #1, lsl #0 +286: and x6, x22, x23 -cbz x6, 2109f -b 2110f -2109: +cbz x6, 2110f +b 2111f +2110: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -98566,15 +96798,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2111f +cbz x4, 2112f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2113: +2114: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2114f +cbz x6, 2115f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98586,26 +96818,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2116f +cbz x4, 2117f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2118f +cbz x4, 2119f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2119f -2118: +b 2120f 2119: -b 2117f -2116: +2120: +b 2118f 2117: +2118: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98617,26 +96849,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2120f +cbz x4, 2121f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2122f +cbz x4, 2123f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2123f -2122: +b 2124f 2123: -b 2121f -2120: +2124: +b 2122f 2121: +2122: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98648,26 +96880,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2124f +cbz x4, 2125f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2126f +cbz x4, 2127f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2127f -2126: +b 2128f 2127: -b 2125f -2124: +2128: +b 2126f 2125: +2126: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98679,33 +96911,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2128f +cbz x4, 2129f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2130f +cbz x4, 2131f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2131f -2130: +b 2132f 2131: -b 2129f -2128: -2129: -cbz x4, 2132f -b 2133f 2132: -movz x5, #0, lsl #0 +b 2130f +2129: +2130: +cbz x4, 2133f +b 2134f 2133: -b 2115f -2114: +movz x5, #0, lsl #0 +2134: +b 2116f +2115: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -98729,34 +96961,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2115: -cbnz x5, 2113b -cbz x4, 2134f +2116: +cbnz x5, 2114b +cbz x4, 2135f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2135f -2134: +b 2136f 2135: +2136: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2112f -2111: -movz x22, #1, lsl #0 +b 2113f 2112: +movz x22, #1, lsl #0 +2113: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2136f +cbz x4, 2137f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2138: +2139: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2139f +cbz x6, 2140f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98768,26 +97000,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2141f +cbz x4, 2142f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2143f +cbz x4, 2144f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2144f -2143: +b 2145f 2144: -b 2142f -2141: +2145: +b 2143f 2142: +2143: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98799,26 +97031,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2145f +cbz x4, 2146f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2147f +cbz x4, 2148f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2148f -2147: +b 2149f 2148: -b 2146f -2145: +2149: +b 2147f 2146: +2147: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98830,26 +97062,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2149f +cbz x4, 2150f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2151f +cbz x4, 2152f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2152f -2151: +b 2153f 2152: -b 2150f -2149: +2153: +b 2151f 2150: +2151: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -98861,33 +97093,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2153f +cbz x4, 2154f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2155f +cbz x4, 2156f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2156f -2155: +b 2157f 2156: -b 2154f -2153: -2154: -cbz x4, 2157f -b 2158f 2157: -movz x5, #0, lsl #0 +b 2155f +2154: +2155: +cbz x4, 2158f +b 2159f 2158: -b 2140f -2139: +movz x5, #0, lsl #0 +2159: +b 2141f +2140: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -98911,51 +97143,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2140: -cbnz x5, 2138b -cbz x4, 2159f +2141: +cbnz x5, 2139b +cbz x4, 2160f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2160f -2159: +b 2161f 2160: +2161: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2137f -2136: -movz x23, #1, lsl #0 +b 2138f 2137: +movz x23, #1, lsl #0 +2138: and x6, x22, x23 -2110: +2111: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2161f -2163: +cbz x4, 2162f +2164: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2163b -b 2162f -2161: +cbnz x4, 2164b +b 2163f 2162: +2163: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2164f -2166: +cbz x4, 2165f +2167: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2166b -b 2165f -2164: +cbnz x4, 2167b +b 2166f 2165: -258: +2166: +259: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -99545,10 +97777,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2167: +2168: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2168f +cbz x6, 2169f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99560,26 +97792,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2170f +cbz x4, 2171f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2172f +cbz x4, 2173f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2173f -2172: +b 2174f 2173: -b 2171f -2170: +2174: +b 2172f 2171: +2172: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99591,26 +97823,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2174f +cbz x4, 2175f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2176f +cbz x4, 2177f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2177f -2176: +b 2178f 2177: -b 2175f -2174: +2178: +b 2176f 2175: +2176: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99622,26 +97854,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2178f +cbz x4, 2179f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2180f +cbz x4, 2181f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2181f -2180: +b 2182f 2181: -b 2179f -2178: +2182: +b 2180f 2179: +2180: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99653,33 +97885,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2182f +cbz x4, 2183f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2184f +cbz x4, 2185f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2185f -2184: +b 2186f 2185: -b 2183f -2182: -2183: -cbz x4, 2186f -b 2187f 2186: -movz x5, #0, lsl #0 +b 2184f +2183: +2184: +cbz x4, 2187f +b 2188f 2187: -b 2169f -2168: +movz x5, #0, lsl #0 +2188: +b 2170f +2169: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -99703,14 +97935,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2169: -cbnz x5, 2167b -cbz x4, 2188f +2170: +cbnz x5, 2168b +cbz x4, 2189f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2189f -2188: +b 2190f 2189: +2190: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -99723,10 +97955,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2190: +2191: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2191f +cbz x6, 2192f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99738,26 +97970,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2193f +cbz x4, 2194f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2195f +cbz x4, 2196f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2196f -2195: +b 2197f 2196: -b 2194f -2193: +2197: +b 2195f 2194: +2195: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99769,26 +98001,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2197f +cbz x4, 2198f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2199f +cbz x4, 2200f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2200f -2199: +b 2201f 2200: -b 2198f -2197: +2201: +b 2199f 2198: +2199: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99800,26 +98032,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2201f +cbz x4, 2202f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2203f +cbz x4, 2204f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2204f -2203: +b 2205f 2204: -b 2202f -2201: +2205: +b 2203f 2202: +2203: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -99831,33 +98063,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2205f +cbz x4, 2206f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2207f +cbz x4, 2208f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2208f -2207: +b 2209f 2208: -b 2206f -2205: -2206: -cbz x4, 2209f -b 2210f 2209: -movz x5, #0, lsl #0 +b 2207f +2206: +2207: +cbz x4, 2210f +b 2211f 2210: -b 2192f -2191: +movz x5, #0, lsl #0 +2211: +b 2193f +2192: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -99881,23 +98113,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2192: -cbnz x5, 2190b -cbz x4, 2211f +2193: +cbnz x5, 2191b +cbz x4, 2212f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2212f -2211: +b 2213f 2212: +2213: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2213f -b 2214f -2213: +cbz x6, 2214f +b 2215f +2214: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -100029,15 +98261,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2215f +cbz x4, 2216f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2217: +2218: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2218f +cbz x6, 2219f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100049,26 +98281,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2220f +cbz x4, 2221f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2222f +cbz x4, 2223f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2223f -2222: +b 2224f 2223: -b 2221f -2220: +2224: +b 2222f 2221: +2222: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100080,26 +98312,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2224f +cbz x4, 2225f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2226f +cbz x4, 2227f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2227f -2226: +b 2228f 2227: -b 2225f -2224: +2228: +b 2226f 2225: +2226: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100111,26 +98343,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2228f +cbz x4, 2229f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2230f +cbz x4, 2231f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2231f -2230: +b 2232f 2231: -b 2229f -2228: +2232: +b 2230f 2229: +2230: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100142,33 +98374,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2232f +cbz x4, 2233f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2234f +cbz x4, 2235f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2235f -2234: +b 2236f 2235: -b 2233f -2232: -2233: -cbz x4, 2236f -b 2237f 2236: -movz x5, #0, lsl #0 +b 2234f +2233: +2234: +cbz x4, 2237f +b 2238f 2237: -b 2219f -2218: +movz x5, #0, lsl #0 +2238: +b 2220f +2219: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -100192,34 +98424,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2219: -cbnz x5, 2217b -cbz x4, 2238f +2220: +cbnz x5, 2218b +cbz x4, 2239f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2239f -2238: +b 2240f 2239: +2240: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2216f -2215: -movz x22, #1, lsl #0 +b 2217f 2216: +movz x22, #1, lsl #0 +2217: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2240f +cbz x4, 2241f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2242: +2243: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2243f +cbz x6, 2244f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100231,26 +98463,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2245f +cbz x4, 2246f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2247f +cbz x4, 2248f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2248f -2247: +b 2249f 2248: -b 2246f -2245: +2249: +b 2247f 2246: +2247: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100262,26 +98494,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2249f +cbz x4, 2250f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2251f +cbz x4, 2252f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2252f -2251: +b 2253f 2252: -b 2250f -2249: +2253: +b 2251f 2250: +2251: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100293,26 +98525,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2253f +cbz x4, 2254f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2255f +cbz x4, 2256f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2256f -2255: +b 2257f 2256: -b 2254f -2253: +2257: +b 2255f 2254: +2255: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100324,33 +98556,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2257f +cbz x4, 2258f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2259f +cbz x4, 2260f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2260f -2259: +b 2261f 2260: -b 2258f -2257: -2258: -cbz x4, 2261f -b 2262f 2261: -movz x5, #0, lsl #0 +b 2259f +2258: +2259: +cbz x4, 2262f +b 2263f 2262: -b 2244f -2243: +movz x5, #0, lsl #0 +2263: +b 2245f +2244: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -100374,27 +98606,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2244: -cbnz x5, 2242b -cbz x4, 2263f +2245: +cbnz x5, 2243b +cbz x4, 2264f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2264f -2263: +b 2265f 2264: +2265: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2241f -2240: -movz x23, #1, lsl #0 +b 2242f 2241: +movz x23, #1, lsl #0 +2242: and x6, x22, x23 -cbz x6, 2265f -b 2266f -2265: +cbz x6, 2266f +b 2267f +2266: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -100526,15 +98758,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2267f +cbz x4, 2268f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2269: +2270: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2270f +cbz x6, 2271f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100546,26 +98778,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2272f +cbz x4, 2273f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2274f +cbz x4, 2275f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2275f -2274: +b 2276f 2275: -b 2273f -2272: +2276: +b 2274f 2273: +2274: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100577,26 +98809,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2276f +cbz x4, 2277f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2278f +cbz x4, 2279f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2279f -2278: +b 2280f 2279: -b 2277f -2276: +2280: +b 2278f 2277: +2278: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100608,26 +98840,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2280f +cbz x4, 2281f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2282f +cbz x4, 2283f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2283f -2282: +b 2284f 2283: -b 2281f -2280: +2284: +b 2282f 2281: +2282: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100639,33 +98871,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2284f +cbz x4, 2285f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2286f +cbz x4, 2287f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2287f -2286: +b 2288f 2287: -b 2285f -2284: -2285: -cbz x4, 2288f -b 2289f 2288: -movz x5, #0, lsl #0 +b 2286f +2285: +2286: +cbz x4, 2289f +b 2290f 2289: -b 2271f -2270: +movz x5, #0, lsl #0 +2290: +b 2272f +2271: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -100689,34 +98921,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2271: -cbnz x5, 2269b -cbz x4, 2290f +2272: +cbnz x5, 2270b +cbz x4, 2291f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2291f -2290: +b 2292f 2291: +2292: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2268f -2267: -movz x22, #1, lsl #0 +b 2269f 2268: +movz x22, #1, lsl #0 +2269: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2292f +cbz x4, 2293f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2294: +2295: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2295f +cbz x6, 2296f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100728,26 +98960,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2297f +cbz x4, 2298f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2299f +cbz x4, 2300f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2300f -2299: +b 2301f 2300: -b 2298f -2297: +2301: +b 2299f 2298: +2299: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100759,26 +98991,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2301f +cbz x4, 2302f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2303f +cbz x4, 2304f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2304f -2303: +b 2305f 2304: -b 2302f -2301: +2305: +b 2303f 2302: +2303: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100790,26 +99022,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2305f +cbz x4, 2306f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2307f +cbz x4, 2308f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2308f -2307: +b 2309f 2308: -b 2306f -2305: +2309: +b 2307f 2306: +2307: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -100821,33 +99053,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2309f +cbz x4, 2310f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2311f +cbz x4, 2312f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2312f -2311: +b 2313f 2312: -b 2310f -2309: -2310: -cbz x4, 2313f -b 2314f 2313: -movz x5, #0, lsl #0 +b 2311f +2310: +2311: +cbz x4, 2314f +b 2315f 2314: -b 2296f -2295: +movz x5, #0, lsl #0 +2315: +b 2297f +2296: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -100871,51 +99103,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2296: -cbnz x5, 2294b -cbz x4, 2315f +2297: +cbnz x5, 2295b +cbz x4, 2316f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2316f -2315: +b 2317f 2316: +2317: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2293f -2292: -movz x23, #1, lsl #0 +b 2294f 2293: +movz x23, #1, lsl #0 +2294: and x6, x22, x23 -2266: +2267: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2317f -2319: +cbz x4, 2318f +2320: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2319b -b 2318f -2317: +cbnz x4, 2320b +b 2319f 2318: +2319: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2320f -2322: +cbz x4, 2321f +2323: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2322b -b 2321f -2320: +cbnz x4, 2323b +b 2322f 2321: -2214: +2322: +2215: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -101505,10 +99737,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2323: +2324: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2324f +cbz x6, 2325f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101520,26 +99752,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2326f +cbz x4, 2327f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2328f +cbz x4, 2329f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2329f -2328: +b 2330f 2329: -b 2327f -2326: +2330: +b 2328f 2327: +2328: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101551,26 +99783,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2330f +cbz x4, 2331f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2332f +cbz x4, 2333f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2333f -2332: +b 2334f 2333: -b 2331f -2330: +2334: +b 2332f 2331: +2332: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101582,26 +99814,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2334f +cbz x4, 2335f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2336f +cbz x4, 2337f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2337f -2336: +b 2338f 2337: -b 2335f -2334: +2338: +b 2336f 2335: +2336: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101613,33 +99845,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2338f +cbz x4, 2339f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2340f +cbz x4, 2341f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2341f -2340: +b 2342f 2341: -b 2339f -2338: -2339: -cbz x4, 2342f -b 2343f 2342: -movz x5, #0, lsl #0 +b 2340f +2339: +2340: +cbz x4, 2343f +b 2344f 2343: -b 2325f -2324: +movz x5, #0, lsl #0 +2344: +b 2326f +2325: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -101663,14 +99895,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2325: -cbnz x5, 2323b -cbz x4, 2344f +2326: +cbnz x5, 2324b +cbz x4, 2345f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2345f -2344: +b 2346f 2345: +2346: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -101683,10 +99915,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2346: +2347: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2347f +cbz x6, 2348f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101698,26 +99930,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2349f +cbz x4, 2350f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2351f +cbz x4, 2352f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2352f -2351: +b 2353f 2352: -b 2350f -2349: +2353: +b 2351f 2350: +2351: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101729,26 +99961,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2353f +cbz x4, 2354f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2355f +cbz x4, 2356f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2356f -2355: +b 2357f 2356: -b 2354f -2353: +2357: +b 2355f 2354: +2355: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101760,26 +99992,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2357f +cbz x4, 2358f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2359f +cbz x4, 2360f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2360f -2359: +b 2361f 2360: -b 2358f -2357: +2361: +b 2359f 2358: +2359: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -101791,33 +100023,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2361f +cbz x4, 2362f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2363f +cbz x4, 2364f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2364f -2363: +b 2365f 2364: -b 2362f -2361: -2362: -cbz x4, 2365f -b 2366f 2365: -movz x5, #0, lsl #0 +b 2363f +2362: +2363: +cbz x4, 2366f +b 2367f 2366: -b 2348f -2347: +movz x5, #0, lsl #0 +2367: +b 2349f +2348: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -101841,23 +100073,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2348: -cbnz x5, 2346b -cbz x4, 2367f +2349: +cbnz x5, 2347b +cbz x4, 2368f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2368f -2367: +b 2369f 2368: +2369: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2369f -b 2370f -2369: +cbz x6, 2370f +b 2371f +2370: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -101989,15 +100221,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2371f +cbz x4, 2372f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2373: +2374: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2374f +cbz x6, 2375f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102009,26 +100241,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2376f +cbz x4, 2377f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2378f +cbz x4, 2379f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2379f -2378: +b 2380f 2379: -b 2377f -2376: +2380: +b 2378f 2377: +2378: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102040,26 +100272,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2380f +cbz x4, 2381f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2382f +cbz x4, 2383f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2383f -2382: +b 2384f 2383: -b 2381f -2380: +2384: +b 2382f 2381: +2382: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102071,26 +100303,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2384f +cbz x4, 2385f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2386f +cbz x4, 2387f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2387f -2386: +b 2388f 2387: -b 2385f -2384: +2388: +b 2386f 2385: +2386: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102102,33 +100334,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2388f +cbz x4, 2389f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2390f +cbz x4, 2391f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2391f -2390: +b 2392f 2391: -b 2389f -2388: -2389: -cbz x4, 2392f -b 2393f 2392: -movz x5, #0, lsl #0 +b 2390f +2389: +2390: +cbz x4, 2393f +b 2394f 2393: -b 2375f -2374: +movz x5, #0, lsl #0 +2394: +b 2376f +2375: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -102152,34 +100384,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2375: -cbnz x5, 2373b -cbz x4, 2394f +2376: +cbnz x5, 2374b +cbz x4, 2395f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2395f -2394: +b 2396f 2395: +2396: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2372f -2371: -movz x22, #1, lsl #0 +b 2373f 2372: +movz x22, #1, lsl #0 +2373: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2396f +cbz x4, 2397f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2398: +2399: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2399f +cbz x6, 2400f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102191,26 +100423,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2401f +cbz x4, 2402f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2403f +cbz x4, 2404f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2404f -2403: +b 2405f 2404: -b 2402f -2401: +2405: +b 2403f 2402: +2403: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102222,26 +100454,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2405f +cbz x4, 2406f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2407f +cbz x4, 2408f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2408f -2407: +b 2409f 2408: -b 2406f -2405: +2409: +b 2407f 2406: +2407: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102253,26 +100485,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2409f +cbz x4, 2410f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2411f +cbz x4, 2412f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2412f -2411: +b 2413f 2412: -b 2410f -2409: +2413: +b 2411f 2410: +2411: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102284,33 +100516,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2413f +cbz x4, 2414f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2415f +cbz x4, 2416f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2416f -2415: +b 2417f 2416: -b 2414f -2413: -2414: -cbz x4, 2417f -b 2418f 2417: -movz x5, #0, lsl #0 +b 2415f +2414: +2415: +cbz x4, 2418f +b 2419f 2418: -b 2400f -2399: +movz x5, #0, lsl #0 +2419: +b 2401f +2400: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -102334,27 +100566,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2400: -cbnz x5, 2398b -cbz x4, 2419f +2401: +cbnz x5, 2399b +cbz x4, 2420f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2420f -2419: +b 2421f 2420: +2421: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2397f -2396: -movz x23, #1, lsl #0 +b 2398f 2397: +movz x23, #1, lsl #0 +2398: and x6, x22, x23 -cbz x6, 2421f -b 2422f -2421: +cbz x6, 2422f +b 2423f +2422: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -102486,15 +100718,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2423f +cbz x4, 2424f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2425: +2426: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2426f +cbz x6, 2427f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102506,26 +100738,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2428f +cbz x4, 2429f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2430f +cbz x4, 2431f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2431f -2430: +b 2432f 2431: -b 2429f -2428: +2432: +b 2430f 2429: +2430: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102537,26 +100769,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2432f +cbz x4, 2433f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2434f +cbz x4, 2435f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2435f -2434: +b 2436f 2435: -b 2433f -2432: +2436: +b 2434f 2433: +2434: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102568,26 +100800,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2436f +cbz x4, 2437f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2438f +cbz x4, 2439f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2439f -2438: +b 2440f 2439: -b 2437f -2436: +2440: +b 2438f 2437: +2438: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102599,33 +100831,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2440f +cbz x4, 2441f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2442f +cbz x4, 2443f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2443f -2442: +b 2444f 2443: -b 2441f -2440: -2441: -cbz x4, 2444f -b 2445f 2444: -movz x5, #0, lsl #0 +b 2442f +2441: +2442: +cbz x4, 2445f +b 2446f 2445: -b 2427f -2426: +movz x5, #0, lsl #0 +2446: +b 2428f +2427: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -102649,34 +100881,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2427: -cbnz x5, 2425b -cbz x4, 2446f +2428: +cbnz x5, 2426b +cbz x4, 2447f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2447f -2446: +b 2448f 2447: +2448: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2424f -2423: -movz x22, #1, lsl #0 +b 2425f 2424: +movz x22, #1, lsl #0 +2425: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2448f +cbz x4, 2449f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2450: +2451: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2451f +cbz x6, 2452f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102688,26 +100920,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2453f +cbz x4, 2454f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2455f +cbz x4, 2456f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2456f -2455: +b 2457f 2456: -b 2454f -2453: +2457: +b 2455f 2454: +2455: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102719,26 +100951,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2457f +cbz x4, 2458f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2459f +cbz x4, 2460f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2460f -2459: +b 2461f 2460: -b 2458f -2457: +2461: +b 2459f 2458: +2459: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102750,26 +100982,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2461f +cbz x4, 2462f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2463f +cbz x4, 2464f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2464f -2463: +b 2465f 2464: -b 2462f -2461: +2465: +b 2463f 2462: +2463: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -102781,33 +101013,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2465f +cbz x4, 2466f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2467f +cbz x4, 2468f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2468f -2467: +b 2469f 2468: -b 2466f -2465: -2466: -cbz x4, 2469f -b 2470f 2469: -movz x5, #0, lsl #0 +b 2467f +2466: +2467: +cbz x4, 2470f +b 2471f 2470: -b 2452f -2451: +movz x5, #0, lsl #0 +2471: +b 2453f +2452: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -102831,51 +101063,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2452: -cbnz x5, 2450b -cbz x4, 2471f +2453: +cbnz x5, 2451b +cbz x4, 2472f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2472f -2471: +b 2473f 2472: +2473: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2449f -2448: -movz x23, #1, lsl #0 +b 2450f 2449: +movz x23, #1, lsl #0 +2450: and x6, x22, x23 -2422: +2423: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2473f -2475: +cbz x4, 2474f +2476: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2475b -b 2474f -2473: +cbnz x4, 2476b +b 2475f 2474: +2475: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2476f -2478: +cbz x4, 2477f +2479: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2478b -b 2477f -2476: +cbnz x4, 2479b +b 2478f 2477: -2370: +2478: +2371: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -103457,10 +101689,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2479: +2480: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2480f +cbz x6, 2481f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103472,26 +101704,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2482f +cbz x4, 2483f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2484f +cbz x4, 2485f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2485f -2484: +b 2486f 2485: -b 2483f -2482: +2486: +b 2484f 2483: +2484: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103503,26 +101735,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2486f +cbz x4, 2487f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2488f +cbz x4, 2489f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2489f -2488: +b 2490f 2489: -b 2487f -2486: +2490: +b 2488f 2487: +2488: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103534,26 +101766,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2490f +cbz x4, 2491f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2492f +cbz x4, 2493f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2493f -2492: +b 2494f 2493: -b 2491f -2490: +2494: +b 2492f 2491: +2492: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103565,33 +101797,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2494f +cbz x4, 2495f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2496f +cbz x4, 2497f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2497f -2496: +b 2498f 2497: -b 2495f -2494: -2495: -cbz x4, 2498f -b 2499f 2498: -movz x5, #0, lsl #0 +b 2496f +2495: +2496: +cbz x4, 2499f +b 2500f 2499: -b 2481f -2480: +movz x5, #0, lsl #0 +2500: +b 2482f +2481: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -103615,14 +101847,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2481: -cbnz x5, 2479b -cbz x4, 2500f +2482: +cbnz x5, 2480b +cbz x4, 2501f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2501f -2500: +b 2502f 2501: +2502: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -103635,10 +101867,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2502: +2503: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2503f +cbz x6, 2504f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103650,26 +101882,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2505f +cbz x4, 2506f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2507f +cbz x4, 2508f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2508f -2507: +b 2509f 2508: -b 2506f -2505: +2509: +b 2507f 2506: +2507: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103681,26 +101913,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2509f +cbz x4, 2510f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2511f +cbz x4, 2512f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2512f -2511: +b 2513f 2512: -b 2510f -2509: +2513: +b 2511f 2510: +2511: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103712,26 +101944,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2513f +cbz x4, 2514f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2515f +cbz x4, 2516f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2516f -2515: +b 2517f 2516: -b 2514f -2513: +2517: +b 2515f 2514: +2515: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103743,33 +101975,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2517f +cbz x4, 2518f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2519f +cbz x4, 2520f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2520f -2519: +b 2521f 2520: -b 2518f -2517: -2518: -cbz x4, 2521f -b 2522f 2521: -movz x5, #0, lsl #0 +b 2519f +2518: +2519: +cbz x4, 2522f +b 2523f 2522: -b 2504f -2503: +movz x5, #0, lsl #0 +2523: +b 2505f +2504: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -103793,23 +102025,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2504: -cbnz x5, 2502b -cbz x4, 2523f +2505: +cbnz x5, 2503b +cbz x4, 2524f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2524f -2523: +b 2525f 2524: +2525: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2525f -b 2526f -2525: +cbz x6, 2526f +b 2527f +2526: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -103941,15 +102173,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2527f +cbz x4, 2528f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2529: +2530: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2530f +cbz x6, 2531f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103961,26 +102193,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2532f +cbz x4, 2533f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2534f +cbz x4, 2535f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2535f -2534: +b 2536f 2535: -b 2533f -2532: +2536: +b 2534f 2533: +2534: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -103992,26 +102224,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2536f +cbz x4, 2537f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2538f +cbz x4, 2539f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2539f -2538: +b 2540f 2539: -b 2537f -2536: +2540: +b 2538f 2537: +2538: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104023,26 +102255,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2540f +cbz x4, 2541f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2542f +cbz x4, 2543f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2543f -2542: +b 2544f 2543: -b 2541f -2540: +2544: +b 2542f 2541: +2542: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104054,33 +102286,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2544f +cbz x4, 2545f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2546f +cbz x4, 2547f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2547f -2546: +b 2548f 2547: -b 2545f -2544: -2545: -cbz x4, 2548f -b 2549f 2548: -movz x5, #0, lsl #0 +b 2546f +2545: +2546: +cbz x4, 2549f +b 2550f 2549: -b 2531f -2530: +movz x5, #0, lsl #0 +2550: +b 2532f +2531: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -104104,34 +102336,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2531: -cbnz x5, 2529b -cbz x4, 2550f +2532: +cbnz x5, 2530b +cbz x4, 2551f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2551f -2550: +b 2552f 2551: +2552: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2528f -2527: -movz x22, #1, lsl #0 +b 2529f 2528: +movz x22, #1, lsl #0 +2529: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2552f +cbz x4, 2553f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2554: +2555: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2555f +cbz x6, 2556f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104143,26 +102375,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2557f +cbz x4, 2558f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2559f +cbz x4, 2560f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2560f -2559: +b 2561f 2560: -b 2558f -2557: +2561: +b 2559f 2558: +2559: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104174,26 +102406,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2561f +cbz x4, 2562f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2563f +cbz x4, 2564f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2564f -2563: +b 2565f 2564: -b 2562f -2561: +2565: +b 2563f 2562: +2563: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104205,26 +102437,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2565f +cbz x4, 2566f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2567f +cbz x4, 2568f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2568f -2567: +b 2569f 2568: -b 2566f -2565: +2569: +b 2567f 2566: +2567: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104236,33 +102468,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2569f +cbz x4, 2570f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2571f +cbz x4, 2572f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2572f -2571: +b 2573f 2572: -b 2570f -2569: -2570: -cbz x4, 2573f -b 2574f 2573: -movz x5, #0, lsl #0 +b 2571f +2570: +2571: +cbz x4, 2574f +b 2575f 2574: -b 2556f -2555: +movz x5, #0, lsl #0 +2575: +b 2557f +2556: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -104286,27 +102518,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2556: -cbnz x5, 2554b -cbz x4, 2575f +2557: +cbnz x5, 2555b +cbz x4, 2576f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2576f -2575: +b 2577f 2576: +2577: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2553f -2552: -movz x23, #1, lsl #0 +b 2554f 2553: +movz x23, #1, lsl #0 +2554: and x6, x22, x23 -cbz x6, 2577f -b 2578f -2577: +cbz x6, 2578f +b 2579f +2578: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -104438,15 +102670,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2579f +cbz x4, 2580f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2581: +2582: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2582f +cbz x6, 2583f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104458,26 +102690,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2584f +cbz x4, 2585f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2586f +cbz x4, 2587f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2587f -2586: +b 2588f 2587: -b 2585f -2584: +2588: +b 2586f 2585: +2586: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104489,26 +102721,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2588f +cbz x4, 2589f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2590f +cbz x4, 2591f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2591f -2590: +b 2592f 2591: -b 2589f -2588: +2592: +b 2590f 2589: +2590: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104520,26 +102752,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2592f +cbz x4, 2593f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2594f +cbz x4, 2595f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2595f -2594: +b 2596f 2595: -b 2593f -2592: +2596: +b 2594f 2593: +2594: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104551,33 +102783,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2596f +cbz x4, 2597f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2598f +cbz x4, 2599f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2599f -2598: +b 2600f 2599: -b 2597f -2596: -2597: -cbz x4, 2600f -b 2601f 2600: -movz x5, #0, lsl #0 +b 2598f +2597: +2598: +cbz x4, 2601f +b 2602f 2601: -b 2583f -2582: +movz x5, #0, lsl #0 +2602: +b 2584f +2583: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -104601,34 +102833,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2583: -cbnz x5, 2581b -cbz x4, 2602f +2584: +cbnz x5, 2582b +cbz x4, 2603f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2603f -2602: +b 2604f 2603: +2604: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2580f -2579: -movz x22, #1, lsl #0 +b 2581f 2580: +movz x22, #1, lsl #0 +2581: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2604f +cbz x4, 2605f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2606: +2607: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2607f +cbz x6, 2608f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104640,26 +102872,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2609f +cbz x4, 2610f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2611f +cbz x4, 2612f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2612f -2611: +b 2613f 2612: -b 2610f -2609: +2613: +b 2611f 2610: +2611: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104671,26 +102903,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2613f +cbz x4, 2614f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2615f +cbz x4, 2616f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2616f -2615: +b 2617f 2616: -b 2614f -2613: +2617: +b 2615f 2614: +2615: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104702,26 +102934,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2617f +cbz x4, 2618f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2619f +cbz x4, 2620f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2620f -2619: +b 2621f 2620: -b 2618f -2617: +2621: +b 2619f 2618: +2619: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -104733,33 +102965,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2621f +cbz x4, 2622f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2623f +cbz x4, 2624f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2624f -2623: +b 2625f 2624: -b 2622f -2621: -2622: -cbz x4, 2625f -b 2626f 2625: -movz x5, #0, lsl #0 +b 2623f +2622: +2623: +cbz x4, 2626f +b 2627f 2626: -b 2608f -2607: +movz x5, #0, lsl #0 +2627: +b 2609f +2608: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -104783,51 +103015,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2608: -cbnz x5, 2606b -cbz x4, 2627f +2609: +cbnz x5, 2607b +cbz x4, 2628f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2628f -2627: +b 2629f 2628: +2629: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2605f -2604: -movz x23, #1, lsl #0 +b 2606f 2605: +movz x23, #1, lsl #0 +2606: and x6, x22, x23 -2578: +2579: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2629f -2631: +cbz x4, 2630f +2632: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2631b -b 2630f -2629: +cbnz x4, 2632b +b 2631f 2630: +2631: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2632f -2634: +cbz x4, 2633f +2635: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2634b -b 2633f -2632: +cbnz x4, 2635b +b 2634f 2633: -2526: +2634: +2527: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -105473,10 +103705,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2635: +2636: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2636f +cbz x6, 2637f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105488,26 +103720,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2638f +cbz x4, 2639f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2640f +cbz x4, 2641f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2641f -2640: +b 2642f 2641: -b 2639f -2638: +2642: +b 2640f 2639: +2640: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105519,26 +103751,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2642f +cbz x4, 2643f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2644f +cbz x4, 2645f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2645f -2644: +b 2646f 2645: -b 2643f -2642: +2646: +b 2644f 2643: +2644: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105550,26 +103782,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2646f +cbz x4, 2647f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2648f +cbz x4, 2649f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2649f -2648: +b 2650f 2649: -b 2647f -2646: +2650: +b 2648f 2647: +2648: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105581,33 +103813,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2650f +cbz x4, 2651f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2652f +cbz x4, 2653f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2653f -2652: +b 2654f 2653: -b 2651f -2650: -2651: -cbz x4, 2654f -b 2655f 2654: -movz x5, #0, lsl #0 +b 2652f +2651: +2652: +cbz x4, 2655f +b 2656f 2655: -b 2637f -2636: +movz x5, #0, lsl #0 +2656: +b 2638f +2637: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -105631,14 +103863,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2637: -cbnz x5, 2635b -cbz x4, 2656f +2638: +cbnz x5, 2636b +cbz x4, 2657f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2657f -2656: +b 2658f 2657: +2658: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -105651,10 +103883,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2658: +2659: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2659f +cbz x6, 2660f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105666,26 +103898,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2661f +cbz x4, 2662f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2663f +cbz x4, 2664f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2664f -2663: +b 2665f 2664: -b 2662f -2661: +2665: +b 2663f 2662: +2663: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105697,26 +103929,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2665f +cbz x4, 2666f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2667f +cbz x4, 2668f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2668f -2667: +b 2669f 2668: -b 2666f -2665: +2669: +b 2667f 2666: +2667: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105728,26 +103960,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2669f +cbz x4, 2670f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2671f +cbz x4, 2672f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2672f -2671: +b 2673f 2672: -b 2670f -2669: +2673: +b 2671f 2670: +2671: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105759,33 +103991,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2673f +cbz x4, 2674f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2675f +cbz x4, 2676f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2676f -2675: +b 2677f 2676: -b 2674f -2673: -2674: -cbz x4, 2677f -b 2678f 2677: -movz x5, #0, lsl #0 +b 2675f +2674: +2675: +cbz x4, 2678f +b 2679f 2678: -b 2660f -2659: +movz x5, #0, lsl #0 +2679: +b 2661f +2660: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -105809,23 +104041,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2660: -cbnz x5, 2658b -cbz x4, 2679f +2661: +cbnz x5, 2659b +cbz x4, 2680f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2680f -2679: +b 2681f 2680: +2681: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2681f -b 2682f -2681: +cbz x6, 2682f +b 2683f +2682: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -105957,15 +104189,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2683f +cbz x4, 2684f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2685: +2686: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2686f +cbz x6, 2687f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -105977,26 +104209,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2688f +cbz x4, 2689f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2690f +cbz x4, 2691f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2691f -2690: +b 2692f 2691: -b 2689f -2688: +2692: +b 2690f 2689: +2690: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106008,26 +104240,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2692f +cbz x4, 2693f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2694f +cbz x4, 2695f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2695f -2694: +b 2696f 2695: -b 2693f -2692: +2696: +b 2694f 2693: +2694: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106039,26 +104271,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2696f +cbz x4, 2697f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2698f +cbz x4, 2699f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2699f -2698: +b 2700f 2699: -b 2697f -2696: +2700: +b 2698f 2697: +2698: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106070,33 +104302,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2700f +cbz x4, 2701f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2702f +cbz x4, 2703f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2703f -2702: +b 2704f 2703: -b 2701f -2700: -2701: -cbz x4, 2704f -b 2705f 2704: -movz x5, #0, lsl #0 +b 2702f +2701: +2702: +cbz x4, 2705f +b 2706f 2705: -b 2687f -2686: +movz x5, #0, lsl #0 +2706: +b 2688f +2687: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -106120,34 +104352,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2687: -cbnz x5, 2685b -cbz x4, 2706f +2688: +cbnz x5, 2686b +cbz x4, 2707f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2707f -2706: +b 2708f 2707: +2708: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2684f -2683: -movz x22, #1, lsl #0 +b 2685f 2684: +movz x22, #1, lsl #0 +2685: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2708f +cbz x4, 2709f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2710: +2711: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2711f +cbz x6, 2712f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106159,26 +104391,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2713f +cbz x4, 2714f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2715f +cbz x4, 2716f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2716f -2715: +b 2717f 2716: -b 2714f -2713: +2717: +b 2715f 2714: +2715: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106190,26 +104422,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2717f +cbz x4, 2718f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2719f +cbz x4, 2720f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2720f -2719: +b 2721f 2720: -b 2718f -2717: +2721: +b 2719f 2718: +2719: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106221,26 +104453,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2721f +cbz x4, 2722f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2723f +cbz x4, 2724f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2724f -2723: +b 2725f 2724: -b 2722f -2721: +2725: +b 2723f 2722: +2723: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106252,33 +104484,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2725f +cbz x4, 2726f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2727f +cbz x4, 2728f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2728f -2727: +b 2729f 2728: -b 2726f -2725: -2726: -cbz x4, 2729f -b 2730f 2729: -movz x5, #0, lsl #0 +b 2727f +2726: +2727: +cbz x4, 2730f +b 2731f 2730: -b 2712f -2711: +movz x5, #0, lsl #0 +2731: +b 2713f +2712: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -106302,27 +104534,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2712: -cbnz x5, 2710b -cbz x4, 2731f +2713: +cbnz x5, 2711b +cbz x4, 2732f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2732f -2731: +b 2733f 2732: +2733: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2709f -2708: -movz x23, #1, lsl #0 +b 2710f 2709: +movz x23, #1, lsl #0 +2710: and x6, x22, x23 -cbz x6, 2733f -b 2734f -2733: +cbz x6, 2734f +b 2735f +2734: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -106454,15 +104686,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2735f +cbz x4, 2736f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2737: +2738: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2738f +cbz x6, 2739f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106474,26 +104706,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2740f +cbz x4, 2741f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2742f +cbz x4, 2743f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2743f -2742: +b 2744f 2743: -b 2741f -2740: +2744: +b 2742f 2741: +2742: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106505,26 +104737,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2744f +cbz x4, 2745f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2746f +cbz x4, 2747f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2747f -2746: +b 2748f 2747: -b 2745f -2744: +2748: +b 2746f 2745: +2746: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106536,26 +104768,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2748f +cbz x4, 2749f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2750f +cbz x4, 2751f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2751f -2750: +b 2752f 2751: -b 2749f -2748: +2752: +b 2750f 2749: +2750: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106567,33 +104799,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2752f +cbz x4, 2753f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2754f +cbz x4, 2755f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2755f -2754: +b 2756f 2755: -b 2753f -2752: -2753: -cbz x4, 2756f -b 2757f 2756: -movz x5, #0, lsl #0 +b 2754f +2753: +2754: +cbz x4, 2757f +b 2758f 2757: -b 2739f -2738: +movz x5, #0, lsl #0 +2758: +b 2740f +2739: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -106617,34 +104849,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2739: -cbnz x5, 2737b -cbz x4, 2758f +2740: +cbnz x5, 2738b +cbz x4, 2759f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2759f -2758: +b 2760f 2759: +2760: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2736f -2735: -movz x22, #1, lsl #0 +b 2737f 2736: +movz x22, #1, lsl #0 +2737: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2760f +cbz x4, 2761f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2762: +2763: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2763f +cbz x6, 2764f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106656,26 +104888,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2765f +cbz x4, 2766f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2767f +cbz x4, 2768f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2768f -2767: +b 2769f 2768: -b 2766f -2765: +2769: +b 2767f 2766: +2767: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106687,26 +104919,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2769f +cbz x4, 2770f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2771f +cbz x4, 2772f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2772f -2771: +b 2773f 2772: -b 2770f -2769: +2773: +b 2771f 2770: +2771: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106718,26 +104950,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2773f +cbz x4, 2774f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2775f +cbz x4, 2776f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2776f -2775: +b 2777f 2776: -b 2774f -2773: +2777: +b 2775f 2774: +2775: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -106749,33 +104981,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2777f +cbz x4, 2778f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2779f +cbz x4, 2780f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2780f -2779: +b 2781f 2780: -b 2778f -2777: -2778: -cbz x4, 2781f -b 2782f 2781: -movz x5, #0, lsl #0 +b 2779f +2778: +2779: +cbz x4, 2782f +b 2783f 2782: -b 2764f -2763: +movz x5, #0, lsl #0 +2783: +b 2765f +2764: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -106799,51 +105031,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2764: -cbnz x5, 2762b -cbz x4, 2783f +2765: +cbnz x5, 2763b +cbz x4, 2784f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2784f -2783: +b 2785f 2784: +2785: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2761f -2760: -movz x23, #1, lsl #0 +b 2762f 2761: +movz x23, #1, lsl #0 +2762: and x6, x22, x23 -2734: +2735: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2785f -2787: +cbz x4, 2786f +2788: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2787b -b 2786f -2785: +cbnz x4, 2788b +b 2787f 2786: +2787: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2788f -2790: +cbz x4, 2789f +2791: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2790b -b 2789f -2788: +cbnz x4, 2791b +b 2790f 2789: -2682: +2790: +2683: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -107481,10 +105713,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2791: +2792: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2792f +cbz x6, 2793f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107496,26 +105728,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2794f +cbz x4, 2795f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2796f +cbz x4, 2797f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2797f -2796: +b 2798f 2797: -b 2795f -2794: +2798: +b 2796f 2795: +2796: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107527,26 +105759,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2798f +cbz x4, 2799f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2800f +cbz x4, 2801f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2801f -2800: +b 2802f 2801: -b 2799f -2798: +2802: +b 2800f 2799: +2800: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107558,26 +105790,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2802f +cbz x4, 2803f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2804f +cbz x4, 2805f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2805f -2804: +b 2806f 2805: -b 2803f -2802: +2806: +b 2804f 2803: +2804: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107589,33 +105821,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2806f +cbz x4, 2807f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2808f +cbz x4, 2809f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2809f -2808: +b 2810f 2809: -b 2807f -2806: -2807: -cbz x4, 2810f -b 2811f 2810: -movz x5, #0, lsl #0 +b 2808f +2807: +2808: +cbz x4, 2811f +b 2812f 2811: -b 2793f -2792: +movz x5, #0, lsl #0 +2812: +b 2794f +2793: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -107639,14 +105871,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2793: -cbnz x5, 2791b -cbz x4, 2812f +2794: +cbnz x5, 2792b +cbz x4, 2813f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2813f -2812: +b 2814f 2813: +2814: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -107659,10 +105891,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2814: +2815: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2815f +cbz x6, 2816f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107674,26 +105906,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2817f +cbz x4, 2818f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2819f +cbz x4, 2820f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2820f -2819: +b 2821f 2820: -b 2818f -2817: +2821: +b 2819f 2818: +2819: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107705,26 +105937,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2821f +cbz x4, 2822f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2823f +cbz x4, 2824f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2824f -2823: +b 2825f 2824: -b 2822f -2821: +2825: +b 2823f 2822: +2823: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107736,26 +105968,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2825f +cbz x4, 2826f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2827f +cbz x4, 2828f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2828f -2827: +b 2829f 2828: -b 2826f -2825: +2829: +b 2827f 2826: +2827: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107767,33 +105999,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2829f +cbz x4, 2830f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2831f +cbz x4, 2832f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2832f -2831: +b 2833f 2832: -b 2830f -2829: -2830: -cbz x4, 2833f -b 2834f 2833: -movz x5, #0, lsl #0 +b 2831f +2830: +2831: +cbz x4, 2834f +b 2835f 2834: -b 2816f -2815: +movz x5, #0, lsl #0 +2835: +b 2817f +2816: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -107817,23 +106049,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2816: -cbnz x5, 2814b -cbz x4, 2835f +2817: +cbnz x5, 2815b +cbz x4, 2836f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2836f -2835: +b 2837f 2836: +2837: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2837f -b 2838f -2837: +cbz x6, 2838f +b 2839f +2838: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -107965,15 +106197,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2839f +cbz x4, 2840f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2841: +2842: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2842f +cbz x6, 2843f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -107985,26 +106217,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2844f +cbz x4, 2845f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2846f +cbz x4, 2847f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2847f -2846: +b 2848f 2847: -b 2845f -2844: +2848: +b 2846f 2845: +2846: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108016,26 +106248,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2848f +cbz x4, 2849f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2850f +cbz x4, 2851f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2851f -2850: +b 2852f 2851: -b 2849f -2848: +2852: +b 2850f 2849: +2850: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108047,26 +106279,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2852f +cbz x4, 2853f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2854f +cbz x4, 2855f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2855f -2854: +b 2856f 2855: -b 2853f -2852: +2856: +b 2854f 2853: +2854: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108078,33 +106310,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2856f +cbz x4, 2857f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2858f +cbz x4, 2859f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2859f -2858: +b 2860f 2859: -b 2857f -2856: -2857: -cbz x4, 2860f -b 2861f 2860: -movz x5, #0, lsl #0 +b 2858f +2857: +2858: +cbz x4, 2861f +b 2862f 2861: -b 2843f -2842: +movz x5, #0, lsl #0 +2862: +b 2844f +2843: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -108128,34 +106360,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2843: -cbnz x5, 2841b -cbz x4, 2862f +2844: +cbnz x5, 2842b +cbz x4, 2863f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2863f -2862: +b 2864f 2863: +2864: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2840f -2839: -movz x22, #1, lsl #0 +b 2841f 2840: +movz x22, #1, lsl #0 +2841: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2864f +cbz x4, 2865f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2866: +2867: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2867f +cbz x6, 2868f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108167,26 +106399,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2869f +cbz x4, 2870f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2871f +cbz x4, 2872f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2872f -2871: +b 2873f 2872: -b 2870f -2869: +2873: +b 2871f 2870: +2871: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108198,26 +106430,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2873f +cbz x4, 2874f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2875f +cbz x4, 2876f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2876f -2875: +b 2877f 2876: -b 2874f -2873: +2877: +b 2875f 2874: +2875: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108229,26 +106461,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2877f +cbz x4, 2878f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2879f +cbz x4, 2880f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2880f -2879: +b 2881f 2880: -b 2878f -2877: +2881: +b 2879f 2878: +2879: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108260,33 +106492,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2881f +cbz x4, 2882f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2883f +cbz x4, 2884f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2884f -2883: +b 2885f 2884: -b 2882f -2881: -2882: -cbz x4, 2885f -b 2886f 2885: -movz x5, #0, lsl #0 +b 2883f +2882: +2883: +cbz x4, 2886f +b 2887f 2886: -b 2868f -2867: +movz x5, #0, lsl #0 +2887: +b 2869f +2868: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -108310,27 +106542,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2868: -cbnz x5, 2866b -cbz x4, 2887f +2869: +cbnz x5, 2867b +cbz x4, 2888f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2888f -2887: +b 2889f 2888: +2889: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2865f -2864: -movz x23, #1, lsl #0 +b 2866f 2865: +movz x23, #1, lsl #0 +2866: and x6, x22, x23 -cbz x6, 2889f -b 2890f -2889: +cbz x6, 2890f +b 2891f +2890: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -108462,15 +106694,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2891f +cbz x4, 2892f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2893: +2894: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2894f +cbz x6, 2895f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108482,26 +106714,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2896f +cbz x4, 2897f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2898f +cbz x4, 2899f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2899f -2898: +b 2900f 2899: -b 2897f -2896: +2900: +b 2898f 2897: +2898: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108513,26 +106745,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2900f +cbz x4, 2901f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2902f +cbz x4, 2903f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2903f -2902: +b 2904f 2903: -b 2901f -2900: +2904: +b 2902f 2901: +2902: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108544,26 +106776,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2904f +cbz x4, 2905f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2906f +cbz x4, 2907f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2907f -2906: +b 2908f 2907: -b 2905f -2904: +2908: +b 2906f 2905: +2906: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108575,33 +106807,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2908f +cbz x4, 2909f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2910f +cbz x4, 2911f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2911f -2910: +b 2912f 2911: -b 2909f -2908: -2909: -cbz x4, 2912f -b 2913f 2912: -movz x5, #0, lsl #0 +b 2910f +2909: +2910: +cbz x4, 2913f +b 2914f 2913: -b 2895f -2894: +movz x5, #0, lsl #0 +2914: +b 2896f +2895: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -108625,34 +106857,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2895: -cbnz x5, 2893b -cbz x4, 2914f +2896: +cbnz x5, 2894b +cbz x4, 2915f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2915f -2914: +b 2916f 2915: +2916: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2892f -2891: -movz x22, #1, lsl #0 +b 2893f 2892: +movz x22, #1, lsl #0 +2893: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2916f +cbz x4, 2917f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2918: +2919: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2919f +cbz x6, 2920f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108664,26 +106896,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2921f +cbz x4, 2922f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2923f +cbz x4, 2924f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2924f -2923: +b 2925f 2924: -b 2922f -2921: +2925: +b 2923f 2922: +2923: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108695,26 +106927,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2925f +cbz x4, 2926f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2927f +cbz x4, 2928f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2928f -2927: +b 2929f 2928: -b 2926f -2925: +2929: +b 2927f 2926: +2927: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108726,26 +106958,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2929f +cbz x4, 2930f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2931f +cbz x4, 2932f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2932f -2931: +b 2933f 2932: -b 2930f -2929: +2933: +b 2931f 2930: +2931: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -108757,33 +106989,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2933f +cbz x4, 2934f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2935f +cbz x4, 2936f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2936f -2935: +b 2937f 2936: -b 2934f -2933: -2934: -cbz x4, 2937f -b 2938f 2937: -movz x5, #0, lsl #0 +b 2935f +2934: +2935: +cbz x4, 2938f +b 2939f 2938: -b 2920f -2919: +movz x5, #0, lsl #0 +2939: +b 2921f +2920: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -108807,51 +107039,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2920: -cbnz x5, 2918b -cbz x4, 2939f +2921: +cbnz x5, 2919b +cbz x4, 2940f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2940f -2939: +b 2941f 2940: +2941: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2917f -2916: -movz x23, #1, lsl #0 +b 2918f 2917: +movz x23, #1, lsl #0 +2918: and x6, x22, x23 -2890: +2891: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2941f -2943: +cbz x4, 2942f +2944: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2943b -b 2942f -2941: +cbnz x4, 2944b +b 2943f 2942: +2943: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2944f -2946: +cbz x4, 2945f +2947: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2946b -b 2945f -2944: +cbnz x4, 2947b +b 2946f 2945: -2838: +2946: +2839: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -109489,10 +107721,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2947: +2948: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2948f +cbz x6, 2949f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109504,26 +107736,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2950f +cbz x4, 2951f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2952f +cbz x4, 2953f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2953f -2952: +b 2954f 2953: -b 2951f -2950: +2954: +b 2952f 2951: +2952: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109535,26 +107767,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2954f +cbz x4, 2955f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2956f +cbz x4, 2957f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2957f -2956: +b 2958f 2957: -b 2955f -2954: +2958: +b 2956f 2955: +2956: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109566,26 +107798,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2958f +cbz x4, 2959f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2960f +cbz x4, 2961f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2961f -2960: +b 2962f 2961: -b 2959f -2958: +2962: +b 2960f 2959: +2960: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109597,33 +107829,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2962f +cbz x4, 2963f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2964f +cbz x4, 2965f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2965f -2964: +b 2966f 2965: -b 2963f -2962: -2963: -cbz x4, 2966f -b 2967f 2966: -movz x5, #0, lsl #0 +b 2964f +2963: +2964: +cbz x4, 2967f +b 2968f 2967: -b 2949f -2948: +movz x5, #0, lsl #0 +2968: +b 2950f +2949: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -109647,14 +107879,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2949: -cbnz x5, 2947b -cbz x4, 2968f +2950: +cbnz x5, 2948b +cbz x4, 2969f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2969f -2968: +b 2970f 2969: +2970: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -109667,10 +107899,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2970: +2971: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2971f +cbz x6, 2972f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109682,26 +107914,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2973f +cbz x4, 2974f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2975f +cbz x4, 2976f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2976f -2975: +b 2977f 2976: -b 2974f -2973: +2977: +b 2975f 2974: +2975: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109713,26 +107945,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2977f +cbz x4, 2978f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2979f +cbz x4, 2980f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2980f -2979: +b 2981f 2980: -b 2978f -2977: +2981: +b 2979f 2978: +2979: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109744,26 +107976,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2981f +cbz x4, 2982f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2983f +cbz x4, 2984f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2984f -2983: +b 2985f 2984: -b 2982f -2981: +2985: +b 2983f 2982: +2983: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109775,33 +108007,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2985f +cbz x4, 2986f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2987f +cbz x4, 2988f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2988f -2987: +b 2989f 2988: -b 2986f -2985: -2986: -cbz x4, 2989f -b 2990f 2989: -movz x5, #0, lsl #0 +b 2987f +2986: +2987: +cbz x4, 2990f +b 2991f 2990: -b 2972f -2971: +movz x5, #0, lsl #0 +2991: +b 2973f +2972: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -109825,23 +108057,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2972: -cbnz x5, 2970b -cbz x4, 2991f +2973: +cbnz x5, 2971b +cbz x4, 2992f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2992f -2991: +b 2993f 2992: +2993: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2993f -b 2994f -2993: +cbz x6, 2994f +b 2995f +2994: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -109973,15 +108205,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2995f +cbz x4, 2996f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2997: +2998: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2998f +cbz x6, 2999f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -109993,26 +108225,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21000f +cbz x4, 21001f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21002f +cbz x4, 21003f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21003f -21002: +b 21004f 21003: -b 21001f -21000: +21004: +b 21002f 21001: +21002: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110024,26 +108256,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21004f +cbz x4, 21005f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21006f +cbz x4, 21007f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21007f -21006: +b 21008f 21007: -b 21005f -21004: +21008: +b 21006f 21005: +21006: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110055,26 +108287,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21008f +cbz x4, 21009f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21010f +cbz x4, 21011f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21011f -21010: +b 21012f 21011: -b 21009f -21008: +21012: +b 21010f 21009: +21010: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110086,33 +108318,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21012f +cbz x4, 21013f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21014f +cbz x4, 21015f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21015f -21014: +b 21016f 21015: -b 21013f -21012: -21013: -cbz x4, 21016f -b 21017f 21016: -movz x5, #0, lsl #0 +b 21014f +21013: +21014: +cbz x4, 21017f +b 21018f 21017: -b 2999f -2998: +movz x5, #0, lsl #0 +21018: +b 21000f +2999: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -110136,34 +108368,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2999: -cbnz x5, 2997b -cbz x4, 21018f +21000: +cbnz x5, 2998b +cbz x4, 21019f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21019f -21018: +b 21020f 21019: +21020: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2996f -2995: -movz x22, #1, lsl #0 +b 2997f 2996: +movz x22, #1, lsl #0 +2997: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 21020f +cbz x4, 21021f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21022: +21023: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21023f +cbz x6, 21024f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110175,26 +108407,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21025f +cbz x4, 21026f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21027f +cbz x4, 21028f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21028f -21027: +b 21029f 21028: -b 21026f -21025: +21029: +b 21027f 21026: +21027: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110206,26 +108438,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21029f +cbz x4, 21030f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21031f +cbz x4, 21032f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21032f -21031: +b 21033f 21032: -b 21030f -21029: +21033: +b 21031f 21030: +21031: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110237,26 +108469,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21033f +cbz x4, 21034f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21035f +cbz x4, 21036f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21036f -21035: +b 21037f 21036: -b 21034f -21033: +21037: +b 21035f 21034: +21035: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110268,33 +108500,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21037f +cbz x4, 21038f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21039f +cbz x4, 21040f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21040f -21039: +b 21041f 21040: -b 21038f -21037: -21038: -cbz x4, 21041f -b 21042f 21041: -movz x5, #0, lsl #0 +b 21039f +21038: +21039: +cbz x4, 21042f +b 21043f 21042: -b 21024f -21023: +movz x5, #0, lsl #0 +21043: +b 21025f +21024: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -110318,27 +108550,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21024: -cbnz x5, 21022b -cbz x4, 21043f +21025: +cbnz x5, 21023b +cbz x4, 21044f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21044f -21043: +b 21045f 21044: +21045: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 21021f -21020: -movz x23, #1, lsl #0 +b 21022f 21021: +movz x23, #1, lsl #0 +21022: and x6, x22, x23 -cbz x6, 21045f -b 21046f -21045: +cbz x6, 21046f +b 21047f +21046: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -110470,15 +108702,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 21047f +cbz x4, 21048f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21049: +21050: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21050f +cbz x6, 21051f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110490,26 +108722,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21052f +cbz x4, 21053f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21054f +cbz x4, 21055f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21055f -21054: +b 21056f 21055: -b 21053f -21052: +21056: +b 21054f 21053: +21054: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110521,26 +108753,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21056f +cbz x4, 21057f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21058f +cbz x4, 21059f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21059f -21058: +b 21060f 21059: -b 21057f -21056: +21060: +b 21058f 21057: +21058: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110552,26 +108784,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21060f +cbz x4, 21061f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21062f +cbz x4, 21063f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21063f -21062: +b 21064f 21063: -b 21061f -21060: +21064: +b 21062f 21061: +21062: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110583,33 +108815,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21064f +cbz x4, 21065f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21066f +cbz x4, 21067f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21067f -21066: +b 21068f 21067: -b 21065f -21064: -21065: -cbz x4, 21068f -b 21069f 21068: -movz x5, #0, lsl #0 +b 21066f +21065: +21066: +cbz x4, 21069f +b 21070f 21069: -b 21051f -21050: +movz x5, #0, lsl #0 +21070: +b 21052f +21051: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -110633,34 +108865,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21051: -cbnz x5, 21049b -cbz x4, 21070f +21052: +cbnz x5, 21050b +cbz x4, 21071f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21071f -21070: +b 21072f 21071: +21072: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 21048f -21047: -movz x22, #1, lsl #0 +b 21049f 21048: +movz x22, #1, lsl #0 +21049: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 21072f +cbz x4, 21073f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21074: +21075: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21075f +cbz x6, 21076f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110672,26 +108904,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21077f +cbz x4, 21078f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21079f +cbz x4, 21080f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21080f -21079: +b 21081f 21080: -b 21078f -21077: +21081: +b 21079f 21078: +21079: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110703,26 +108935,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21081f +cbz x4, 21082f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21083f +cbz x4, 21084f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21084f -21083: +b 21085f 21084: -b 21082f -21081: +21085: +b 21083f 21082: +21083: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110734,26 +108966,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21085f +cbz x4, 21086f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21087f +cbz x4, 21088f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21088f -21087: +b 21089f 21088: -b 21086f -21085: +21089: +b 21087f 21086: +21087: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -110765,33 +108997,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21089f +cbz x4, 21090f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21091f +cbz x4, 21092f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21092f -21091: +b 21093f 21092: -b 21090f -21089: -21090: -cbz x4, 21093f -b 21094f 21093: -movz x5, #0, lsl #0 +b 21091f +21090: +21091: +cbz x4, 21094f +b 21095f 21094: -b 21076f -21075: +movz x5, #0, lsl #0 +21095: +b 21077f +21076: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -110815,51 +109047,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21076: -cbnz x5, 21074b -cbz x4, 21095f +21077: +cbnz x5, 21075b +cbz x4, 21096f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21096f -21095: +b 21097f 21096: +21097: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 21073f -21072: -movz x23, #1, lsl #0 +b 21074f 21073: +movz x23, #1, lsl #0 +21074: and x6, x22, x23 -21046: +21047: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 21097f -21099: +cbz x4, 21098f +21100: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 21099b -b 21098f -21097: +cbnz x4, 21100b +b 21099f 21098: +21099: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 21100f -21102: +cbz x4, 21101f +21103: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 21102b -b 21101f -21100: +cbnz x4, 21103b +b 21102f 21101: -2994: +21102: +2995: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -110925,7 +109157,7 @@ str q0, [x19, #320] add x20, x19, #400 add x21, x19, #904 movz x22, #3, lsl #0 -21103: +21104: add x1, x19, #768 add x1, x1, #768 str x30, [x1, #128] @@ -111019,7 +109251,7 @@ str x7, [x21, #160] add x20, x20, #168 add x21, x21, #168 sub x22, x22, #1 -cbnz x22, 21103b +cbnz x22, 21104b movz x9, #256, lsl #0 movk x9, #513, lsl #16 movk x9, #1027, lsl #32 @@ -111064,10 +109296,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21104: +21105: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21105f +cbz x6, 21106f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111079,26 +109311,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21107f +cbz x4, 21108f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21109f +cbz x4, 21110f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21110f -21109: +b 21111f 21110: -b 21108f -21107: +21111: +b 21109f 21108: +21109: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111110,26 +109342,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21111f +cbz x4, 21112f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21113f +cbz x4, 21114f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21114f -21113: +b 21115f 21114: -b 21112f -21111: +21115: +b 21113f 21112: +21113: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111141,26 +109373,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21115f +cbz x4, 21116f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21117f +cbz x4, 21118f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21118f -21117: +b 21119f 21118: -b 21116f -21115: +21119: +b 21117f 21116: +21117: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111172,33 +109404,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21119f +cbz x4, 21120f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21121f +cbz x4, 21122f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21122f -21121: +b 21123f 21122: -b 21120f -21119: -21120: -cbz x4, 21123f -b 21124f 21123: -movz x5, #0, lsl #0 +b 21121f +21120: +21121: +cbz x4, 21124f +b 21125f 21124: -b 21106f -21105: +movz x5, #0, lsl #0 +21125: +b 21107f +21106: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -111222,14 +109454,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21106: -cbnz x5, 21104b -cbz x4, 21125f +21107: +cbnz x5, 21105b +cbz x4, 21126f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21126f -21125: +b 21127f 21126: +21127: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -111242,10 +109474,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21127: +21128: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21128f +cbz x6, 21129f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111257,26 +109489,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21130f +cbz x4, 21131f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21132f +cbz x4, 21133f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21133f -21132: +b 21134f 21133: -b 21131f -21130: +21134: +b 21132f 21131: +21132: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111288,26 +109520,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21134f +cbz x4, 21135f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21136f +cbz x4, 21137f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21137f -21136: +b 21138f 21137: -b 21135f -21134: +21138: +b 21136f 21135: +21136: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111319,26 +109551,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21138f +cbz x4, 21139f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21140f +cbz x4, 21141f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21141f -21140: +b 21142f 21141: -b 21139f -21138: +21142: +b 21140f 21139: +21140: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111350,33 +109582,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21142f +cbz x4, 21143f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21144f +cbz x4, 21145f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21145f -21144: +b 21146f 21145: -b 21143f -21142: -21143: -cbz x4, 21146f -b 21147f 21146: -movz x5, #0, lsl #0 +b 21144f +21143: +21144: +cbz x4, 21147f +b 21148f 21147: -b 21129f -21128: +movz x5, #0, lsl #0 +21148: +b 21130f +21129: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -111400,23 +109632,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21129: -cbnz x5, 21127b -cbz x4, 21148f +21130: +cbnz x5, 21128b +cbz x4, 21149f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21149f -21148: +b 21150f 21149: +21150: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 21150f -b 21151f -21150: +cbz x6, 21151f +b 21152f +21151: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -111548,15 +109780,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 21152f +cbz x4, 21153f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21154: +21155: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21155f +cbz x6, 21156f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111568,26 +109800,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21157f +cbz x4, 21158f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21159f +cbz x4, 21160f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21160f -21159: +b 21161f 21160: -b 21158f -21157: +21161: +b 21159f 21158: +21159: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111599,26 +109831,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21161f +cbz x4, 21162f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21163f +cbz x4, 21164f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21164f -21163: +b 21165f 21164: -b 21162f -21161: +21165: +b 21163f 21162: +21163: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111630,26 +109862,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21165f +cbz x4, 21166f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21167f +cbz x4, 21168f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21168f -21167: +b 21169f 21168: -b 21166f -21165: +21169: +b 21167f 21166: +21167: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111661,33 +109893,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21169f +cbz x4, 21170f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21171f +cbz x4, 21172f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21172f -21171: +b 21173f 21172: -b 21170f -21169: -21170: -cbz x4, 21173f -b 21174f 21173: -movz x5, #0, lsl #0 +b 21171f +21170: +21171: +cbz x4, 21174f +b 21175f 21174: -b 21156f -21155: +movz x5, #0, lsl #0 +21175: +b 21157f +21156: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -111711,34 +109943,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21156: -cbnz x5, 21154b -cbz x4, 21175f +21157: +cbnz x5, 21155b +cbz x4, 21176f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21176f -21175: +b 21177f 21176: +21177: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 21153f -21152: -movz x22, #1, lsl #0 +b 21154f 21153: +movz x22, #1, lsl #0 +21154: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 21177f +cbz x4, 21178f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21179: +21180: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21180f +cbz x6, 21181f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111750,26 +109982,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21182f +cbz x4, 21183f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21184f +cbz x4, 21185f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21185f -21184: +b 21186f 21185: -b 21183f -21182: +21186: +b 21184f 21183: +21184: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111781,26 +110013,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21186f +cbz x4, 21187f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21188f +cbz x4, 21189f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21189f -21188: +b 21190f 21189: -b 21187f -21186: +21190: +b 21188f 21187: +21188: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111812,26 +110044,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21190f +cbz x4, 21191f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21192f +cbz x4, 21193f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21193f -21192: +b 21194f 21193: -b 21191f -21190: +21194: +b 21192f 21191: +21192: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -111843,33 +110075,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21194f +cbz x4, 21195f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21196f +cbz x4, 21197f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21197f -21196: +b 21198f 21197: -b 21195f -21194: -21195: -cbz x4, 21198f -b 21199f 21198: -movz x5, #0, lsl #0 +b 21196f +21195: +21196: +cbz x4, 21199f +b 21200f 21199: -b 21181f -21180: +movz x5, #0, lsl #0 +21200: +b 21182f +21181: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -111893,27 +110125,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21181: -cbnz x5, 21179b -cbz x4, 21200f +21182: +cbnz x5, 21180b +cbz x4, 21201f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21201f -21200: +b 21202f 21201: +21202: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 21178f -21177: -movz x23, #1, lsl #0 +b 21179f 21178: +movz x23, #1, lsl #0 +21179: and x6, x22, x23 -cbz x6, 21202f -b 21203f -21202: +cbz x6, 21203f +b 21204f +21203: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -112045,15 +110277,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 21204f +cbz x4, 21205f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -21206: +21207: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 21207f +cbz x6, 21208f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -112065,26 +110297,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21209f +cbz x4, 21210f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21211f +cbz x4, 21212f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21212f -21211: +b 21213f 21212: -b 21210f -21209: +21213: +b 21211f 21210: +21211: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -112096,26 +110328,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21213f +cbz x4, 21214f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21215f +cbz x4, 21216f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21216f -21215: +b 21217f 21216: -b 21214f -21213: +21217: +b 21215f 21214: +21215: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -112127,26 +110359,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21217f +cbz x4, 21218f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21219f +cbz x4, 21220f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21220f -21219: +b 21221f 21220: -b 21218f -21217: +21221: +b 21219f 21218: +21219: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -112158,215 +110390,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 21221f +cbz x4, 21222f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 21223f +cbz x4, 21224f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 21224f -21223: +b 21225f 21224: -b 21222f -21221: -21222: -cbz x4, 21225f -b 21226f 21225: -movz x5, #0, lsl #0 +b 21223f +21222: +21223: +cbz x4, 21226f +b 21227f 21226: -b 21208f -21207: -ldr q0, [x2, #0] -tbl v0.16b, {{v0.16b}}, v16.16b -ushr v1.8h, v0.8h, #4 -bif v0.16b, v1.16b, v17.16b -cmhi v1.8h, v19.8h, v0.8h -and v1.16b, v1.16b, v20.16b -addv h1, v1.8h -umov w6, v1.s[0] -movz x7, #255, lsl #0 -and x7, x6, x7 -lsl x7, x7, #4 -add x7, x16, x7 -ldr q2, [x7, #0] -tbl v0.16b, {{v0.16b}}, v2.16b -str q0, [x3, #0] -lsr x6, x6, #8 -movz x7, #15, lsl #0 -and x6, x6, x7 -sub x4, x4, x6 -lsl x6, x6, #1 -add x3, x3, x6 -add x2, x2, #12 -sub x5, x5, #4 -21208: -cbnz x5, 21206b -cbz x4, 21227f -movz x13, #0, lsl #0 -str w13, [x3, #0] -b 21228f -21227: -21228: -sub x0, x4, #1 -lsr x0, x0, #63 -add x22, x0, #0 -str x3, [x19, #1680] -str x4, [x19, #1688] -b 21205f -21204: -movz x22, #1, lsl #0 -21205: -ldr x3, [x19, #1696] -ldr x4, [x19, #1704] -cbz x4, 21229f -add x2, x19, #904 -movz x5, #56, lsl #0 -movz x9, #3329, lsl #0 -movz x10, #15, lsl #0 -21231: -sub x6, x4, #8 -lsr x6, x6, #63 -cbz x6, 21232f -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 21234f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 21236f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 21237f -21236: -21237: -b 21235f -21234: -21235: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 21238f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 21240f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 21241f -21240: -21241: -b 21239f -21238: -21239: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 21242f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 21244f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 21245f -21244: -21245: -b 21243f -21242: -21243: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 21246f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 21248f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 21249f -21248: -21249: -b 21247f -21246: -21247: -cbz x4, 21250f -b 21251f -21250: movz x5, #0, lsl #0 -21251: -b 21233f -21232: +21227: +b 21209f +21208: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -112390,51 +110440,233 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -21233: -cbnz x5, 21231b -cbz x4, 21252f +21209: +cbnz x5, 21207b +cbz x4, 21228f movz x13, #0, lsl #0 str w13, [x3, #0] -b 21253f +b 21229f +21228: +21229: +sub x0, x4, #1 +lsr x0, x0, #63 +add x22, x0, #0 +str x3, [x19, #1680] +str x4, [x19, #1688] +b 21206f +21205: +movz x22, #1, lsl #0 +21206: +ldr x3, [x19, #1696] +ldr x4, [x19, #1704] +cbz x4, 21230f +add x2, x19, #904 +movz x5, #56, lsl #0 +movz x9, #3329, lsl #0 +movz x10, #15, lsl #0 +21232: +sub x6, x4, #8 +lsr x6, x6, #63 +cbz x6, 21233f +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 21235f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 21237f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 21238f +21237: +21238: +b 21236f +21235: +21236: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 21239f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 21241f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 21242f +21241: +21242: +b 21240f +21239: +21240: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 21243f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 21245f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 21246f +21245: +21246: +b 21244f +21243: +21244: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 21247f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 21249f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 21250f +21249: +21250: +b 21248f +21247: +21248: +cbz x4, 21251f +b 21252f +21251: +movz x5, #0, lsl #0 21252: +b 21234f +21233: +ldr q0, [x2, #0] +tbl v0.16b, {{v0.16b}}, v16.16b +ushr v1.8h, v0.8h, #4 +bif v0.16b, v1.16b, v17.16b +cmhi v1.8h, v19.8h, v0.8h +and v1.16b, v1.16b, v20.16b +addv h1, v1.8h +umov w6, v1.s[0] +movz x7, #255, lsl #0 +and x7, x6, x7 +lsl x7, x7, #4 +add x7, x16, x7 +ldr q2, [x7, #0] +tbl v0.16b, {{v0.16b}}, v2.16b +str q0, [x3, #0] +lsr x6, x6, #8 +movz x7, #15, lsl #0 +and x6, x6, x7 +sub x4, x4, x6 +lsl x6, x6, #1 +add x3, x3, x6 +add x2, x2, #12 +sub x5, x5, #4 +21234: +cbnz x5, 21232b +cbz x4, 21253f +movz x13, #0, lsl #0 +str w13, [x3, #0] +b 21254f 21253: +21254: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 21230f -21229: -movz x23, #1, lsl #0 +b 21231f 21230: +movz x23, #1, lsl #0 +21231: and x6, x22, x23 -21203: +21204: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 21254f -21256: +cbz x4, 21255f +21257: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 21256b -b 21255f -21254: +cbnz x4, 21257b +b 21256f 21255: +21256: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 21257f -21259: +cbz x4, 21258f +21260: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 21259b -b 21258f -21257: +cbnz x4, 21260b +b 21259f 21258: -21151: +21259: +21152: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -112515,7 +110747,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -21260: +21261: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -112644,7 +110876,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 21260b +cbnz x11, 21261b movz x0, #25752, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -112658,7 +110890,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -21261: +21262: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -112755,7 +110987,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 21261b +cbnz x5, 21262b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -112791,7 +111023,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -21262: +21263: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -112951,7 +111183,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 21262b +cbnz x5, 21263b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #45056, lsl #0 @@ -112959,7 +111191,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -21263: +21264: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -112970,14 +111202,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21263b +cbnz x11, 21264b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #26776, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -21264: +21265: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -112986,7 +111218,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 21264b +cbnz x10, 21265b movz x0, #26776, lsl #0 add x0, x28, x0 movz x1, #11, lsl #0 @@ -113007,7 +111239,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -21265: +21266: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -113136,7 +111368,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 21265b +cbnz x11, 21266b movz x0, #25752, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -113150,7 +111382,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -21266: +21267: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -113247,7 +111479,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 21266b +cbnz x5, 21267b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -113283,7 +111515,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -21267: +21268: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -113443,7 +111675,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 21267b +cbnz x5, 21268b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #46080, lsl #0 @@ -113451,7 +111683,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -21268: +21269: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -113462,14 +111694,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21268b +cbnz x11, 21269b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #26776, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -21269: +21270: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -113478,7 +111710,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 21269b +cbnz x10, 21270b movz x0, #26776, lsl #0 add x0, x28, x0 movz x1, #11, lsl #0 @@ -113499,7 +111731,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -21270: +21271: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -113628,7 +111860,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 21270b +cbnz x11, 21271b movz x0, #25752, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -113642,7 +111874,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -21271: +21272: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -113739,7 +111971,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 21271b +cbnz x5, 21272b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -113775,7 +112007,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -21272: +21273: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -113935,7 +112167,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 21272b +cbnz x5, 21273b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #47104, lsl #0 @@ -113943,7 +112175,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -21273: +21274: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -113954,14 +112186,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21273b +cbnz x11, 21274b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #26776, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -21274: +21275: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -113970,7 +112202,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 21274b +cbnz x10, 21275b movz x0, #26776, lsl #0 add x0, x28, x0 movz x1, #11, lsl #0 @@ -113991,7 +112223,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -21275: +21276: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -114120,7 +112352,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 21275b +cbnz x11, 21276b movz x0, #25752, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -114134,7 +112366,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -21276: +21277: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -114231,7 +112463,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 21276b +cbnz x5, 21277b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -114267,7 +112499,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -21277: +21278: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -114427,7 +112659,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 21277b +cbnz x5, 21278b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #48128, lsl #0 @@ -114435,7 +112667,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -21278: +21279: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -114446,14 +112678,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21278b +cbnz x11, 21279b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #26776, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -21279: +21280: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -114462,7 +112694,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 21279b +cbnz x10, 21280b movz x0, #26776, lsl #0 add x0, x28, x0 movz x1, #11, lsl #0 @@ -114488,7 +112720,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -21280: +21281: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -114507,7 +112739,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21280b +cbnz x11, 21281b add x0, x25, #1920 movz x1, #5272, lsl #0 add x1, x28, x1 @@ -114526,7 +112758,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -21281: +21282: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -114545,7 +112777,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21281b +cbnz x11, 21282b add x0, x25, #2304 movz x1, #6296, lsl #0 add x1, x28, x1 @@ -114564,7 +112796,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -21282: +21283: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -114583,7 +112815,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21282b +cbnz x11, 21283b add x0, x25, #2688 movz x1, #7320, lsl #0 add x1, x28, x1 @@ -114602,7 +112834,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -21283: +21284: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -114621,7 +112853,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21283b +cbnz x11, 21284b movz x0, #25752, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -114635,7 +112867,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -21284: +21285: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -114764,7 +112996,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 21284b +cbnz x11, 21285b movz x0, #25752, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -114778,7 +113010,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -21285: +21286: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -114875,7 +113107,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 21285b +cbnz x5, 21286b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -114911,7 +113143,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -21286: +21287: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -115071,7 +113303,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 21286b +cbnz x5, 21287b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #24728, lsl #0 @@ -115079,7 +113311,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -21287: +21288: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -115090,14 +113322,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 21287b +cbnz x11, 21288b movz x0, #25752, lsl #0 add x0, x28, x0 movz x1, #26776, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -21288: +21289: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -115106,7 +113338,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 21288b +cbnz x10, 21289b add x0, x28, #912 movz x1, #32, lsl #0 movz x2, #1, lsl #0 @@ -115130,7 +113362,7 @@ movz x1, #28824, lsl #0 add x1, x28, x1 movz x2, #196, lsl #0 movz x10, #0, lsl #0 -21289: +21290: ldr x9, [x0, #0] ldr x11, [x1, #0] eor x9, x9, x11 @@ -115138,7 +113370,7 @@ orr x10, x10, x9 add x0, x0, #8 add x1, x1, #8 sub x2, x2, #1 -cbnz x2, 21289b +cbnz x2, 21290b movz x11, #0, lsl #0 sub x11, x11, x10 orr x10, x10, x11 @@ -115179,18 +113411,16 @@ ret .arch_extension nosha3 .p2align 6", vg_mlkem1024_decode_decompress = sym super::mlkem1024::vg_mlkem1024_decode_decompress, - vg_mlkem_decode12 = sym super::mlkem::vg_mlkem_decode12, + vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem_sub = sym super::mlkem::vg_mlkem_sub, vg_mlkem_compress_encode = sym super::mlkem::vg_mlkem_compress_encode, vg_keccak_f1600_sha3 = sym super::sha3::vg_keccak_f1600_sha3, vg_keccak_f1600_x3_sha3 = sym super::sha3::vg_keccak_f1600_x3_sha3, vg_keccak_f1600_x2_sha3 = sym super::sha3::vg_keccak_f1600_x2_sha3, vg_mlkem16_cbd2 = sym super::mlkem16::vg_mlkem16_cbd2, - vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem1024_compress_encode = sym super::mlkem1024::vg_mlkem1024_compress_encode, vg_mlkem_decode_decompress = sym super::mlkem::vg_mlkem_decode_decompress, vg_mlkem_add = sym super::mlkem::vg_mlkem_add, - VG_MLKEM_NTT16_ZETAS = sym super::consts::VG_MLKEM_NTT16_ZETAS, VG_MLKEM_GAMMA = sym super::consts::VG_MLKEM_GAMMA, VG_MLKEM_INTT16_ZETAS = sym super::consts::VG_MLKEM_INTT16_ZETAS, VG_MLKEM_REJ_SHUFFLE = sym super::consts::VG_MLKEM_REJ_SHUFFLE, diff --git a/src/asm/aarch64/mlkem768.rs b/src/asm/aarch64/mlkem768.rs index ca171f7eee..fce21038be 100644 --- a/src/asm/aarch64/mlkem768.rs +++ b/src/asm/aarch64/mlkem768.rs @@ -23865,864 +23865,189 @@ movz x24, #1, lsl #0 add x0, x26, #0 movz x1, #320, lsl #0 movz x2, #10, lsl #0 -movz x3, #13464, lsl #0 +movz x3, #18584, lsl #0 add x3, x28, x3 bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #13464, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +20: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 20b movz x0, #13464, lsl #0 add x0, x28, x0 add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -20: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 20b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -21: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 21b +bl {vg_mlkem16_ntt} add x0, x26, #320 movz x1, #320, lsl #0 movz x2, #10, lsl #0 -movz x3, #14488, lsl #0 +movz x3, #18584, lsl #0 add x3, x28, x3 bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #14488, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +21: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 21b movz x0, #14488, lsl #0 add x0, x28, x0 add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -22: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 22b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -23: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 23b +bl {vg_mlkem16_ntt} add x0, x26, #640 movz x1, #320, lsl #0 movz x2, #10, lsl #0 -movz x3, #15512, lsl #0 +movz x3, #18584, lsl #0 add x3, x28, x3 bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #15512, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +22: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 22b movz x0, #15512, lsl #0 add x0, x28, x0 add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -24: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 24b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -25: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 25b +bl {vg_mlkem16_ntt} add x0, x25, #0 movz x1, #4248, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +23: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 23b add x0, x25, #384 movz x1, #5272, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +24: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 24b add x0, x25, #768 movz x1, #6296, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +25: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 25b movz x0, #17560, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -24734,12 +24059,15 @@ dup v16.4s, w9 dup v17.4s, w10", concat!("adrp x3, ", vg_sym_page!("{VG_MLKEM_GAMMA}")), concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), - r"movz x11, #32, lsl #0 + r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 26: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 +movi v25.2d, #0 +movi v26.2d, #0 +movi v27.2d, #0 movz x1, #4248, lsl #0 add x1, x28, x1 movz x2, #13464, lsl #0 @@ -24750,14 +24078,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #5272, lsl #0 add x1, x28, x1 movz x2, #14488, lsl #0 @@ -24768,14 +24100,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #6296, lsl #0 add x1, x28, x1 movz x2, #15512, lsl #0 @@ -24786,15 +24122,20 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h ldr q18, [x3, #0] +ldr q28, [x3, #16] sqdmulh v24.4s, v22.4s, v17.4s mls v22.4s, v24.4s, v16.4s mla v21.4s, v22.4s, v18.4s @@ -24806,12 +24147,25 @@ sqdmulh v24.4s, v23.4s, v17.4s mls v23.4s, v24.4s, v16.4s sub v24.4s, v23.4s, v16.4s umin v23.4s, v23.4s, v24.4s -zip1 v0.4s, v21.4s, v23.4s -zip2 v1.4s, v21.4s, v23.4s -str q0, [x0, #0] -str q1, [x0, #16] +sqdmulh v24.4s, v26.4s, v17.4s +mls v26.4s, v24.4s, v16.4s +mla v25.4s, v26.4s, v28.4s +sqdmulh v24.4s, v25.4s, v17.4s +mls v25.4s, v24.4s, v16.4s +sub v24.4s, v25.4s, v16.4s +umin v25.4s, v25.4s, v24.4s +sqdmulh v24.4s, v27.4s, v17.4s +mls v27.4s, v24.4s, v16.4s +sub v24.4s, v27.4s, v16.4s +umin v27.4s, v27.4s, v24.4s +uzp1 v0.8h, v21.8h, v25.8h +uzp1 v1.8h, v23.8h, v27.8h +zip1 v2.8h, v0.8h, v1.8h +zip2 v3.8h, v0.8h, v1.8h +str q2, [x0, #0] +str q3, [x0, #16] add x0, x0, #32 -add x3, x3, #16 +add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 cbnz x11, 26b @@ -24829,18 +24183,10 @@ add x3, x1, #0", concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 27: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #32] -ldr q21, [x2, #48] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #64] -ldr q21, [x2, #80] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #96] -ldr q21, [x2, #112] -uzp1 v3.8h, v20.8h, v21.8h +ldr q0, [x2, #0] +ldr q1, [x2, #16] +ldr q2, [x2, #32] +ldr q3, [x2, #48] ldr q20, [x6, #0] ldr q21, [x6, #16] ldr q22, [x6, #32] @@ -24929,7 +24275,7 @@ str q0, [x3, #0] str q1, [x3, #16] str q2, [x3, #32] str q3, [x3, #48] -add x2, x2, #128 +add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 @@ -25076,84 +24422,76 @@ mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #0] -str q25, [x3, #16] +str q16, [x3, #0] mul v16.8h, v1.8h, v22.8h sqrdmulh v17.8h, v1.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #128] -str q25, [x3, #144] +str q16, [x3, #64] mul v16.8h, v2.8h, v22.8h sqrdmulh v17.8h, v2.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #256] -str q25, [x3, #272] +str q16, [x3, #128] mul v16.8h, v3.8h, v22.8h sqrdmulh v17.8h, v3.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #384] -str q25, [x3, #400] +str q16, [x3, #192] mul v16.8h, v4.8h, v22.8h sqrdmulh v17.8h, v4.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #512] -str q25, [x3, #528] +str q16, [x3, #256] mul v16.8h, v5.8h, v22.8h sqrdmulh v17.8h, v5.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #640] -str q25, [x3, #656] +str q16, [x3, #320] mul v16.8h, v6.8h, v22.8h sqrdmulh v17.8h, v6.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #768] -str q25, [x3, #784] +str q16, [x3, #384] mul v16.8h, v7.8h, v22.8h sqrdmulh v17.8h, v7.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #896] -str q25, [x3, #912] +str q16, [x3, #448] add x2, x2, #16 -add x3, x3, #32 +add x3, x3, #16 sub x5, x5, #1 cbnz x5, 28b +movz x0, #17560, lsl #0 +add x0, x28, x0 +movz x1, #18584, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +movi v29.2d, #0 +29: +ldr q0, [x0, #0] +zip1 v1.8h, v0.8h, v29.8h +zip2 v2.8h, v0.8h, v29.8h +str q1, [x1, #0] +str q2, [x1, #16] +add x0, x0, #16 +add x1, x1, #32 +sub x10, x10, #1 +cbnz x10, 29b add x0, x26, #960 movz x1, #128, lsl #0 movz x2, #4, lsl #0 @@ -25162,7 +24500,7 @@ add x3, x28, x3 bl {vg_mlkem_decode_decompress} movz x0, #16536, lsl #0 add x0, x28, x0 -movz x1, #17560, lsl #0 +movz x1, #18584, lsl #0 add x1, x28, x1 bl {vg_mlkem_sub} movz x0, #16536, lsl #0 @@ -26173,10 +25511,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -29: +210: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 210f +cbz x6, 211f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26188,26 +25526,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 212f +cbz x4, 213f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 214f +cbz x4, 215f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 215f -214: +b 216f 215: -b 213f -212: +216: +b 214f 213: +214: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26219,26 +25557,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 216f +cbz x4, 217f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 218f +cbz x4, 219f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 219f -218: +b 220f 219: -b 217f -216: +220: +b 218f 217: +218: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26250,26 +25588,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 220f +cbz x4, 221f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 222f +cbz x4, 223f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 223f -222: +b 224f 223: -b 221f -220: +224: +b 222f 221: +222: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26281,33 +25619,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 224f +cbz x4, 225f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 226f +cbz x4, 227f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 227f -226: +b 228f 227: -b 225f -224: -225: -cbz x4, 228f -b 229f 228: -movz x5, #0, lsl #0 +b 226f +225: +226: +cbz x4, 229f +b 230f 229: -b 211f -210: +movz x5, #0, lsl #0 +230: +b 212f +211: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -26331,14 +25669,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -211: -cbnz x5, 29b -cbz x4, 230f +212: +cbnz x5, 210b +cbz x4, 231f movz x13, #0, lsl #0 str w13, [x3, #0] -b 231f -230: +b 232f 231: +232: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -26351,10 +25689,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -232: +233: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 233f +cbz x6, 234f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26366,26 +25704,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 235f +cbz x4, 236f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 237f +cbz x4, 238f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 238f -237: +b 239f 238: -b 236f -235: +239: +b 237f 236: +237: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26397,26 +25735,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 239f +cbz x4, 240f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 241f +cbz x4, 242f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 242f -241: +b 243f 242: -b 240f -239: +243: +b 241f 240: +241: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26428,26 +25766,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 243f +cbz x4, 244f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 245f +cbz x4, 246f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 246f -245: +b 247f 246: -b 244f -243: +247: +b 245f 244: +245: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26459,33 +25797,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 247f +cbz x4, 248f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 249f +cbz x4, 250f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 250f -249: +b 251f 250: -b 248f -247: -248: -cbz x4, 251f -b 252f 251: -movz x5, #0, lsl #0 +b 249f +248: +249: +cbz x4, 252f +b 253f 252: -b 234f -233: +movz x5, #0, lsl #0 +253: +b 235f +234: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -26509,23 +25847,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -234: -cbnz x5, 232b -cbz x4, 253f +235: +cbnz x5, 233b +cbz x4, 254f movz x13, #0, lsl #0 str w13, [x3, #0] -b 254f -253: +b 255f 254: +255: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 255f -b 256f -255: +cbz x6, 256f +b 257f +256: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -26657,15 +25995,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 257f +cbz x4, 258f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -259: +260: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 260f +cbz x6, 261f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26677,26 +26015,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 262f +cbz x4, 263f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 264f +cbz x4, 265f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 265f -264: +b 266f 265: -b 263f -262: +266: +b 264f 263: +264: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26708,26 +26046,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 266f +cbz x4, 267f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 268f +cbz x4, 269f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 269f -268: +b 270f 269: -b 267f -266: +270: +b 268f 267: +268: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26739,26 +26077,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 270f +cbz x4, 271f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 272f +cbz x4, 273f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 273f -272: +b 274f 273: -b 271f -270: +274: +b 272f 271: +272: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26770,33 +26108,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 274f +cbz x4, 275f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 276f +cbz x4, 277f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 277f -276: +b 278f 277: -b 275f -274: -275: -cbz x4, 278f -b 279f 278: -movz x5, #0, lsl #0 +b 276f +275: +276: +cbz x4, 279f +b 280f 279: -b 261f -260: +movz x5, #0, lsl #0 +280: +b 262f +261: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -26820,34 +26158,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -261: -cbnz x5, 259b -cbz x4, 280f +262: +cbnz x5, 260b +cbz x4, 281f movz x13, #0, lsl #0 str w13, [x3, #0] -b 281f -280: +b 282f 281: +282: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 258f -257: -movz x22, #1, lsl #0 +b 259f 258: +movz x22, #1, lsl #0 +259: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 282f +cbz x4, 283f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -284: +285: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 285f +cbz x6, 286f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26859,26 +26197,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 287f +cbz x4, 288f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 289f +cbz x4, 290f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 290f -289: +b 291f 290: -b 288f -287: +291: +b 289f 288: +289: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26890,26 +26228,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 291f +cbz x4, 292f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 293f +cbz x4, 294f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 294f -293: +b 295f 294: -b 292f -291: +295: +b 293f 292: +293: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26921,26 +26259,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 295f +cbz x4, 296f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 297f +cbz x4, 298f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 298f -297: +b 299f 298: -b 296f -295: +299: +b 297f 296: +297: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -26952,33 +26290,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 299f +cbz x4, 2100f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2101f +cbz x4, 2102f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2102f -2101: +b 2103f 2102: -b 2100f -299: -2100: -cbz x4, 2103f -b 2104f 2103: -movz x5, #0, lsl #0 +b 2101f +2100: +2101: +cbz x4, 2104f +b 2105f 2104: -b 286f -285: +movz x5, #0, lsl #0 +2105: +b 287f +286: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -27002,27 +26340,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -286: -cbnz x5, 284b -cbz x4, 2105f +287: +cbnz x5, 285b +cbz x4, 2106f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2106f -2105: +b 2107f 2106: +2107: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 283f -282: -movz x23, #1, lsl #0 +b 284f 283: +movz x23, #1, lsl #0 +284: and x6, x22, x23 -cbz x6, 2107f -b 2108f -2107: +cbz x6, 2108f +b 2109f +2108: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -27154,15 +26492,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2109f +cbz x4, 2110f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2111: +2112: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2112f +cbz x6, 2113f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27174,26 +26512,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2114f +cbz x4, 2115f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2116f +cbz x4, 2117f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2117f -2116: +b 2118f 2117: -b 2115f -2114: +2118: +b 2116f 2115: +2116: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27205,26 +26543,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2118f +cbz x4, 2119f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2120f +cbz x4, 2121f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2121f -2120: +b 2122f 2121: -b 2119f -2118: +2122: +b 2120f 2119: +2120: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27236,26 +26574,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2122f +cbz x4, 2123f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2124f +cbz x4, 2125f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2125f -2124: +b 2126f 2125: -b 2123f -2122: +2126: +b 2124f 2123: +2124: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27267,33 +26605,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2126f +cbz x4, 2127f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2128f +cbz x4, 2129f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2129f -2128: +b 2130f 2129: -b 2127f -2126: -2127: -cbz x4, 2130f -b 2131f 2130: -movz x5, #0, lsl #0 +b 2128f +2127: +2128: +cbz x4, 2131f +b 2132f 2131: -b 2113f -2112: +movz x5, #0, lsl #0 +2132: +b 2114f +2113: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -27317,34 +26655,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2113: -cbnz x5, 2111b -cbz x4, 2132f +2114: +cbnz x5, 2112b +cbz x4, 2133f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2133f -2132: +b 2134f 2133: +2134: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2110f -2109: -movz x22, #1, lsl #0 +b 2111f 2110: +movz x22, #1, lsl #0 +2111: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2134f +cbz x4, 2135f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2136: +2137: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2137f +cbz x6, 2138f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27356,26 +26694,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2139f +cbz x4, 2140f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2141f +cbz x4, 2142f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2142f -2141: +b 2143f 2142: -b 2140f -2139: +2143: +b 2141f 2140: +2141: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27387,26 +26725,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2143f +cbz x4, 2144f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2145f +cbz x4, 2146f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2146f -2145: +b 2147f 2146: -b 2144f -2143: +2147: +b 2145f 2144: +2145: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27418,26 +26756,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2147f +cbz x4, 2148f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2149f +cbz x4, 2150f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2150f -2149: +b 2151f 2150: -b 2148f -2147: +2151: +b 2149f 2148: +2149: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -27449,33 +26787,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2151f +cbz x4, 2152f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2153f +cbz x4, 2154f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2154f -2153: +b 2155f 2154: -b 2152f -2151: -2152: -cbz x4, 2155f -b 2156f 2155: -movz x5, #0, lsl #0 +b 2153f +2152: +2153: +cbz x4, 2156f +b 2157f 2156: -b 2138f -2137: +movz x5, #0, lsl #0 +2157: +b 2139f +2138: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -27499,51 +26837,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2138: -cbnz x5, 2136b -cbz x4, 2157f +2139: +cbnz x5, 2137b +cbz x4, 2158f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2158f -2157: +b 2159f 2158: +2159: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2135f -2134: -movz x23, #1, lsl #0 +b 2136f 2135: +movz x23, #1, lsl #0 +2136: and x6, x22, x23 -2108: +2109: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2159f -2161: +cbz x4, 2160f +2162: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2161b -b 2160f -2159: +cbnz x4, 2162b +b 2161f 2160: +2161: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2162f -2164: +cbz x4, 2163f +2165: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2164b -b 2163f -2162: +cbnz x4, 2165b +b 2164f 2163: -256: +2164: +257: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -28133,10 +27471,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2165: +2166: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2166f +cbz x6, 2167f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28148,26 +27486,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2168f +cbz x4, 2169f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2170f +cbz x4, 2171f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2171f -2170: +b 2172f 2171: -b 2169f -2168: +2172: +b 2170f 2169: +2170: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28179,26 +27517,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2172f +cbz x4, 2173f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2174f +cbz x4, 2175f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2175f -2174: +b 2176f 2175: -b 2173f -2172: +2176: +b 2174f 2173: +2174: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28210,26 +27548,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2176f +cbz x4, 2177f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2178f +cbz x4, 2179f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2179f -2178: +b 2180f 2179: -b 2177f -2176: +2180: +b 2178f 2177: +2178: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28241,33 +27579,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2180f +cbz x4, 2181f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2182f +cbz x4, 2183f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2183f -2182: +b 2184f 2183: -b 2181f -2180: -2181: -cbz x4, 2184f -b 2185f 2184: -movz x5, #0, lsl #0 +b 2182f +2181: +2182: +cbz x4, 2185f +b 2186f 2185: -b 2167f -2166: +movz x5, #0, lsl #0 +2186: +b 2168f +2167: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -28291,14 +27629,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2167: -cbnz x5, 2165b -cbz x4, 2186f +2168: +cbnz x5, 2166b +cbz x4, 2187f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2187f -2186: +b 2188f 2187: +2188: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -28311,10 +27649,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2188: +2189: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2189f +cbz x6, 2190f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28326,26 +27664,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2191f +cbz x4, 2192f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2193f +cbz x4, 2194f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2194f -2193: +b 2195f 2194: -b 2192f -2191: +2195: +b 2193f 2192: +2193: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28357,26 +27695,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2195f +cbz x4, 2196f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2197f +cbz x4, 2198f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2198f -2197: +b 2199f 2198: -b 2196f -2195: +2199: +b 2197f 2196: +2197: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28388,26 +27726,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2199f +cbz x4, 2200f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2201f +cbz x4, 2202f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2202f -2201: +b 2203f 2202: -b 2200f -2199: +2203: +b 2201f 2200: +2201: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28419,33 +27757,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2203f +cbz x4, 2204f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2205f +cbz x4, 2206f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2206f -2205: +b 2207f 2206: -b 2204f -2203: -2204: -cbz x4, 2207f -b 2208f 2207: -movz x5, #0, lsl #0 +b 2205f +2204: +2205: +cbz x4, 2208f +b 2209f 2208: -b 2190f -2189: +movz x5, #0, lsl #0 +2209: +b 2191f +2190: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -28469,23 +27807,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2190: -cbnz x5, 2188b -cbz x4, 2209f +2191: +cbnz x5, 2189b +cbz x4, 2210f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2210f -2209: +b 2211f 2210: +2211: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2211f -b 2212f -2211: +cbz x6, 2212f +b 2213f +2212: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -28617,15 +27955,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2213f +cbz x4, 2214f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2215: +2216: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2216f +cbz x6, 2217f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28637,26 +27975,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2218f +cbz x4, 2219f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2220f +cbz x4, 2221f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2221f -2220: +b 2222f 2221: -b 2219f -2218: +2222: +b 2220f 2219: +2220: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28668,26 +28006,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2222f +cbz x4, 2223f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2224f +cbz x4, 2225f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2225f -2224: +b 2226f 2225: -b 2223f -2222: +2226: +b 2224f 2223: +2224: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28699,26 +28037,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2226f +cbz x4, 2227f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2228f +cbz x4, 2229f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2229f -2228: +b 2230f 2229: -b 2227f -2226: +2230: +b 2228f 2227: +2228: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28730,33 +28068,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2230f +cbz x4, 2231f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2232f +cbz x4, 2233f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2233f -2232: +b 2234f 2233: -b 2231f -2230: -2231: -cbz x4, 2234f -b 2235f 2234: -movz x5, #0, lsl #0 +b 2232f +2231: +2232: +cbz x4, 2235f +b 2236f 2235: -b 2217f -2216: +movz x5, #0, lsl #0 +2236: +b 2218f +2217: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -28780,34 +28118,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2217: -cbnz x5, 2215b -cbz x4, 2236f +2218: +cbnz x5, 2216b +cbz x4, 2237f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2237f -2236: +b 2238f 2237: +2238: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2214f -2213: -movz x22, #1, lsl #0 +b 2215f 2214: +movz x22, #1, lsl #0 +2215: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2238f +cbz x4, 2239f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2240: +2241: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2241f +cbz x6, 2242f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28819,26 +28157,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2243f +cbz x4, 2244f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2245f +cbz x4, 2246f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2246f -2245: +b 2247f 2246: -b 2244f -2243: +2247: +b 2245f 2244: +2245: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28850,26 +28188,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2247f +cbz x4, 2248f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2249f +cbz x4, 2250f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2250f -2249: +b 2251f 2250: -b 2248f -2247: +2251: +b 2249f 2248: +2249: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28881,26 +28219,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2251f +cbz x4, 2252f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2253f +cbz x4, 2254f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2254f -2253: +b 2255f 2254: -b 2252f -2251: +2255: +b 2253f 2252: +2253: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -28912,33 +28250,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2255f +cbz x4, 2256f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2257f +cbz x4, 2258f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2258f -2257: +b 2259f 2258: -b 2256f -2255: -2256: -cbz x4, 2259f -b 2260f 2259: -movz x5, #0, lsl #0 +b 2257f +2256: +2257: +cbz x4, 2260f +b 2261f 2260: -b 2242f -2241: +movz x5, #0, lsl #0 +2261: +b 2243f +2242: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -28962,27 +28300,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2242: -cbnz x5, 2240b -cbz x4, 2261f +2243: +cbnz x5, 2241b +cbz x4, 2262f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2262f -2261: +b 2263f 2262: +2263: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2239f -2238: -movz x23, #1, lsl #0 +b 2240f 2239: +movz x23, #1, lsl #0 +2240: and x6, x22, x23 -cbz x6, 2263f -b 2264f -2263: +cbz x6, 2264f +b 2265f +2264: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -29114,15 +28452,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2265f +cbz x4, 2266f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2267: +2268: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2268f +cbz x6, 2269f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29134,26 +28472,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2270f +cbz x4, 2271f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2272f +cbz x4, 2273f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2273f -2272: +b 2274f 2273: -b 2271f -2270: +2274: +b 2272f 2271: +2272: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29165,26 +28503,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2274f +cbz x4, 2275f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2276f +cbz x4, 2277f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2277f -2276: +b 2278f 2277: -b 2275f -2274: +2278: +b 2276f 2275: +2276: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29196,26 +28534,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2278f +cbz x4, 2279f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2280f +cbz x4, 2281f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2281f -2280: +b 2282f 2281: -b 2279f -2278: +2282: +b 2280f 2279: +2280: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29227,33 +28565,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2282f +cbz x4, 2283f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2284f +cbz x4, 2285f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2285f -2284: +b 2286f 2285: -b 2283f -2282: -2283: -cbz x4, 2286f -b 2287f 2286: -movz x5, #0, lsl #0 +b 2284f +2283: +2284: +cbz x4, 2287f +b 2288f 2287: -b 2269f -2268: +movz x5, #0, lsl #0 +2288: +b 2270f +2269: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -29277,34 +28615,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2269: -cbnz x5, 2267b -cbz x4, 2288f +2270: +cbnz x5, 2268b +cbz x4, 2289f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2289f -2288: +b 2290f 2289: +2290: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2266f -2265: -movz x22, #1, lsl #0 +b 2267f 2266: +movz x22, #1, lsl #0 +2267: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2290f +cbz x4, 2291f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2292: +2293: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2293f +cbz x6, 2294f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29316,26 +28654,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2295f +cbz x4, 2296f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2297f +cbz x4, 2298f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2298f -2297: +b 2299f 2298: -b 2296f -2295: +2299: +b 2297f 2296: +2297: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29347,26 +28685,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2299f +cbz x4, 2300f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2301f +cbz x4, 2302f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2302f -2301: +b 2303f 2302: -b 2300f -2299: +2303: +b 2301f 2300: +2301: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29378,26 +28716,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2303f +cbz x4, 2304f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2305f +cbz x4, 2306f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2306f -2305: +b 2307f 2306: -b 2304f -2303: +2307: +b 2305f 2304: +2305: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -29409,33 +28747,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2307f +cbz x4, 2308f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2309f +cbz x4, 2310f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2310f -2309: +b 2311f 2310: -b 2308f -2307: -2308: -cbz x4, 2311f -b 2312f 2311: -movz x5, #0, lsl #0 +b 2309f +2308: +2309: +cbz x4, 2312f +b 2313f 2312: -b 2294f -2293: +movz x5, #0, lsl #0 +2313: +b 2295f +2294: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -29459,51 +28797,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2294: -cbnz x5, 2292b -cbz x4, 2313f +2295: +cbnz x5, 2293b +cbz x4, 2314f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2314f -2313: +b 2315f 2314: +2315: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2291f -2290: -movz x23, #1, lsl #0 +b 2292f 2291: +movz x23, #1, lsl #0 +2292: and x6, x22, x23 -2264: +2265: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2315f -2317: +cbz x4, 2316f +2318: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2317b -b 2316f -2315: +cbnz x4, 2318b +b 2317f 2316: +2317: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2318f -2320: +cbz x4, 2319f +2321: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2320b -b 2319f -2318: +cbnz x4, 2321b +b 2320f 2319: -2212: +2320: +2213: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -30049,10 +29387,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2321: +2322: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2322f +cbz x6, 2323f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30064,26 +29402,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2324f +cbz x4, 2325f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2326f +cbz x4, 2327f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2327f -2326: +b 2328f 2327: -b 2325f -2324: +2328: +b 2326f 2325: +2326: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30095,26 +29433,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2328f +cbz x4, 2329f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2330f +cbz x4, 2331f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2331f -2330: +b 2332f 2331: -b 2329f -2328: +2332: +b 2330f 2329: +2330: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30126,26 +29464,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2332f +cbz x4, 2333f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2334f +cbz x4, 2335f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2335f -2334: +b 2336f 2335: -b 2333f -2332: +2336: +b 2334f 2333: +2334: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30157,33 +29495,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2336f +cbz x4, 2337f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2338f +cbz x4, 2339f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2339f -2338: +b 2340f 2339: -b 2337f -2336: -2337: -cbz x4, 2340f -b 2341f 2340: -movz x5, #0, lsl #0 +b 2338f +2337: +2338: +cbz x4, 2341f +b 2342f 2341: -b 2323f -2322: +movz x5, #0, lsl #0 +2342: +b 2324f +2323: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -30207,14 +29545,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2323: -cbnz x5, 2321b -cbz x4, 2342f +2324: +cbnz x5, 2322b +cbz x4, 2343f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2343f -2342: +b 2344f 2343: +2344: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -30227,10 +29565,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2344: +2345: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2345f +cbz x6, 2346f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30242,26 +29580,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2347f +cbz x4, 2348f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2349f +cbz x4, 2350f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2350f -2349: +b 2351f 2350: -b 2348f -2347: +2351: +b 2349f 2348: +2349: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30273,26 +29611,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2351f +cbz x4, 2352f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2353f +cbz x4, 2354f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2354f -2353: +b 2355f 2354: -b 2352f -2351: +2355: +b 2353f 2352: +2353: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30304,26 +29642,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2355f +cbz x4, 2356f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2357f +cbz x4, 2358f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2358f -2357: +b 2359f 2358: -b 2356f -2355: +2359: +b 2357f 2356: +2357: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30335,33 +29673,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2359f +cbz x4, 2360f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2361f +cbz x4, 2362f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2362f -2361: +b 2363f 2362: -b 2360f -2359: -2360: -cbz x4, 2363f -b 2364f 2363: -movz x5, #0, lsl #0 +b 2361f +2360: +2361: +cbz x4, 2364f +b 2365f 2364: -b 2346f -2345: +movz x5, #0, lsl #0 +2365: +b 2347f +2346: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -30385,23 +29723,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2346: -cbnz x5, 2344b -cbz x4, 2365f +2347: +cbnz x5, 2345b +cbz x4, 2366f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2366f -2365: +b 2367f 2366: +2367: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2367f -b 2368f -2367: +cbz x6, 2368f +b 2369f +2368: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -30533,15 +29871,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2369f +cbz x4, 2370f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2371: +2372: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2372f +cbz x6, 2373f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30553,26 +29891,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2374f +cbz x4, 2375f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2376f +cbz x4, 2377f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2377f -2376: +b 2378f 2377: -b 2375f -2374: +2378: +b 2376f 2375: +2376: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30584,26 +29922,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2378f +cbz x4, 2379f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2380f +cbz x4, 2381f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2381f -2380: +b 2382f 2381: -b 2379f -2378: +2382: +b 2380f 2379: +2380: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30615,26 +29953,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2382f +cbz x4, 2383f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2384f +cbz x4, 2385f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2385f -2384: +b 2386f 2385: -b 2383f -2382: +2386: +b 2384f 2383: +2384: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30646,33 +29984,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2386f +cbz x4, 2387f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2388f +cbz x4, 2389f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2389f -2388: +b 2390f 2389: -b 2387f -2386: -2387: -cbz x4, 2390f -b 2391f 2390: -movz x5, #0, lsl #0 +b 2388f +2387: +2388: +cbz x4, 2391f +b 2392f 2391: -b 2373f -2372: +movz x5, #0, lsl #0 +2392: +b 2374f +2373: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -30696,34 +30034,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2373: -cbnz x5, 2371b -cbz x4, 2392f +2374: +cbnz x5, 2372b +cbz x4, 2393f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2393f -2392: +b 2394f 2393: +2394: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2370f -2369: -movz x22, #1, lsl #0 +b 2371f 2370: +movz x22, #1, lsl #0 +2371: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2394f +cbz x4, 2395f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2396: +2397: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2397f +cbz x6, 2398f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30735,26 +30073,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2399f +cbz x4, 2400f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2401f +cbz x4, 2402f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2402f -2401: +b 2403f 2402: -b 2400f -2399: +2403: +b 2401f 2400: +2401: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30766,26 +30104,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2403f +cbz x4, 2404f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2405f +cbz x4, 2406f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2406f -2405: +b 2407f 2406: -b 2404f -2403: +2407: +b 2405f 2404: +2405: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30797,26 +30135,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2407f +cbz x4, 2408f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2409f +cbz x4, 2410f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2410f -2409: +b 2411f 2410: -b 2408f -2407: +2411: +b 2409f 2408: +2409: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -30828,33 +30166,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2411f +cbz x4, 2412f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2413f +cbz x4, 2414f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2414f -2413: +b 2415f 2414: -b 2412f -2411: -2412: -cbz x4, 2415f -b 2416f 2415: -movz x5, #0, lsl #0 +b 2413f +2412: +2413: +cbz x4, 2416f +b 2417f 2416: -b 2398f -2397: +movz x5, #0, lsl #0 +2417: +b 2399f +2398: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -30878,27 +30216,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2398: -cbnz x5, 2396b -cbz x4, 2417f +2399: +cbnz x5, 2397b +cbz x4, 2418f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2418f -2417: +b 2419f 2418: +2419: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2395f -2394: -movz x23, #1, lsl #0 +b 2396f 2395: +movz x23, #1, lsl #0 +2396: and x6, x22, x23 -cbz x6, 2419f -b 2420f -2419: +cbz x6, 2420f +b 2421f +2420: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -31030,15 +30368,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2421f +cbz x4, 2422f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2423: +2424: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2424f +cbz x6, 2425f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31050,26 +30388,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2426f +cbz x4, 2427f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2428f +cbz x4, 2429f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2429f -2428: +b 2430f 2429: -b 2427f -2426: +2430: +b 2428f 2427: +2428: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31081,26 +30419,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2430f +cbz x4, 2431f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2432f +cbz x4, 2433f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2433f -2432: +b 2434f 2433: -b 2431f -2430: +2434: +b 2432f 2431: +2432: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31112,26 +30450,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2434f +cbz x4, 2435f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2436f +cbz x4, 2437f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2437f -2436: +b 2438f 2437: -b 2435f -2434: +2438: +b 2436f 2435: +2436: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31143,33 +30481,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2438f +cbz x4, 2439f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2440f +cbz x4, 2441f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2441f -2440: +b 2442f 2441: -b 2439f -2438: -2439: -cbz x4, 2442f -b 2443f 2442: -movz x5, #0, lsl #0 +b 2440f +2439: +2440: +cbz x4, 2443f +b 2444f 2443: -b 2425f -2424: +movz x5, #0, lsl #0 +2444: +b 2426f +2425: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -31193,34 +30531,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2425: -cbnz x5, 2423b -cbz x4, 2444f +2426: +cbnz x5, 2424b +cbz x4, 2445f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2445f -2444: +b 2446f 2445: +2446: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2422f -2421: -movz x22, #1, lsl #0 +b 2423f 2422: +movz x22, #1, lsl #0 +2423: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2446f +cbz x4, 2447f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2448: +2449: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2449f +cbz x6, 2450f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31232,26 +30570,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2451f +cbz x4, 2452f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2453f +cbz x4, 2454f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2454f -2453: +b 2455f 2454: -b 2452f -2451: +2455: +b 2453f 2452: +2453: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31263,26 +30601,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2455f +cbz x4, 2456f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2457f +cbz x4, 2458f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2458f -2457: +b 2459f 2458: -b 2456f -2455: +2459: +b 2457f 2456: +2457: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31294,26 +30632,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2459f +cbz x4, 2460f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2461f +cbz x4, 2462f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2462f -2461: +b 2463f 2462: -b 2460f -2459: +2463: +b 2461f 2460: +2461: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -31325,33 +30663,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2463f +cbz x4, 2464f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2465f +cbz x4, 2466f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2466f -2465: +b 2467f 2466: -b 2464f -2463: -2464: -cbz x4, 2467f -b 2468f 2467: -movz x5, #0, lsl #0 +b 2465f +2464: +2465: +cbz x4, 2468f +b 2469f 2468: -b 2450f -2449: +movz x5, #0, lsl #0 +2469: +b 2451f +2450: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -31375,51 +30713,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2450: -cbnz x5, 2448b -cbz x4, 2469f +2451: +cbnz x5, 2449b +cbz x4, 2470f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2470f -2469: +b 2471f 2470: +2471: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2447f -2446: -movz x23, #1, lsl #0 +b 2448f 2447: +movz x23, #1, lsl #0 +2448: and x6, x22, x23 -2420: +2421: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2471f -2473: +cbz x4, 2472f +2474: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2473b -b 2472f -2471: +cbnz x4, 2474b +b 2473f 2472: +2473: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2474f -2476: +cbz x4, 2475f +2477: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2476b -b 2475f -2474: +cbnz x4, 2477b +b 2476f 2475: -2368: +2476: +2369: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -32065,10 +31403,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2477: +2478: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2478f +cbz x6, 2479f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32080,26 +31418,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2480f +cbz x4, 2481f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2482f +cbz x4, 2483f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2483f -2482: +b 2484f 2483: -b 2481f -2480: +2484: +b 2482f 2481: +2482: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32111,26 +31449,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2484f +cbz x4, 2485f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2486f +cbz x4, 2487f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2487f -2486: +b 2488f 2487: -b 2485f -2484: +2488: +b 2486f 2485: +2486: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32142,26 +31480,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2488f +cbz x4, 2489f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2490f +cbz x4, 2491f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2491f -2490: +b 2492f 2491: -b 2489f -2488: +2492: +b 2490f 2489: +2490: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32173,33 +31511,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2492f +cbz x4, 2493f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2494f +cbz x4, 2495f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2495f -2494: +b 2496f 2495: -b 2493f -2492: -2493: -cbz x4, 2496f -b 2497f 2496: -movz x5, #0, lsl #0 +b 2494f +2493: +2494: +cbz x4, 2497f +b 2498f 2497: -b 2479f -2478: +movz x5, #0, lsl #0 +2498: +b 2480f +2479: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -32223,14 +31561,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2479: -cbnz x5, 2477b -cbz x4, 2498f +2480: +cbnz x5, 2478b +cbz x4, 2499f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2499f -2498: +b 2500f 2499: +2500: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -32243,10 +31581,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2500: +2501: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2501f +cbz x6, 2502f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32258,26 +31596,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2503f +cbz x4, 2504f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2505f +cbz x4, 2506f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2506f -2505: +b 2507f 2506: -b 2504f -2503: +2507: +b 2505f 2504: +2505: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32289,26 +31627,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2507f +cbz x4, 2508f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2509f +cbz x4, 2510f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2510f -2509: +b 2511f 2510: -b 2508f -2507: +2511: +b 2509f 2508: +2509: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32320,26 +31658,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2511f +cbz x4, 2512f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2513f +cbz x4, 2514f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2514f -2513: +b 2515f 2514: -b 2512f -2511: +2515: +b 2513f 2512: +2513: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32351,33 +31689,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2515f +cbz x4, 2516f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2517f +cbz x4, 2518f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2518f -2517: +b 2519f 2518: -b 2516f -2515: -2516: -cbz x4, 2519f -b 2520f 2519: -movz x5, #0, lsl #0 +b 2517f +2516: +2517: +cbz x4, 2520f +b 2521f 2520: -b 2502f -2501: +movz x5, #0, lsl #0 +2521: +b 2503f +2502: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -32401,23 +31739,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2502: -cbnz x5, 2500b -cbz x4, 2521f +2503: +cbnz x5, 2501b +cbz x4, 2522f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2522f -2521: +b 2523f 2522: +2523: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2523f -b 2524f -2523: +cbz x6, 2524f +b 2525f +2524: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -32549,15 +31887,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2525f +cbz x4, 2526f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2527: +2528: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2528f +cbz x6, 2529f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32569,26 +31907,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2530f +cbz x4, 2531f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2532f +cbz x4, 2533f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2533f -2532: +b 2534f 2533: -b 2531f -2530: +2534: +b 2532f 2531: +2532: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32600,26 +31938,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2534f +cbz x4, 2535f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2536f +cbz x4, 2537f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2537f -2536: +b 2538f 2537: -b 2535f -2534: +2538: +b 2536f 2535: +2536: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32631,26 +31969,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2538f +cbz x4, 2539f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2540f +cbz x4, 2541f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2541f -2540: +b 2542f 2541: -b 2539f -2538: +2542: +b 2540f 2539: +2540: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32662,33 +32000,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2542f +cbz x4, 2543f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2544f +cbz x4, 2545f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2545f -2544: +b 2546f 2545: -b 2543f -2542: -2543: -cbz x4, 2546f -b 2547f 2546: -movz x5, #0, lsl #0 +b 2544f +2543: +2544: +cbz x4, 2547f +b 2548f 2547: -b 2529f -2528: +movz x5, #0, lsl #0 +2548: +b 2530f +2529: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -32712,34 +32050,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2529: -cbnz x5, 2527b -cbz x4, 2548f +2530: +cbnz x5, 2528b +cbz x4, 2549f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2549f -2548: +b 2550f 2549: +2550: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2526f -2525: -movz x22, #1, lsl #0 +b 2527f 2526: +movz x22, #1, lsl #0 +2527: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2550f +cbz x4, 2551f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2552: +2553: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2553f +cbz x6, 2554f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32751,26 +32089,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2555f +cbz x4, 2556f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2557f +cbz x4, 2558f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2558f -2557: +b 2559f 2558: -b 2556f -2555: +2559: +b 2557f 2556: +2557: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32782,26 +32120,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2559f +cbz x4, 2560f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2561f +cbz x4, 2562f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2562f -2561: +b 2563f 2562: -b 2560f -2559: +2563: +b 2561f 2560: +2561: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32813,26 +32151,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2563f +cbz x4, 2564f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2565f +cbz x4, 2566f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2566f -2565: +b 2567f 2566: -b 2564f -2563: +2567: +b 2565f 2564: +2565: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -32844,33 +32182,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2567f +cbz x4, 2568f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2569f +cbz x4, 2570f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2570f -2569: +b 2571f 2570: -b 2568f -2567: -2568: -cbz x4, 2571f -b 2572f 2571: -movz x5, #0, lsl #0 +b 2569f +2568: +2569: +cbz x4, 2572f +b 2573f 2572: -b 2554f -2553: +movz x5, #0, lsl #0 +2573: +b 2555f +2554: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -32894,27 +32232,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2554: -cbnz x5, 2552b -cbz x4, 2573f +2555: +cbnz x5, 2553b +cbz x4, 2574f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2574f -2573: +b 2575f 2574: +2575: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2551f -2550: -movz x23, #1, lsl #0 +b 2552f 2551: +movz x23, #1, lsl #0 +2552: and x6, x22, x23 -cbz x6, 2575f -b 2576f -2575: +cbz x6, 2576f +b 2577f +2576: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -33046,15 +32384,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2577f +cbz x4, 2578f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2579: +2580: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2580f +cbz x6, 2581f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -33066,26 +32404,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2582f +cbz x4, 2583f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2584f +cbz x4, 2585f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2585f -2584: +b 2586f 2585: -b 2583f -2582: +2586: +b 2584f 2583: +2584: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -33097,26 +32435,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2586f +cbz x4, 2587f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2588f +cbz x4, 2589f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2589f -2588: +b 2590f 2589: -b 2587f -2586: +2590: +b 2588f 2587: +2588: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -33128,26 +32466,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2590f +cbz x4, 2591f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2592f +cbz x4, 2593f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2593f -2592: +b 2594f 2593: -b 2591f -2590: +2594: +b 2592f 2591: +2592: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -33159,33 +32497,215 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2594f +cbz x4, 2595f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2596f +cbz x4, 2597f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2597f -2596: +b 2598f 2597: -b 2595f -2594: -2595: -cbz x4, 2598f -b 2599f 2598: -movz x5, #0, lsl #0 +b 2596f +2595: +2596: +cbz x4, 2599f +b 2600f 2599: -b 2581f -2580: +movz x5, #0, lsl #0 +2600: +b 2582f +2581: +ldr q0, [x2, #0] +tbl v0.16b, {{v0.16b}}, v16.16b +ushr v1.8h, v0.8h, #4 +bif v0.16b, v1.16b, v17.16b +cmhi v1.8h, v19.8h, v0.8h +and v1.16b, v1.16b, v20.16b +addv h1, v1.8h +umov w6, v1.s[0] +movz x7, #255, lsl #0 +and x7, x6, x7 +lsl x7, x7, #4 +add x7, x16, x7 +ldr q2, [x7, #0] +tbl v0.16b, {{v0.16b}}, v2.16b +str q0, [x3, #0] +lsr x6, x6, #8 +movz x7, #15, lsl #0 +and x6, x6, x7 +sub x4, x4, x6 +lsl x6, x6, #1 +add x3, x3, x6 +add x2, x2, #12 +sub x5, x5, #4 +2582: +cbnz x5, 2580b +cbz x4, 2601f +movz x13, #0, lsl #0 +str w13, [x3, #0] +b 2602f +2601: +2602: +sub x0, x4, #1 +lsr x0, x0, #63 +add x22, x0, #0 +str x3, [x19, #1680] +str x4, [x19, #1688] +b 2579f +2578: +movz x22, #1, lsl #0 +2579: +ldr x3, [x19, #1696] +ldr x4, [x19, #1704] +cbz x4, 2603f +add x2, x19, #904 +movz x5, #56, lsl #0 +movz x9, #3329, lsl #0 +movz x10, #15, lsl #0 +2605: +sub x6, x4, #8 +lsr x6, x6, #63 +cbz x6, 2606f +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2608f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2610f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2611f +2610: +2611: +b 2609f +2608: +2609: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2612f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2614f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2615f +2614: +2615: +b 2613f +2612: +2613: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2616f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2618f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2619f +2618: +2619: +b 2617f +2616: +2617: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2620f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2622f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2623f +2622: +2623: +b 2621f +2620: +2621: +cbz x4, 2624f +b 2625f +2624: +movz x5, #0, lsl #0 +2625: +b 2607f +2606: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -33209,233 +32729,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2581: -cbnz x5, 2579b -cbz x4, 2600f -movz x13, #0, lsl #0 -str w13, [x3, #0] -b 2601f -2600: -2601: -sub x0, x4, #1 -lsr x0, x0, #63 -add x22, x0, #0 -str x3, [x19, #1680] -str x4, [x19, #1688] -b 2578f -2577: -movz x22, #1, lsl #0 -2578: -ldr x3, [x19, #1696] -ldr x4, [x19, #1704] -cbz x4, 2602f -add x2, x19, #904 -movz x5, #56, lsl #0 -movz x9, #3329, lsl #0 -movz x10, #15, lsl #0 -2604: -sub x6, x4, #8 -lsr x6, x6, #63 -cbz x6, 2605f -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2607f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2609f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2610f -2609: -2610: -b 2608f 2607: -2608: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2611f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2613f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2614f -2613: -2614: -b 2612f -2611: -2612: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2615f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2617f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2618f -2617: -2618: -b 2616f -2615: -2616: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2619f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2621f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2622f -2621: -2622: -b 2620f -2619: -2620: -cbz x4, 2623f -b 2624f -2623: -movz x5, #0, lsl #0 -2624: -b 2606f -2605: -ldr q0, [x2, #0] -tbl v0.16b, {{v0.16b}}, v16.16b -ushr v1.8h, v0.8h, #4 -bif v0.16b, v1.16b, v17.16b -cmhi v1.8h, v19.8h, v0.8h -and v1.16b, v1.16b, v20.16b -addv h1, v1.8h -umov w6, v1.s[0] -movz x7, #255, lsl #0 -and x7, x6, x7 -lsl x7, x7, #4 -add x7, x16, x7 -ldr q2, [x7, #0] -tbl v0.16b, {{v0.16b}}, v2.16b -str q0, [x3, #0] -lsr x6, x6, #8 -movz x7, #15, lsl #0 -and x6, x6, x7 -sub x4, x4, x6 -lsl x6, x6, #1 -add x3, x3, x6 -add x2, x2, #12 -sub x5, x5, #4 -2606: -cbnz x5, 2604b -cbz x4, 2625f +cbnz x5, 2605b +cbz x4, 2626f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2626f -2625: +b 2627f 2626: +2627: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2603f -2602: -movz x23, #1, lsl #0 +b 2604f 2603: +movz x23, #1, lsl #0 +2604: and x6, x22, x23 -2576: +2577: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2627f -2629: +cbz x4, 2628f +2630: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2629b -b 2628f -2627: +cbnz x4, 2630b +b 2629f 2628: +2629: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2630f -2632: +cbz x4, 2631f +2633: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2632b -b 2631f -2630: +cbnz x4, 2633b +b 2632f 2631: -2524: +2632: +2525: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -33973,10 +33311,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2633: +2634: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2634f +cbz x6, 2635f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -33988,26 +33326,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2636f +cbz x4, 2637f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2638f +cbz x4, 2639f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2639f -2638: +b 2640f 2639: -b 2637f -2636: +2640: +b 2638f 2637: +2638: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34019,26 +33357,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2640f +cbz x4, 2641f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2642f +cbz x4, 2643f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2643f -2642: +b 2644f 2643: -b 2641f -2640: +2644: +b 2642f 2641: +2642: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34050,26 +33388,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2644f +cbz x4, 2645f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2646f +cbz x4, 2647f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2647f -2646: +b 2648f 2647: -b 2645f -2644: +2648: +b 2646f 2645: +2646: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34081,33 +33419,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2648f +cbz x4, 2649f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2650f +cbz x4, 2651f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2651f -2650: +b 2652f 2651: -b 2649f -2648: -2649: -cbz x4, 2652f -b 2653f 2652: -movz x5, #0, lsl #0 +b 2650f +2649: +2650: +cbz x4, 2653f +b 2654f 2653: -b 2635f -2634: +movz x5, #0, lsl #0 +2654: +b 2636f +2635: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -34131,20 +33469,20 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2635: -cbnz x5, 2633b -cbz x4, 2654f +2636: +cbnz x5, 2634b +cbz x4, 2655f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2655f -2654: +b 2656f 2655: +2656: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 -cbz x22, 2656f -b 2657f -2656: +cbz x22, 2657f +b 2658f +2657: str x22, [x19, #1848] str x23, [x19, #1856] add x2, x28, #1176 @@ -34201,7 +33539,7 @@ add x20, x19, #400 add x21, x19, #904 movz x22, #3, lsl #0 movz x22, #5, lsl #0 -2658: +2659: add x1, x19, #768 add x1, x1, #768 str x30, [x1, #128] @@ -34252,7 +33590,7 @@ ldr x6, [x19, #320] str x6, [x20, #160] add x20, x20, #168 sub x22, x22, #1 -cbnz x22, 2658b +cbnz x22, 2659b movz x9, #256, lsl #0 movk x9, #513, lsl #16 movk x9, #1027, lsl #32 @@ -34299,10 +33637,10 @@ movz x4, #256, lsl #0 movz x5, #280, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2659: +2660: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2660f +cbz x6, 2661f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34314,26 +33652,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2662f +cbz x4, 2663f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2664f +cbz x4, 2665f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2665f -2664: +b 2666f 2665: -b 2663f -2662: +2666: +b 2664f 2663: +2664: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34345,26 +33683,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2666f +cbz x4, 2667f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2668f +cbz x4, 2669f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2669f -2668: +b 2670f 2669: -b 2667f -2666: +2670: +b 2668f 2667: +2668: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34376,26 +33714,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2670f +cbz x4, 2671f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2672f +cbz x4, 2673f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2673f -2672: +b 2674f 2673: -b 2671f -2670: +2674: +b 2672f 2671: +2672: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -34407,33 +33745,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2674f +cbz x4, 2675f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2676f +cbz x4, 2677f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2677f -2676: +b 2678f 2677: -b 2675f -2674: -2675: -cbz x4, 2678f -b 2679f 2678: -movz x5, #0, lsl #0 +b 2676f +2675: +2676: +cbz x4, 2679f +b 2680f 2679: -b 2661f -2660: +movz x5, #0, lsl #0 +2680: +b 2662f +2661: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -34457,30 +33795,30 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2661: -cbnz x5, 2659b -cbz x4, 2680f +2662: +cbnz x5, 2660b +cbz x4, 2681f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2681f -2680: +b 2682f 2681: +2682: sub x0, x4, #1 lsr x0, x0, #63 and x24, x24, x0 -cbz x4, 2682f -2684: +cbz x4, 2683f +2685: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2684b -b 2683f -2682: +cbnz x4, 2685b +b 2684f 2683: +2684: ldr x22, [x19, #1848] ldr x23, [x19, #1856] -2657: +2658: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -34542,7 +33880,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2685: +2686: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -34649,7 +33987,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2685b +cbnz x11, 2686b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -34663,7 +34001,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2686: +2687: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -34760,7 +34098,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2686b +cbnz x5, 2687b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -34796,7 +34134,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2687: +2688: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -34956,7 +34294,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2687b +cbnz x5, 2688b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #29696, lsl #0 @@ -34964,7 +34302,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2688: +2689: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -34975,14 +34313,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2688b +cbnz x11, 2689b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2689: +2690: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -34991,7 +34329,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2689b +cbnz x10, 2690b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -35012,7 +34350,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2690: +2691: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -35119,7 +34457,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2690b +cbnz x11, 2691b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -35133,7 +34471,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2691: +2692: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -35230,7 +34568,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2691b +cbnz x5, 2692b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -35266,7 +34604,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2692: +2693: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -35426,7 +34764,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2692b +cbnz x5, 2693b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #30720, lsl #0 @@ -35434,7 +34772,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2693: +2694: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -35445,14 +34783,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2693b +cbnz x11, 2694b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2694: +2695: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -35461,7 +34799,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2694b +cbnz x10, 2695b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -35482,7 +34820,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2695: +2696: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -35589,7 +34927,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2695b +cbnz x11, 2696b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -35603,7 +34941,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2696: +2697: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -35700,7 +35038,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2696b +cbnz x5, 2697b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -35736,7 +35074,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2697: +2698: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -35896,7 +35234,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2697b +cbnz x5, 2698b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #31744, lsl #0 @@ -35904,7 +35242,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2698: +2699: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -35915,14 +35253,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2698b +cbnz x11, 2699b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2699: +2700: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -35931,7 +35269,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2699b +cbnz x10, 2700b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -35957,7 +35295,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2700: +2701: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -35976,7 +35314,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2700b +cbnz x11, 2701b add x0, x25, #1536 movz x1, #5272, lsl #0 add x1, x28, x1 @@ -35995,7 +35333,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2701: +2702: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -36014,7 +35352,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2701b +cbnz x11, 2702b add x0, x25, #1920 movz x1, #6296, lsl #0 add x1, x28, x1 @@ -36033,7 +35371,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2702: +2703: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -36052,7 +35390,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2702b +cbnz x11, 2703b movz x0, #17560, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -36066,7 +35404,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2703: +2704: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -36173,7 +35511,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2703b +cbnz x11, 2704b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -36187,7 +35525,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2704: +2705: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -36284,7 +35622,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2704b +cbnz x5, 2705b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -36320,7 +35658,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2705: +2706: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -36480,7 +35818,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2705b +cbnz x5, 2706b movz x9, #6, lsl #0 strb w9, [x28, #976] add x0, x28, #0 @@ -36607,7 +35945,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2706: +2707: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -36618,14 +35956,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2706b +cbnz x11, 2707b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2707: +2708: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -36634,7 +35972,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2707b +cbnz x10, 2708b add x0, x28, #912 movz x1, #32, lsl #0 movz x2, #1, lsl #0 @@ -36658,7 +35996,7 @@ movz x1, #20632, lsl #0 add x1, x28, x1 movz x2, #136, lsl #0 movz x10, #0, lsl #0 -2708: +2709: ldr x9, [x0, #0] ldr x11, [x1, #0] eor x9, x9, x11 @@ -36666,7 +36004,7 @@ orr x10, x10, x9 add x0, x0, #8 add x1, x1, #8 sub x2, x2, #1 -cbnz x2, 2708b +cbnz x2, 2709b movz x11, #0, lsl #0 sub x11, x11, x10 orr x10, x10, x11 @@ -36707,16 +36045,14 @@ ret .arch_extension nosha3 .p2align 6", vg_mlkem_decode_decompress = sym super::mlkem::vg_mlkem_decode_decompress, - vg_mlkem_decode12 = sym super::mlkem::vg_mlkem_decode12, + vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem_sub = sym super::mlkem::vg_mlkem_sub, vg_mlkem_compress_encode = sym super::mlkem::vg_mlkem_compress_encode, vg_keccak_f1600 = sym super::sha3::vg_keccak_f1600, vg_keccak_f1600_x3_sha3 = sym super::sha3::vg_keccak_f1600_x3_sha3, vg_keccak_f1600_x2_sha3 = sym super::sha3::vg_keccak_f1600_x2_sha3, vg_mlkem16_cbd2 = sym super::mlkem16::vg_mlkem16_cbd2, - vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem_add = sym super::mlkem::vg_mlkem_add, - VG_MLKEM_NTT16_ZETAS = sym super::consts::VG_MLKEM_NTT16_ZETAS, VG_MLKEM_GAMMA = sym super::consts::VG_MLKEM_GAMMA, VG_MLKEM_INTT16_ZETAS = sym super::consts::VG_MLKEM_INTT16_ZETAS, VG_MLKEM_REJ_SHUFFLE = sym super::consts::VG_MLKEM_REJ_SHUFFLE, @@ -57881,868 +57217,193 @@ add x25, x0, #0 add x26, x1, #0 add x27, x2, #0 add x28, x3, #0 -movz x24, #1, lsl #0 -add x0, x26, #0 -movz x1, #320, lsl #0 -movz x2, #10, lsl #0 -movz x3, #13464, lsl #0 -add x3, x28, x3 -bl {vg_mlkem_decode_decompress} -movz x0, #13464, lsl #0 -add x0, x28, x0 -add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -20: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 20b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -21: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 21b +movz x24, #1, lsl #0 +add x0, x26, #0 +movz x1, #320, lsl #0 +movz x2, #10, lsl #0 +movz x3, #18584, lsl #0 +add x3, x28, x3 +bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #13464, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +20: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 20b +movz x0, #13464, lsl #0 +add x0, x28, x0 +add x1, x28, #3224 +bl {vg_mlkem16_ntt} add x0, x26, #320 movz x1, #320, lsl #0 movz x2, #10, lsl #0 -movz x3, #14488, lsl #0 +movz x3, #18584, lsl #0 add x3, x28, x3 bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #14488, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +21: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 21b movz x0, #14488, lsl #0 add x0, x28, x0 add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -22: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 22b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -23: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 23b +bl {vg_mlkem16_ntt} add x0, x26, #640 movz x1, #320, lsl #0 movz x2, #10, lsl #0 -movz x3, #15512, lsl #0 +movz x3, #18584, lsl #0 add x3, x28, x3 bl {vg_mlkem_decode_decompress} +movz x0, #18584, lsl #0 +add x0, x28, x0 +movz x1, #15512, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +22: +ldr q0, [x0, #0] +ldr q1, [x0, #16] +uzp1 v2.8h, v0.8h, v1.8h +str q2, [x1, #0] +add x0, x0, #32 +add x1, x1, #16 +sub x10, x10, #1 +cbnz x10, 22b movz x0, #15512, lsl #0 add x0, x28, x0 add x1, x28, #3224 -movz x9, #3329, lsl #0 -dup v31.8h, w9 -movz x9, #10, lsl #0 -dup v30.8h, w9 -movi v29.2d, #0 -movz x9, #1729, lsl #0 -mov v18.h[0], w9 -movz x9, #17019, lsl #0 -mov v19.h[0], w9 -movz x9, #2580, lsl #0 -mov v18.h[1], w9 -movz x9, #25395, lsl #0 -mov v19.h[1], w9 -movz x9, #3289, lsl #0 -mov v18.h[2], w9 -movz x9, #32374, lsl #0 -mov v19.h[2], w9 -movz x9, #2642, lsl #0 -mov v18.h[3], w9 -movz x9, #26006, lsl #0 -mov v19.h[3], w9 -movz x9, #630, lsl #0 -mov v18.h[4], w9 -movz x9, #6201, lsl #0 -mov v19.h[4], w9 -movz x9, #1897, lsl #0 -mov v18.h[5], w9 -movz x9, #18673, lsl #0 -mov v19.h[5], w9 -movz x9, #848, lsl #0 -mov v18.h[6], w9 -movz x9, #8347, lsl #0 -mov v19.h[6], w9 -add x2, x0, #0 -add x3, x1, #0 -movz x5, #4, lsl #0 -24: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #128] -ldr q21, [x2, #144] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #256] -ldr q21, [x2, #272] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #384] -ldr q21, [x2, #400] -uzp1 v3.8h, v20.8h, v21.8h -ldr q20, [x2, #512] -ldr q21, [x2, #528] -uzp1 v4.8h, v20.8h, v21.8h -ldr q20, [x2, #640] -ldr q21, [x2, #656] -uzp1 v5.8h, v20.8h, v21.8h -ldr q20, [x2, #768] -ldr q21, [x2, #784] -uzp1 v6.8h, v20.8h, v21.8h -ldr q20, [x2, #896] -ldr q21, [x2, #912] -uzp1 v7.8h, v20.8h, v21.8h -dup v22.8h, v18.h[0] -dup v23.8h, v19.h[0] -mul v16.8h, v4.8h, v22.8h -sqrdmulh v17.8h, v4.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v4.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v3.8h, v16.8h -add v3.8h, v3.8h, v16.8h -dup v22.8h, v18.h[1] -dup v23.8h, v19.h[1] -mul v16.8h, v2.8h, v22.8h -sqrdmulh v17.8h, v2.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v22.8h, v18.h[2] -dup v23.8h, v19.h[2] -mul v16.8h, v6.8h, v22.8h -sqrdmulh v17.8h, v6.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v6.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v5.8h, v16.8h -add v5.8h, v5.8h, v16.8h -dup v22.8h, v18.h[3] -dup v23.8h, v19.h[3] -mul v16.8h, v1.8h, v22.8h -sqrdmulh v17.8h, v1.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v22.8h, v18.h[4] -dup v23.8h, v19.h[4] -mul v16.8h, v3.8h, v22.8h -sqrdmulh v17.8h, v3.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -dup v22.8h, v18.h[5] -dup v23.8h, v19.h[5] -mul v16.8h, v5.8h, v22.8h -sqrdmulh v17.8h, v5.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -dup v22.8h, v18.h[6] -dup v23.8h, v19.h[6] -mul v16.8h, v7.8h, v22.8h -sqrdmulh v17.8h, v7.8h, v23.8h -mls v16.8h, v17.8h, v31.8h -sub v7.8h, v6.8h, v16.8h -add v6.8h, v6.8h, v16.8h -str q0, [x3, #0] -str q1, [x3, #64] -str q2, [x3, #128] -str q3, [x3, #192] -str q4, [x3, #256] -str q5, [x3, #320] -str q6, [x3, #384] -str q7, [x3, #448] -add x2, x2, #32 -add x3, x3, #16 -sub x5, x5, #1 -cbnz x5, 24b -add x2, x1, #0", - concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_NTT16_ZETAS}")), - concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_NTT16_ZETAS}")), - r"add x4, x0, #0 -movz x5, #8, lsl #0 -25: -ldr q0, [x2, #0] -ldr q1, [x2, #16] -ldr q2, [x2, #32] -ldr q3, [x2, #48] -ldr q20, [x6, #0] -ldr q21, [x6, #16] -ldr q22, [x6, #32] -ldr q23, [x6, #48] -dup v24.8h, v20.h[0] -dup v25.8h, v21.h[0] -mul v16.8h, v2.8h, v24.8h -sqrdmulh v17.8h, v2.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v2.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v1.8h, v16.8h -add v1.8h, v1.8h, v16.8h -dup v24.8h, v20.h[1] -dup v25.8h, v21.h[1] -mul v16.8h, v1.8h, v24.8h -sqrdmulh v17.8h, v1.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -dup v24.8h, v20.h[2] -dup v25.8h, v21.h[2] -mul v16.8h, v3.8h, v24.8h -sqrdmulh v17.8h, v3.8h, v25.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -zip2 v24.8h, v20.8h, v20.8h -zip2 v25.8h, v21.8h, v21.8h -zip1 v26.4s, v24.4s, v24.4s -zip1 v27.4s, v25.4s, v25.4s -zip1 v28.8h, v22.8h, v22.8h -zip1 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v0.2d, v1.2d -trn2 v5.2d, v0.2d, v1.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v0.4s, v4.4s, v5.4s -trn2 v1.4s, v4.4s, v5.4s -mul v16.8h, v1.8h, v28.8h -sqrdmulh v17.8h, v1.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v1.8h, v0.8h, v16.8h -add v0.8h, v0.8h, v16.8h -trn1 v4.4s, v0.4s, v1.4s -trn2 v5.4s, v0.4s, v1.4s -trn1 v0.2d, v4.2d, v5.2d -trn2 v1.2d, v4.2d, v5.2d -zip2 v26.4s, v24.4s, v24.4s -zip2 v27.4s, v25.4s, v25.4s -zip2 v28.8h, v22.8h, v22.8h -zip2 v19.8h, v23.8h, v23.8h -trn1 v4.2d, v2.2d, v3.2d -trn2 v5.2d, v2.2d, v3.2d -mul v16.8h, v5.8h, v26.8h -sqrdmulh v17.8h, v5.8h, v27.8h -mls v16.8h, v17.8h, v31.8h -sub v5.8h, v4.8h, v16.8h -add v4.8h, v4.8h, v16.8h -trn1 v2.4s, v4.4s, v5.4s -trn2 v3.4s, v4.4s, v5.4s -mul v16.8h, v3.8h, v28.8h -sqrdmulh v17.8h, v3.8h, v19.8h -mls v16.8h, v17.8h, v31.8h -sub v3.8h, v2.8h, v16.8h -add v2.8h, v2.8h, v16.8h -trn1 v4.4s, v2.4s, v3.4s -trn2 v5.4s, v2.4s, v3.4s -trn1 v2.2d, v4.2d, v5.2d -trn2 v3.2d, v4.2d, v5.2d -sqrdmulh v16.8h, v0.8h, v30.8h -mls v0.8h, v16.8h, v31.8h -sshr v16.8h, v0.8h, #15 -and v16.16b, v16.16b, v31.16b -add v0.8h, v0.8h, v16.8h -sqrdmulh v16.8h, v1.8h, v30.8h -mls v1.8h, v16.8h, v31.8h -sshr v16.8h, v1.8h, #15 -and v16.16b, v16.16b, v31.16b -add v1.8h, v1.8h, v16.8h -sqrdmulh v16.8h, v2.8h, v30.8h -mls v2.8h, v16.8h, v31.8h -sshr v16.8h, v2.8h, #15 -and v16.16b, v16.16b, v31.16b -add v2.8h, v2.8h, v16.8h -sqrdmulh v16.8h, v3.8h, v30.8h -mls v3.8h, v16.8h, v31.8h -sshr v16.8h, v3.8h, #15 -and v16.16b, v16.16b, v31.16b -add v3.8h, v3.8h, v16.8h -zip1 v24.8h, v0.8h, v29.8h -zip2 v25.8h, v0.8h, v29.8h -str q24, [x4, #0] -str q25, [x4, #16] -zip1 v24.8h, v1.8h, v29.8h -zip2 v25.8h, v1.8h, v29.8h -str q24, [x4, #32] -str q25, [x4, #48] -zip1 v24.8h, v2.8h, v29.8h -zip2 v25.8h, v2.8h, v29.8h -str q24, [x4, #64] -str q25, [x4, #80] -zip1 v24.8h, v3.8h, v29.8h -zip2 v25.8h, v3.8h, v29.8h -str q24, [x4, #96] -str q25, [x4, #112] -add x2, x2, #64 -add x6, x6, #64 -add x4, x4, #128 -sub x5, x5, #1 -cbnz x5, 25b +bl {vg_mlkem16_ntt} add x0, x25, #0 movz x1, #4248, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +23: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 23b add x0, x25, #384 movz x1, #5272, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +24: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 24b add x0, x25, #768 movz x1, #6296, lsl #0 add x1, x28, x1 -bl {vg_mlkem_decode12} +movz x9, #256, lsl #0 +movk x9, #65282, lsl #16 +movk x9, #1027, lsl #32 +movk x9, #65285, lsl #48 +dup v16.2d, x9 +movz x9, #1798, lsl #0 +movk x9, #65288, lsl #16 +movk x9, #2569, lsl #32 +movk x9, #65291, lsl #48 +mov v16.d[1], x9 +movz x9, #4095, lsl #0 +dup v17.4s, w9 +movz x9, #3329, lsl #0 +dup v18.4s, w9 +movz x11, #32, lsl #0 +25: +ldr x9, [x0, #0] +ldr w10, [x0, #8] +dup v0.2d, x9 +mov v0.d[1], x10 +tbl v1.16b, {{v0.16b}}, v16.16b +and v2.16b, v1.16b, v17.16b +ushr v3.4s, v1.4s, #12 +sub v4.4s, v2.4s, v18.4s +umin v2.4s, v2.4s, v4.4s +sub v4.4s, v3.4s, v18.4s +umin v3.4s, v3.4s, v4.4s +zip1 v0.4s, v2.4s, v3.4s +zip2 v1.4s, v2.4s, v3.4s +uzp1 v0.8h, v0.8h, v1.8h +str q0, [x1, #0] +add x0, x0, #12 +add x1, x1, #16 +sub x11, x11, #1 +cbnz x11, 25b movz x0, #17560, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -58754,12 +57415,15 @@ dup v16.4s, w9 dup v17.4s, w10", concat!("adrp x3, ", vg_sym_page!("{VG_MLKEM_GAMMA}")), concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), - r"movz x11, #32, lsl #0 + r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 26: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 +movi v25.2d, #0 +movi v26.2d, #0 +movi v27.2d, #0 movz x1, #4248, lsl #0 add x1, x28, x1 movz x2, #13464, lsl #0 @@ -58770,14 +57434,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #5272, lsl #0 add x1, x28, x1 movz x2, #14488, lsl #0 @@ -58788,14 +57456,18 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h movz x1, #6296, lsl #0 add x1, x28, x1 movz x2, #15512, lsl #0 @@ -58806,15 +57478,20 @@ ldr q0, [x1, #0] ldr q1, [x1, #16] ldr q4, [x2, #0] ldr q5, [x2, #16] -uzp1 v6.4s, v0.4s, v1.4s -uzp2 v7.4s, v0.4s, v1.4s -uzp1 v19.4s, v4.4s, v5.4s -uzp2 v20.4s, v4.4s, v5.4s -mla v21.4s, v6.4s, v19.4s -mla v22.4s, v7.4s, v20.4s -mla v23.4s, v6.4s, v20.4s -mla v23.4s, v7.4s, v19.4s +uzp1 v6.8h, v0.8h, v1.8h +uzp2 v7.8h, v0.8h, v1.8h +uzp1 v19.8h, v4.8h, v5.8h +uzp2 v20.8h, v4.8h, v5.8h +umlal v21.4s, v6.4h, v19.4h +umlal2 v25.4s, v6.8h, v19.8h +umlal v22.4s, v7.4h, v20.4h +umlal2 v26.4s, v7.8h, v20.8h +umlal v23.4s, v6.4h, v20.4h +umlal2 v27.4s, v6.8h, v20.8h +umlal v23.4s, v7.4h, v19.4h +umlal2 v27.4s, v7.8h, v19.8h ldr q18, [x3, #0] +ldr q28, [x3, #16] sqdmulh v24.4s, v22.4s, v17.4s mls v22.4s, v24.4s, v16.4s mla v21.4s, v22.4s, v18.4s @@ -58826,12 +57503,25 @@ sqdmulh v24.4s, v23.4s, v17.4s mls v23.4s, v24.4s, v16.4s sub v24.4s, v23.4s, v16.4s umin v23.4s, v23.4s, v24.4s -zip1 v0.4s, v21.4s, v23.4s -zip2 v1.4s, v21.4s, v23.4s -str q0, [x0, #0] -str q1, [x0, #16] +sqdmulh v24.4s, v26.4s, v17.4s +mls v26.4s, v24.4s, v16.4s +mla v25.4s, v26.4s, v28.4s +sqdmulh v24.4s, v25.4s, v17.4s +mls v25.4s, v24.4s, v16.4s +sub v24.4s, v25.4s, v16.4s +umin v25.4s, v25.4s, v24.4s +sqdmulh v24.4s, v27.4s, v17.4s +mls v27.4s, v24.4s, v16.4s +sub v24.4s, v27.4s, v16.4s +umin v27.4s, v27.4s, v24.4s +uzp1 v0.8h, v21.8h, v25.8h +uzp1 v1.8h, v23.8h, v27.8h +zip1 v2.8h, v0.8h, v1.8h +zip2 v3.8h, v0.8h, v1.8h +str q2, [x0, #0] +str q3, [x0, #16] add x0, x0, #32 -add x3, x3, #16 +add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 cbnz x11, 26b @@ -58849,18 +57539,10 @@ add x3, x1, #0", concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 27: -ldr q20, [x2, #0] -ldr q21, [x2, #16] -uzp1 v0.8h, v20.8h, v21.8h -ldr q20, [x2, #32] -ldr q21, [x2, #48] -uzp1 v1.8h, v20.8h, v21.8h -ldr q20, [x2, #64] -ldr q21, [x2, #80] -uzp1 v2.8h, v20.8h, v21.8h -ldr q20, [x2, #96] -ldr q21, [x2, #112] -uzp1 v3.8h, v20.8h, v21.8h +ldr q0, [x2, #0] +ldr q1, [x2, #16] +ldr q2, [x2, #32] +ldr q3, [x2, #48] ldr q20, [x6, #0] ldr q21, [x6, #16] ldr q22, [x6, #32] @@ -58949,7 +57631,7 @@ str q0, [x3, #0] str q1, [x3, #16] str q2, [x3, #32] str q3, [x3, #48] -add x2, x2, #128 +add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 @@ -59096,84 +57778,76 @@ mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #0] -str q25, [x3, #16] +str q16, [x3, #0] mul v16.8h, v1.8h, v22.8h sqrdmulh v17.8h, v1.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #128] -str q25, [x3, #144] +str q16, [x3, #64] mul v16.8h, v2.8h, v22.8h sqrdmulh v17.8h, v2.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #256] -str q25, [x3, #272] +str q16, [x3, #128] mul v16.8h, v3.8h, v22.8h sqrdmulh v17.8h, v3.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #384] -str q25, [x3, #400] +str q16, [x3, #192] mul v16.8h, v4.8h, v22.8h sqrdmulh v17.8h, v4.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #512] -str q25, [x3, #528] +str q16, [x3, #256] mul v16.8h, v5.8h, v22.8h sqrdmulh v17.8h, v5.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #640] -str q25, [x3, #656] +str q16, [x3, #320] mul v16.8h, v6.8h, v22.8h sqrdmulh v17.8h, v6.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #768] -str q25, [x3, #784] +str q16, [x3, #384] mul v16.8h, v7.8h, v22.8h sqrdmulh v17.8h, v7.8h, v23.8h mls v16.8h, v17.8h, v31.8h sshr v17.8h, v16.8h, #15 and v17.16b, v17.16b, v31.16b add v16.8h, v16.8h, v17.8h -zip1 v24.8h, v16.8h, v29.8h -zip2 v25.8h, v16.8h, v29.8h -str q24, [x3, #896] -str q25, [x3, #912] +str q16, [x3, #448] add x2, x2, #16 -add x3, x3, #32 +add x3, x3, #16 sub x5, x5, #1 cbnz x5, 28b +movz x0, #17560, lsl #0 +add x0, x28, x0 +movz x1, #18584, lsl #0 +add x1, x28, x1 +movz x10, #32, lsl #0 +movi v29.2d, #0 +29: +ldr q0, [x0, #0] +zip1 v1.8h, v0.8h, v29.8h +zip2 v2.8h, v0.8h, v29.8h +str q1, [x1, #0] +str q2, [x1, #16] +add x0, x0, #16 +add x1, x1, #32 +sub x10, x10, #1 +cbnz x10, 29b add x0, x26, #960 movz x1, #128, lsl #0 movz x2, #4, lsl #0 @@ -59182,7 +57856,7 @@ add x3, x28, x3 bl {vg_mlkem_decode_decompress} movz x0, #16536, lsl #0 add x0, x28, x0 -movz x1, #17560, lsl #0 +movz x1, #18584, lsl #0 add x1, x28, x1 bl {vg_mlkem_sub} movz x0, #16536, lsl #0 @@ -60193,10 +58867,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -29: +210: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 210f +cbz x6, 211f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60208,26 +58882,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 212f +cbz x4, 213f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 214f +cbz x4, 215f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 215f -214: +b 216f 215: -b 213f -212: +216: +b 214f 213: +214: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60239,26 +58913,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 216f +cbz x4, 217f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 218f +cbz x4, 219f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 219f -218: +b 220f 219: -b 217f -216: +220: +b 218f 217: +218: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60270,26 +58944,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 220f +cbz x4, 221f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 222f +cbz x4, 223f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 223f -222: +b 224f 223: -b 221f -220: +224: +b 222f 221: +222: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60301,33 +58975,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 224f +cbz x4, 225f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 226f +cbz x4, 227f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 227f -226: +b 228f 227: -b 225f -224: -225: -cbz x4, 228f -b 229f 228: -movz x5, #0, lsl #0 +b 226f +225: +226: +cbz x4, 229f +b 230f 229: -b 211f -210: +movz x5, #0, lsl #0 +230: +b 212f +211: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -60351,14 +59025,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -211: -cbnz x5, 29b -cbz x4, 230f +212: +cbnz x5, 210b +cbz x4, 231f movz x13, #0, lsl #0 str w13, [x3, #0] -b 231f -230: +b 232f 231: +232: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -60371,10 +59045,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -232: +233: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 233f +cbz x6, 234f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60386,26 +59060,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 235f +cbz x4, 236f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 237f +cbz x4, 238f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 238f -237: +b 239f 238: -b 236f -235: +239: +b 237f 236: +237: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60417,26 +59091,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 239f +cbz x4, 240f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 241f +cbz x4, 242f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 242f -241: +b 243f 242: -b 240f -239: +243: +b 241f 240: +241: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60448,26 +59122,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 243f +cbz x4, 244f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 245f +cbz x4, 246f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 246f -245: +b 247f 246: -b 244f -243: +247: +b 245f 244: +245: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60479,33 +59153,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 247f +cbz x4, 248f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 249f +cbz x4, 250f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 250f -249: +b 251f 250: -b 248f -247: -248: -cbz x4, 251f -b 252f 251: -movz x5, #0, lsl #0 +b 249f +248: +249: +cbz x4, 252f +b 253f 252: -b 234f -233: +movz x5, #0, lsl #0 +253: +b 235f +234: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -60529,23 +59203,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -234: -cbnz x5, 232b -cbz x4, 253f +235: +cbnz x5, 233b +cbz x4, 254f movz x13, #0, lsl #0 str w13, [x3, #0] -b 254f -253: +b 255f 254: +255: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 255f -b 256f -255: +cbz x6, 256f +b 257f +256: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -60677,15 +59351,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 257f +cbz x4, 258f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -259: +260: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 260f +cbz x6, 261f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60697,26 +59371,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 262f +cbz x4, 263f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 264f +cbz x4, 265f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 265f -264: +b 266f 265: -b 263f -262: +266: +b 264f 263: +264: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60728,26 +59402,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 266f +cbz x4, 267f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 268f +cbz x4, 269f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 269f -268: +b 270f 269: -b 267f -266: +270: +b 268f 267: +268: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60759,26 +59433,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 270f +cbz x4, 271f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 272f +cbz x4, 273f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 273f -272: +b 274f 273: -b 271f -270: +274: +b 272f 271: +272: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60790,33 +59464,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 274f +cbz x4, 275f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 276f +cbz x4, 277f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 277f -276: +b 278f 277: -b 275f -274: -275: -cbz x4, 278f -b 279f 278: -movz x5, #0, lsl #0 +b 276f +275: +276: +cbz x4, 279f +b 280f 279: -b 261f -260: +movz x5, #0, lsl #0 +280: +b 262f +261: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -60840,34 +59514,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -261: -cbnz x5, 259b -cbz x4, 280f +262: +cbnz x5, 260b +cbz x4, 281f movz x13, #0, lsl #0 str w13, [x3, #0] -b 281f -280: +b 282f 281: +282: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 258f -257: -movz x22, #1, lsl #0 +b 259f 258: +movz x22, #1, lsl #0 +259: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 282f +cbz x4, 283f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -284: +285: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 285f +cbz x6, 286f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60879,26 +59553,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 287f +cbz x4, 288f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 289f +cbz x4, 290f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 290f -289: +b 291f 290: -b 288f -287: +291: +b 289f 288: +289: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60910,26 +59584,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 291f +cbz x4, 292f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 293f +cbz x4, 294f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 294f -293: +b 295f 294: -b 292f -291: +295: +b 293f 292: +293: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60941,26 +59615,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 295f +cbz x4, 296f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 297f +cbz x4, 298f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 298f -297: +b 299f 298: -b 296f -295: +299: +b 297f 296: +297: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -60972,33 +59646,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 299f +cbz x4, 2100f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2101f +cbz x4, 2102f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2102f -2101: +b 2103f 2102: -b 2100f -299: -2100: -cbz x4, 2103f -b 2104f 2103: -movz x5, #0, lsl #0 +b 2101f +2100: +2101: +cbz x4, 2104f +b 2105f 2104: -b 286f -285: +movz x5, #0, lsl #0 +2105: +b 287f +286: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -61022,27 +59696,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -286: -cbnz x5, 284b -cbz x4, 2105f +287: +cbnz x5, 285b +cbz x4, 2106f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2106f -2105: +b 2107f 2106: +2107: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 283f -282: -movz x23, #1, lsl #0 +b 284f 283: +movz x23, #1, lsl #0 +284: and x6, x22, x23 -cbz x6, 2107f -b 2108f -2107: +cbz x6, 2108f +b 2109f +2108: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -61174,15 +59848,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2109f +cbz x4, 2110f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2111: +2112: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2112f +cbz x6, 2113f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61194,26 +59868,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2114f +cbz x4, 2115f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2116f +cbz x4, 2117f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2117f -2116: +b 2118f 2117: -b 2115f -2114: +2118: +b 2116f 2115: +2116: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61225,26 +59899,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2118f +cbz x4, 2119f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2120f +cbz x4, 2121f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2121f -2120: +b 2122f 2121: -b 2119f -2118: +2122: +b 2120f 2119: +2120: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61256,26 +59930,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2122f +cbz x4, 2123f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2124f +cbz x4, 2125f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2125f -2124: +b 2126f 2125: -b 2123f -2122: +2126: +b 2124f 2123: +2124: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61287,33 +59961,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2126f +cbz x4, 2127f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2128f +cbz x4, 2129f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2129f -2128: +b 2130f 2129: -b 2127f -2126: -2127: -cbz x4, 2130f -b 2131f 2130: -movz x5, #0, lsl #0 +b 2128f +2127: +2128: +cbz x4, 2131f +b 2132f 2131: -b 2113f -2112: +movz x5, #0, lsl #0 +2132: +b 2114f +2113: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -61337,34 +60011,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2113: -cbnz x5, 2111b -cbz x4, 2132f +2114: +cbnz x5, 2112b +cbz x4, 2133f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2133f -2132: +b 2134f 2133: +2134: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2110f -2109: -movz x22, #1, lsl #0 +b 2111f 2110: +movz x22, #1, lsl #0 +2111: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2134f +cbz x4, 2135f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2136: +2137: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2137f +cbz x6, 2138f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61376,26 +60050,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2139f +cbz x4, 2140f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2141f +cbz x4, 2142f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2142f -2141: +b 2143f 2142: -b 2140f -2139: +2143: +b 2141f 2140: +2141: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61407,26 +60081,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2143f +cbz x4, 2144f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2145f +cbz x4, 2146f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2146f -2145: +b 2147f 2146: -b 2144f -2143: +2147: +b 2145f 2144: +2145: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61438,26 +60112,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2147f +cbz x4, 2148f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2149f +cbz x4, 2150f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2150f -2149: +b 2151f 2150: -b 2148f -2147: +2151: +b 2149f 2148: +2149: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -61469,33 +60143,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2151f +cbz x4, 2152f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2153f +cbz x4, 2154f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2154f -2153: +b 2155f 2154: -b 2152f -2151: -2152: -cbz x4, 2155f -b 2156f 2155: -movz x5, #0, lsl #0 +b 2153f +2152: +2153: +cbz x4, 2156f +b 2157f 2156: -b 2138f -2137: +movz x5, #0, lsl #0 +2157: +b 2139f +2138: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -61519,51 +60193,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2138: -cbnz x5, 2136b -cbz x4, 2157f +2139: +cbnz x5, 2137b +cbz x4, 2158f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2158f -2157: +b 2159f 2158: +2159: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2135f -2134: -movz x23, #1, lsl #0 +b 2136f 2135: +movz x23, #1, lsl #0 +2136: and x6, x22, x23 -2108: +2109: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2159f -2161: +cbz x4, 2160f +2162: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2161b -b 2160f -2159: +cbnz x4, 2162b +b 2161f 2160: +2161: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2162f -2164: +cbz x4, 2163f +2165: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2164b -b 2163f -2162: +cbnz x4, 2165b +b 2164f 2163: -256: +2164: +257: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -62153,10 +60827,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2165: +2166: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2166f +cbz x6, 2167f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62168,26 +60842,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2168f +cbz x4, 2169f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2170f +cbz x4, 2171f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2171f -2170: +b 2172f 2171: -b 2169f -2168: +2172: +b 2170f 2169: +2170: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62199,26 +60873,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2172f +cbz x4, 2173f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2174f +cbz x4, 2175f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2175f -2174: +b 2176f 2175: -b 2173f -2172: +2176: +b 2174f 2173: +2174: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62230,26 +60904,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2176f +cbz x4, 2177f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2178f +cbz x4, 2179f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2179f -2178: +b 2180f 2179: -b 2177f -2176: +2180: +b 2178f 2177: +2178: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62261,33 +60935,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2180f +cbz x4, 2181f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2182f +cbz x4, 2183f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2183f -2182: +b 2184f 2183: -b 2181f -2180: -2181: -cbz x4, 2184f -b 2185f 2184: -movz x5, #0, lsl #0 +b 2182f +2181: +2182: +cbz x4, 2185f +b 2186f 2185: -b 2167f -2166: +movz x5, #0, lsl #0 +2186: +b 2168f +2167: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -62311,14 +60985,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2167: -cbnz x5, 2165b -cbz x4, 2186f +2168: +cbnz x5, 2166b +cbz x4, 2187f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2187f -2186: +b 2188f 2187: +2188: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -62331,10 +61005,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2188: +2189: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2189f +cbz x6, 2190f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62346,26 +61020,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2191f +cbz x4, 2192f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2193f +cbz x4, 2194f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2194f -2193: +b 2195f 2194: -b 2192f -2191: +2195: +b 2193f 2192: +2193: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62377,26 +61051,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2195f +cbz x4, 2196f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2197f +cbz x4, 2198f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2198f -2197: +b 2199f 2198: -b 2196f -2195: +2199: +b 2197f 2196: +2197: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62408,26 +61082,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2199f +cbz x4, 2200f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2201f +cbz x4, 2202f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2202f -2201: +b 2203f 2202: -b 2200f -2199: +2203: +b 2201f 2200: +2201: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62439,33 +61113,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2203f +cbz x4, 2204f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2205f +cbz x4, 2206f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2206f -2205: +b 2207f 2206: -b 2204f -2203: -2204: -cbz x4, 2207f -b 2208f 2207: -movz x5, #0, lsl #0 +b 2205f +2204: +2205: +cbz x4, 2208f +b 2209f 2208: -b 2190f -2189: +movz x5, #0, lsl #0 +2209: +b 2191f +2190: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -62489,23 +61163,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2190: -cbnz x5, 2188b -cbz x4, 2209f +2191: +cbnz x5, 2189b +cbz x4, 2210f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2210f -2209: +b 2211f 2210: +2211: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2211f -b 2212f -2211: +cbz x6, 2212f +b 2213f +2212: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -62637,15 +61311,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2213f +cbz x4, 2214f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2215: +2216: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2216f +cbz x6, 2217f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62657,26 +61331,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2218f +cbz x4, 2219f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2220f +cbz x4, 2221f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2221f -2220: +b 2222f 2221: -b 2219f -2218: +2222: +b 2220f 2219: +2220: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62688,26 +61362,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2222f +cbz x4, 2223f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2224f +cbz x4, 2225f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2225f -2224: +b 2226f 2225: -b 2223f -2222: +2226: +b 2224f 2223: +2224: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62719,26 +61393,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2226f +cbz x4, 2227f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2228f +cbz x4, 2229f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2229f -2228: +b 2230f 2229: -b 2227f -2226: +2230: +b 2228f 2227: +2228: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62750,33 +61424,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2230f +cbz x4, 2231f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2232f +cbz x4, 2233f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2233f -2232: +b 2234f 2233: -b 2231f -2230: -2231: -cbz x4, 2234f -b 2235f 2234: -movz x5, #0, lsl #0 +b 2232f +2231: +2232: +cbz x4, 2235f +b 2236f 2235: -b 2217f -2216: +movz x5, #0, lsl #0 +2236: +b 2218f +2217: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -62800,34 +61474,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2217: -cbnz x5, 2215b -cbz x4, 2236f +2218: +cbnz x5, 2216b +cbz x4, 2237f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2237f -2236: +b 2238f 2237: +2238: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2214f -2213: -movz x22, #1, lsl #0 +b 2215f 2214: +movz x22, #1, lsl #0 +2215: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2238f +cbz x4, 2239f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2240: +2241: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2241f +cbz x6, 2242f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62839,26 +61513,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2243f +cbz x4, 2244f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2245f +cbz x4, 2246f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2246f -2245: +b 2247f 2246: -b 2244f -2243: +2247: +b 2245f 2244: +2245: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62870,26 +61544,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2247f +cbz x4, 2248f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2249f +cbz x4, 2250f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2250f -2249: +b 2251f 2250: -b 2248f -2247: +2251: +b 2249f 2248: +2249: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62901,26 +61575,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2251f +cbz x4, 2252f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2253f +cbz x4, 2254f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2254f -2253: +b 2255f 2254: -b 2252f -2251: +2255: +b 2253f 2252: +2253: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -62932,33 +61606,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2255f +cbz x4, 2256f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2257f +cbz x4, 2258f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2258f -2257: +b 2259f 2258: -b 2256f -2255: -2256: -cbz x4, 2259f -b 2260f 2259: -movz x5, #0, lsl #0 +b 2257f +2256: +2257: +cbz x4, 2260f +b 2261f 2260: -b 2242f -2241: +movz x5, #0, lsl #0 +2261: +b 2243f +2242: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -62982,27 +61656,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2242: -cbnz x5, 2240b -cbz x4, 2261f +2243: +cbnz x5, 2241b +cbz x4, 2262f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2262f -2261: +b 2263f 2262: +2263: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2239f -2238: -movz x23, #1, lsl #0 +b 2240f 2239: +movz x23, #1, lsl #0 +2240: and x6, x22, x23 -cbz x6, 2263f -b 2264f -2263: +cbz x6, 2264f +b 2265f +2264: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -63134,15 +61808,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2265f +cbz x4, 2266f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2267: +2268: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2268f +cbz x6, 2269f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63154,26 +61828,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2270f +cbz x4, 2271f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2272f +cbz x4, 2273f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2273f -2272: +b 2274f 2273: -b 2271f -2270: +2274: +b 2272f 2271: +2272: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63185,26 +61859,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2274f +cbz x4, 2275f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2276f +cbz x4, 2277f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2277f -2276: +b 2278f 2277: -b 2275f -2274: +2278: +b 2276f 2275: +2276: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63216,26 +61890,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2278f +cbz x4, 2279f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2280f +cbz x4, 2281f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2281f -2280: +b 2282f 2281: -b 2279f -2278: +2282: +b 2280f 2279: +2280: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63247,33 +61921,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2282f +cbz x4, 2283f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2284f +cbz x4, 2285f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2285f -2284: +b 2286f 2285: -b 2283f -2282: -2283: -cbz x4, 2286f -b 2287f 2286: -movz x5, #0, lsl #0 +b 2284f +2283: +2284: +cbz x4, 2287f +b 2288f 2287: -b 2269f -2268: +movz x5, #0, lsl #0 +2288: +b 2270f +2269: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -63297,34 +61971,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2269: -cbnz x5, 2267b -cbz x4, 2288f +2270: +cbnz x5, 2268b +cbz x4, 2289f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2289f -2288: +b 2290f 2289: +2290: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2266f -2265: -movz x22, #1, lsl #0 +b 2267f 2266: +movz x22, #1, lsl #0 +2267: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2290f +cbz x4, 2291f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2292: +2293: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2293f +cbz x6, 2294f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63336,26 +62010,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2295f +cbz x4, 2296f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2297f +cbz x4, 2298f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2298f -2297: +b 2299f 2298: -b 2296f -2295: +2299: +b 2297f 2296: +2297: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63367,26 +62041,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2299f +cbz x4, 2300f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2301f +cbz x4, 2302f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2302f -2301: +b 2303f 2302: -b 2300f -2299: +2303: +b 2301f 2300: +2301: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63398,26 +62072,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2303f +cbz x4, 2304f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2305f +cbz x4, 2306f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2306f -2305: +b 2307f 2306: -b 2304f -2303: +2307: +b 2305f 2304: +2305: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -63429,33 +62103,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2307f +cbz x4, 2308f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2309f +cbz x4, 2310f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2310f -2309: +b 2311f 2310: -b 2308f -2307: -2308: -cbz x4, 2311f -b 2312f 2311: -movz x5, #0, lsl #0 +b 2309f +2308: +2309: +cbz x4, 2312f +b 2313f 2312: -b 2294f -2293: +movz x5, #0, lsl #0 +2313: +b 2295f +2294: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -63479,51 +62153,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2294: -cbnz x5, 2292b -cbz x4, 2313f +2295: +cbnz x5, 2293b +cbz x4, 2314f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2314f -2313: +b 2315f 2314: +2315: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2291f -2290: -movz x23, #1, lsl #0 +b 2292f 2291: +movz x23, #1, lsl #0 +2292: and x6, x22, x23 -2264: +2265: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2315f -2317: +cbz x4, 2316f +2318: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2317b -b 2316f -2315: +cbnz x4, 2318b +b 2317f 2316: +2317: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2318f -2320: +cbz x4, 2319f +2321: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2320b -b 2319f -2318: +cbnz x4, 2321b +b 2320f 2319: -2212: +2320: +2213: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -64069,10 +62743,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2321: +2322: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2322f +cbz x6, 2323f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64084,26 +62758,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2324f +cbz x4, 2325f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2326f +cbz x4, 2327f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2327f -2326: +b 2328f 2327: -b 2325f -2324: +2328: +b 2326f 2325: +2326: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64115,26 +62789,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2328f +cbz x4, 2329f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2330f +cbz x4, 2331f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2331f -2330: +b 2332f 2331: -b 2329f -2328: +2332: +b 2330f 2329: +2330: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64146,26 +62820,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2332f +cbz x4, 2333f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2334f +cbz x4, 2335f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2335f -2334: +b 2336f 2335: -b 2333f -2332: +2336: +b 2334f 2333: +2334: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64177,33 +62851,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2336f +cbz x4, 2337f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2338f +cbz x4, 2339f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2339f -2338: +b 2340f 2339: -b 2337f -2336: -2337: -cbz x4, 2340f -b 2341f 2340: -movz x5, #0, lsl #0 +b 2338f +2337: +2338: +cbz x4, 2341f +b 2342f 2341: -b 2323f -2322: +movz x5, #0, lsl #0 +2342: +b 2324f +2323: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -64227,14 +62901,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2323: -cbnz x5, 2321b -cbz x4, 2342f +2324: +cbnz x5, 2322b +cbz x4, 2343f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2343f -2342: +b 2344f 2343: +2344: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -64247,10 +62921,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2344: +2345: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2345f +cbz x6, 2346f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64262,26 +62936,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2347f +cbz x4, 2348f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2349f +cbz x4, 2350f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2350f -2349: +b 2351f 2350: -b 2348f -2347: +2351: +b 2349f 2348: +2349: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64293,26 +62967,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2351f +cbz x4, 2352f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2353f +cbz x4, 2354f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2354f -2353: +b 2355f 2354: -b 2352f -2351: +2355: +b 2353f 2352: +2353: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64324,26 +62998,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2355f +cbz x4, 2356f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2357f +cbz x4, 2358f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2358f -2357: +b 2359f 2358: -b 2356f -2355: +2359: +b 2357f 2356: +2357: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64355,33 +63029,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2359f +cbz x4, 2360f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2361f +cbz x4, 2362f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2362f -2361: +b 2363f 2362: -b 2360f -2359: -2360: -cbz x4, 2363f -b 2364f 2363: -movz x5, #0, lsl #0 +b 2361f +2360: +2361: +cbz x4, 2364f +b 2365f 2364: -b 2346f -2345: +movz x5, #0, lsl #0 +2365: +b 2347f +2346: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -64405,23 +63079,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2346: -cbnz x5, 2344b -cbz x4, 2365f +2347: +cbnz x5, 2345b +cbz x4, 2366f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2366f -2365: +b 2367f 2366: +2367: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2367f -b 2368f -2367: +cbz x6, 2368f +b 2369f +2368: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -64553,15 +63227,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2369f +cbz x4, 2370f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2371: +2372: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2372f +cbz x6, 2373f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64573,26 +63247,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2374f +cbz x4, 2375f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2376f +cbz x4, 2377f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2377f -2376: +b 2378f 2377: -b 2375f -2374: +2378: +b 2376f 2375: +2376: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64604,26 +63278,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2378f +cbz x4, 2379f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2380f +cbz x4, 2381f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2381f -2380: +b 2382f 2381: -b 2379f -2378: +2382: +b 2380f 2379: +2380: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64635,26 +63309,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2382f +cbz x4, 2383f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2384f +cbz x4, 2385f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2385f -2384: +b 2386f 2385: -b 2383f -2382: +2386: +b 2384f 2383: +2384: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64666,33 +63340,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2386f +cbz x4, 2387f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2388f +cbz x4, 2389f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2389f -2388: +b 2390f 2389: -b 2387f -2386: -2387: -cbz x4, 2390f -b 2391f 2390: -movz x5, #0, lsl #0 +b 2388f +2387: +2388: +cbz x4, 2391f +b 2392f 2391: -b 2373f -2372: +movz x5, #0, lsl #0 +2392: +b 2374f +2373: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -64716,34 +63390,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2373: -cbnz x5, 2371b -cbz x4, 2392f +2374: +cbnz x5, 2372b +cbz x4, 2393f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2393f -2392: +b 2394f 2393: +2394: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2370f -2369: -movz x22, #1, lsl #0 +b 2371f 2370: +movz x22, #1, lsl #0 +2371: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2394f +cbz x4, 2395f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2396: +2397: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2397f +cbz x6, 2398f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64755,26 +63429,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2399f +cbz x4, 2400f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2401f +cbz x4, 2402f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2402f -2401: +b 2403f 2402: -b 2400f -2399: +2403: +b 2401f 2400: +2401: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64786,26 +63460,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2403f +cbz x4, 2404f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2405f +cbz x4, 2406f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2406f -2405: +b 2407f 2406: -b 2404f -2403: +2407: +b 2405f 2404: +2405: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64817,26 +63491,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2407f +cbz x4, 2408f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2409f +cbz x4, 2410f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2410f -2409: +b 2411f 2410: -b 2408f -2407: +2411: +b 2409f 2408: +2409: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -64848,33 +63522,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2411f +cbz x4, 2412f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2413f +cbz x4, 2414f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2414f -2413: +b 2415f 2414: -b 2412f -2411: -2412: -cbz x4, 2415f -b 2416f 2415: -movz x5, #0, lsl #0 +b 2413f +2412: +2413: +cbz x4, 2416f +b 2417f 2416: -b 2398f -2397: +movz x5, #0, lsl #0 +2417: +b 2399f +2398: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -64898,27 +63572,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2398: -cbnz x5, 2396b -cbz x4, 2417f +2399: +cbnz x5, 2397b +cbz x4, 2418f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2418f -2417: +b 2419f 2418: +2419: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2395f -2394: -movz x23, #1, lsl #0 +b 2396f 2395: +movz x23, #1, lsl #0 +2396: and x6, x22, x23 -cbz x6, 2419f -b 2420f -2419: +cbz x6, 2420f +b 2421f +2420: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -65050,15 +63724,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2421f +cbz x4, 2422f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2423: +2424: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2424f +cbz x6, 2425f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -65070,26 +63744,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2426f +cbz x4, 2427f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2428f +cbz x4, 2429f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2429f -2428: +b 2430f 2429: -b 2427f -2426: +2430: +b 2428f 2427: +2428: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -65101,26 +63775,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2430f +cbz x4, 2431f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2432f +cbz x4, 2433f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2433f -2432: +b 2434f 2433: -b 2431f -2430: +2434: +b 2432f 2431: +2432: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -65132,26 +63806,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2434f +cbz x4, 2435f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2436f +cbz x4, 2437f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2437f -2436: +b 2438f 2437: -b 2435f -2434: +2438: +b 2436f 2435: +2436: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -65163,33 +63837,215 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2438f +cbz x4, 2439f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2440f +cbz x4, 2441f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2441f -2440: +b 2442f 2441: -b 2439f -2438: -2439: -cbz x4, 2442f -b 2443f 2442: -movz x5, #0, lsl #0 +b 2440f +2439: +2440: +cbz x4, 2443f +b 2444f 2443: -b 2425f -2424: +movz x5, #0, lsl #0 +2444: +b 2426f +2425: +ldr q0, [x2, #0] +tbl v0.16b, {{v0.16b}}, v16.16b +ushr v1.8h, v0.8h, #4 +bif v0.16b, v1.16b, v17.16b +cmhi v1.8h, v19.8h, v0.8h +and v1.16b, v1.16b, v20.16b +addv h1, v1.8h +umov w6, v1.s[0] +movz x7, #255, lsl #0 +and x7, x6, x7 +lsl x7, x7, #4 +add x7, x16, x7 +ldr q2, [x7, #0] +tbl v0.16b, {{v0.16b}}, v2.16b +str q0, [x3, #0] +lsr x6, x6, #8 +movz x7, #15, lsl #0 +and x6, x6, x7 +sub x4, x4, x6 +lsl x6, x6, #1 +add x3, x3, x6 +add x2, x2, #12 +sub x5, x5, #4 +2426: +cbnz x5, 2424b +cbz x4, 2445f +movz x13, #0, lsl #0 +str w13, [x3, #0] +b 2446f +2445: +2446: +sub x0, x4, #1 +lsr x0, x0, #63 +add x22, x0, #0 +str x3, [x19, #1680] +str x4, [x19, #1688] +b 2423f +2422: +movz x22, #1, lsl #0 +2423: +ldr x3, [x19, #1696] +ldr x4, [x19, #1704] +cbz x4, 2447f +add x2, x19, #904 +movz x5, #56, lsl #0 +movz x9, #3329, lsl #0 +movz x10, #15, lsl #0 +2449: +sub x6, x4, #8 +lsr x6, x6, #63 +cbz x6, 2450f +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2452f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2454f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2455f +2454: +2455: +b 2453f +2452: +2453: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2456f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2458f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2459f +2458: +2459: +b 2457f +2456: +2457: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2460f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2462f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2463f +2462: +2463: +b 2461f +2460: +2461: +ldrb w6, [x2, #0] +ldrb w7, [x2, #1] +ldrb w8, [x2, #2] +add x2, x2, #3 +sub x5, x5, #1 +and x11, x7, x10 +lsl x11, x11, #8 +add x11, x11, x6 +lsr x12, x7, #4 +lsl x13, x8, #4 +add x12, x12, x13 +cbz x4, 2464f +sub x13, x11, x9 +lsr x14, x13, #63 +str w11, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +cbz x4, 2466f +sub x13, x12, x9 +lsr x14, x13, #63 +str w12, [x3, #0] +lsl x15, x14, #1 +add x3, x3, x15 +sub x4, x4, x14 +b 2467f +2466: +2467: +b 2465f +2464: +2465: +cbz x4, 2468f +b 2469f +2468: +movz x5, #0, lsl #0 +2469: +b 2451f +2450: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -65213,233 +64069,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2425: -cbnz x5, 2423b -cbz x4, 2444f -movz x13, #0, lsl #0 -str w13, [x3, #0] -b 2445f -2444: -2445: -sub x0, x4, #1 -lsr x0, x0, #63 -add x22, x0, #0 -str x3, [x19, #1680] -str x4, [x19, #1688] -b 2422f -2421: -movz x22, #1, lsl #0 -2422: -ldr x3, [x19, #1696] -ldr x4, [x19, #1704] -cbz x4, 2446f -add x2, x19, #904 -movz x5, #56, lsl #0 -movz x9, #3329, lsl #0 -movz x10, #15, lsl #0 -2448: -sub x6, x4, #8 -lsr x6, x6, #63 -cbz x6, 2449f -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2451f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2453f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2454f -2453: -2454: -b 2452f 2451: -2452: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2455f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2457f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2458f -2457: -2458: -b 2456f -2455: -2456: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2459f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2461f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2462f -2461: -2462: -b 2460f -2459: -2460: -ldrb w6, [x2, #0] -ldrb w7, [x2, #1] -ldrb w8, [x2, #2] -add x2, x2, #3 -sub x5, x5, #1 -and x11, x7, x10 -lsl x11, x11, #8 -add x11, x11, x6 -lsr x12, x7, #4 -lsl x13, x8, #4 -add x12, x12, x13 -cbz x4, 2463f -sub x13, x11, x9 -lsr x14, x13, #63 -str w11, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -cbz x4, 2465f -sub x13, x12, x9 -lsr x14, x13, #63 -str w12, [x3, #0] -lsl x15, x14, #1 -add x3, x3, x15 -sub x4, x4, x14 -b 2466f -2465: -2466: -b 2464f -2463: -2464: -cbz x4, 2467f -b 2468f -2467: -movz x5, #0, lsl #0 -2468: -b 2450f -2449: -ldr q0, [x2, #0] -tbl v0.16b, {{v0.16b}}, v16.16b -ushr v1.8h, v0.8h, #4 -bif v0.16b, v1.16b, v17.16b -cmhi v1.8h, v19.8h, v0.8h -and v1.16b, v1.16b, v20.16b -addv h1, v1.8h -umov w6, v1.s[0] -movz x7, #255, lsl #0 -and x7, x6, x7 -lsl x7, x7, #4 -add x7, x16, x7 -ldr q2, [x7, #0] -tbl v0.16b, {{v0.16b}}, v2.16b -str q0, [x3, #0] -lsr x6, x6, #8 -movz x7, #15, lsl #0 -and x6, x6, x7 -sub x4, x4, x6 -lsl x6, x6, #1 -add x3, x3, x6 -add x2, x2, #12 -sub x5, x5, #4 -2450: -cbnz x5, 2448b -cbz x4, 2469f +cbnz x5, 2449b +cbz x4, 2470f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2470f -2469: +b 2471f 2470: +2471: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2447f -2446: -movz x23, #1, lsl #0 +b 2448f 2447: +movz x23, #1, lsl #0 +2448: and x6, x22, x23 -2420: +2421: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2471f -2473: +cbz x4, 2472f +2474: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2473b -b 2472f -2471: +cbnz x4, 2474b +b 2473f 2472: +2473: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2474f -2476: +cbz x4, 2475f +2477: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2476b -b 2475f -2474: +cbnz x4, 2477b +b 2476f 2475: -2368: +2476: +2369: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -66085,10 +64759,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2477: +2478: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2478f +cbz x6, 2479f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66100,26 +64774,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2480f +cbz x4, 2481f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2482f +cbz x4, 2483f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2483f -2482: +b 2484f 2483: -b 2481f -2480: +2484: +b 2482f 2481: +2482: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66131,26 +64805,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2484f +cbz x4, 2485f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2486f +cbz x4, 2487f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2487f -2486: +b 2488f 2487: -b 2485f -2484: +2488: +b 2486f 2485: +2486: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66162,26 +64836,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2488f +cbz x4, 2489f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2490f +cbz x4, 2491f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2491f -2490: +b 2492f 2491: -b 2489f -2488: +2492: +b 2490f 2489: +2490: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66193,33 +64867,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2492f +cbz x4, 2493f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2494f +cbz x4, 2495f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2495f -2494: +b 2496f 2495: -b 2493f -2492: -2493: -cbz x4, 2496f -b 2497f 2496: -movz x5, #0, lsl #0 +b 2494f +2493: +2494: +cbz x4, 2497f +b 2498f 2497: -b 2479f -2478: +movz x5, #0, lsl #0 +2498: +b 2480f +2479: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -66243,14 +64917,14 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2479: -cbnz x5, 2477b -cbz x4, 2498f +2480: +cbnz x5, 2478b +cbz x4, 2499f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2499f -2498: +b 2500f 2499: +2500: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 @@ -66263,10 +64937,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2500: +2501: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2501f +cbz x6, 2502f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66278,26 +64952,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2503f +cbz x4, 2504f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2505f +cbz x4, 2506f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2506f -2505: +b 2507f 2506: -b 2504f -2503: +2507: +b 2505f 2504: +2505: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66309,26 +64983,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2507f +cbz x4, 2508f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2509f +cbz x4, 2510f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2510f -2509: +b 2511f 2510: -b 2508f -2507: +2511: +b 2509f 2508: +2509: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66340,26 +65014,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2511f +cbz x4, 2512f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2513f +cbz x4, 2514f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2514f -2513: +b 2515f 2514: -b 2512f -2511: +2515: +b 2513f 2512: +2513: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66371,33 +65045,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2515f +cbz x4, 2516f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2517f +cbz x4, 2518f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2518f -2517: +b 2519f 2518: -b 2516f -2515: -2516: -cbz x4, 2519f -b 2520f 2519: -movz x5, #0, lsl #0 +b 2517f +2516: +2517: +cbz x4, 2520f +b 2521f 2520: -b 2502f -2501: +movz x5, #0, lsl #0 +2521: +b 2503f +2502: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -66421,23 +65095,23 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2502: -cbnz x5, 2500b -cbz x4, 2521f +2503: +cbnz x5, 2501b +cbz x4, 2522f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2522f -2521: +b 2523f 2522: +2523: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] and x6, x22, x23 -cbz x6, 2523f -b 2524f -2523: +cbz x6, 2524f +b 2525f +2524: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -66569,15 +65243,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2525f +cbz x4, 2526f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2527: +2528: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2528f +cbz x6, 2529f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66589,26 +65263,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2530f +cbz x4, 2531f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2532f +cbz x4, 2533f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2533f -2532: +b 2534f 2533: -b 2531f -2530: +2534: +b 2532f 2531: +2532: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66620,26 +65294,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2534f +cbz x4, 2535f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2536f +cbz x4, 2537f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2537f -2536: +b 2538f 2537: -b 2535f -2534: +2538: +b 2536f 2535: +2536: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66651,26 +65325,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2538f +cbz x4, 2539f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2540f +cbz x4, 2541f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2541f -2540: +b 2542f 2541: -b 2539f -2538: +2542: +b 2540f 2539: +2540: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66682,33 +65356,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2542f +cbz x4, 2543f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2544f +cbz x4, 2545f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2545f -2544: +b 2546f 2545: -b 2543f -2542: -2543: -cbz x4, 2546f -b 2547f 2546: -movz x5, #0, lsl #0 +b 2544f +2543: +2544: +cbz x4, 2547f +b 2548f 2547: -b 2529f -2528: +movz x5, #0, lsl #0 +2548: +b 2530f +2529: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -66732,34 +65406,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2529: -cbnz x5, 2527b -cbz x4, 2548f +2530: +cbnz x5, 2528b +cbz x4, 2549f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2549f -2548: +b 2550f 2549: +2550: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2526f -2525: -movz x22, #1, lsl #0 +b 2527f 2526: +movz x22, #1, lsl #0 +2527: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2550f +cbz x4, 2551f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2552: +2553: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2553f +cbz x6, 2554f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66771,26 +65445,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2555f +cbz x4, 2556f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2557f +cbz x4, 2558f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2558f -2557: +b 2559f 2558: -b 2556f -2555: +2559: +b 2557f 2556: +2557: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66802,26 +65476,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2559f +cbz x4, 2560f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2561f +cbz x4, 2562f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2562f -2561: +b 2563f 2562: -b 2560f -2559: +2563: +b 2561f 2560: +2561: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66833,26 +65507,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2563f +cbz x4, 2564f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2565f +cbz x4, 2566f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2566f -2565: +b 2567f 2566: -b 2564f -2563: +2567: +b 2565f 2564: +2565: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -66864,33 +65538,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2567f +cbz x4, 2568f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2569f +cbz x4, 2570f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2570f -2569: +b 2571f 2570: -b 2568f -2567: -2568: -cbz x4, 2571f -b 2572f 2571: -movz x5, #0, lsl #0 +b 2569f +2568: +2569: +cbz x4, 2572f +b 2573f 2572: -b 2554f -2553: +movz x5, #0, lsl #0 +2573: +b 2555f +2554: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -66914,27 +65588,27 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2554: -cbnz x5, 2552b -cbz x4, 2573f +2555: +cbnz x5, 2553b +cbz x4, 2574f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2574f -2573: +b 2575f 2574: +2575: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2551f -2550: -movz x23, #1, lsl #0 +b 2552f 2551: +movz x23, #1, lsl #0 +2552: and x6, x22, x23 -cbz x6, 2575f -b 2576f -2575: +cbz x6, 2576f +b 2577f +2576: add x20, x19, #400 add x21, x19, #904 add x1, x19, #768 @@ -67066,15 +65740,15 @@ str x9, [x19, #1408]", concat!("add x16, x16, ", vg_sym_pageoff!("{VG_MLKEM_REJ_SHUFFLE}")), r"ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2577f +cbz x4, 2578f add x2, x19, #400 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2579: +2580: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2580f +cbz x6, 2581f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67086,26 +65760,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2582f +cbz x4, 2583f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2584f +cbz x4, 2585f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2585f -2584: +b 2586f 2585: -b 2583f -2582: +2586: +b 2584f 2583: +2584: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67117,26 +65791,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2586f +cbz x4, 2587f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2588f +cbz x4, 2589f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2589f -2588: +b 2590f 2589: -b 2587f -2586: +2590: +b 2588f 2587: +2588: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67148,26 +65822,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2590f +cbz x4, 2591f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2592f +cbz x4, 2593f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2593f -2592: +b 2594f 2593: -b 2591f -2590: +2594: +b 2592f 2591: +2592: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67179,33 +65853,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2594f +cbz x4, 2595f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2596f +cbz x4, 2597f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2597f -2596: +b 2598f 2597: -b 2595f -2594: -2595: -cbz x4, 2598f -b 2599f 2598: -movz x5, #0, lsl #0 +b 2596f +2595: +2596: +cbz x4, 2599f +b 2600f 2599: -b 2581f -2580: +movz x5, #0, lsl #0 +2600: +b 2582f +2581: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -67229,34 +65903,34 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2581: -cbnz x5, 2579b -cbz x4, 2600f +2582: +cbnz x5, 2580b +cbz x4, 2601f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2601f -2600: +b 2602f 2601: +2602: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 str x3, [x19, #1680] str x4, [x19, #1688] -b 2578f -2577: -movz x22, #1, lsl #0 +b 2579f 2578: +movz x22, #1, lsl #0 +2579: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2602f +cbz x4, 2603f add x2, x19, #904 movz x5, #56, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2604: +2605: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2605f +cbz x6, 2606f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67268,26 +65942,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2607f +cbz x4, 2608f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2609f +cbz x4, 2610f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2610f -2609: +b 2611f 2610: -b 2608f -2607: +2611: +b 2609f 2608: +2609: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67299,26 +65973,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2611f +cbz x4, 2612f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2613f +cbz x4, 2614f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2614f -2613: +b 2615f 2614: -b 2612f -2611: +2615: +b 2613f 2612: +2613: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67330,26 +66004,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2615f +cbz x4, 2616f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2617f +cbz x4, 2618f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2618f -2617: +b 2619f 2618: -b 2616f -2615: +2619: +b 2617f 2616: +2617: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -67361,33 +66035,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2619f +cbz x4, 2620f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2621f +cbz x4, 2622f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2622f -2621: +b 2623f 2622: -b 2620f -2619: -2620: -cbz x4, 2623f -b 2624f 2623: -movz x5, #0, lsl #0 +b 2621f +2620: +2621: +cbz x4, 2624f +b 2625f 2624: -b 2606f -2605: +movz x5, #0, lsl #0 +2625: +b 2607f +2606: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -67411,51 +66085,51 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2606: -cbnz x5, 2604b -cbz x4, 2625f +2607: +cbnz x5, 2605b +cbz x4, 2626f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2626f -2625: +b 2627f 2626: +2627: sub x0, x4, #1 lsr x0, x0, #63 add x23, x0, #0 str x3, [x19, #1696] str x4, [x19, #1704] -b 2603f -2602: -movz x23, #1, lsl #0 +b 2604f 2603: +movz x23, #1, lsl #0 +2604: and x6, x22, x23 -2576: +2577: and x24, x24, x6 ldr x3, [x19, #1680] ldr x4, [x19, #1688] -cbz x4, 2627f -2629: +cbz x4, 2628f +2630: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2629b -b 2628f -2627: +cbnz x4, 2630b +b 2629f 2628: +2629: ldr x3, [x19, #1696] ldr x4, [x19, #1704] -cbz x4, 2630f -2632: +cbz x4, 2631f +2633: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2632b -b 2631f -2630: +cbnz x4, 2633b +b 2632f 2631: -2524: +2632: +2525: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -67993,10 +66667,10 @@ movz x4, #256, lsl #0 movz x5, #168, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2633: +2634: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2634f +cbz x6, 2635f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68008,26 +66682,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2636f +cbz x4, 2637f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2638f +cbz x4, 2639f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2639f -2638: +b 2640f 2639: -b 2637f -2636: +2640: +b 2638f 2637: +2638: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68039,26 +66713,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2640f +cbz x4, 2641f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2642f +cbz x4, 2643f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2643f -2642: +b 2644f 2643: -b 2641f -2640: +2644: +b 2642f 2641: +2642: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68070,26 +66744,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2644f +cbz x4, 2645f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2646f +cbz x4, 2647f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2647f -2646: +b 2648f 2647: -b 2645f -2644: +2648: +b 2646f 2645: +2646: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68101,33 +66775,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2648f +cbz x4, 2649f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2650f +cbz x4, 2651f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2651f -2650: +b 2652f 2651: -b 2649f -2648: -2649: -cbz x4, 2652f -b 2653f 2652: -movz x5, #0, lsl #0 +b 2650f +2649: +2650: +cbz x4, 2653f +b 2654f 2653: -b 2635f -2634: +movz x5, #0, lsl #0 +2654: +b 2636f +2635: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -68151,20 +66825,20 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2635: -cbnz x5, 2633b -cbz x4, 2654f +2636: +cbnz x5, 2634b +cbz x4, 2655f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2655f -2654: +b 2656f 2655: +2656: sub x0, x4, #1 lsr x0, x0, #63 add x22, x0, #0 -cbz x22, 2656f -b 2657f -2656: +cbz x22, 2657f +b 2658f +2657: str x22, [x19, #1848] str x23, [x19, #1856] add x2, x28, #1176 @@ -68221,7 +66895,7 @@ add x20, x19, #400 add x21, x19, #904 movz x22, #3, lsl #0 movz x22, #5, lsl #0 -2658: +2659: add x1, x19, #768 add x1, x1, #768 str x30, [x1, #128] @@ -68272,7 +66946,7 @@ ldr x6, [x19, #320] str x6, [x20, #160] add x20, x20, #168 sub x22, x22, #1 -cbnz x22, 2658b +cbnz x22, 2659b movz x9, #256, lsl #0 movk x9, #513, lsl #16 movk x9, #1027, lsl #32 @@ -68319,10 +66993,10 @@ movz x4, #256, lsl #0 movz x5, #280, lsl #0 movz x9, #3329, lsl #0 movz x10, #15, lsl #0 -2659: +2660: sub x6, x4, #8 lsr x6, x6, #63 -cbz x6, 2660f +cbz x6, 2661f ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68334,26 +67008,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2662f +cbz x4, 2663f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2664f +cbz x4, 2665f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2665f -2664: +b 2666f 2665: -b 2663f -2662: +2666: +b 2664f 2663: +2664: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68365,26 +67039,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2666f +cbz x4, 2667f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2668f +cbz x4, 2669f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2669f -2668: +b 2670f 2669: -b 2667f -2666: +2670: +b 2668f 2667: +2668: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68396,26 +67070,26 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2670f +cbz x4, 2671f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2672f +cbz x4, 2673f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2673f -2672: +b 2674f 2673: -b 2671f -2670: +2674: +b 2672f 2671: +2672: ldrb w6, [x2, #0] ldrb w7, [x2, #1] ldrb w8, [x2, #2] @@ -68427,33 +67101,33 @@ add x11, x11, x6 lsr x12, x7, #4 lsl x13, x8, #4 add x12, x12, x13 -cbz x4, 2674f +cbz x4, 2675f sub x13, x11, x9 lsr x14, x13, #63 str w11, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -cbz x4, 2676f +cbz x4, 2677f sub x13, x12, x9 lsr x14, x13, #63 str w12, [x3, #0] lsl x15, x14, #1 add x3, x3, x15 sub x4, x4, x14 -b 2677f -2676: +b 2678f 2677: -b 2675f -2674: -2675: -cbz x4, 2678f -b 2679f 2678: -movz x5, #0, lsl #0 +b 2676f +2675: +2676: +cbz x4, 2679f +b 2680f 2679: -b 2661f -2660: +movz x5, #0, lsl #0 +2680: +b 2662f +2661: ldr q0, [x2, #0] tbl v0.16b, {{v0.16b}}, v16.16b ushr v1.8h, v0.8h, #4 @@ -68477,30 +67151,30 @@ lsl x6, x6, #1 add x3, x3, x6 add x2, x2, #12 sub x5, x5, #4 -2661: -cbnz x5, 2659b -cbz x4, 2680f +2662: +cbnz x5, 2660b +cbz x4, 2681f movz x13, #0, lsl #0 str w13, [x3, #0] -b 2681f -2680: +b 2682f 2681: +2682: sub x0, x4, #1 lsr x0, x0, #63 and x24, x24, x0 -cbz x4, 2682f -2684: +cbz x4, 2683f +2685: movz x13, #0, lsl #0 str w13, [x3, #0] add x3, x3, #2 sub x4, x4, #1 -cbnz x4, 2684b -b 2683f -2682: +cbnz x4, 2685b +b 2684f 2683: +2684: ldr x22, [x19, #1848] ldr x23, [x19, #1856] -2657: +2658: ldr x30, [x19, #1840] ldr x23, [x19, #1832] ldr x22, [x19, #1824] @@ -68562,7 +67236,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2685: +2686: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -68669,7 +67343,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2685b +cbnz x11, 2686b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -68683,7 +67357,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2686: +2687: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -68780,7 +67454,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2686b +cbnz x5, 2687b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -68816,7 +67490,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2687: +2688: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -68976,7 +67650,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2687b +cbnz x5, 2688b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #29696, lsl #0 @@ -68984,7 +67658,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2688: +2689: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -68995,14 +67669,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2688b +cbnz x11, 2689b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2689: +2690: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -69011,7 +67685,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2689b +cbnz x10, 2690b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -69032,7 +67706,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2690: +2691: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -69139,7 +67813,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2690b +cbnz x11, 2691b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -69153,7 +67827,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2691: +2692: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -69250,7 +67924,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2691b +cbnz x5, 2692b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -69286,7 +67960,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2692: +2693: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -69446,7 +68120,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2692b +cbnz x5, 2693b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #30720, lsl #0 @@ -69454,7 +68128,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2693: +2694: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -69465,14 +68139,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2693b +cbnz x11, 2694b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2694: +2695: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -69481,7 +68155,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2694b +cbnz x10, 2695b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -69502,7 +68176,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2695: +2696: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -69609,7 +68283,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2695b +cbnz x11, 2696b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -69623,7 +68297,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2696: +2697: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -69720,7 +68394,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2696b +cbnz x5, 2697b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -69756,7 +68430,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2697: +2698: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -69916,7 +68590,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2697b +cbnz x5, 2698b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #31744, lsl #0 @@ -69924,7 +68598,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2698: +2699: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -69935,14 +68609,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2698b +cbnz x11, 2699b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2699: +2700: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -69951,7 +68625,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2699b +cbnz x10, 2700b movz x0, #18584, lsl #0 add x0, x28, x0 movz x1, #10, lsl #0 @@ -69977,7 +68651,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2700: +2701: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -69996,7 +68670,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2700b +cbnz x11, 2701b add x0, x25, #1536 movz x1, #5272, lsl #0 add x1, x28, x1 @@ -70015,7 +68689,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2701: +2702: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -70034,7 +68708,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2701b +cbnz x11, 2702b add x0, x25, #1920 movz x1, #6296, lsl #0 add x1, x28, x1 @@ -70053,7 +68727,7 @@ dup v17.4s, w9 movz x9, #3329, lsl #0 dup v18.4s, w9 movz x11, #32, lsl #0 -2702: +2703: ldr x9, [x0, #0] ldr w10, [x0, #8] dup v0.2d, x9 @@ -70072,7 +68746,7 @@ str q0, [x1, #0] add x0, x0, #12 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2702b +cbnz x11, 2703b movz x0, #17560, lsl #0 add x0, x28, x0 movz x9, #3329, lsl #0 @@ -70086,7 +68760,7 @@ dup v17.4s, w10", concat!("add x3, x3, ", vg_sym_pageoff!("{VG_MLKEM_GAMMA}")), r"movz x11, #16, lsl #0 movz x12, #0, lsl #0 -2703: +2704: movi v21.2d, #0 movi v22.2d, #0 movi v23.2d, #0 @@ -70193,7 +68867,7 @@ add x0, x0, #32 add x3, x3, #32 add x12, x12, #32 sub x11, x11, #1 -cbnz x11, 2703b +cbnz x11, 2704b movz x0, #17560, lsl #0 add x0, x28, x0 add x1, x28, #3224 @@ -70207,7 +68881,7 @@ add x3, x1, #0", concat!("adrp x6, ", vg_sym_page!("{VG_MLKEM_INTT16_ZETAS}")), concat!("add x6, x6, ", vg_sym_pageoff!("{VG_MLKEM_INTT16_ZETAS}")), r"movz x5, #8, lsl #0 -2704: +2705: ldr q0, [x2, #0] ldr q1, [x2, #16] ldr q2, [x2, #32] @@ -70304,7 +68978,7 @@ add x2, x2, #64 add x3, x3, #64 add x6, x6, #64 sub x5, x5, #1 -cbnz x5, 2704b +cbnz x5, 2705b movz x9, #848, lsl #0 mov v18.h[0], w9 movz x9, #8347, lsl #0 @@ -70340,7 +69014,7 @@ mov v19.h[7], w9 add x2, x1, #0 add x3, x0, #0 movz x5, #4, lsl #0 -2705: +2706: ldr q0, [x2, #0] ldr q1, [x2, #64] ldr q2, [x2, #128] @@ -70500,7 +69174,7 @@ str q16, [x3, #448] add x2, x2, #16 add x3, x3, #16 sub x5, x5, #1 -cbnz x5, 2705b +cbnz x5, 2706b movz x9, #6, lsl #0 strb w9, [x28, #976] add x0, x28, #0 @@ -70627,7 +69301,7 @@ add x1, x28, x1 movz x9, #3329, lsl #0 dup v16.8h, w9 movz x11, #32, lsl #0 -2706: +2707: ldr q0, [x0, #0] ldr q1, [x1, #0] add v2.8h, v0.8h, v1.8h @@ -70638,14 +69312,14 @@ str q2, [x0, #0] add x0, x0, #16 add x1, x1, #16 sub x11, x11, #1 -cbnz x11, 2706b +cbnz x11, 2707b movz x0, #17560, lsl #0 add x0, x28, x0 movz x1, #18584, lsl #0 add x1, x28, x1 movz x10, #32, lsl #0 movi v29.2d, #0 -2707: +2708: ldr q0, [x0, #0] zip1 v1.8h, v0.8h, v29.8h zip2 v2.8h, v0.8h, v29.8h @@ -70654,7 +69328,7 @@ str q2, [x1, #16] add x0, x0, #16 add x1, x1, #32 sub x10, x10, #1 -cbnz x10, 2707b +cbnz x10, 2708b add x0, x28, #912 movz x1, #32, lsl #0 movz x2, #1, lsl #0 @@ -70678,7 +69352,7 @@ movz x1, #20632, lsl #0 add x1, x28, x1 movz x2, #136, lsl #0 movz x10, #0, lsl #0 -2708: +2709: ldr x9, [x0, #0] ldr x11, [x1, #0] eor x9, x9, x11 @@ -70686,7 +69360,7 @@ orr x10, x10, x9 add x0, x0, #8 add x1, x1, #8 sub x2, x2, #1 -cbnz x2, 2708b +cbnz x2, 2709b movz x11, #0, lsl #0 sub x11, x11, x10 orr x10, x10, x11 @@ -70727,16 +69401,14 @@ ret .arch_extension nosha3 .p2align 6", vg_mlkem_decode_decompress = sym super::mlkem::vg_mlkem_decode_decompress, - vg_mlkem_decode12 = sym super::mlkem::vg_mlkem_decode12, + vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem_sub = sym super::mlkem::vg_mlkem_sub, vg_mlkem_compress_encode = sym super::mlkem::vg_mlkem_compress_encode, vg_keccak_f1600_sha3 = sym super::sha3::vg_keccak_f1600_sha3, vg_keccak_f1600_x3_sha3 = sym super::sha3::vg_keccak_f1600_x3_sha3, vg_keccak_f1600_x2_sha3 = sym super::sha3::vg_keccak_f1600_x2_sha3, vg_mlkem16_cbd2 = sym super::mlkem16::vg_mlkem16_cbd2, - vg_mlkem16_ntt = sym super::mlkem16::vg_mlkem16_ntt, vg_mlkem_add = sym super::mlkem::vg_mlkem_add, - VG_MLKEM_NTT16_ZETAS = sym super::consts::VG_MLKEM_NTT16_ZETAS, VG_MLKEM_GAMMA = sym super::consts::VG_MLKEM_GAMMA, VG_MLKEM_INTT16_ZETAS = sym super::consts::VG_MLKEM_INTT16_ZETAS, VG_MLKEM_REJ_SHUFFLE = sym super::consts::VG_MLKEM_REJ_SHUFFLE,