diff --git a/src/hotspot/cpu/aarch64/register_aarch64.cpp b/src/hotspot/cpu/aarch64/register_aarch64.cpp index b61627f6b91e..3a46e38a72a7 100644 --- a/src/hotspot/cpu/aarch64/register_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/register_aarch64.cpp @@ -59,23 +59,3 @@ const char* PRegister::PRegisterImpl::name() const { }; return is_valid() ? names[encoding()] : "pnoreg"; } - -// convenience methods for splitting 8-way vector register sequences -// in half -- needed because vector operations can normally only be -// benefit from 4-way instruction parallelism - -VSeq<4> vs_front(const VSeq<8>& v) { - return VSeq<4>(v.base(), v.delta()); -} - -VSeq<4> vs_back(const VSeq<8>& v) { - return VSeq<4>(v.base() + 4 * v.delta(), v.delta()); -} - -VSeq<4> vs_even(const VSeq<8>& v) { - return VSeq<4>(v.base(), v.delta() * 2); -} - -VSeq<4> vs_odd(const VSeq<8>& v) { - return VSeq<4>(v.base() + 1, v.delta() * 2); -} diff --git a/src/hotspot/cpu/aarch64/register_aarch64.hpp b/src/hotspot/cpu/aarch64/register_aarch64.hpp index 45351bb39451..118b8c8b2bb0 100644 --- a/src/hotspot/cpu/aarch64/register_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/register_aarch64.hpp @@ -432,19 +432,20 @@ inline Register as_Register(FloatRegister reg) { // inputs into front and back halves or odd and even halves (see // convenience methods below). +// helper macro for computing register masks +#define VS_MASK_BIT(base, delta, i) (1 << (base + delta * i)) + template class VSeq { static_assert(N >= 2, "vector sequence length must be greater than 1"); - static_assert(N <= 8, "vector sequence length must not exceed 8"); - static_assert((N & (N - 1)) == 0, "vector sequence length must be power of two"); private: int _base; // index of first register in sequence int _delta; // increment to derive successive indices public: VSeq(FloatRegister base_reg, int delta = 1) : VSeq(base_reg->encoding(), delta) { } VSeq(int base, int delta = 1) : _base(base), _delta(delta) { - assert (_base >= 0, "invalid base register"); - assert (_delta >= 0, "invalid register delta"); - assert ((_base + (N - 1) * _delta) < 32, "range exceeded"); + assert (_base >= 0 && _base <= 31, "invalid base register"); + assert ((_base + (N - 1) * _delta) >= 0, "register range underflow"); + assert ((_base + (N - 1) * _delta) < 32, "register range overflow"); } // indexed access to sequence FloatRegister operator [](int i) const { @@ -453,27 +454,89 @@ template class VSeq { } int mask() const { int m = 0; - int bit = 1 << _base; for (int i = 0; i < N; i++) { - m |= bit << (i * _delta); + m |= VS_MASK_BIT(_base, _delta, i); } return m; } int base() const { return _base; } int delta() const { return _delta; } + bool is_constant() const { return _delta == 0; } }; -// declare convenience methods for splitting vector register sequences - -VSeq<4> vs_front(const VSeq<8>& v); -VSeq<4> vs_back(const VSeq<8>& v); -VSeq<4> vs_even(const VSeq<8>& v); -VSeq<4> vs_odd(const VSeq<8>& v); - -// methods for use in asserts to check VSeq inputs and oupts are +// methods for use in asserts to check VSeq inputs and outputs are // either disjoint or equal template bool vs_disjoint(const VSeq& n, const VSeq& m) { return (n.mask() & m.mask()) == 0; } template bool vs_same(const VSeq& n, const VSeq& m) { return n.mask() == m.mask(); } +// method for use in asserts to check whether registers appearing in +// an output sequence will be written before they are read from an +// input sequence. + +template bool vs_write_before_read(const VSeq& vout, const VSeq& vin) { + int b_in = vin.base(); + int d_in = vin.delta(); + int b_out = vout.base(); + int d_out = vout.delta(); + int bit_in = 1 << b_in; + int bit_out = 1 << b_out; + int mask_read = vin.mask(); // all pending reads + int mask_write = 0; // no writes as yet + + + for (int i = 0; i < N - 1; i++) { + // check whether a pending read clashes with a write + if ((mask_write & mask_read) != 0) { + return true; + } + // remove the pending input (so long as this is a constant + // sequence) + if (d_in != 0) { + mask_read ^= VS_MASK_BIT(b_in, d_in, i); + } + // record the next write + mask_write |= VS_MASK_BIT(b_out, d_out, i); + } + // no write before read + return false; +} + +// convenience methods for splitting 8-way or 4-way vector register +// sequences in half -- needed because vector operations can normally +// benefit from 4-way instruction parallelism or, occasionally, 2-way +// parallelism + +template +VSeq vs_front(const VSeq& v) { + static_assert(N > 0 && ((N & 1) == 0), "sequence length must be even"); + return VSeq(v.base(), v.delta()); +} + +template +VSeq vs_back(const VSeq& v) { + static_assert(N > 0 && ((N & 1) == 0), "sequence length must be even"); + return VSeq(v.base() + N / 2 * v.delta(), v.delta()); +} + +template +VSeq vs_even(const VSeq& v) { + static_assert(N > 0 && ((N & 1) == 0), "sequence length must be even"); + return VSeq(v.base(), v.delta() * 2); +} + +template +VSeq vs_odd(const VSeq& v) { + static_assert(N > 0 && ((N & 1) == 0), "sequence length must be even"); + return VSeq(v.base() + v.delta(), v.delta() * 2); +} + +// convenience method to construct a vector register sequence that +// indexes its elements in reverse order to the original + +template +VSeq vs_reverse(const VSeq& v) { + return VSeq(v.base() + (N - 1) * v.delta(), -v.delta()); +} + #endif // CPU_AARCH64_REGISTER_AARCH64_HPP diff --git a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp index 77cea7998832..d003533d9f4e 100644 --- a/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp @@ -4486,6 +4486,11 @@ class StubGenerator: public StubCodeGenerator { template void vs_addv(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); for (int i = 0; i < N; i++) { __ addv(v[i], T, v1[i], v2[i]); } @@ -4494,6 +4499,11 @@ class StubGenerator: public StubCodeGenerator { template void vs_subv(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); for (int i = 0; i < N; i++) { __ subv(v[i], T, v1[i], v2[i]); } @@ -4502,6 +4512,11 @@ class StubGenerator: public StubCodeGenerator { template void vs_mulv(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); for (int i = 0; i < N; i++) { __ mulv(v[i], T, v1[i], v2[i]); } @@ -4509,6 +4524,10 @@ class StubGenerator: public StubCodeGenerator { template void vs_negr(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); for (int i = 0; i < N; i++) { __ negr(v[i], T, v1[i]); } @@ -4517,6 +4536,10 @@ class StubGenerator: public StubCodeGenerator { template void vs_sshr(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, int shift) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); for (int i = 0; i < N; i++) { __ sshr(v[i], T, v1[i], shift); } @@ -4524,6 +4547,11 @@ class StubGenerator: public StubCodeGenerator { template void vs_andr(const VSeq& v, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); for (int i = 0; i < N; i++) { __ andr(v[i], __ T16B, v1[i], v2[i]); } @@ -4531,18 +4559,51 @@ class StubGenerator: public StubCodeGenerator { template void vs_orr(const VSeq& v, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); for (int i = 0; i < N; i++) { __ orr(v[i], __ T16B, v1[i], v2[i]); } } template - void vs_notr(const VSeq& v, const VSeq& v1) { + void vs_notr(const VSeq& v, const VSeq& v1) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); for (int i = 0; i < N; i++) { __ notr(v[i], __ T16B, v1[i]); } } + template + void vs_sqdmulh(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, const VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); + for (int i = 0; i < N; i++) { + __ sqdmulh(v[i], T, v1[i], v2[i]); + } + } + + template + void vs_mlsv(const VSeq& v, Assembler::SIMD_Arrangement T, const VSeq& v1, VSeq& v2) { + // output must not be constant + assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector"); + // output cannot overwrite pending inputs + assert(!vs_write_before_read(v, v1), "output overwrites input"); + assert(!vs_write_before_read(v, v2), "output overwrites input"); + for (int i = 0; i < N; i++) { + __ mlsv(v[i], T, v1[i], v2[i]); + } + } + // load N/2 successive pairs of quadword values from memory in order // into N successive vector registers of the sequence via the // address supplied in base. @@ -4558,6 +4619,7 @@ class StubGenerator: public StubCodeGenerator { // in base using post-increment addressing template void vs_ldpq_post(const VSeq& v, Register base) { + static_assert((N & (N - 1)) == 0, "sequence length must be even"); for (int i = 0; i < N; i += 2) { __ ldpq(v[i], v[i+1], __ post(base, 32)); } @@ -4568,11 +4630,55 @@ class StubGenerator: public StubCodeGenerator { // supplied in base using post-increment addressing template void vs_stpq_post(const VSeq& v, Register base) { + static_assert((N & (N - 1)) == 0, "sequence length must be even"); for (int i = 0; i < N; i += 2) { __ stpq(v[i], v[i+1], __ post(base, 32)); } } + // load N/2 pairs of quadword values from memory de-interleaved into + // N vector registers 2 at a time via the address supplied in base + // using post-increment addressing. + template + void vs_ld2_post(const VSeq& v, Assembler::SIMD_Arrangement T, Register base) { + static_assert((N & (N - 1)) == 0, "sequence length must be even"); + for (int i = 0; i < N; i += 2) { + __ ld2(v[i], v[i+1], T, __ post(base, 32)); + } + } + + // store N vector registers interleaved into N/2 pairs of quadword + // memory locations via the address supplied in base using + // post-increment addressing. + template + void vs_st2_post(const VSeq& v, Assembler::SIMD_Arrangement T, Register base) { + static_assert((N & (N - 1)) == 0, "sequence length must be even"); + for (int i = 0; i < N; i += 2) { + __ st2(v[i], v[i+1], T, __ post(base, 32)); + } + } + + // load N quadword values from memory de-interleaved into N vector + // registers 3 elements at a time via the address supplied in base. + template + void vs_ld3(const VSeq& v, Assembler::SIMD_Arrangement T, Register base) { + static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3"); + for (int i = 0; i < N; i += 3) { + __ ld3(v[i], v[i+1], v[i+2], T, base); + } + } + + // load N quadword values from memory de-interleaved into N vector + // registers 3 elements at a time via the address supplied in base + // using post-increment addressing. + template + void vs_ld3_post(const VSeq& v, Assembler::SIMD_Arrangement T, Register base) { + static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3"); + for (int i = 0; i < N; i += 3) { + __ ld3(v[i], v[i+1], v[i+2], T, __ post(base, 48)); + } + } + // load N/2 pairs of quadword values from memory into N vector // registers via the address supplied in base with each pair indexed // using the the start offset plus the corresponding entry in the @@ -4645,23 +4751,29 @@ class StubGenerator: public StubCodeGenerator { } } - // Helper routines for various flavours of dilithium montgomery - // multiply + // Helper routines for various flavours of Montgomery multiply - // Perform 16 32-bit Montgomery multiplications in parallel - // See the montMul() method of the sun.security.provider.ML_DSA class. - // - // Computes 4x4S results - // a = b * c * 2^-32 mod MONT_Q - // Inputs: vb, vc - 4x4S vector register sequences - // vq - 2x4S constants - // Temps: vtmp - 4x4S vector sequence trashed after call - // Outputs: va - 4x4S vector register sequences + // Perform 16 32-bit (4x4S) or 32 16-bit (4 x 8H) Montgomery + // multiplications in parallel + // + + // See the montMul() method of the sun.security.provider.ML_DSA + // class. + // + // Computes 4x4S results or 8x8H results + // a = b * c * 2^MONT_R_BITS mod MONT_Q + // Inputs: vb, vc - 4x4S or 4x8H vector register sequences + // vq - 2x4S or 2x8H constants + // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call + // Outputs: va - 4x4S or 4x8H vector register sequences // vb, vc, vtmp and vq must all be disjoint // va must be disjoint from all other inputs/temps or must equal vc - // n.b. MONT_R_BITS is 32, so the right shift by it is implicit. - void dilithium_montmul16(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc, - const VSeq<4>& vtmp, const VSeq<2>& vq) { + // va must have a non-zero delta i.e. it must not be a constant vseq. + // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit. + void vs_montmul4(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc, + Assembler::SIMD_Arrangement T, + const VSeq<4>& vtmp, const VSeq<2>& vq) { + assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul"); assert(vs_disjoint(vb, vc), "vb and vc overlap"); assert(vs_disjoint(vb, vq), "vb and vq overlap"); assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap"); @@ -4675,40 +4787,107 @@ class StubGenerator: public StubCodeGenerator { assert(vs_disjoint(va, vb), "va and vb overlap"); assert(vs_disjoint(va, vq), "va and vq overlap"); assert(vs_disjoint(va, vtmp), "va and vtmp overlap"); + assert(!va.is_constant(), "output vector must identify 4 different registers"); // schedule 4 streams of instructions across the vector sequences for (int i = 0; i < 4; i++) { - __ sqdmulh(vtmp[i], __ T4S, vb[i], vc[i]); // aHigh = hi32(2 * b * c) - __ mulv(va[i], __ T4S, vb[i], vc[i]); // aLow = lo32(b * c) + __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c) + __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c) } for (int i = 0; i < 4; i++) { - __ mulv(va[i], __ T4S, va[i], vq[0]); // m = aLow * qinv + __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv } for (int i = 0; i < 4; i++) { - __ sqdmulh(va[i], __ T4S, va[i], vq[1]); // n = hi32(2 * m * q) + __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q) } for (int i = 0; i < 4; i++) { - __ shsubv(va[i], __ T4S, vtmp[i], va[i]); // a = (aHigh - n) / 2 + __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2 } } - // Perform 2x16 32-bit Montgomery multiplications in parallel - // See the montMul() method of the sun.security.provider.ML_DSA class. - // - // Computes 8x4S results - // a = b * c * 2^-32 mod MONT_Q - // Inputs: vb, vc - 8x4S vector register sequences - // vq - 2x4S constants - // Temps: vtmp - 4x4S vector sequence trashed after call - // Outputs: va - 8x4S vector register sequences + // Perform 8 32-bit (4x4S) or 16 16-bit (2 x 8H) Montgomery + // multiplications in parallel + // + + // See the montMul() method of the sun.security.provider.ML_DSA + // class. + // + // Computes 4x4S results or 8x8H results + // a = b * c * 2^MONT_R_BITS mod MONT_Q + // Inputs: vb, vc - 4x4S or 4x8H vector register sequences + // vq - 2x4S or 2x8H constants + // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call + // Outputs: va - 4x4S or 4x8H vector register sequences // vb, vc, vtmp and vq must all be disjoint // va must be disjoint from all other inputs/temps or must equal vc - // n.b. MONT_R_BITS is 32, so the right shift by it is implicit. - void vs_montmul32(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc, - const VSeq<4>& vtmp, const VSeq<2>& vq) { + // va must have a non-zero delta i.e. it must not be a constant vseq. + // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit. + void vs_montmul2(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc, + Assembler::SIMD_Arrangement T, + const VSeq<2>& vtmp, const VSeq<2>& vq) { + assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul"); + assert(vs_disjoint(vb, vc), "vb and vc overlap"); + assert(vs_disjoint(vb, vq), "vb and vq overlap"); + assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap"); + + assert(vs_disjoint(vc, vq), "vc and vq overlap"); + assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap"); + + assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap"); + + assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal"); + assert(vs_disjoint(va, vb), "va and vb overlap"); + assert(vs_disjoint(va, vq), "va and vq overlap"); + assert(vs_disjoint(va, vtmp), "va and vtmp overlap"); + assert(!va.is_constant(), "output vector must identify 2 different registers"); + + // schedule 2 streams of instructions across the vector sequences + for (int i = 0; i < 2; i++) { + __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c) + __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c) + } + + for (int i = 0; i < 2; i++) { + __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv + } + + for (int i = 0; i < 2; i++) { + __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q) + } + + for (int i = 0; i < 2; i++) { + __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2 + } + } + + // Perform 16 16-bit Montgomery multiplications in parallel. + void kyber_montmul16(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc, + const VSeq<2>& vtmp, const VSeq<2>& vq) { + // Use the helper routine to schedule a 2x8H Montgomery multiply. + // It will assert that the register use is valid + vs_montmul2(va, vb, vc, __ T8H, vtmp, vq); + } + + // Perform 32 16-bit Montgomery multiplications in parallel. + void kyber_montmul32(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc, + const VSeq<4>& vtmp, const VSeq<2>& vq) { + // Use the helper routine to schedule a 4x8H Montgomery multiply. + // It will assert that the register use is valid + vs_montmul4(va, vb, vc, __ T8H, vtmp, vq); + } + + // Perform 64 16-bit Montgomery multiplications in parallel. + void kyber_montmul64(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc, + const VSeq<4>& vtmp, const VSeq<2>& vq) { + // Schedule two successive 4x8H multiplies via the montmul helper + // on the front and back halves of va, vb and vc. The helper will + // assert that the register use has no overlap conflicts on each + // individual call but we also need to ensure that the necessary + // disjoint/equality constraints are met across both calls. + // vb, vc, vtmp and vq must be disjoint. va must either be // disjoint from all other registers or equal vc @@ -4726,8 +4905,8 @@ class StubGenerator: public StubCodeGenerator { assert(vs_disjoint(va, vq), "va and vq overlap"); assert(vs_disjoint(va, vtmp), "va and vtmp overlap"); - // we need to multiply the front and back halves of each sequence - // 4x4S at a time because + // we multiply the front and back halves of each sequence 4 at a + // time because // // 1) we are currently only able to get 4-way instruction // parallelism at best @@ -4736,14 +4915,1229 @@ class StubGenerator: public StubCodeGenerator { // scratch registers to hold intermediate results so vtmp can only // be a VSeq<4> which means we only have 4 scratch slots - dilithium_montmul16(vs_front(va), vs_front(vb), vs_front(vc), vtmp, vq); - dilithium_montmul16(vs_back(va), vs_back(vb), vs_back(vc), vtmp, vq); + vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T8H, vtmp, vq); + vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T8H, vtmp, vq); + } + + void kyber_montmul32_sub_add(const VSeq<4>& va0, const VSeq<4>& va1, + const VSeq<4>& vc, + const VSeq<4>& vtmp, + const VSeq<2>& vq) { + // compute a = montmul(a1, c) + kyber_montmul32(vc, va1, vc, vtmp, vq); + // ouptut a1 = a0 - a + vs_subv(va1, __ T8H, va0, vc); + // and a0 = a0 + a + vs_addv(va0, __ T8H, va0, vc); + } + + void kyber_sub_add_montmul32(const VSeq<4>& va0, const VSeq<4>& va1, + const VSeq<4>& vb, + const VSeq<4>& vtmp1, + const VSeq<4>& vtmp2, + const VSeq<2>& vq) { + // compute c = a0 - a1 + vs_subv(vtmp1, __ T8H, va0, va1); + // output a0 = a0 + a1 + vs_addv(va0, __ T8H, va0, va1); + // output a1 = b montmul c + kyber_montmul32(va1, vtmp1, vb, vtmp2, vq); + } + + void load64shorts(const VSeq<8>& v, Register shorts) { + vs_ldpq_post(v, shorts); + } + + void load32shorts(const VSeq<4>& v, Register shorts) { + vs_ldpq_post(v, shorts); } - // perform combined montmul then add/sub on 4x4S vectors + void store64shorts(VSeq<8> v, Register tmpAddr) { + vs_stpq_post(v, tmpAddr); + } + + // Kyber NTT function. + // Implements + // static int implKyberNtt(short[] poly, short[] ntt_zetas) {} + // + // coeffs (short[256]) = c_rarg0 + // ntt_zetas (short[256]) = c_rarg1 + address generate_kyberNtt() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberNtt"); + address start = __ pc(); + __ enter(); + + const Register coeffs = c_rarg0; + const Register zetas = c_rarg1; + + const Register kyberConsts = r10; + const Register tmpAddr = r11; + + VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs + VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3 + VSeq<2> vq(30); // n.b. constants overlap vs3 + + __ lea(kyberConsts, ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + // load the montmul constants + vs_ldpq(vq, kyberConsts); + + // Each level corresponds to an iteration of the outermost loop of the + // Java method seilerNTT(int[] coeffs). There are some differences + // from what is done in the seilerNTT() method, though: + // 1. The computation is using 16-bit signed values, we do not convert them + // to ints here. + // 2. The zetas are delivered in a bigger array, 128 zetas are stored in + // this array for each level, it is easier that way to fill up the vector + // registers. + // 3. In the seilerNTT() method we use R = 2^20 for the Montgomery + // multiplications (this is because that way there should not be any + // overflow during the inverse NTT computation), here we usr R = 2^16 so + // that we can use the 16-bit arithmetic in the vector unit. + // + // On each level, we fill up the vector registers in such a way that the + // array elements that need to be multiplied by the zetas go into one + // set of vector registers while the corresponding ones that don't need to + // be multiplied, go into another set. + // We can do 32 Montgomery multiplications in parallel, using 12 vector + // registers interleaving the steps of 4 identical computations, + // each done on 8 16-bit values per register. + + // At levels 0-3 the coefficients multiplied by or added/subtracted + // to the zetas occur in discrete blocks whose size is some multiple + // of 32. + + // level 0 + __ add(tmpAddr, coeffs, 256); + load64shorts(vs1, tmpAddr); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 0); + load64shorts(vs1, tmpAddr); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 0); + vs_stpq_post(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 256); + vs_stpq_post(vs3, tmpAddr); + // restore montmul constants + vs_ldpq(vq, kyberConsts); + load64shorts(vs1, tmpAddr); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 128); + load64shorts(vs1, tmpAddr); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 128); + store64shorts(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 384); + store64shorts(vs3, tmpAddr); + + // level 1 + // restore montmul constants + vs_ldpq(vq, kyberConsts); + __ add(tmpAddr, coeffs, 128); + load64shorts(vs1, tmpAddr); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 0); + load64shorts(vs1, tmpAddr); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 0); + store64shorts(vs1, tmpAddr); + store64shorts(vs3, tmpAddr); + vs_ldpq(vq, kyberConsts); + __ add(tmpAddr, coeffs, 384); + load64shorts(vs1, tmpAddr); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 256); + load64shorts(vs1, tmpAddr); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 256); + store64shorts(vs1, tmpAddr); + store64shorts(vs3, tmpAddr); + + // level 2 + vs_ldpq(vq, kyberConsts); + int offsets1[4] = { 0, 32, 128, 160 }; + vs_ldpq_indexed(vs1, coeffs, 64, offsets1); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldpq_indexed(vs1, coeffs, 0, offsets1); + // kyber_subv_addv64(); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 0); + vs_stpq_post(vs_front(vs1), tmpAddr); + vs_stpq_post(vs_front(vs3), tmpAddr); + vs_stpq_post(vs_back(vs1), tmpAddr); + vs_stpq_post(vs_back(vs3), tmpAddr); + vs_ldpq(vq, kyberConsts); + vs_ldpq_indexed(vs1, tmpAddr, 64, offsets1); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldpq_indexed(vs1, coeffs, 256, offsets1); + // kyber_subv_addv64(); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 256); + vs_stpq_post(vs_front(vs1), tmpAddr); + vs_stpq_post(vs_front(vs3), tmpAddr); + vs_stpq_post(vs_back(vs1), tmpAddr); + vs_stpq_post(vs_back(vs3), tmpAddr); + + // level 3 + vs_ldpq(vq, kyberConsts); + int offsets2[4] = { 0, 64, 128, 192 }; + vs_ldpq_indexed(vs1, coeffs, 32, offsets2); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldpq_indexed(vs1, coeffs, 0, offsets2); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs1, coeffs, 0, offsets2); + vs_stpq_indexed(vs3, coeffs, 32, offsets2); + + vs_ldpq(vq, kyberConsts); + vs_ldpq_indexed(vs1, coeffs, 256 + 32, offsets2); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldpq_indexed(vs1, coeffs, 256, offsets2); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs1, coeffs, 256, offsets2); + vs_stpq_indexed(vs3, coeffs, 256 + 32, offsets2); + + // level 4 + // At level 4 coefficients occur in 8 discrete blocks of size 16 + // so they are loaded using employing an ldr at 8 distinct offsets. + + vs_ldpq(vq, kyberConsts); + int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 }; + vs_ldr_indexed(vs1, __ Q, coeffs, 16, offsets3); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3); + vs_str_indexed(vs3, __ Q, coeffs, 16, offsets3); + + vs_ldpq(vq, kyberConsts); + vs_ldr_indexed(vs1, __ Q, coeffs, 256 + 16, offsets3); + load64shorts(vs2, zetas); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3); + vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_addv(vs1, __ T8H, vs1, vs2); + vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3); + vs_str_indexed(vs3, __ Q, coeffs, 256 + 16, offsets3); + + // level 5 + // At level 5 related coefficients occur in discrete blocks of size 8 so + // need to be loaded interleaved using an ld2 operation with arrangement 2D. + + vs_ldpq(vq, kyberConsts); + int offsets4[4] = { 0, 32, 64, 96 }; + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4); + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4); + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4); + + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4); + + // level 6 + // At level 6 related coefficients occur in discrete blocks of size 4 so + // need to be loaded interleaved using an ld2 operation with arrangement 4S. + + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4); + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4); + // __ ldpq(v18, v19, __ post(zetas, 32)); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4); + + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4); + + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber Inverse NTT function + // Implements + // static int implKyberInverseNtt(short[] poly, short[] zetas) {} + // + // coeffs (short[256]) = c_rarg0 + // ntt_zetas (short[256]) = c_rarg1 + address generate_kyberInverseNtt() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberInverseNtt"); + address start = __ pc(); + __ enter(); + + const Register coeffs = c_rarg0; + const Register zetas = c_rarg1; + + const Register kyberConsts = r10; + const Register tmpAddr = r11; + const Register tmpAddr2 = c_rarg2; + + VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs + VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3 + VSeq<2> vq(30); // n.b. constants overlap vs3 + + __ lea(kyberConsts, + ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + + // level 0 + // At level 0 related coefficients occur in discrete blocks of size 4 so + // need to be loaded interleaved using an ld2 operation with arrangement 4S. + + vs_ldpq(vq, kyberConsts); + int offsets4[4] = { 0, 32, 64, 96 }; + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4); + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4); + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4); + vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4); + + // level 1 + // At level 1 related coefficients occur in discrete blocks of size 8 so + // need to be loaded interleaved using an ld2 operation with arrangement 2D. + + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4); + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4); + + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4); + vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4); + load32shorts(vs_front(vs2), zetas); + kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1), + vs_front(vs2), vs_back(vs2), vtmp, vq); + vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4); + + // level 2 + // At level 2 coefficients occur in 8 discrete blocks of size 16 + // so they are loaded using employing an ldr at 8 distinct offsets. + + int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 }; + vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3); + vs_ldr_indexed(vs2, __ Q, coeffs, 16, offsets3); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_str_indexed(vs3, __ Q, coeffs, 0, offsets3); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_str_indexed(vs2, __ Q, coeffs, 16, offsets3); + + vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3); + vs_ldr_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_str_indexed(vs3, __ Q, coeffs, 256, offsets3); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_str_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3); + + // Barrett reduction at indexes where overflow may happen + + // load q and the multiplier for the Barrett reduction + __ add(tmpAddr, kyberConsts, 16); + vs_ldpq(vq, tmpAddr); + + VSeq<8> vq1 = VSeq<8>(vq[0], 0); // 2 constant 8 sequences + VSeq<8> vq2 = VSeq<8>(vq[1], 0); // for above two kyber constants + VSeq<8> vq3 = VSeq<8>(v29, 0); // 3rd sequence for const montmul + vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3); + vs_sqdmulh(vs2, __ T8H, vs1, vq2); + vs_sshr(vs2, __ T8H, vs2, 11); + vs_mlsv(vs1, __ T8H, vs2, vq1); + vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3); + vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3); + vs_sqdmulh(vs2, __ T8H, vs1, vq2); + vs_sshr(vs2, __ T8H, vs2, 11); + vs_mlsv(vs1, __ T8H, vs2, vq1); + vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3); + + // level 3 + // From level 3 upwards coefficients occur in discrete blocks whose size is + // some multiple of 32 so can be loaded using ldpq and suitable indexes. + + int offsets2[4] = { 0, 64, 128, 192 }; + vs_ldpq_indexed(vs1, coeffs, 0, offsets2); + vs_ldpq_indexed(vs2, coeffs, 32, offsets2); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs3, coeffs, 0, offsets2); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_stpq_indexed(vs2, coeffs, 32, offsets2); + + vs_ldpq_indexed(vs1, coeffs, 256, offsets2); + vs_ldpq_indexed(vs2, coeffs, 256 + 32, offsets2); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs3, coeffs, 256, offsets2); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_stpq_indexed(vs2, coeffs, 256 + 32, offsets2); + + // level 4 + + int offsets1[4] = { 0, 32, 128, 160 }; + vs_ldpq_indexed(vs1, coeffs, 0, offsets1); + vs_ldpq_indexed(vs2, coeffs, 64, offsets1); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs3, coeffs, 0, offsets1); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_stpq_indexed(vs2, coeffs, 64, offsets1); + + vs_ldpq_indexed(vs1, coeffs, 256, offsets1); + vs_ldpq_indexed(vs2, coeffs, 256 + 64, offsets1); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + vs_stpq_indexed(vs3, coeffs, 256, offsets1); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + vs_stpq_indexed(vs2, coeffs, 256 + 64, offsets1); + + // level 5 + + __ add(tmpAddr, coeffs, 0); + load64shorts(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 128); + load64shorts(vs2, tmpAddr); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 0); + store64shorts(vs3, tmpAddr); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 128); + store64shorts(vs2, tmpAddr); + + load64shorts(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 384); + load64shorts(vs2, tmpAddr); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 256); + store64shorts(vs3, tmpAddr); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 384); + store64shorts(vs2, tmpAddr); + + // Barrett reduction at indexes where overflow may happen + + // load q and the multiplier for the Barrett reduction + __ add(tmpAddr, kyberConsts, 16); + vs_ldpq(vq, tmpAddr); + + int offsets0[2] = { 0, 256 }; + vs_ldpq_indexed(vs_front(vs1), coeffs, 0, offsets0); + vs_sqdmulh(vs2, __ T8H, vs1, vq2); + vs_sshr(vs2, __ T8H, vs2, 11); + vs_mlsv(vs1, __ T8H, vs2, vq1); + vs_stpq_indexed(vs_front(vs1), coeffs, 0, offsets0); + + // level 6 + + __ add(tmpAddr, coeffs, 0); + load64shorts(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 256); + load64shorts(vs2, tmpAddr); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 0); + store64shorts(vs3, tmpAddr); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 256); + store64shorts(vs2, tmpAddr); + + __ add(tmpAddr, coeffs, 128); + load64shorts(vs1, tmpAddr); + __ add(tmpAddr, coeffs, 384); + load64shorts(vs2, tmpAddr); + vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq + vs_subv(vs1, __ T8H, vs1, vs2); + __ add(tmpAddr, coeffs, 128); + store64shorts(vs3, tmpAddr); + load64shorts(vs2, zetas); + vs_ldpq(vq, kyberConsts); + kyber_montmul64(vs2, vs1, vs2, vtmp, vq); + __ add(tmpAddr, coeffs, 384); + store64shorts(vs2, tmpAddr); + + // multiply by 2^-n + + // load toMont(2^-n mod q) + __ add(tmpAddr, kyberConsts, 48); + __ ldr(v29, __ Q, tmpAddr); + + vs_ldpq(vq, kyberConsts); + __ add(tmpAddr, coeffs, 0); + load64shorts(vs1, tmpAddr); + kyber_montmul64(vs2, vs1, vq3, vtmp, vq); + __ add(tmpAddr, coeffs, 0); + store64shorts(vs2, tmpAddr); + + // now tmpAddr contains coeffs + 128 because store64shorts adjusted it so + load64shorts(vs1, tmpAddr); + kyber_montmul64(vs2, vs1, vq3, vtmp, vq); + __ add(tmpAddr, coeffs, 128); + store64shorts(vs2, tmpAddr); + + // now tmpAddr contains coeffs + 256 + load64shorts(vs1, tmpAddr); + kyber_montmul64(vs2, vs1, vq3, vtmp, vq); + __ add(tmpAddr, coeffs, 256); + store64shorts(vs2, tmpAddr); + + // now tmpAddr contains coeffs + 384 + load64shorts(vs1, tmpAddr); + kyber_montmul64(vs2, vs1, vq3, vtmp, vq); + __ add(tmpAddr, coeffs, 384); + store64shorts(vs2, tmpAddr); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber multiply polynomials in the NTT domain. + // Implements + // static int implKyberNttMult( + // short[] result, short[] ntta, short[] nttb, short[] zetas) {} + // + // result (short[256]) = c_rarg0 + // ntta (short[256]) = c_rarg1 + // nttb (short[256]) = c_rarg2 + // zetas (short[128]) = c_rarg3 + address generate_kyberNttMult() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberNttMult"); + address start = __ pc(); + __ enter(); + + const Register result = c_rarg0; + const Register ntta = c_rarg1; + const Register nttb = c_rarg2; + const Register zetas = c_rarg3; + + const Register kyberConsts = r10; + const Register limit = r11; + + VSeq<4> vs1(0), vs2(4); // 4 sets of 8x8H inputs/outputs/tmps + VSeq<4> vs3(16), vs4(20); + VSeq<2> vq(30); // pair of constants for montmul: q, qinv + VSeq<2> vz(28); // pair of zetas + VSeq<4> vc(27, 0); // constant sequence for montmul: montRSquareModQ + + __ lea(kyberConsts, + ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + + Label kyberNttMult_loop; - void dilithium_montmul16_sub_add(const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vc, - const VSeq<4>& vtmp, const VSeq<2>& vq) { + __ add(limit, result, 512); + + // load q and qinv + vs_ldpq(vq, kyberConsts); + + // load R^2 mod q (to convert back from Montgomery representation) + __ add(kyberConsts, kyberConsts, 64); + __ ldr(v27, __ Q, kyberConsts); + + __ BIND(kyberNttMult_loop); + + // load 16 zetas + vs_ldpq_post(vz, zetas); + + // load 2 sets of 32 coefficients from the two input arrays + // interleaved as shorts. i.e. pairs of shorts adjacent in memory + // are striped across pairs of vector registers + vs_ld2_post(vs_front(vs1), __ T8H, ntta); // x 8H + vs_ld2_post(vs_back(vs1), __ T8H, nttb); // x 8H + vs_ld2_post(vs_front(vs4), __ T8H, ntta); // x 8H + vs_ld2_post(vs_back(vs4), __ T8H, nttb); // x 8H + + // compute 4 montmul cross-products for pairs (a0,a1) and (b0,b1) + // i.e. montmul the first and second halves of vs1 in order and + // then with one sequence reversed storing the two results in vs3 + // + // vs3[0] <- montmul(a0, b0) + // vs3[1] <- montmul(a1, b1) + // vs3[2] <- montmul(a0, b1) + // vs3[3] <- montmul(a1, b0) + kyber_montmul16(vs_front(vs3), vs_front(vs1), vs_back(vs1), vs_front(vs2), vq); + kyber_montmul16(vs_back(vs3), + vs_front(vs1), vs_reverse(vs_back(vs1)), vs_back(vs2), vq); + + // compute 4 montmul cross-products for pairs (a2,a3) and (b2,b3) + // i.e. montmul the first and second halves of vs4 in order and + // then with one sequence reversed storing the two results in vs1 + // + // vs1[0] <- montmul(a2, b2) + // vs1[1] <- montmul(a3, b3) + // vs1[2] <- montmul(a2, b3) + // vs1[3] <- montmul(a3, b2) + kyber_montmul16(vs_front(vs1), vs_front(vs4), vs_back(vs4), vs_front(vs2), vq); + kyber_montmul16(vs_back(vs1), + vs_front(vs4), vs_reverse(vs_back(vs4)), vs_back(vs2), vq); + + // montmul result 2 of each cross-product i.e. (a1*b1, a3*b3) by a zeta. + // We can schedule two montmuls at a time if we use a suitable vector + // sequence . + int delta = vs1[1]->encoding() - vs3[1]->encoding(); + VSeq<2> vs5(vs3[1], delta); + + // vs3[1] <- montmul(montmul(a1, b1), z0) + // vs1[1] <- montmul(montmul(a3, b3), z1) + kyber_montmul16(vs5, vz, vs5, vs_front(vs2), vq); + + // add results in pairs storing in vs3 + // vs3[0] <- montmul(a0, b0) + montmul(montmul(a1, b1), z0); + // vs3[1] <- montmul(a0, b1) + montmul(a1, b0); + vs_addv(vs_front(vs3), __ T8H, vs_even(vs3), vs_odd(vs3)); + + // vs3[2] <- montmul(a2, b2) + montmul(montmul(a3, b3), z1); + // vs3[3] <- montmul(a2, b3) + montmul(a3, b2); + vs_addv(vs_back(vs3), __ T8H, vs_even(vs1), vs_odd(vs1)); + + // vs1 <- montmul(vs3, montRSquareModQ) + kyber_montmul32(vs1, vs3, vc, vs2, vq); + + // store back the two pairs of result vectors de-interleaved as 8H elements + // i.e. storing each pairs of shorts striped across a register pair adjacent + // in memory + vs_st2_post(vs1, __ T8H, result); + + __ cmp(result, limit); + __ br(Assembler::NE, kyberNttMult_loop); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber add 2 polynomials. + // Implements + // static int implKyberAddPoly(short[] result, short[] a, short[] b) {} + // + // result (short[256]) = c_rarg0 + // a (short[256]) = c_rarg1 + // b (short[256]) = c_rarg2 + address generate_kyberAddPoly_2() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberAddPoly_2"); + address start = __ pc(); + __ enter(); + + const Register result = c_rarg0; + const Register a = c_rarg1; + const Register b = c_rarg2; + + const Register kyberConsts = r11; + + // We sum 256 sets of values in total i.e. 32 x 8H quadwords. + // So, we can load, add and store the data in 3 groups of 11, + // 11 and 10 at a time i.e. we need to map sets of 10 or 11 + // registers. A further constraint is that the mapping needs + // to skip callee saves. So, we allocate the register + // sequences using two 8 sequences, two 2 sequences and two + // single registers. + VSeq<8> vs1_1(0); + VSeq<2> vs1_2(16); + FloatRegister vs1_3 = v28; + VSeq<8> vs2_1(18); + VSeq<2> vs2_2(26); + FloatRegister vs2_3 = v29; + + // two constant vector sequences + VSeq<8> vc_1(31, 0); + VSeq<2> vc_2(31, 0); + + FloatRegister vc_3 = v31; + __ lea(kyberConsts, + ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + + __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q + for (int i = 0; i < 3; i++) { + // load 80 or 88 values from a into vs1_1/2/3 + vs_ldpq_post(vs1_1, a); + vs_ldpq_post(vs1_2, a); + if (i < 2) { + __ ldr(vs1_3, __ Q, __ post(a, 16)); + } + // load 80 or 88 values from b into vs2_1/2/3 + vs_ldpq_post(vs2_1, b); + vs_ldpq_post(vs2_2, b); + if (i < 2) { + __ ldr(vs2_3, __ Q, __ post(b, 16)); + } + // sum 80 or 88 values across vs1 and vs2 into vs1 + vs_addv(vs1_1, __ T8H, vs1_1, vs2_1); + vs_addv(vs1_2, __ T8H, vs1_2, vs2_2); + if (i < 2) { + __ addv(vs1_3, __ T8H, vs1_3, vs2_3); + } + // add constant to all 80 or 88 results + vs_addv(vs1_1, __ T8H, vs1_1, vc_1); + vs_addv(vs1_2, __ T8H, vs1_2, vc_2); + if (i < 2) { + __ addv(vs1_3, __ T8H, vs1_3, vc_3); + } + // store 80 or 88 values + vs_stpq_post(vs1_1, result); + vs_stpq_post(vs1_2, result); + if (i < 2) { + __ str(vs1_3, __ Q, __ post(result, 16)); + } + } + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber add 3 polynomials. + // Implements + // static int implKyberAddPoly(short[] result, short[] a, short[] b, short[] c) {} + // + // result (short[256]) = c_rarg0 + // a (short[256]) = c_rarg1 + // b (short[256]) = c_rarg2 + // c (short[256]) = c_rarg3 + address generate_kyberAddPoly_3() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberAddPoly_3"); + address start = __ pc(); + __ enter(); + + const Register result = c_rarg0; + const Register a = c_rarg1; + const Register b = c_rarg2; + const Register c = c_rarg3; + + const Register kyberConsts = r11; + + // As above we sum 256 sets of values in total i.e. 32 x 8H + // quadwords. So, we can load, add and store the data in 3 + // groups of 11, 11 and 10 at a time i.e. we need to map sets + // of 10 or 11 registers. A further constraint is that the + // mapping needs to skip callee saves. So, we allocate the + // register sequences using two 8 sequences, two 2 sequences + // and two single registers. + VSeq<8> vs1_1(0); + VSeq<2> vs1_2(16); + FloatRegister vs1_3 = v28; + VSeq<8> vs2_1(18); + VSeq<2> vs2_2(26); + FloatRegister vs2_3 = v29; + + // two constant vector sequences + VSeq<8> vc_1(31, 0); + VSeq<2> vc_2(31, 0); + + FloatRegister vc_3 = v31; + + __ lea(kyberConsts, + ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + + __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q + for (int i = 0; i < 3; i++) { + // load 80 or 88 values from a into vs1_1/2/3 + vs_ldpq_post(vs1_1, a); + vs_ldpq_post(vs1_2, a); + if (i < 2) { + __ ldr(vs1_3, __ Q, __ post(a, 16)); + } + // load 80 or 88 values from b into vs2_1/2/3 + vs_ldpq_post(vs2_1, b); + vs_ldpq_post(vs2_2, b); + if (i < 2) { + __ ldr(vs2_3, __ Q, __ post(b, 16)); + } + // sum 80 or 88 values across vs1 and vs2 into vs1 + vs_addv(vs1_1, __ T8H, vs1_1, vs2_1); + vs_addv(vs1_2, __ T8H, vs1_2, vs2_2); + if (i < 2) { + __ addv(vs1_3, __ T8H, vs1_3, vs2_3); + } + // load 80 or 88 values from c into vs2_1/2/3 + vs_ldpq_post(vs2_1, c); + vs_ldpq_post(vs2_2, c); + if (i < 2) { + __ ldr(vs2_3, __ Q, __ post(c, 16)); + } + // sum 80 or 88 values across vs1 and vs2 into vs1 + vs_addv(vs1_1, __ T8H, vs1_1, vs2_1); + vs_addv(vs1_2, __ T8H, vs1_2, vs2_2); + if (i < 2) { + __ addv(vs1_3, __ T8H, vs1_3, vs2_3); + } + // add constant to all 80 or 88 results + vs_addv(vs1_1, __ T8H, vs1_1, vc_1); + vs_addv(vs1_2, __ T8H, vs1_2, vc_2); + if (i < 2) { + __ addv(vs1_3, __ T8H, vs1_3, vc_3); + } + // store 80 or 88 values + vs_stpq_post(vs1_1, result); + vs_stpq_post(vs1_2, result); + if (i < 2) { + __ str(vs1_3, __ Q, __ post(result, 16)); + } + } + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber parse XOF output to polynomial coefficient candidates + // or decodePoly(12, ...). + // Implements + // static int implKyber12To16( + // byte[] condensed, int index, short[] parsed, int parsedLength) {} + // + // (parsedLength or (parsedLength - 48) must be divisible by 64.) + // + // condensed (byte[]) = c_rarg0 + // condensedIndex = c_rarg1 + // parsed (short[112 or 256]) = c_rarg2 + // parsedLength (112 or 256) = c_rarg3 + address generate_kyber12To16() { + Label L_F00, L_loop, L_end; + + __ BIND(L_F00); + __ emit_int64(0x0f000f000f000f00); + __ emit_int64(0x0f000f000f000f00); + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyber12To16"); + address start = __ pc(); + __ enter(); + + const Register condensed = c_rarg0; + const Register condensedOffs = c_rarg1; + const Register parsed = c_rarg2; + const Register parsedLength = c_rarg3; + + const Register tmpAddr = r11; + + // Data is input 96 bytes at a time i.e. in groups of 6 x 16B + // quadwords so we need a 6 vector sequence for the inputs. + // Parsing produces 64 shorts, employing two 8 vector + // sequences to store and combine the intermediate data. + VSeq<6> vin(24); + VSeq<8> va(0), vb(16); + + __ adr(tmpAddr, L_F00); + __ ldr(v31, __ Q, tmpAddr); // 8H times 0x0f00 + __ add(condensed, condensed, condensedOffs); + + __ BIND(L_loop); + // load 96 (6 x 16B) byte values + vs_ld3_post(vin, __ T16B, condensed); + + // The front half of sequence vin (vin[0], vin[1] and vin[2]) + // holds 48 (16x3) contiguous bytes from memory striped + // horizontally across each of the 16 byte lanes. Equivalently, + // that is 16 pairs of 12-bit integers. Likewise the back half + // holds the next 48 bytes in the same arrangement. + + // Each vector in the front half can also be viewed as a vertical + // strip across the 16 pairs of 12 bit integers. Each byte in + // vin[0] stores the low 8 bits of the first int in a pair. Each + // byte in vin[1] stores the high 4 bits of the first int and the + // low 4 bits of the second int. Each byte in vin[2] stores the + // high 8 bits of the second int. Likewise the vectors in second + // half. + + // Converting the data to 16-bit shorts requires first of all + // expanding each of the 6 x 16B vectors into 6 corresponding + // pairs of 8H vectors. Mask, shift and add operations on the + // resulting vector pairs can be used to combine 4 and 8 bit + // parts of related 8H vector elements. + // + // The middle vectors (vin[2] and vin[5]) are actually expanded + // twice, one copy manipulated to provide the lower 4 bits + // belonging to the first short in a pair and another copy + // manipulated to provide the higher 4 bits belonging to the + // second short in a pair. This is why the the vector sequences va + // and vb used to hold the expanded 8H elements are of length 8. + + // Expand vin[0] into va[0:1], and vin[1] into va[2:3] and va[4:5] + // n.b. target elements 2 and 3 duplicate elements 4 and 5 + __ ushll(va[0], __ T8H, vin[0], __ T8B, 0); + __ ushll2(va[1], __ T8H, vin[0], __ T16B, 0); + __ ushll(va[2], __ T8H, vin[1], __ T8B, 0); + __ ushll2(va[3], __ T8H, vin[1], __ T16B, 0); + __ ushll(va[4], __ T8H, vin[1], __ T8B, 0); + __ ushll2(va[5], __ T8H, vin[1], __ T16B, 0); + + // likewise expand vin[3] into vb[0:1], and vin[4] into vb[2:3] + // and vb[4:5] + __ ushll(vb[0], __ T8H, vin[3], __ T8B, 0); + __ ushll2(vb[1], __ T8H, vin[3], __ T16B, 0); + __ ushll(vb[2], __ T8H, vin[4], __ T8B, 0); + __ ushll2(vb[3], __ T8H, vin[4], __ T16B, 0); + __ ushll(vb[4], __ T8H, vin[4], __ T8B, 0); + __ ushll2(vb[5], __ T8H, vin[4], __ T16B, 0); + + // shift lo byte of copy 1 of the middle stripe into the high byte + __ shl(va[2], __ T8H, va[2], 8); + __ shl(va[3], __ T8H, va[3], 8); + __ shl(vb[2], __ T8H, vb[2], 8); + __ shl(vb[3], __ T8H, vb[3], 8); + + // expand vin[2] into va[6:7] and vin[5] into vb[6:7] but this + // time pre-shifted by 4 to ensure top bits of input 12-bit int + // are in bit positions [4..11]. + __ ushll(va[6], __ T8H, vin[2], __ T8B, 4); + __ ushll2(va[7], __ T8H, vin[2], __ T16B, 4); + __ ushll(vb[6], __ T8H, vin[5], __ T8B, 4); + __ ushll2(vb[7], __ T8H, vin[5], __ T16B, 4); + + // mask hi 4 bits of the 1st 12-bit int in a pair from copy1 and + // shift lo 4 bits of the 2nd 12-bit int in a pair to the bottom of + // copy2 + __ andr(va[2], __ T16B, va[2], v31); + __ andr(va[3], __ T16B, va[3], v31); + __ ushr(va[4], __ T8H, va[4], 4); + __ ushr(va[5], __ T8H, va[5], 4); + __ andr(vb[2], __ T16B, vb[2], v31); + __ andr(vb[3], __ T16B, vb[3], v31); + __ ushr(vb[4], __ T8H, vb[4], 4); + __ ushr(vb[5], __ T8H, vb[5], 4); + + // sum hi 4 bits and lo 8 bits of the 1st 12-bit int in each pair and + // hi 8 bits plus lo 4 bits of the 2nd 12-bit int in each pair + // n.b. the ordering ensures: i) inputs are consumed before they + // are overwritten ii) the order of 16-bit results across successive + // pairs of vectors in va and then vb reflects the order of the + // corresponding 12-bit inputs + __ addv(va[0], __ T8H, va[0], va[2]); + __ addv(va[2], __ T8H, va[1], va[3]); + __ addv(va[1], __ T8H, va[4], va[6]); + __ addv(va[3], __ T8H, va[5], va[7]); + __ addv(vb[0], __ T8H, vb[0], vb[2]); + __ addv(vb[2], __ T8H, vb[1], vb[3]); + __ addv(vb[1], __ T8H, vb[4], vb[6]); + __ addv(vb[3], __ T8H, vb[5], vb[7]); + + // store 64 results interleaved as shorts + vs_st2_post(vs_front(va), __ T8H, parsed); + vs_st2_post(vs_front(vb), __ T8H, parsed); + + __ sub(parsedLength, parsedLength, 64); + __ cmp(parsedLength, (u1)64); + __ br(Assembler::GE, L_loop); + __ cbz(parsedLength, L_end); + + // if anything is left it should be a final 72 bytes of input + // i.e. a final 48 12-bit values. so we handle this by loading + // 48 bytes into all 16B lanes of front(vin) and only 24 + // bytes into the lower 8B lane of back(vin) + vs_ld3_post(vs_front(vin), __ T16B, condensed); + vs_ld3(vs_back(vin), __ T8B, condensed); + + // Expand vin[0] into va[0:1], and vin[1] into va[2:3] and va[4:5] + // n.b. target elements 2 and 3 of va duplicate elements 4 and + // 5 and target element 2 of vb duplicates element 4. + __ ushll(va[0], __ T8H, vin[0], __ T8B, 0); + __ ushll2(va[1], __ T8H, vin[0], __ T16B, 0); + __ ushll(va[2], __ T8H, vin[1], __ T8B, 0); + __ ushll2(va[3], __ T8H, vin[1], __ T16B, 0); + __ ushll(va[4], __ T8H, vin[1], __ T8B, 0); + __ ushll2(va[5], __ T8H, vin[1], __ T16B, 0); + + // This time expand just the lower 8 lanes + __ ushll(vb[0], __ T8H, vin[3], __ T8B, 0); + __ ushll(vb[2], __ T8H, vin[4], __ T8B, 0); + __ ushll(vb[4], __ T8H, vin[4], __ T8B, 0); + + // shift lo byte of copy 1 of the middle stripe into the high byte + __ shl(va[2], __ T8H, va[2], 8); + __ shl(va[3], __ T8H, va[3], 8); + __ shl(vb[2], __ T8H, vb[2], 8); + + // expand vin[2] into va[6:7] and lower 8 lanes of vin[5] into + // vb[6] pre-shifted by 4 to ensure top bits of the input 12-bit + // int are in bit positions [4..11]. + __ ushll(va[6], __ T8H, vin[2], __ T8B, 4); + __ ushll2(va[7], __ T8H, vin[2], __ T16B, 4); + __ ushll(vb[6], __ T8H, vin[5], __ T8B, 4); + + // mask hi 4 bits of each 1st 12-bit int in pair from copy1 and + // shift lo 4 bits of each 2nd 12-bit int in pair to bottom of + // copy2 + __ andr(va[2], __ T16B, va[2], v31); + __ andr(va[3], __ T16B, va[3], v31); + __ ushr(va[4], __ T8H, va[4], 4); + __ ushr(va[5], __ T8H, va[5], 4); + __ andr(vb[2], __ T16B, vb[2], v31); + __ ushr(vb[4], __ T8H, vb[4], 4); + + + + // sum hi 4 bits and lo 8 bits of each 1st 12-bit int in pair and + // hi 8 bits plus lo 4 bits of each 2nd 12-bit int in pair + + // n.b. ordering ensures: i) inputs are consumed before they are + // overwritten ii) order of 16-bit results across succsessive + // pairs of vectors in va and then lower half of vb reflects order + // of corresponding 12-bit inputs + __ addv(va[0], __ T8H, va[0], va[2]); + __ addv(va[2], __ T8H, va[1], va[3]); + __ addv(va[1], __ T8H, va[4], va[6]); + __ addv(va[3], __ T8H, va[5], va[7]); + __ addv(vb[0], __ T8H, vb[0], vb[2]); + __ addv(vb[1], __ T8H, vb[4], vb[6]); + + // store 48 results interleaved as shorts + vs_st2_post(vs_front(va), __ T8H, parsed); + vs_st2_post(vs_front(vs_front(vb)), __ T8H, parsed); + + __ BIND(L_end); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + // Kyber Barrett reduce function. + // Implements + // static int implKyberBarrettReduce(short[] coeffs) {} + // + // coeffs (short[256]) = c_rarg0 + address generate_kyberBarrettReduce() { + + __ align(CodeEntryAlignment); + StubCodeMark mark(this, "StubRoutines", "kyberBarrettReduce"); + address start = __ pc(); + __ enter(); + + const Register coeffs = c_rarg0; + + const Register kyberConsts = r10; + const Register result = r11; + + // As above we process 256 sets of values in total i.e. 32 x + // 8H quadwords. So, we can load, add and store the data in 3 + // groups of 11, 11 and 10 at a time i.e. we need to map sets + // of 10 or 11 registers. A further constraint is that the + // mapping needs to skip callee saves. So, we allocate the + // register sequences using two 8 sequences, two 2 sequences + // and two single registers. + VSeq<8> vs1_1(0); + VSeq<2> vs1_2(16); + FloatRegister vs1_3 = v28; + VSeq<8> vs2_1(18); + VSeq<2> vs2_2(26); + FloatRegister vs2_3 = v29; + + // we also need a pair of corresponding constant sequences + + VSeq<8> vc1_1(30, 0); + VSeq<2> vc1_2(30, 0); + FloatRegister vc1_3 = v30; // for kyber_q + + VSeq<8> vc2_1(31, 0); + VSeq<2> vc2_2(31, 0); + FloatRegister vc2_3 = v31; // for kyberBarrettMultiplier + + __ add(result, coeffs, 0); + __ lea(kyberConsts, + ExternalAddress((address) StubRoutines::aarch64::_kyberConsts)); + + // load q and the multiplier for the Barrett reduction + __ add(kyberConsts, kyberConsts, 16); + __ ldpq(vc1_3, vc2_3, kyberConsts); + + for (int i = 0; i < 3; i++) { + // load 80 or 88 coefficients + vs_ldpq_post(vs1_1, coeffs); + vs_ldpq_post(vs1_2, coeffs); + if (i < 2) { + __ ldr(vs1_3, __ Q, __ post(coeffs, 16)); + } + + // vs2 <- (2 * vs1 * kyberBarrettMultiplier) >> 16 + vs_sqdmulh(vs2_1, __ T8H, vs1_1, vc2_1); + vs_sqdmulh(vs2_2, __ T8H, vs1_2, vc2_2); + if (i < 2) { + __ sqdmulh(vs2_3, __ T8H, vs1_3, vc2_3); + } + + // vs2 <- (vs1 * kyberBarrettMultiplier) >> 26 + vs_sshr(vs2_1, __ T8H, vs2_1, 11); + vs_sshr(vs2_2, __ T8H, vs2_2, 11); + if (i < 2) { + __ sshr(vs2_3, __ T8H, vs2_3, 11); + } + + // vs1 <- vs1 - vs2 * kyber_q + vs_mlsv(vs1_1, __ T8H, vs2_1, vc1_1); + vs_mlsv(vs1_2, __ T8H, vs2_2, vc1_2); + if (i < 2) { + __ mlsv(vs1_3, __ T8H, vs2_3, vc1_3); + } + + vs_stpq_post(vs1_1, result); + vs_stpq_post(vs1_2, result); + if (i < 2) { + __ str(vs1_3, __ Q, __ post(result, 16)); + } + } + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov(r0, zr); // return 0 + __ ret(lr); + + return start; + } + + + // Dilithium-specific montmul helper routines that generate parallel + // code for, respectively, a single 4x4s vector sequence montmul or + // two such multiplies in a row. + + // Perform 16 32-bit Montgomery multiplications in parallel + void dilithium_montmul16(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc, + const VSeq<4>& vtmp, const VSeq<2>& vq) { + // Use the helper routine to schedule a 4x4S Montgomery multiply. + // It will assert that the register use is valid + vs_montmul4(va, vb, vc, __ T4S, vtmp, vq); + } + + // Perform 2x16 32-bit Montgomery multiplications in parallel + void dilithium_montmul32(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc, + const VSeq<4>& vtmp, const VSeq<2>& vq) { + // Schedule two successive 4x4S multiplies via the montmul helper + // on the front and back halves of va, vb and vc. The helper will + // assert that the register use has no overlap conflicts on each + // individual call but we also need to ensure that the necessary + // disjoint/equality constraints are met across both calls. + + // vb, vc, vtmp and vq must be disjoint. va must either be + // disjoint from all other registers or equal vc + + assert(vs_disjoint(vb, vc), "vb and vc overlap"); + assert(vs_disjoint(vb, vq), "vb and vq overlap"); + assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap"); + + assert(vs_disjoint(vc, vq), "vc and vq overlap"); + assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap"); + + assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap"); + + assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal"); + assert(vs_disjoint(va, vb), "va and vb overlap"); + assert(vs_disjoint(va, vq), "va and vq overlap"); + assert(vs_disjoint(va, vtmp), "va and vtmp overlap"); + + // We multiply the front and back halves of each sequence 4 at a + // time because + // + // 1) we are currently only able to get 4-way instruction + // parallelism at best + // + // 2) we need registers for the constants in vq and temporary + // scratch registers to hold intermediate results so vtmp can only + // be a VSeq<4> which means we only have 4 scratch slots. + + vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T4S, vtmp, vq); + vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T4S, vtmp, vq); + } + + // Perform combined montmul then add/sub on 4x4S vectors. + void dilithium_montmul16_sub_add( + const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vc, + const VSeq<4>& vtmp, const VSeq<2>& vq) { // compute a = montmul(a1, c) dilithium_montmul16(vc, va1, vc, vtmp, vq); // ouptut a1 = a0 - a @@ -4752,10 +6146,10 @@ class StubGenerator: public StubCodeGenerator { vs_addv(va0, __ T4S, va0, vc); } - // perform combined add/sub then montul on 4x4S vectors - - void dilithium_sub_add_montmul16(const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vb, - const VSeq<4>& vtmp1, const VSeq<4>& vtmp2, const VSeq<2>& vq) { + // Perform combined add/sub then montul on 4x4S vectors. + void dilithium_sub_add_montmul16( + const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vb, + const VSeq<4>& vtmp1, const VSeq<4>& vtmp2, const VSeq<2>& vq) { // compute c = a0 - a1 vs_subv(vtmp1, __ T4S, va0, va1); // output a0 = a0 + a1 @@ -4798,10 +6192,10 @@ class StubGenerator: public StubCodeGenerator { offsets[3] = 192; } - // for levels 1 - 4 we simply load 2 x 4 adjacent values at a + // For levels 1 - 4 we simply load 2 x 4 adjacent values at a // time at 4 different offsets and multiply them in order by the // next set of input values. So we employ indexed load and store - // pair instructions with arrangement 4S + // pair instructions with arrangement 4S. for (int i = 0; i < 4; i++) { // reload q and qinv vs_ldpq(vq, dilithiumConsts); // qInv, q @@ -4810,7 +6204,7 @@ class StubGenerator: public StubCodeGenerator { // load next 8x4S inputs == b vs_ldpq_post(vs2, zetas); // compute a == c2 * b mod MONT_Q - vs_montmul32(vs2, vs1, vs2, vtmp, vq); + dilithium_montmul32(vs2, vs1, vs2, vtmp, vq); // load 8x4s coefficients via first start pos == c1 vs_ldpq_indexed(vs1, coeffs, c1Start, offsets); // compute a1 = c1 + a @@ -4863,20 +6257,21 @@ class StubGenerator: public StubCodeGenerator { VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3 VSeq<2> vq(30); // n.b. constants overlap vs3 - int offsets[4] = {0, 32, 64, 96}; - int offsets1[8] = {16, 48, 80, 112, 144, 176, 208, 240 }; + int offsets[4] = { 0, 32, 64, 96}; + int offsets1[8] = { 16, 48, 80, 112, 144, 176, 208, 240 }; int offsets2[8] = { 0, 32, 64, 96, 128, 160, 192, 224 }; __ add(result, coeffs, 0); - __ lea(dilithiumConsts, ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); + __ lea(dilithiumConsts, + ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); - // Each level represents one iteration of the outer for loop of the Java version + // Each level represents one iteration of the outer for loop of the Java version. // level 0-4 dilithiumNttLevel0_4(dilithiumConsts, coeffs, zetas); // level 5 - // at level 5 the coefficients we need to combine with the zetas + // At level 5 the coefficients we need to combine with the zetas // are grouped in memory in blocks of size 4. So, for both sets of // coefficients we load 4 adjacent values at 8 different offsets // using an indexed ldr with register variant Q and multiply them @@ -4890,7 +6285,7 @@ class StubGenerator: public StubCodeGenerator { // load next 32 (8x4S) inputs = b vs_ldpq_post(vs2, zetas); // a = b montul c1 - vs_montmul32(vs2, vs1, vs2, vtmp, vq); + dilithium_montmul32(vs2, vs1, vs2, vtmp, vq); // load 32 (8x4S) coefficients via second offsets = c2 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets2); // add/sub with result of multiply @@ -4902,7 +6297,7 @@ class StubGenerator: public StubCodeGenerator { } // level 6 - // at level 6 the coefficients we need to combine with the zetas + // At level 6 the coefficients we need to combine with the zetas // are grouped in memory in pairs, the first two being montmul // inputs and the second add/sub inputs. We can still implement // the montmul+sub+add using 4-way parallelism but only if we @@ -4930,7 +6325,7 @@ class StubGenerator: public StubCodeGenerator { } // level 7 - // at level 7 the coefficients we need to combine with the zetas + // At level 7 the coefficients we need to combine with the zetas // occur singly with montmul inputs alterating with add/sub // inputs. Once again we can use 4-way parallelism to combine 16 // zetas at a time. However, we have to load 8 adjacent values at @@ -5002,10 +6397,10 @@ class StubGenerator: public StubCodeGenerator { offsets[3] = 96; } - // for levels 3 - 7 we simply load 2 x 4 adjacent values at a + // For levels 3 - 7 we simply load 2 x 4 adjacent values at a // time at 4 different offsets and multiply them in order by the // next set of input values. So we employ indexed load and store - // pair instructions with arrangement 4S + // pair instructions with arrangement 4S. for (int i = 0; i < 4; i++) { // load v1 32 (8x4S) coefficients relative to first start index vs_ldpq_indexed(vs1, coeffs, c1Start, offsets); @@ -5022,7 +6417,7 @@ class StubGenerator: public StubCodeGenerator { // load b next 32 (8x4S) inputs vs_ldpq_post(vs2, zetas); // a = a1 montmul b - vs_montmul32(vs2, vs1, vs2, vtmp, vq); + dilithium_montmul32(vs2, vs1, vs2, vtmp, vq); // save a relative to second start index vs_stpq_indexed(vs2, coeffs, c2Start, offsets); @@ -5072,16 +6467,16 @@ class StubGenerator: public StubCodeGenerator { int offsets2[8] = { 16, 48, 80, 112, 144, 176, 208, 240 }; __ add(result, coeffs, 0); - __ lea(dilithiumConsts, ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); + __ lea(dilithiumConsts, + ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); // Each level represents one iteration of the outer for loop of the Java version - // level0 // level 0 // At level 0 we need to interleave adjacent quartets of // coefficients before we multiply and add/sub by the next 16 // zetas just as we did for level 7 in the multiply code. So we - // load and store the values using an ld2/st2 with arrangement 4S + // load and store the values using an ld2/st2 with arrangement 4S. for (int i = 0; i < 1024; i += 128) { // load constants q, qinv // n.b. this can be moved out of the loop as they do not get @@ -5103,7 +6498,7 @@ class StubGenerator: public StubCodeGenerator { // At level 1 we need to interleave pairs of adjacent pairs of // coefficients before we multiply by the next 16 zetas just as we // did for level 6 in the multiply code. So we load and store the - // values an ld2/st2 with arrangement 2D + // values an ld2/st2 with arrangement 2D. for (int i = 0; i < 1024; i += 128) { // a0/a1 load interleaved 32 (8x2D) coefficients vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets); @@ -5139,7 +6534,7 @@ class StubGenerator: public StubCodeGenerator { // reload constants q, qinv -- they were clobbered earlier vs_ldpq(vq, dilithiumConsts); // qInv, q // compute a1 = b montmul c - vs_montmul32(vs2, vs1, vs2, vtmp, vq); + dilithium_montmul32(vs2, vs1, vs2, vtmp, vq); // store a1 32 (8x4S) coefficients via second offsets vs_str_indexed(vs2, __ Q, coeffs, i, offsets2); } @@ -5152,7 +6547,6 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); return start; - } // Dilithium multiply polynomials in the NTT domain. @@ -5185,7 +6579,8 @@ class StubGenerator: public StubCodeGenerator { VSeq<2> vq(30); // n.b. constants overlap vs3 VSeq<8> vrsquare(29, 0); // for montmul by constant RSQUARE - __ lea(dilithiumConsts, ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); + __ lea(dilithiumConsts, + ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); // load constants q, qinv vs_ldpq(vq, dilithiumConsts); // qInv, q @@ -5202,9 +6597,9 @@ class StubGenerator: public StubCodeGenerator { // c load 32 (8x4S) next inputs from poly2 vs_ldpq_post(vs2, poly2); // compute a = b montmul c - vs_montmul32(vs2, vs1, vs2, vtmp, vq); + dilithium_montmul32(vs2, vs1, vs2, vtmp, vq); // compute a = rsquare montmul a - vs_montmul32(vs2, vrsquare, vs2, vtmp, vq); + dilithium_montmul32(vs2, vrsquare, vs2, vtmp, vq); // save a 32 (8x4S) results vs_stpq_post(vs2, result); @@ -5217,7 +6612,6 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); return start; - } // Dilithium Motgomery multiply an array by a constant. @@ -5244,13 +6638,14 @@ class StubGenerator: public StubCodeGenerator { const Register len = r12; VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs - VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3 + VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3 VSeq<2> vq(30); // n.b. constants overlap vs3 VSeq<8> vconst(29, 0); // for montmul by constant // results track inputs __ add(result, coeffs, 0); - __ lea(dilithiumConsts, ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); + __ lea(dilithiumConsts, + ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); // load constants q, qinv -- they do not get clobbered by first two loops vs_ldpq(vq, dilithiumConsts); // qInv, q @@ -5264,7 +6659,7 @@ class StubGenerator: public StubCodeGenerator { // load next 32 inputs vs_ldpq_post(vs2, coeffs); // mont mul by constant - vs_montmul32(vs2, vconst, vs2, vtmp, vq); + dilithium_montmul32(vs2, vconst, vs2, vtmp, vq); // write next 32 results vs_stpq_post(vs2, result); @@ -5277,7 +6672,6 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); return start; - } // Dilithium decompose poly. @@ -5307,9 +6701,12 @@ class StubGenerator: public StubCodeGenerator { const Register dilithiumConsts = r10; const Register tmp = r11; - VSeq<4> vs1(0), vs2(4), vs3(8); // 6 independent sets of 4x4s values + // 6 independent sets of 4x4s values + VSeq<4> vs1(0), vs2(4), vs3(8); VSeq<4> vs4(12), vs5(16), vtmp(20); - VSeq<4> one(25, 0); // 7 constants for cross-multiplying + + // 7 constants for cross-multiplying + VSeq<4> one(25, 0); VSeq<4> qminus1(26, 0); VSeq<4> g2(27, 0); VSeq<4> twog2(28, 0); @@ -5319,7 +6716,8 @@ class StubGenerator: public StubCodeGenerator { __ enter(); - __ lea(dilithiumConsts, ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); + __ lea(dilithiumConsts, + ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts)); // save callee-saved registers __ stpd(v8, v9, __ pre(sp, -64)); @@ -5416,7 +6814,6 @@ class StubGenerator: public StubCodeGenerator { __ st4(vs3[0], vs3[1], vs3[2], vs3[3], __ T4S, __ post(lowPart, 64)); __ st4(vs1[0], vs1[1], vs1[2], vs1[3], __ T4S, __ post(highPart, 64)); - __ sub(len, len, 64); __ cmp(len, (u1)64); __ br(Assembler::GE, L_loop); @@ -5432,7 +6829,6 @@ class StubGenerator: public StubCodeGenerator { __ ret(lr); return start; - } /** @@ -9884,6 +11280,16 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_chacha20Block = generate_chacha20Block_blockpar(); } + if (UseKyberIntrinsics) { + StubRoutines::_kyberNtt = generate_kyberNtt(); + StubRoutines::_kyberInverseNtt = generate_kyberInverseNtt(); + StubRoutines::_kyberNttMult = generate_kyberNttMult(); + StubRoutines::_kyberAddPoly_2 = generate_kyberAddPoly_2(); + StubRoutines::_kyberAddPoly_3 = generate_kyberAddPoly_3(); + StubRoutines::_kyber12To16 = generate_kyber12To16(); + StubRoutines::_kyberBarrettReduce = generate_kyberBarrettReduce(); + } + if (UseDilithiumIntrinsics) { StubRoutines::_dilithiumAlmostNtt = generate_dilithiumAlmostNtt(); StubRoutines::_dilithiumAlmostInverseNtt = generate_dilithiumAlmostInverseNtt(); diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp index f7f836cab9f6..538090d93ac4 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.cpp @@ -68,6 +68,17 @@ address StubRoutines::aarch64::_spin_wait = CAST_FROM_FN_PTR(address, empty_spin bool StubRoutines::aarch64::_completed = false; +ATTRIBUTE_ALIGNED(64) uint16_t StubRoutines::aarch64::_kyberConsts[] = +{ + // Because we sometimes load these in pairs, montQInvModR, kyber_q + // and kyberBarrettMultiplier should stay together and in this order. + 0xF301, 0xF301, 0xF301, 0xF301, 0xF301, 0xF301, 0xF301, 0xF301, // montQInvModR + 0x0D01, 0x0D01, 0x0D01, 0x0D01, 0x0D01, 0x0D01, 0x0D01, 0x0D01, // kyber_q + 0x4EBF, 0x4EBF, 0x4EBF, 0x4EBF, 0x4EBF, 0x4EBF, 0x4EBF, 0x4EBF, // kyberBarrettMultiplier + 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, 0x0200, // toMont((kyber_n / 2)^-1 (mod kyber_q)) + 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549, 0x0549 // montRSquareModQ +}; + ATTRIBUTE_ALIGNED(64) uint32_t StubRoutines::aarch64::_dilithiumConsts[] = { 58728449, 58728449, 58728449, 58728449, // montQInvModR diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp index d4e206d1b38d..57adae386336 100644 --- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp @@ -220,6 +220,7 @@ class aarch64 { } private: + static uint16_t _kyberConsts[]; static uint32_t _dilithiumConsts[]; static juint _crc_table[]; static jubyte _adler_table[]; diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index 50ea4b3d2c47..63ec48f55c97 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -411,13 +411,24 @@ void VM_Version::initialize() { FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false); } + if (_features & CPU_ASIMD) { + if (FLAG_IS_DEFAULT(UseKyberIntrinsics)) { + UseKyberIntrinsics = true; + } + } else if (UseKyberIntrinsics) { + if (!FLAG_IS_DEFAULT(UseKyberIntrinsics)) { + warning("Kyber intrinsics require ASIMD instructions"); + } + FLAG_SET_DEFAULT(UseKyberIntrinsics, false); + } + if (_features & CPU_ASIMD) { if (FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) { UseDilithiumIntrinsics = true; } } else if (UseDilithiumIntrinsics) { if (!FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) { - warning("Dilithium intrinsic requires ASIMD instructions"); + warning("Dilithium intrinsics require ASIMD instructions"); } FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false); } @@ -669,6 +680,7 @@ void VM_Version::initialize_cpu_information(void) { get_compatible_board(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len); desc_len = (int)strlen(_cpu_desc); snprintf(_cpu_desc + desc_len, CPU_DETAILED_DESC_BUF_SIZE - desc_len, " %s", _features_string); + fprintf(stderr, "_features_string = \"%s\"", _features_string); _initialized = true; } diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp index a018eaf7b576..beca282a2b71 100644 --- a/src/hotspot/cpu/x86/assembler_x86.cpp +++ b/src/hotspot/cpu/x86/assembler_x86.cpp @@ -4316,22 +4316,6 @@ void Assembler::vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_l emit_int24(0x01, (0xC0 | encode), imm8); } -void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { - assert(VM_Version::supports_evex(), ""); - InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); - attributes.set_is_evex_instruction(); - int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); - emit_int16(0x76, (0xC0 | encode)); -} - -void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { - assert(VM_Version::supports_avx512_vbmi(), ""); - InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); - attributes.set_is_evex_instruction(); - int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); - emit_int16(0x7D, (0xC0 | encode)); -} - void Assembler::evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len) { assert(VM_Version::supports_avx512_vbmi(), ""); InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); @@ -14130,3 +14114,58 @@ void Assembler::evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int emit_int16(0x7E, (0xC0 | encode)); } +void Assembler::evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_avx512_vbmi() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x75, (0xC0 | encode)); +} + +void Assembler::evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_avx512bw() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x75, (0xC0 | encode)); +} + +void Assembler::evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x76, (0xC0 | encode)); +} + +void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x76, (0xC0 | encode)); +} + +void Assembler::evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x77, (0xC0 | encode)); +} + +void Assembler::evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_evex() && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl()), ""); + InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x77, (0xC0 | encode)); +} + +void Assembler::evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { + assert(VM_Version::supports_avx512_vbmi(), ""); + InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x7D, (0xC0 | encode)); +} diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp index fe7e1b20e8be..cdf879a6e5d8 100644 --- a/src/hotspot/cpu/x86/assembler_x86.hpp +++ b/src/hotspot/cpu/x86/assembler_x86.hpp @@ -1776,12 +1776,17 @@ class Assembler : public AbstractAssembler { void vpermilps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void vpermilpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len); void vpermpd(XMMRegister dst, XMMRegister src, int imm8, int vector_len); + void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len); + void evpermi2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); + void evpermi2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); + void evpermi2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); + void evpermi2ps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); + void evpermi2pd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpermt2b(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpermt2w(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpermt2d(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); void evpermt2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len); - void evpmultishiftqb(XMMRegister dst, XMMRegister ctl, XMMRegister src, int vector_len); void pause(); diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp index 957ec4139aff..8b0fde90e7ca 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp @@ -4206,6 +4206,8 @@ void StubGenerator::generate_compiler_stubs() { StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock(); } + generate_dilithium_stubs(); + generate_sha3_stubs(); #ifdef COMPILER2 diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp index 5a3d6fe917cc..0ee9ae67c58b 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.hpp @@ -425,8 +425,9 @@ class StubGenerator: public StubCodeGenerator { // SHA3 stubs void generate_sha3_stubs(); - address generate_sha3_implCompress(bool multiBlock, const char *name); + // Dilithium stubs and helper functions + void generate_dilithium_stubs(); // BASE64 stubs address base64_shuffle_addr(); diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp new file mode 100644 index 000000000000..669223640fc1 --- /dev/null +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_dilithium.cpp @@ -0,0 +1,1030 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "asm/assembler.hpp" +#include "asm/assembler.inline.hpp" +#include "runtime/stubRoutines.hpp" +#include "macroAssembler_x86.hpp" +#include "stubGenerator_x86_64.hpp" + +#define __ _masm-> + +#define xmm(i) as_XMMRegister(i) + +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif // PRODUCT + +#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") + +#define XMMBYTES 64 + +// Constants +// +ATTRIBUTE_ALIGNED(64) static const uint32_t dilithiumAvx512Consts[] = { + 58728449, // montQInvModR + 8380417, // dilithium_q + 2365951, // montRSquareModQ + 5373807 // Barrett addend for modular reduction +}; + +const int montQInvModRIdx = 0; +const int dilithium_qIdx = 4; +const int montRSquareModQIdx = 8; +const int barrettAddendIdx = 12; + +static address dilithiumAvx512ConstsAddr(int offset) { + return ((address) dilithiumAvx512Consts) + offset; +} + +const Register scratch = r10; +const XMMRegister montMulPerm = xmm28; +const XMMRegister montQInvModR = xmm30; +const XMMRegister dilithium_q = xmm31; + + +ATTRIBUTE_ALIGNED(64) static const uint32_t dilithiumAvx512Perms[] = { + // collect montmul results into the destination register + 17, 1, 19, 3, 21, 5, 23, 7, 25, 9, 27, 11, 29, 13, 31, 15, + // ntt + // level 4 + 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, + 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31, + // level 5 + 0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27, + 4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31, + // level 6 + 0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29, + 2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31, + // level 7 + 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30, + 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31, + 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23, + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + + // ntt inverse + // level 0 + 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, + 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, + // level 1 + 0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30, + 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31, + // level 2 + 0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29, + 2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31, + // level 3 + 0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27, + 4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31, + // level 4 + 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, + 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 +}; + +const int montMulPermsIdx = 0; +const int nttL4PermsIdx = 64; +const int nttL5PermsIdx = 192; +const int nttL6PermsIdx = 320; +const int nttL7PermsIdx = 448; +const int nttInvL0PermsIdx = 704; +const int nttInvL1PermsIdx = 832; +const int nttInvL2PermsIdx = 960; +const int nttInvL3PermsIdx = 1088; +const int nttInvL4PermsIdx = 1216; + +static address dilithiumAvx512PermsAddr() { + return (address) dilithiumAvx512Perms; +} + +// We do Montgomery multiplications of two vectors of 16 ints each in 4 steps: +// 1. Do the multiplications of the corresponding even numbered slots into +// the odd numbered slots of a third register. +// 2. Swap the even and odd numbered slots of the original input registers. +// 3. Similar to step 1, but into a different output register. +// 4. Combine the outputs of step 1 and step 3 into the output of the Montgomery +// multiplication. +// (For levels 0-6 in the Ntt and levels 1-7 of the inverse Ntt we only swap the +// odd-even slots of the first multiplicand as in the second (zetas) the +// odd slots contain the same number as the corresponding even one.) +// The indexes of the registers to be multiplied +// are in inputRegs1[] and inputRegs[2]. +// The results go to the registers whose indexes are in outputRegs. +// scratchRegs should contain 12 different register indexes. +// The set in outputRegs should not overlap with the set of the middle four +// scratch registers. +// The sets in inputRegs1 and inputRegs2 cannot overlap with the set of the +// first eight scratch registers. +// In most of the cases, the odd and the corresponding even slices of the +// registers indexed by the numbers in inputRegs2 will contain the same number, +// this should be indicated by calling this function with +// input2NeedsShuffle=false . +// +static void montMul64(int outputRegs[], int inputRegs1[], int inputRegs2[], + int scratchRegs[], bool input2NeedsShuffle, + MacroAssembler *_masm) { + + for (int i = 0; i < 4; i++) { + __ vpmuldq(xmm(scratchRegs[i]), xmm(inputRegs1[i]), xmm(inputRegs2[i]), + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ vpmulld(xmm(scratchRegs[i + 4]), xmm(scratchRegs[i]), montQInvModR, + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ vpmuldq(xmm(scratchRegs[i + 4]), xmm(scratchRegs[i + 4]), dilithium_q, + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ evpsubd(xmm(scratchRegs[i + 4]), k0, xmm(scratchRegs[i]), + xmm(scratchRegs[i + 4]), false, Assembler::AVX_512bit); + } + + for (int i = 0; i < 4; i++) { + __ vpshufd(xmm(inputRegs1[i]), xmm(inputRegs1[i]), 0xB1, + Assembler::AVX_512bit); + if (input2NeedsShuffle) { + __ vpshufd(xmm(inputRegs2[i]), xmm(inputRegs2[i]), 0xB1, + Assembler::AVX_512bit); + } + } + + for (int i = 0; i < 4; i++) { + __ vpmuldq(xmm(scratchRegs[i]), xmm(inputRegs1[i]), xmm(inputRegs2[i]), + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ vpmulld(xmm(scratchRegs[i + 8]), xmm(scratchRegs[i]), montQInvModR, + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ vpmuldq(xmm(scratchRegs[i + 8]), xmm(scratchRegs[i + 8]), dilithium_q, + Assembler::AVX_512bit); + } + for (int i = 0; i < 4; i++) { + __ evpsubd(xmm(outputRegs[i]), k0, xmm(scratchRegs[i]), + xmm(scratchRegs[i + 8]), false, Assembler::AVX_512bit); + } + + for (int i = 0; i < 4; i++) { + __ evpermt2d(xmm(outputRegs[i]), montMulPerm, xmm(scratchRegs[i + 4]), + Assembler::AVX_512bit); + } +} + +static void montMul64(int outputRegs[], int inputRegs1[], int inputRegs2[], + int scratchRegs[], MacroAssembler *_masm) { + montMul64(outputRegs, inputRegs1, inputRegs2, scratchRegs, false, _masm); +} + +static void sub_add(int subResult[], int addResult[], + int input1[], int input2[], MacroAssembler *_masm) { + + for (int i = 0; i < 4; i++) { + __ evpsubd(xmm(subResult[i]), k0, xmm(input1[i]), xmm(input2[i]), false, + Assembler::AVX_512bit); + } + + for (int i = 0; i < 4; i++) { + __ evpaddd(xmm(addResult[i]), k0, xmm(input1[i]), xmm(input2[i]), false, + Assembler::AVX_512bit); + } +} + +static void loadPerm(int destinationRegs[], Register perms, + int offset, MacroAssembler *_masm) { + __ evmovdqul(xmm(destinationRegs[0]), Address(perms, offset), + Assembler::AVX_512bit); + for (int i = 1; i < 4; i++) { + __ evmovdqul(xmm(destinationRegs[i]), xmm(destinationRegs[0]), + Assembler::AVX_512bit); + } +} + +static void load4Xmms(int destinationRegs[], Register source, int offset, + MacroAssembler *_masm) { + for (int i = 0; i < 4; i++) { + __ evmovdqul(xmm(destinationRegs[i]), Address(source, offset + i * XMMBYTES), + Assembler::AVX_512bit); + } +} + +static void loadXmm29(Register source, int offset, MacroAssembler *_masm) { + __ evmovdqul(xmm29, Address(source, offset), Assembler::AVX_512bit); +} + +static void store4Xmms(Register destination, int offset, int xmmRegs[], + MacroAssembler *_masm) { + for (int i = 0; i < 4; i++) { + __ evmovdqul(Address(destination, offset + i * XMMBYTES), xmm(xmmRegs[i]), + Assembler::AVX_512bit); + } +} + +static int xmm0_3[] = {0, 1, 2, 3}; +static int xmm0145[] = {0, 1, 4, 5}; +static int xmm0246[] = {0, 2, 4, 6}; +static int xmm0426[] = {0, 4, 2, 6}; +static int xmm1357[] = {1, 3, 5, 7}; +static int xmm1537[] = {1, 5, 3, 7}; +static int xmm2367[] = {2, 3, 6, 7}; +static int xmm4_7[] = {4, 5, 6, 7}; +static int xmm8_11[] = {8, 9, 10, 11}; +static int xmm12_15[] = {12, 13, 14, 15}; +static int xmm16_19[] = {16, 17, 18, 19}; +static int xmm20_23[] = {20, 21, 22, 23}; +static int xmm20222426[] = {20, 22, 24, 26}; +static int xmm21232527[] = {21, 23, 25, 27}; +static int xmm24_27[] = {24, 25, 26, 27}; +static int xmm4_20_24[] = {4, 5, 6, 7, 20, 21, 22, 23, 24, 25, 26, 27}; +static int xmm16_27[] = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27}; +static int xmm29_29[] = {29, 29, 29, 29}; + +// Dilithium NTT function except for the final "normalization" to |coeff| < Q. +// Implements +// static int implDilithiumAlmostNtt(int[] coeffs, int zetas[]) {} +// +// coeffs (int[256]) = c_rarg0 +// zetas (int[256]) = c_rarg1 +// +// +static address generate_dilithiumAlmostNtt_avx512(StubGenerator *stubgen, + MacroAssembler *_masm) { + + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "dilithiumAlmostNtt"); + address start = __ pc(); + __ enter(); + + Label L_loop, L_end; + + const Register coeffs = c_rarg0; + const Register zetas = c_rarg1; + const Register iterations = c_rarg2; + + const Register perms = r11; + + __ lea(perms, ExternalAddress(dilithiumAvx512PermsAddr())); + + __ evmovdqul(montMulPerm, Address(perms, montMulPermsIdx), Assembler::AVX_512bit); + + // Each level represents one iteration of the outer for loop of the Java version + // In each of these iterations half of the coefficients are (Montgomery) + // multiplied by a zeta corresponding to the coefficient and then these + // products will be added to and subtracted from the other half of the + // coefficients. In each level we just collect the coefficients (using + // evpermi2d() instructions where necessary, i.e. in levels 4-7) that need to + // be multiplied by the zetas in one set, the rest to another set of vector + // registers, then redistribute the addition/substraction results. + + // For levels 0 and 1 the zetas are not different within the 4 xmm registers + // that we would use for them, so we use only one, xmm29. + loadXmm29(zetas, 0, _masm); + __ vpbroadcastd(montQInvModR, + ExternalAddress(dilithiumAvx512ConstsAddr(montQInvModRIdx)), + Assembler::AVX_512bit, scratch); // q^-1 mod 2^32 + __ vpbroadcastd(dilithium_q, + ExternalAddress(dilithiumAvx512ConstsAddr(dilithium_qIdx)), + Assembler::AVX_512bit, scratch); // q + + // load all coefficients into the vector registers Zmm_0-Zmm_15, + // 16 coefficients into each + load4Xmms(xmm0_3, coeffs, 0, _masm); + load4Xmms(xmm4_7, coeffs, 4 * XMMBYTES, _masm); + load4Xmms(xmm8_11, coeffs, 8 * XMMBYTES, _masm); + load4Xmms(xmm12_15, coeffs, 12 * XMMBYTES, _masm); + + // level 0 and 1 can be done entirely in registers as the zetas on these + // levels are the same for all the montmuls that we can do in parallel + + // level 0 + montMul64(xmm16_19, xmm8_11, xmm29_29, xmm16_27, _masm); + sub_add(xmm8_11, xmm0_3, xmm0_3, xmm16_19, _masm); + montMul64(xmm16_19, xmm12_15, xmm29_29, xmm16_27, _masm); + loadXmm29(zetas, 512, _masm); // for level 1 + sub_add(xmm12_15, xmm4_7, xmm4_7, xmm16_19, _masm); + + // level 1 + + montMul64(xmm16_19, xmm4_7, xmm29_29, xmm16_27, _masm); + loadXmm29(zetas, 768, _masm); + sub_add(xmm4_7, xmm0_3, xmm0_3, xmm16_19, _masm); + montMul64(xmm16_19, xmm12_15, xmm29_29, xmm16_27, _masm); + sub_add(xmm12_15, xmm8_11, xmm8_11, xmm16_19, _masm); + + // levels 2 to 7 are done in 2 batches, by first saving half of the coefficients + // from level 1 into memory, doing all the level 2 to level 7 computations + // on the remaining half in the vector registers, saving the result to + // memory after level 7, then loading back the coefficients that we saved after + // level 1 and do the same computation with those + + store4Xmms(coeffs, 8 * XMMBYTES, xmm8_11, _masm); + store4Xmms(coeffs, 12 * XMMBYTES, xmm12_15, _masm); + + __ movl(iterations, 2); + + __ align(OptoLoopAlignment); + __ BIND(L_loop); + + __ subl(iterations, 1); + + // level 2 + load4Xmms(xmm12_15, zetas, 2 * 512, _masm); + montMul64(xmm16_19, xmm2367, xmm12_15, xmm16_27, _masm); + load4Xmms(xmm12_15, zetas, 3 * 512, _masm); // for level 3 + sub_add(xmm2367, xmm0145, xmm0145, xmm16_19, _masm); + + // level 3 + + montMul64(xmm16_19, xmm1357, xmm12_15, xmm16_27, _masm); + sub_add(xmm1357, xmm0246, xmm0246, xmm16_19, _masm); + + // level 4 + loadPerm(xmm16_19, perms, nttL4PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttL4PermsIdx + 64, _masm); + load4Xmms(xmm24_27, zetas, 4 * 512, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i/2 + 16), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 12), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + + montMul64(xmm12_15, xmm12_15, xmm24_27, xmm4_20_24, _masm); + sub_add(xmm1357, xmm0246, xmm16_19, xmm12_15, _masm); + + // level 5 + loadPerm(xmm16_19, perms, nttL5PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttL5PermsIdx + 64, _masm); + load4Xmms(xmm24_27, zetas, 5 * 512, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i/2 + 16), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 12), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + + montMul64(xmm12_15, xmm12_15, xmm24_27, xmm4_20_24, _masm); + sub_add(xmm1357, xmm0246, xmm16_19, xmm12_15, _masm); + + // level 6 + loadPerm(xmm16_19, perms, nttL6PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttL6PermsIdx + 64, _masm); + load4Xmms(xmm24_27, zetas, 6 * 512, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i/2 + 16), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 12), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + + montMul64(xmm12_15, xmm12_15, xmm24_27, xmm4_20_24, _masm); + sub_add(xmm1357, xmm0246, xmm16_19, xmm12_15, _masm); + + // level 7 + loadPerm(xmm16_19, perms, nttL7PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttL7PermsIdx + 64, _masm); + load4Xmms(xmm24_27, zetas, 7 * 512, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 16), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 12), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + + montMul64(xmm12_15, xmm12_15, xmm24_27, xmm4_20_24, true, _masm); + loadPerm(xmm0246, perms, nttL7PermsIdx + 2 * XMMBYTES, _masm); + loadPerm(xmm1357, perms, nttL7PermsIdx + 3 * XMMBYTES, _masm); + sub_add(xmm21232527, xmm20222426, xmm16_19, xmm12_15, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i), xmm(i + 20), xmm(i + 21), Assembler::AVX_512bit); + __ evpermi2d(xmm(i + 1), xmm(i + 20), xmm(i + 21), Assembler::AVX_512bit); + } + + __ cmpl(iterations, 0); + __ jcc(Assembler::equal, L_end); + + store4Xmms(coeffs, 0, xmm0_3, _masm); + store4Xmms(coeffs, 4 * XMMBYTES, xmm4_7, _masm); + + load4Xmms(xmm0_3, coeffs, 8 * XMMBYTES, _masm); + load4Xmms(xmm4_7, coeffs, 12 * XMMBYTES, _masm); + + __ addptr(zetas, 4 * XMMBYTES); + + __ jmp(L_loop); + + __ BIND(L_end); + + store4Xmms(coeffs, 8 * XMMBYTES, xmm0_3, _masm); + store4Xmms(coeffs, 12 * XMMBYTES, xmm4_7, _masm); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov64(rax, 0); // return 0 + __ ret(0); + + return start; +} + +// Dilithium Inverse NTT function except the final mod Q division by 2^256. +// Implements +// static int implDilithiumAlmostInverseNtt(int[] coeffs, int[] zetas) {} +// +// coeffs (int[256]) = c_rarg0 +// zetas (int[256]) = c_rarg1 +static address generate_dilithiumAlmostInverseNtt_avx512(StubGenerator *stubgen, + MacroAssembler *_masm) { + + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "dilithiumAlmostInverseNtt"); + address start = __ pc(); + __ enter(); + + Label L_loop, L_end; + + const Register coeffs = c_rarg0; + const Register zetas = c_rarg1; + + const Register iterations = c_rarg2; + + const Register perms = r11; + + __ lea(perms, ExternalAddress(dilithiumAvx512PermsAddr())); + + __ evmovdqul(montMulPerm, Address(perms, montMulPermsIdx), Assembler::AVX_512bit); + __ vpbroadcastd(montQInvModR, + ExternalAddress(dilithiumAvx512ConstsAddr(montQInvModRIdx)), + Assembler::AVX_512bit, scratch); // q^-1 mod 2^32 + __ vpbroadcastd(dilithium_q, + ExternalAddress(dilithiumAvx512ConstsAddr(dilithium_qIdx)), + Assembler::AVX_512bit, scratch); // q + + // Each level represents one iteration of the outer for loop of the + // Java version. + // In each of these iterations half of the coefficients are added to and + // subtracted from the other half of the coefficients then the result of + // the substartion is (Montgomery) multiplied by the corresponding zetas. + // In each level we just collect the coefficients (using evpermi2d() + // instructions where necessary, i.e. on levels 0-4) so that the results of + // the additions and subtractions go to the vector registers so that they + // align with each other and the zetas. + + // We do levels 0-6 in two batches, each batch entirely in the vector registers + load4Xmms(xmm0_3, coeffs, 0, _masm); + load4Xmms(xmm4_7, coeffs, 4 * XMMBYTES, _masm); + + __ movl(iterations, 2); + + __ align(OptoLoopAlignment); + __ BIND(L_loop); + + __ subl(iterations, 1); + + // level 0 + loadPerm(xmm8_11, perms, nttInvL0PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttInvL0PermsIdx + 64, _masm); + + for (int i = 0; i < 8; i += 2) { + __ evpermi2d(xmm(i / 2 + 8), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + __ evpermi2d(xmm(i / 2 + 12), xmm(i), xmm(i + 1), Assembler::AVX_512bit); + } + + load4Xmms(xmm4_7, zetas, 0, _masm); + sub_add(xmm24_27, xmm0_3, xmm8_11, xmm12_15, _masm); + montMul64(xmm4_7, xmm4_7, xmm24_27, xmm16_27, true, _masm); + + // level 1 + loadPerm(xmm8_11, perms, nttInvL1PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttInvL1PermsIdx + 64, _masm); + + for (int i = 0; i < 4; i++) { + __ evpermi2d(xmm(i + 8), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + __ evpermi2d(xmm(i + 12), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + } + + load4Xmms(xmm4_7, zetas, 512, _masm); + sub_add(xmm24_27, xmm0_3, xmm8_11, xmm12_15, _masm); + montMul64(xmm4_7, xmm24_27, xmm4_7, xmm16_27, _masm); + + // level 2 + loadPerm(xmm8_11, perms, nttInvL2PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttInvL2PermsIdx + 64, _masm); + + for (int i = 0; i < 4; i++) { + __ evpermi2d(xmm(i + 8), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + __ evpermi2d(xmm(i + 12), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + } + + load4Xmms(xmm4_7, zetas, 2 * 512, _masm); + sub_add(xmm24_27, xmm0_3, xmm8_11, xmm12_15, _masm); + montMul64(xmm4_7, xmm24_27, xmm4_7, xmm16_27, _masm); + + // level 3 + loadPerm(xmm8_11, perms, nttInvL3PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttInvL3PermsIdx + 64, _masm); + + for (int i = 0; i < 4; i++) { + __ evpermi2d(xmm(i + 8), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + __ evpermi2d(xmm(i + 12), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + } + + load4Xmms(xmm4_7, zetas, 3 * 512, _masm); + sub_add(xmm24_27, xmm0_3, xmm8_11, xmm12_15, _masm); + montMul64(xmm4_7, xmm24_27, xmm4_7, xmm16_27, _masm); + + // level 4 + loadPerm(xmm8_11, perms, nttInvL4PermsIdx, _masm); + loadPerm(xmm12_15, perms, nttInvL4PermsIdx + 64, _masm); + + for (int i = 0; i < 4; i++) { + __ evpermi2d(xmm(i + 8), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + __ evpermi2d(xmm(i + 12), xmm(i), xmm(i + 4), Assembler::AVX_512bit); + } + + load4Xmms(xmm4_7, zetas, 4 * 512, _masm); + sub_add(xmm24_27, xmm0_3, xmm8_11, xmm12_15, _masm); + montMul64(xmm4_7, xmm24_27, xmm4_7, xmm16_27, _masm); + + // level 5 + load4Xmms(xmm12_15, zetas, 5 * 512, _masm); + sub_add(xmm8_11, xmm0_3, xmm0426, xmm1537, _masm); + montMul64(xmm4_7, xmm8_11, xmm12_15, xmm16_27, _masm); + + // level 6 + load4Xmms(xmm12_15, zetas, 6 * 512, _masm); + sub_add(xmm8_11, xmm0_3, xmm0145, xmm2367, _masm); + montMul64(xmm4_7, xmm8_11, xmm12_15, xmm16_27, _masm); + + __ cmpl(iterations, 0); + __ jcc(Assembler::equal, L_end); + + // save the coefficients of the first batch, adjust the zetas + // and load the second batch of coefficients + store4Xmms(coeffs, 0, xmm0_3, _masm); + store4Xmms(coeffs, 4 * XMMBYTES, xmm4_7, _masm); + + __ addptr(zetas, 4 * XMMBYTES); + + load4Xmms(xmm0_3, coeffs, 8 * XMMBYTES, _masm); + load4Xmms(xmm4_7, coeffs, 12 * XMMBYTES, _masm); + + __ jmp(L_loop); + + __ BIND(L_end); + + // load the coeffs of the first batch of coefficients that were saved after + // level 6 into Zmm_8-Zmm_15 and do the last level entirely in the vector + // registers + load4Xmms(xmm8_11, coeffs, 0, _masm); + load4Xmms(xmm12_15, coeffs, 4 * XMMBYTES, _masm); + + // level 7 + + loadXmm29(zetas, 7 * 512, _masm); + + for (int i = 0; i < 8; i++) { + __ evpaddd(xmm(i + 16), k0, xmm(i), xmm(i + 8), false, Assembler::AVX_512bit); + } + + for (int i = 0; i < 8; i++) { + __ evpsubd(xmm(i), k0, xmm(i + 8), xmm(i), false, Assembler::AVX_512bit); + } + + store4Xmms(coeffs, 0, xmm16_19, _masm); + store4Xmms(coeffs, 4 * XMMBYTES, xmm20_23, _masm); + montMul64(xmm0_3, xmm0_3, xmm29_29, xmm16_27, _masm); + montMul64(xmm4_7, xmm4_7, xmm29_29, xmm16_27, _masm); + store4Xmms(coeffs, 8 * XMMBYTES, xmm0_3, _masm); + store4Xmms(coeffs, 12 * XMMBYTES, xmm4_7, _masm); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov64(rax, 0); // return 0 + __ ret(0); + + return start; +} + +// Dilithium multiply polynomials in the NTT domain. +// Implements +// static int implDilithiumNttMult( +// int[] result, int[] ntta, int[] nttb {} +// +// result (int[256]) = c_rarg0 +// poly1 (int[256]) = c_rarg1 +// poly2 (int[256]) = c_rarg2 +static address generate_dilithiumNttMult_avx512(StubGenerator *stubgen, + MacroAssembler *_masm) { + + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "dilithiumNttMult"); + address start = __ pc(); + __ enter(); + + Label L_loop; + + const Register result = c_rarg0; + const Register poly1 = c_rarg1; + const Register poly2 = c_rarg2; + + const Register perms = r10; // scratch reused after not needed any more + const Register len = r11; + + const XMMRegister montRSquareModQ = xmm29; + + __ vpbroadcastd(montQInvModR, + ExternalAddress(dilithiumAvx512ConstsAddr(montQInvModRIdx)), + Assembler::AVX_512bit, scratch); // q^-1 mod 2^32 + __ vpbroadcastd(dilithium_q, + ExternalAddress(dilithiumAvx512ConstsAddr(dilithium_qIdx)), + Assembler::AVX_512bit, scratch); // q + __ vpbroadcastd(montRSquareModQ, + ExternalAddress(dilithiumAvx512ConstsAddr(montRSquareModQIdx)), + Assembler::AVX_512bit, scratch); // 2^64 mod q + + __ lea(perms, ExternalAddress(dilithiumAvx512PermsAddr())); + __ evmovdqul(montMulPerm, Address(perms, montMulPermsIdx), Assembler::AVX_512bit); + + __ movl(len, 4); + + __ align(OptoLoopAlignment); + __ BIND(L_loop); + + load4Xmms(xmm4_7, poly2, 0, _masm); + load4Xmms(xmm0_3, poly1, 0, _masm); + montMul64(xmm4_7, xmm4_7, xmm29_29, xmm16_27, _masm); + montMul64(xmm0_3, xmm0_3, xmm4_7, xmm16_27, true, _masm); + store4Xmms(result, 0, xmm0_3, _masm); + + __ subl(len, 1); + __ addptr(poly1, 4 * XMMBYTES); + __ addptr(poly2, 4 * XMMBYTES); + __ addptr(result, 4 * XMMBYTES); + __ cmpl(len, 0); + __ jcc(Assembler::notEqual, L_loop); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov64(rax, 0); // return 0 + __ ret(0); + + return start; +} + +// Dilithium Motgomery multiply an array by a constant. +// Implements +// static int implDilithiumMontMulByConstant(int[] coeffs, int constant) {} +// +// coeffs (int[256]) = c_rarg0 +// constant (int) = c_rarg1 +static address generate_dilithiumMontMulByConstant_avx512(StubGenerator *stubgen, + MacroAssembler *_masm) { + + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "dilithiumMontMulByConstant"); + address start = __ pc(); + __ enter(); + + Label L_loop; + + const Register coeffs = c_rarg0; + const Register rConstant = c_rarg1; + + const Register perms = c_rarg2; // not used for argument + const Register len = r11; + + const XMMRegister constant = xmm29; + + __ lea(perms, ExternalAddress(dilithiumAvx512PermsAddr())); + + // the following four vector registers are used in montMul64 + __ vpbroadcastd(montQInvModR, + ExternalAddress(dilithiumAvx512ConstsAddr(montQInvModRIdx)), + Assembler::AVX_512bit, scratch); // q^-1 mod 2^32 + __ vpbroadcastd(dilithium_q, + ExternalAddress(dilithiumAvx512ConstsAddr(dilithium_qIdx)), + Assembler::AVX_512bit, scratch); // q + __ evmovdqul(montMulPerm, Address(perms, montMulPermsIdx), Assembler::AVX_512bit); + __ evpbroadcastd(constant, rConstant, Assembler::AVX_512bit); // constant multiplier + + __ movl(len, 2); + + __ align(OptoLoopAlignment); + __ BIND(L_loop); + + load4Xmms(xmm0_3, coeffs, 0, _masm); + load4Xmms(xmm4_7, coeffs, 4 * XMMBYTES, _masm); + montMul64(xmm0_3, xmm0_3, xmm29_29, xmm16_27, _masm); + montMul64(xmm4_7, xmm4_7, xmm29_29, xmm16_27, _masm); + store4Xmms(coeffs, 0, xmm0_3, _masm); + store4Xmms(coeffs, 4 * XMMBYTES, xmm4_7, _masm); + + __ subl(len, 1); + __ addptr(coeffs, 512); + __ cmpl(len, 0); + __ jcc(Assembler::notEqual, L_loop); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov64(rax, 0); // return 0 + __ ret(0); + + return start; +} + +// Dilithium decompose poly. +// Implements +// static int implDilithiumDecomposePoly(int[] coeffs, int constant) {} +// +// input (int[256]) = c_rarg0 +// lowPart (int[256]) = c_rarg1 +// highPart (int[256]) = c_rarg2 +// twoGamma2 (int) = c_rarg3 +// multiplier (int) = c_rarg4 +static address generate_dilithiumDecomposePoly_avx512(StubGenerator *stubgen, + MacroAssembler *_masm) { + + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "dilithiumDecomposePoly"); + address start = __ pc(); + __ enter(); + + Label L_loop; + + const Register input = c_rarg0; + const Register lowPart = c_rarg1; + const Register highPart = c_rarg2; + const Register rTwoGamma2 = c_rarg3; + + const Register len = r11; + const XMMRegister zero = xmm24; + const XMMRegister one = xmm25; + const XMMRegister qMinus1 = xmm26; + const XMMRegister gamma2 = xmm27; + const XMMRegister twoGamma2 = xmm28; + const XMMRegister barrettMultiplier = xmm29; + const XMMRegister barrettAddend = xmm30; + + __ vpxor(zero, zero, zero, Assembler::AVX_512bit); // 0 + __ vpternlogd(xmm0, 0xff, xmm0, xmm0, Assembler::AVX_512bit); // -1 + __ vpsubd(one, zero, xmm0, Assembler::AVX_512bit); // 1 + __ vpbroadcastd(dilithium_q, + ExternalAddress(dilithiumAvx512ConstsAddr(dilithium_qIdx)), + Assembler::AVX_512bit, scratch); // q + __ vpbroadcastd(barrettAddend, + ExternalAddress(dilithiumAvx512ConstsAddr(barrettAddendIdx)), + Assembler::AVX_512bit, scratch); // addend for Barrett reduction + + __ evpbroadcastd(twoGamma2, rTwoGamma2, Assembler::AVX_512bit); // 2 * gamma2 + + #ifndef _WIN64 + const Register rMultiplier = c_rarg4; + #else + const Address multiplier_mem(rbp, 6 * wordSize); + const Register rMultiplier = c_rarg3; // arg3 is already consumed, reused here + __ movptr(rMultiplier, multiplier_mem); + #endif + __ evpbroadcastd(barrettMultiplier, rMultiplier, + Assembler::AVX_512bit); // multiplier for mod 2 * gamma2 reduce + + __ evpsubd(qMinus1, k0, dilithium_q, one, false, Assembler::AVX_512bit); // q - 1 + __ evpsrad(gamma2, k0, twoGamma2, 1, false, Assembler::AVX_512bit); // gamma2 + + __ movl(len, 1024); + + __ align(OptoLoopAlignment); + __ BIND(L_loop); + + load4Xmms(xmm0_3, input, 0, _masm); + + __ addptr(input, 4 * XMMBYTES); + + // rplus in xmm0 + // rplus = rplus - ((rplus + 5373807) >> 23) * dilithium_q; + __ evpaddd(xmm4, k0, xmm0, barrettAddend, false, Assembler::AVX_512bit); + __ evpaddd(xmm5, k0, xmm1, barrettAddend, false, Assembler::AVX_512bit); + __ evpaddd(xmm6, k0, xmm2, barrettAddend, false, Assembler::AVX_512bit); + __ evpaddd(xmm7, k0, xmm3, barrettAddend, false, Assembler::AVX_512bit); + + __ evpsrad(xmm4, k0, xmm4, 23, false, Assembler::AVX_512bit); + __ evpsrad(xmm5, k0, xmm5, 23, false, Assembler::AVX_512bit); + __ evpsrad(xmm6, k0, xmm6, 23, false, Assembler::AVX_512bit); + __ evpsrad(xmm7, k0, xmm7, 23, false, Assembler::AVX_512bit); + + __ evpmulld(xmm4, k0, xmm4, dilithium_q, false, Assembler::AVX_512bit); + __ evpmulld(xmm5, k0, xmm5, dilithium_q, false, Assembler::AVX_512bit); + __ evpmulld(xmm6, k0, xmm6, dilithium_q, false, Assembler::AVX_512bit); + __ evpmulld(xmm7, k0, xmm7, dilithium_q, false, Assembler::AVX_512bit); + + __ evpsubd(xmm0, k0, xmm0, xmm4, false, Assembler::AVX_512bit); + __ evpsubd(xmm1, k0, xmm1, xmm5, false, Assembler::AVX_512bit); + __ evpsubd(xmm2, k0, xmm2, xmm6, false, Assembler::AVX_512bit); + __ evpsubd(xmm3, k0, xmm3, xmm7, false, Assembler::AVX_512bit); + // rplus in xmm0 + // rplus = rplus + ((rplus >> 31) & dilithium_q); + __ evpsrad(xmm4, k0, xmm0, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm5, k0, xmm1, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm6, k0, xmm2, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm7, k0, xmm3, 31, false, Assembler::AVX_512bit); + + __ evpandd(xmm4, k0, xmm4, dilithium_q, false, Assembler::AVX_512bit); + __ evpandd(xmm5, k0, xmm5, dilithium_q, false, Assembler::AVX_512bit); + __ evpandd(xmm6, k0, xmm6, dilithium_q, false, Assembler::AVX_512bit); + __ evpandd(xmm7, k0, xmm7, dilithium_q, false, Assembler::AVX_512bit); + + __ evpaddd(xmm0, k0, xmm0, xmm4, false, Assembler::AVX_512bit); + __ evpaddd(xmm1, k0, xmm1, xmm5, false, Assembler::AVX_512bit); + __ evpaddd(xmm2, k0, xmm2, xmm6, false, Assembler::AVX_512bit); + __ evpaddd(xmm3, k0, xmm3, xmm7, false, Assembler::AVX_512bit); + // rplus in xmm0 + // int quotient = (rplus * barrettMultiplier) >> 22; + __ evpmulld(xmm4, k0, xmm0, barrettMultiplier, false, Assembler::AVX_512bit); + __ evpmulld(xmm5, k0, xmm1, barrettMultiplier, false, Assembler::AVX_512bit); + __ evpmulld(xmm6, k0, xmm2, barrettMultiplier, false, Assembler::AVX_512bit); + __ evpmulld(xmm7, k0, xmm3, barrettMultiplier, false, Assembler::AVX_512bit); + + __ evpsrad(xmm4, k0, xmm4, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm5, k0, xmm5, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm6, k0, xmm6, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm7, k0, xmm7, 22, false, Assembler::AVX_512bit); + // quotient in xmm4 + // int r0 = rplus - quotient * twoGamma2; + __ evpmulld(xmm8, k0, xmm4, twoGamma2, false, Assembler::AVX_512bit); + __ evpmulld(xmm9, k0, xmm5, twoGamma2, false, Assembler::AVX_512bit); + __ evpmulld(xmm10, k0, xmm6, twoGamma2, false, Assembler::AVX_512bit); + __ evpmulld(xmm11, k0, xmm7, twoGamma2, false, Assembler::AVX_512bit); + + __ evpsubd(xmm8, k0, xmm0, xmm8, false, Assembler::AVX_512bit); + __ evpsubd(xmm9, k0, xmm1, xmm9, false, Assembler::AVX_512bit); + __ evpsubd(xmm10, k0, xmm2, xmm10, false, Assembler::AVX_512bit); + __ evpsubd(xmm11, k0, xmm3, xmm11, false, Assembler::AVX_512bit); + // r0 in xmm8 + // int mask = (twoGamma2 - r0) >> 22; + __ evpsubd(xmm12, k0, twoGamma2, xmm8, false, Assembler::AVX_512bit); + __ evpsubd(xmm13, k0, twoGamma2, xmm9, false, Assembler::AVX_512bit); + __ evpsubd(xmm14, k0, twoGamma2, xmm10, false, Assembler::AVX_512bit); + __ evpsubd(xmm15, k0, twoGamma2, xmm11, false, Assembler::AVX_512bit); + + __ evpsrad(xmm12, k0, xmm12, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm13, k0, xmm13, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm14, k0, xmm14, 22, false, Assembler::AVX_512bit); + __ evpsrad(xmm15, k0, xmm15, 22, false, Assembler::AVX_512bit); + // mask in xmm12 + // r0 -= (mask & twoGamma2); + __ evpandd(xmm16, k0, xmm12, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm17, k0, xmm13, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm18, k0, xmm14, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm19, k0, xmm15, twoGamma2, false, Assembler::AVX_512bit); + + __ evpsubd(xmm8, k0, xmm8, xmm16, false, Assembler::AVX_512bit); + __ evpsubd(xmm9, k0, xmm9, xmm17, false, Assembler::AVX_512bit); + __ evpsubd(xmm10, k0, xmm10, xmm18, false, Assembler::AVX_512bit); + __ evpsubd(xmm11, k0, xmm11, xmm19, false, Assembler::AVX_512bit); + // r0 in xmm8 + // quotient += (mask & 1); + __ evpandd(xmm16, k0, xmm12, one, false, Assembler::AVX_512bit); + __ evpandd(xmm17, k0, xmm13, one, false, Assembler::AVX_512bit); + __ evpandd(xmm18, k0, xmm14, one, false, Assembler::AVX_512bit); + __ evpandd(xmm19, k0, xmm15, one, false, Assembler::AVX_512bit); + + __ evpaddd(xmm4, k0, xmm4, xmm16, false, Assembler::AVX_512bit); + __ evpaddd(xmm5, k0, xmm5, xmm17, false, Assembler::AVX_512bit); + __ evpaddd(xmm6, k0, xmm6, xmm18, false, Assembler::AVX_512bit); + __ evpaddd(xmm7, k0, xmm7, xmm19, false, Assembler::AVX_512bit); + + // mask = (twoGamma2 / 2 - r0) >> 31; + __ evpsubd(xmm12, k0, gamma2, xmm8, false, Assembler::AVX_512bit); + __ evpsubd(xmm13, k0, gamma2, xmm9, false, Assembler::AVX_512bit); + __ evpsubd(xmm14, k0, gamma2, xmm10, false, Assembler::AVX_512bit); + __ evpsubd(xmm15, k0, gamma2, xmm11, false, Assembler::AVX_512bit); + + __ evpsrad(xmm12, k0, xmm12, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm13, k0, xmm13, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm14, k0, xmm14, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm15, k0, xmm15, 31, false, Assembler::AVX_512bit); + + // r0 -= (mask & twoGamma2); + __ evpandd(xmm16, k0, xmm12, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm17, k0, xmm13, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm18, k0, xmm14, twoGamma2, false, Assembler::AVX_512bit); + __ evpandd(xmm19, k0, xmm15, twoGamma2, false, Assembler::AVX_512bit); + + __ evpsubd(xmm8, k0, xmm8, xmm16, false, Assembler::AVX_512bit); + __ evpsubd(xmm9, k0, xmm9, xmm17, false, Assembler::AVX_512bit); + __ evpsubd(xmm10, k0, xmm10, xmm18, false, Assembler::AVX_512bit); + __ evpsubd(xmm11, k0, xmm11, xmm19, false, Assembler::AVX_512bit); + // r0 in xmm8 + // quotient += (mask & 1); + __ evpandd(xmm16, k0, xmm12, one, false, Assembler::AVX_512bit); + __ evpandd(xmm17, k0, xmm13, one, false, Assembler::AVX_512bit); + __ evpandd(xmm18, k0, xmm14, one, false, Assembler::AVX_512bit); + __ evpandd(xmm19, k0, xmm15, one, false, Assembler::AVX_512bit); + + __ evpaddd(xmm4, k0, xmm4, xmm16, false, Assembler::AVX_512bit); + __ evpaddd(xmm5, k0, xmm5, xmm17, false, Assembler::AVX_512bit); + __ evpaddd(xmm6, k0, xmm6, xmm18, false, Assembler::AVX_512bit); + __ evpaddd(xmm7, k0, xmm7, xmm19, false, Assembler::AVX_512bit); + // quotient in xmm4 + // int r1 = rplus - r0 - (dilithium_q - 1); + __ evpsubd(xmm16, k0, xmm0, xmm8, false, Assembler::AVX_512bit); + __ evpsubd(xmm17, k0, xmm1, xmm9, false, Assembler::AVX_512bit); + __ evpsubd(xmm18, k0, xmm2, xmm10, false, Assembler::AVX_512bit); + __ evpsubd(xmm19, k0, xmm3, xmm11, false, Assembler::AVX_512bit); + + __ evpsubd(xmm16, k0, xmm16, xmm26, false, Assembler::AVX_512bit); + __ evpsubd(xmm17, k0, xmm17, xmm26, false, Assembler::AVX_512bit); + __ evpsubd(xmm18, k0, xmm18, xmm26, false, Assembler::AVX_512bit); + __ evpsubd(xmm19, k0, xmm19, xmm26, false, Assembler::AVX_512bit); + // r1 in xmm16 + // r1 = (r1 | (-r1)) >> 31; // 0 if rplus - r0 == (dilithium_q - 1), -1 otherwise + __ evpsubd(xmm20, k0, zero, xmm16, false, Assembler::AVX_512bit); + __ evpsubd(xmm21, k0, zero, xmm17, false, Assembler::AVX_512bit); + __ evpsubd(xmm22, k0, zero, xmm18, false, Assembler::AVX_512bit); + __ evpsubd(xmm23, k0, zero, xmm19, false, Assembler::AVX_512bit); + + __ evporq(xmm16, k0, xmm16, xmm20, false, Assembler::AVX_512bit); + __ evporq(xmm17, k0, xmm17, xmm21, false, Assembler::AVX_512bit); + __ evporq(xmm18, k0, xmm18, xmm22, false, Assembler::AVX_512bit); + __ evporq(xmm19, k0, xmm19, xmm23, false, Assembler::AVX_512bit); + + __ evpsubd(xmm12, k0, zero, one, false, Assembler::AVX_512bit); // -1 + + __ evpsrad(xmm0, k0, xmm16, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm1, k0, xmm17, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm2, k0, xmm18, 31, false, Assembler::AVX_512bit); + __ evpsrad(xmm3, k0, xmm19, 31, false, Assembler::AVX_512bit); + // r1 in xmm0 + // r0 += ~r1; + __ evpxorq(xmm20, k0, xmm0, xmm12, false, Assembler::AVX_512bit); + __ evpxorq(xmm21, k0, xmm1, xmm12, false, Assembler::AVX_512bit); + __ evpxorq(xmm22, k0, xmm2, xmm12, false, Assembler::AVX_512bit); + __ evpxorq(xmm23, k0, xmm3, xmm12, false, Assembler::AVX_512bit); + + __ evpaddd(xmm8, k0, xmm8, xmm20, false, Assembler::AVX_512bit); + __ evpaddd(xmm9, k0, xmm9, xmm21, false, Assembler::AVX_512bit); + __ evpaddd(xmm10, k0, xmm10, xmm22, false, Assembler::AVX_512bit); + __ evpaddd(xmm11, k0, xmm11, xmm23, false, Assembler::AVX_512bit); + // r0 in xmm8 + // r1 = r1 & quotient; + __ evpandd(xmm0, k0, xmm4, xmm0, false, Assembler::AVX_512bit); + __ evpandd(xmm1, k0, xmm5, xmm1, false, Assembler::AVX_512bit); + __ evpandd(xmm2, k0, xmm6, xmm2, false, Assembler::AVX_512bit); + __ evpandd(xmm3, k0, xmm7, xmm3, false, Assembler::AVX_512bit); + // r1 in xmm0 + // lowPart[m] = r0; + // highPart[m] = r1; + store4Xmms(highPart, 0, xmm0_3, _masm); + store4Xmms(lowPart, 0, xmm8_11, _masm); + + __ addptr(highPart, 4 * XMMBYTES); + __ addptr(lowPart, 4 * XMMBYTES); + __ subl(len, 4 * XMMBYTES); + __ jcc(Assembler::notEqual, L_loop); + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ mov64(rax, 0); // return 0 + __ ret(0); + + return start; +} + +void StubGenerator::generate_dilithium_stubs() { + // Generate Dilithium intrinsics code + if (UseDilithiumIntrinsics) { + StubRoutines::_dilithiumAlmostNtt = + generate_dilithiumAlmostNtt_avx512(this, _masm); + StubRoutines::_dilithiumAlmostInverseNtt = + generate_dilithiumAlmostInverseNtt_avx512(this, _masm); + StubRoutines::_dilithiumNttMult = + generate_dilithiumNttMult_avx512(this, _masm); + StubRoutines::_dilithiumMontMulByConstant = + generate_dilithiumMontMulByConstant_avx512(this, _masm); + StubRoutines::_dilithiumDecomposePoly = + generate_dilithiumDecomposePoly_avx512(this, _masm); + } +} diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp index 49c39226708e..42d9114715d1 100644 --- a/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp +++ b/src/hotspot/cpu/x86/stubGenerator_x86_64_sha3.cpp @@ -39,6 +39,8 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +#define xmm(i) as_XMMRegister(i) + // Constants ATTRIBUTE_ALIGNED(64) static const uint64_t round_consts_arr[24] = { 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL, @@ -80,13 +82,6 @@ static address permsAndRotsAddr() { return (address) permsAndRots; } -void StubGenerator::generate_sha3_stubs() { - if (UseSHA3Intrinsics) { - StubRoutines::_sha3_implCompress = generate_sha3_implCompress(false,"sha3_implCompress"); - StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(true, "sha3_implCompressMB"); - } -} - // Arguments: // // Inputs: @@ -96,9 +91,11 @@ void StubGenerator::generate_sha3_stubs() { // c_rarg3 - int offset // c_rarg4 - int limit // -address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *name) { +static address generate_sha3_implCompress(bool multiBlock, const char *name, + StubGenerator *stubgen, + MacroAssembler *_masm) { __ align(CodeEntryAlignment); - StubCodeMark mark(this, "StubRoutines", name); + StubCodeMark mark(stubgen, "StubRoutines", name); address start = __ pc(); const Register buf = c_rarg0; @@ -143,29 +140,16 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n __ kshiftrwl(k1, k5, 4); // load the state - __ evmovdquq(xmm0, k5, Address(state, 0), false, Assembler::AVX_512bit); - __ evmovdquq(xmm1, k5, Address(state, 40), false, Assembler::AVX_512bit); - __ evmovdquq(xmm2, k5, Address(state, 80), false, Assembler::AVX_512bit); - __ evmovdquq(xmm3, k5, Address(state, 120), false, Assembler::AVX_512bit); - __ evmovdquq(xmm4, k5, Address(state, 160), false, Assembler::AVX_512bit); + for (int i = 0; i < 5; i++) { + __ evmovdquq(xmm(i), k5, Address(state, i * 40), false, Assembler::AVX_512bit); + } // load the permutation and rotation constants - __ evmovdquq(xmm17, Address(permsAndRots, 0), Assembler::AVX_512bit); - __ evmovdquq(xmm18, Address(permsAndRots, 64), Assembler::AVX_512bit); - __ evmovdquq(xmm19, Address(permsAndRots, 128), Assembler::AVX_512bit); - __ evmovdquq(xmm20, Address(permsAndRots, 192), Assembler::AVX_512bit); - __ evmovdquq(xmm21, Address(permsAndRots, 256), Assembler::AVX_512bit); - __ evmovdquq(xmm22, Address(permsAndRots, 320), Assembler::AVX_512bit); - __ evmovdquq(xmm23, Address(permsAndRots, 384), Assembler::AVX_512bit); - __ evmovdquq(xmm24, Address(permsAndRots, 448), Assembler::AVX_512bit); - __ evmovdquq(xmm25, Address(permsAndRots, 512), Assembler::AVX_512bit); - __ evmovdquq(xmm26, Address(permsAndRots, 576), Assembler::AVX_512bit); - __ evmovdquq(xmm27, Address(permsAndRots, 640), Assembler::AVX_512bit); - __ evmovdquq(xmm28, Address(permsAndRots, 704), Assembler::AVX_512bit); - __ evmovdquq(xmm29, Address(permsAndRots, 768), Assembler::AVX_512bit); - __ evmovdquq(xmm30, Address(permsAndRots, 832), Assembler::AVX_512bit); - __ evmovdquq(xmm31, Address(permsAndRots, 896), Assembler::AVX_512bit); + for (int i = 0; i < 15; i++) { + __ evmovdquq(xmm(i + 17), Address(permsAndRots, i * 64), Assembler::AVX_512bit); + } + __ align(OptoLoopAlignment); __ BIND(sha3_loop); // there will be 24 keccak rounds @@ -220,6 +204,7 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n // The implementation closely follows the Java version, with the state // array "rows" in the lowest 5 64-bit slots of zmm0 - zmm4, i.e. // each row of the SHA3 specification is located in one zmm register. + __ align(OptoLoopAlignment); __ BIND(rounds24_loop); __ subl(roundsLeft, 1); @@ -246,7 +231,7 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n // Do the cyclical permutation of the 24 moving state elements // and the required rotations within each element (the combined - // rho and sigma steps). + // rho and pi steps). __ evpermt2q(xmm4, xmm17, xmm3, Assembler::AVX_512bit); __ evpermt2q(xmm3, xmm18, xmm2, Assembler::AVX_512bit); __ evpermt2q(xmm2, xmm17, xmm1, Assembler::AVX_512bit); @@ -268,7 +253,7 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n __ evpermt2q(xmm2, xmm24, xmm4, Assembler::AVX_512bit); __ evpermt2q(xmm3, xmm25, xmm4, Assembler::AVX_512bit); __ evpermt2q(xmm4, xmm26, xmm5, Assembler::AVX_512bit); - // The combined rho and sigma steps are done. + // The combined rho and pi steps are done. // Do the chi step (the same operation on all 5 rows). // vpternlogq(x, 180, y, z) does x = x ^ (y & ~z). @@ -309,11 +294,9 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n } // store the state - __ evmovdquq(Address(state, 0), k5, xmm0, true, Assembler::AVX_512bit); - __ evmovdquq(Address(state, 40), k5, xmm1, true, Assembler::AVX_512bit); - __ evmovdquq(Address(state, 80), k5, xmm2, true, Assembler::AVX_512bit); - __ evmovdquq(Address(state, 120), k5, xmm3, true, Assembler::AVX_512bit); - __ evmovdquq(Address(state, 160), k5, xmm4, true, Assembler::AVX_512bit); + for (int i = 0; i < 5; i++) { + __ evmovdquq(Address(state, i * 40), k5, xmm(i), true, Assembler::AVX_512bit); + } __ pop(r14); __ pop(r13); @@ -324,3 +307,194 @@ address StubGenerator::generate_sha3_implCompress(bool multiBlock, const char *n return start; } + +// Inputs: +// c_rarg0 - long[] state0 +// c_rarg1 - long[] state1 +// +// Performs two keccak() computations in parallel. The steps of the +// two computations are executed interleaved. +static address generate_double_keccak(StubGenerator *stubgen, MacroAssembler *_masm) { + __ align(CodeEntryAlignment); + StubCodeMark mark(stubgen, "StubRoutines", "double_keccak"); + address start = __ pc(); + + const Register state0 = c_rarg0; + const Register state1 = c_rarg1; + + const Register permsAndRots = c_rarg2; + const Register round_consts = c_rarg3; + const Register constant2use = r10; + const Register roundsLeft = r11; + + Label rounds24_loop; + + __ enter(); + + __ lea(permsAndRots, ExternalAddress(permsAndRotsAddr())); + __ lea(round_consts, ExternalAddress(round_constsAddr())); + + // set up the masks + __ movl(rax, 0x1F); + __ kmovwl(k5, rax); + __ kshiftrwl(k4, k5, 1); + __ kshiftrwl(k3, k5, 2); + __ kshiftrwl(k2, k5, 3); + __ kshiftrwl(k1, k5, 4); + + // load the states + for (int i = 0; i < 5; i++) { + __ evmovdquq(xmm(i), k5, Address(state0, i * 40), false, Assembler::AVX_512bit); + } + for (int i = 0; i < 5; i++) { + __ evmovdquq(xmm(10 + i), k5, Address(state1, i * 40), false, Assembler::AVX_512bit); + } + + // load the permutation and rotation constants + + for (int i = 0; i < 15; i++) { + __ evmovdquq(xmm(17 + i), Address(permsAndRots, i * 64), Assembler::AVX_512bit); + } + + // there will be 24 keccak rounds + // The same operations as the ones in generate_sha3_implCompress are + // performed, but in parallel for two states: one in regs z0-z5, using z6 + // as the scratch register and the other in z10-z15, using z16 as the + // scratch register. + // The permutation and rotation constants, that are loaded into z17-z31, + // are shared between the two computations. + __ movl(roundsLeft, 24); + // load round_constants base + __ movptr(constant2use, round_consts); + + __ align(OptoLoopAlignment); + __ BIND(rounds24_loop); + __ subl( roundsLeft, 1); + + __ evmovdquw(xmm5, xmm0, Assembler::AVX_512bit); + __ evmovdquw(xmm15, xmm10, Assembler::AVX_512bit); + __ vpternlogq(xmm5, 150, xmm1, xmm2, Assembler::AVX_512bit); + __ vpternlogq(xmm15, 150, xmm11, xmm12, Assembler::AVX_512bit); + __ vpternlogq(xmm5, 150, xmm3, xmm4, Assembler::AVX_512bit); + __ vpternlogq(xmm15, 150, xmm13, xmm14, Assembler::AVX_512bit); + __ evprolq(xmm6, xmm5, 1, Assembler::AVX_512bit); + __ evprolq(xmm16, xmm15, 1, Assembler::AVX_512bit); + __ evpermt2q(xmm5, xmm30, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm30, xmm15, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm6, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm16, Assembler::AVX_512bit); + __ vpternlogq(xmm0, 150, xmm5, xmm6, Assembler::AVX_512bit); + __ vpternlogq(xmm10, 150, xmm15, xmm16, Assembler::AVX_512bit); + __ vpternlogq(xmm1, 150, xmm5, xmm6, Assembler::AVX_512bit); + __ vpternlogq(xmm11, 150, xmm15, xmm16, Assembler::AVX_512bit); + __ vpternlogq(xmm2, 150, xmm5, xmm6, Assembler::AVX_512bit); + __ vpternlogq(xmm12, 150, xmm15, xmm16, Assembler::AVX_512bit); + __ vpternlogq(xmm3, 150, xmm5, xmm6, Assembler::AVX_512bit); + __ vpternlogq(xmm13, 150, xmm15, xmm16, Assembler::AVX_512bit); + __ vpternlogq(xmm4, 150, xmm5, xmm6, Assembler::AVX_512bit); + __ vpternlogq(xmm14, 150, xmm15, xmm16, Assembler::AVX_512bit); + __ evpermt2q(xmm4, xmm17, xmm3, Assembler::AVX_512bit); + __ evpermt2q(xmm14, xmm17, xmm13, Assembler::AVX_512bit); + __ evpermt2q(xmm3, xmm18, xmm2, Assembler::AVX_512bit); + __ evpermt2q(xmm13, xmm18, xmm12, Assembler::AVX_512bit); + __ evpermt2q(xmm2, xmm17, xmm1, Assembler::AVX_512bit); + __ evpermt2q(xmm12, xmm17, xmm11, Assembler::AVX_512bit); + __ evpermt2q(xmm1, xmm19, xmm0, Assembler::AVX_512bit); + __ evpermt2q(xmm11, xmm19, xmm10, Assembler::AVX_512bit); + __ evpermt2q(xmm4, xmm20, xmm2, Assembler::AVX_512bit); + __ evpermt2q(xmm14, xmm20, xmm12, Assembler::AVX_512bit); + __ evprolvq(xmm1, xmm1, xmm27, Assembler::AVX_512bit); + __ evprolvq(xmm11, xmm11, xmm27, Assembler::AVX_512bit); + __ evprolvq(xmm3, xmm3, xmm28, Assembler::AVX_512bit); + __ evprolvq(xmm13, xmm13, xmm28, Assembler::AVX_512bit); + __ evprolvq(xmm4, xmm4, xmm29, Assembler::AVX_512bit); + __ evprolvq(xmm14, xmm14, xmm29, Assembler::AVX_512bit); + __ evmovdquw(xmm2, xmm1, Assembler::AVX_512bit); + __ evmovdquw(xmm12, xmm11, Assembler::AVX_512bit); + __ evmovdquw(xmm5, xmm3, Assembler::AVX_512bit); + __ evmovdquw(xmm15, xmm13, Assembler::AVX_512bit); + __ evpermt2q(xmm0, xmm21, xmm4, Assembler::AVX_512bit); + __ evpermt2q(xmm10, xmm21, xmm14, Assembler::AVX_512bit); + __ evpermt2q(xmm1, xmm22, xmm3, Assembler::AVX_512bit); + __ evpermt2q(xmm11, xmm22, xmm13, Assembler::AVX_512bit); + __ evpermt2q(xmm5, xmm22, xmm2, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm22, xmm12, Assembler::AVX_512bit); + __ evmovdquw(xmm3, xmm1, Assembler::AVX_512bit); + __ evmovdquw(xmm13, xmm11, Assembler::AVX_512bit); + __ evmovdquw(xmm2, xmm5, Assembler::AVX_512bit); + __ evmovdquw(xmm12, xmm15, Assembler::AVX_512bit); + __ evpermt2q(xmm1, xmm23, xmm4, Assembler::AVX_512bit); + __ evpermt2q(xmm11, xmm23, xmm14, Assembler::AVX_512bit); + __ evpermt2q(xmm2, xmm24, xmm4, Assembler::AVX_512bit); + __ evpermt2q(xmm12, xmm24, xmm14, Assembler::AVX_512bit); + __ evpermt2q(xmm3, xmm25, xmm4, Assembler::AVX_512bit); + __ evpermt2q(xmm13, xmm25, xmm14, Assembler::AVX_512bit); + __ evpermt2q(xmm4, xmm26, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm14, xmm26, xmm15, Assembler::AVX_512bit); + + __ evpermt2q(xmm5, xmm31, xmm0, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm31, xmm10, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm15, Assembler::AVX_512bit); + __ vpternlogq(xmm0, 180, xmm6, xmm5, Assembler::AVX_512bit); + __ vpternlogq(xmm10, 180, xmm16, xmm15, Assembler::AVX_512bit); + + __ evpermt2q(xmm5, xmm31, xmm1, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm31, xmm11, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm15, Assembler::AVX_512bit); + __ vpternlogq(xmm1, 180, xmm6, xmm5, Assembler::AVX_512bit); + __ vpternlogq(xmm11, 180, xmm16, xmm15, Assembler::AVX_512bit); + + __ evpxorq(xmm0, k1, xmm0, Address(constant2use, 0), true, Assembler::AVX_512bit); + __ evpxorq(xmm10, k1, xmm10, Address(constant2use, 0), true, Assembler::AVX_512bit); + __ addptr(constant2use, 8); + + __ evpermt2q(xmm5, xmm31, xmm2, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm31, xmm12, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm15, Assembler::AVX_512bit); + __ vpternlogq(xmm2, 180, xmm6, xmm5, Assembler::AVX_512bit); + __ vpternlogq(xmm12, 180, xmm16, xmm15, Assembler::AVX_512bit); + + __ evpermt2q(xmm5, xmm31, xmm3, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm31, xmm13, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm15, Assembler::AVX_512bit); + __ vpternlogq(xmm3, 180, xmm6, xmm5, Assembler::AVX_512bit); + __ vpternlogq(xmm13, 180, xmm16, xmm15, Assembler::AVX_512bit); + __ evpermt2q(xmm5, xmm31, xmm4, Assembler::AVX_512bit); + __ evpermt2q(xmm15, xmm31, xmm14, Assembler::AVX_512bit); + __ evpermt2q(xmm6, xmm31, xmm5, Assembler::AVX_512bit); + __ evpermt2q(xmm16, xmm31, xmm15, Assembler::AVX_512bit); + __ vpternlogq(xmm4, 180, xmm6, xmm5, Assembler::AVX_512bit); + __ vpternlogq(xmm14, 180, xmm16, xmm15, Assembler::AVX_512bit); + __ cmpl(roundsLeft, 0); + __ jcc(Assembler::notEqual, rounds24_loop); + + __ xorq(rax, rax); // return 0 + + // store the states + for (int i = 0; i < 5; i++) { + __ evmovdquq(Address(state0, i * 40), k5, xmm(i), true, Assembler::AVX_512bit); + } + for (int i = 0; i < 5; i++) { + __ evmovdquq(Address(state1, i * 40), k5, xmm(10 + i), true, Assembler::AVX_512bit); + } + + __ leave(); // required for proper stackwalking of RuntimeStub frame + __ ret(0); + + return start; +} + +void StubGenerator::generate_sha3_stubs() { + if (UseSHA3Intrinsics) { + StubRoutines::_sha3_implCompress = + generate_sha3_implCompress(false, "sha3_implCompress", this, _masm); + StubRoutines::_double_keccak = + generate_double_keccak(this, _masm); + StubRoutines::_sha3_implCompressMB = + generate_sha3_implCompress(true, "sha3_implCompressMB", this, _masm); + } +} diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp index a5246860c0f6..96cc36a92080 100644 --- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp +++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp @@ -37,7 +37,7 @@ enum platform_dependent_constants { _continuation_stubs_code_size = 1000 LP64_ONLY(+1000), // AVX512 intrinsics add more code in 64-bit VM, // Windows have more code to save/restore registers - _compiler_stubs_code_size = 20000 LP64_ONLY(+30000) WINDOWS_ONLY(+2000), + _compiler_stubs_code_size = 20000 LP64_ONLY(+55000) WINDOWS_ONLY(+2000), _final_stubs_code_size = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000) ZGC_ONLY(+20000) }; diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp index c28afa1e44f5..80b7bd7aa52b 100644 --- a/src/hotspot/cpu/x86/vm_version_x86.cpp +++ b/src/hotspot/cpu/x86/vm_version_x86.cpp @@ -1148,6 +1148,20 @@ void VM_Version::get_processor_features() { } #endif // _LP64 + // Dilithium Intrinsics + // Currently we only have them for AVX512 +#ifdef _LP64 + if (supports_evex() && supports_avx512bw()) { + if (FLAG_IS_DEFAULT(UseDilithiumIntrinsics)) { + UseDilithiumIntrinsics = true; + } + } else +#endif + if (UseDilithiumIntrinsics) { + warning("Intrinsics for ML-DSA are not available on this CPU."); + FLAG_SET_DEFAULT(UseDilithiumIntrinsics, false); + } + // Base64 Intrinsics (Check the condition for which the intrinsic will be active) if (UseAVX >= 2) { if (FLAG_IS_DEFAULT(UseBASE64Intrinsics)) { diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp index 63cc1fa8baf8..cf6923b848fe 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.cpp +++ b/src/hotspot/share/classfile/vmIntrinsics.cpp @@ -485,6 +485,14 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) { case vmIntrinsics::_chacha20Block: if (!UseChaCha20Intrinsics) return true; break; + case vmIntrinsics::_kyberNtt: + case vmIntrinsics::_kyberInverseNtt: + case vmIntrinsics::_kyberNttMult: + case vmIntrinsics::_kyberAddPoly_2: + case vmIntrinsics::_kyberAddPoly_3: + case vmIntrinsics::_kyber12To16: + case vmIntrinsics::_kyberBarrettReduce: + if (!UseKyberIntrinsics) return true; case vmIntrinsics::_dilithiumAlmostNtt: case vmIntrinsics::_dilithiumAlmostInverseNtt: case vmIntrinsics::_dilithiumNttMult: diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp index 2c4b9da5bc2d..a1e1aba273b1 100644 --- a/src/hotspot/share/classfile/vmIntrinsics.hpp +++ b/src/hotspot/share/classfile/vmIntrinsics.hpp @@ -554,8 +554,29 @@ class methodHandle; do_name(chacha20Block_name, "implChaCha20Block") \ do_signature(chacha20Block_signature, "([I[B)I") \ \ + /* support for com.sun.crypto.provider.ML_KEM */ \ + do_class(com_sun_crypto_provider_ML_KEM, "com/sun/crypto/provider/ML_KEM") \ + do_signature(SaSaSaSaI_signature, "([S[S[S[S)I") \ + do_signature(BaISaII_signature, "([BI[SI)I") \ + do_signature(SaSaSaI_signature, "([S[S[S)I") \ + do_signature(SaSaI_signature, "([S[S)I") \ + do_signature(SaI_signature, "([S)I") \ + do_name(kyberAddPoly_name, "implKyberAddPoly") \ + do_intrinsic(_kyberNtt, com_sun_crypto_provider_ML_KEM, kyberNtt_name, SaSaI_signature, F_S) \ + do_name(kyberNtt_name, "implKyberNtt") \ + do_intrinsic(_kyberInverseNtt, com_sun_crypto_provider_ML_KEM, kyberInverseNtt_name, SaSaI_signature, F_S) \ + do_name(kyberInverseNtt_name, "implKyberInverseNtt") \ + do_intrinsic(_kyberNttMult, com_sun_crypto_provider_ML_KEM, kyberNttMult_name, SaSaSaSaI_signature, F_S) \ + do_name(kyberNttMult_name, "implKyberNttMult") \ + do_intrinsic(_kyberAddPoly_2, com_sun_crypto_provider_ML_KEM, kyberAddPoly_name, SaSaSaI_signature, F_S) \ + do_intrinsic(_kyberAddPoly_3, com_sun_crypto_provider_ML_KEM, kyberAddPoly_name, SaSaSaSaI_signature, F_S) \ + do_intrinsic(_kyber12To16, com_sun_crypto_provider_ML_KEM, kyber12To16_name, BaISaII_signature, F_S) \ + do_name(kyber12To16_name, "implKyber12To16") \ + do_intrinsic(_kyberBarrettReduce, com_sun_crypto_provider_ML_KEM, kyberBarrettReduce_name, SaI_signature, F_S) \ + do_name(kyberBarrettReduce_name, "implKyberBarrettReduce") \ + \ /* support for sun.security.provider.ML_DSA */ \ - do_class(sun_security_provider_ML_DSA, "sun/security/provider/ML_DSA") \ + do_class(sun_security_provider_ML_DSA, "sun/security/provider/ML_DSA") \ do_signature(IaII_signature, "([II)I") \ do_signature(IaIaI_signature, "([I[I)I") \ do_signature(IaIaIaI_signature, "([I[I[I)I") \ diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp index b058f31fec9b..9ee14ae3199b 100644 --- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp +++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp @@ -352,6 +352,13 @@ static_field(StubRoutines, _sha3_implCompress, address) \ static_field(StubRoutines, _double_keccak, address) \ static_field(StubRoutines, _sha3_implCompressMB, address) \ + static_field(StubRoutines, _kyberNtt, address) \ + static_field(StubRoutines, _kyberInverseNtt, address) \ + static_field(StubRoutines, _kyberNttMult, address) \ + static_field(StubRoutines, _kyberAddPoly_2, address) \ + static_field(StubRoutines, _kyberAddPoly_3, address) \ + static_field(StubRoutines, _kyber12To16, address) \ + static_field(StubRoutines, _kyberBarrettReduce, address) \ static_field(StubRoutines, _dilithiumAlmostNtt, address) \ static_field(StubRoutines, _dilithiumAlmostInverseNtt, address) \ static_field(StubRoutines, _dilithiumNttMult, address) \ diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index eb9cd8e95402..1a43d3edad42 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -735,6 +735,13 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method) { case vmIntrinsics::_vectorizedMismatch: case vmIntrinsics::_ghash_processBlocks: case vmIntrinsics::_chacha20Block: + case vmIntrinsics::_kyberNtt: + case vmIntrinsics::_kyberInverseNtt: + case vmIntrinsics::_kyberNttMult: + case vmIntrinsics::_kyberAddPoly_2: + case vmIntrinsics::_kyberAddPoly_3: + case vmIntrinsics::_kyber12To16: + case vmIntrinsics::_kyberBarrettReduce: case vmIntrinsics::_dilithiumAlmostNtt: case vmIntrinsics::_dilithiumAlmostInverseNtt: case vmIntrinsics::_dilithiumNttMult: diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp index f71b43ed1525..81a94988ec93 100644 --- a/src/hotspot/share/opto/escape.cpp +++ b/src/hotspot/share/opto/escape.cpp @@ -1172,6 +1172,13 @@ void ConnectionGraph::process_call_arguments(CallNode *call) { strcmp(call->as_CallLeaf()->_name, "poly1305_processBlocks") == 0 || strcmp(call->as_CallLeaf()->_name, "ghash_processBlocks") == 0 || strcmp(call->as_CallLeaf()->_name, "chacha20Block") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberNtt") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberInverseNtt") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberNttMult") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberAddPoly_2") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberAddPoly_3") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyber12To16") == 0 || + strcmp(call->as_CallLeaf()->_name, "kyberBarrettReduce") == 0 || strcmp(call->as_CallLeaf()->_name, "dilithiumAlmostNtt") == 0 || strcmp(call->as_CallLeaf()->_name, "dilithiumAlmostInverseNtt") == 0 || strcmp(call->as_CallLeaf()->_name, "dilithiumNttMult") == 0 || diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index a2f341826ce9..7a0d0a6ca84c 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -629,6 +629,20 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_ghash_processBlocks(); case vmIntrinsics::_chacha20Block: return inline_chacha20Block(); + case vmIntrinsics::_kyberNtt: + return inline_kyberNtt(); + case vmIntrinsics::_kyberInverseNtt: + return inline_kyberInverseNtt(); + case vmIntrinsics::_kyberNttMult: + return inline_kyberNttMult(); + case vmIntrinsics::_kyberAddPoly_2: + return inline_kyberAddPoly_2(); + case vmIntrinsics::_kyberAddPoly_3: + return inline_kyberAddPoly_3(); + case vmIntrinsics::_kyber12To16: + return inline_kyber12To16(); + case vmIntrinsics::_kyberBarrettReduce: + return inline_kyberBarrettReduce(); case vmIntrinsics::_dilithiumAlmostNtt: return inline_dilithiumAlmostNtt(); case vmIntrinsics::_dilithiumAlmostInverseNtt: @@ -7270,6 +7284,245 @@ bool LibraryCallKit::inline_chacha20Block() { return true; } +//------------------------------inline_kyberNtt +bool LibraryCallKit::inline_kyberNtt() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 2, "kyberNtt has 2 parameters"); + + stubAddr = StubRoutines::kyberNtt(); + stubName = "kyberNtt"; + if (!stubAddr) return false; + + Node* coeffs = argument(0); + Node* ntt_zetas = argument(1); + + coeffs = must_be_not_null(coeffs, true); + ntt_zetas = must_be_not_null(ntt_zetas, true); + + Node* coeffs_start = array_element_address(coeffs, intcon(0), T_SHORT); + assert(coeffs_start, "coeffs is null"); + Node* ntt_zetas_start = array_element_address(ntt_zetas, intcon(0), T_SHORT); + assert(ntt_zetas_start, "ntt_zetas is null"); + Node* kyberNtt = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberNtt_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + coeffs_start, ntt_zetas_start); + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberNtt, TypeFunc::Parms)); + set_result(retvalue); + return true; +} + +//------------------------------inline_kyberInverseNtt +bool LibraryCallKit::inline_kyberInverseNtt() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 2, "kyberInverseNtt has 2 parameters"); + + stubAddr = StubRoutines::kyberInverseNtt(); + stubName = "kyberInverseNtt"; + if (!stubAddr) return false; + + Node* coeffs = argument(0); + Node* zetas = argument(1); + + coeffs = must_be_not_null(coeffs, true); + zetas = must_be_not_null(zetas, true); + + Node* coeffs_start = array_element_address(coeffs, intcon(0), T_SHORT); + assert(coeffs_start, "coeffs is null"); + Node* zetas_start = array_element_address(zetas, intcon(0), T_SHORT); + assert(zetas_start, "inverseNtt_zetas is null"); + Node* kyberInverseNtt = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberInverseNtt_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + coeffs_start, zetas_start); + + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberInverseNtt, TypeFunc::Parms)); + set_result(retvalue); + return true; +} + +//------------------------------inline_kyberNttMult +bool LibraryCallKit::inline_kyberNttMult() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 4, "kyberNttMult has 4 parameters"); + + stubAddr = StubRoutines::kyberNttMult(); + stubName = "kyberNttMult"; + if (!stubAddr) return false; + + Node* result = argument(0); + Node* ntta = argument(1); + Node* nttb = argument(2); + Node* zetas = argument(3); + + result = must_be_not_null(result, true); + ntta = must_be_not_null(ntta, true); + nttb = must_be_not_null(nttb, true); + zetas = must_be_not_null(zetas, true); + Node* result_start = array_element_address(result, intcon(0), T_SHORT); + assert(result_start, "result is null"); + Node* ntta_start = array_element_address(ntta, intcon(0), T_SHORT); + assert(ntta_start, "ntta is null"); + Node* nttb_start = array_element_address(nttb, intcon(0), T_SHORT); + assert(nttb_start, "nttb is null"); + Node* zetas_start = array_element_address(zetas, intcon(0), T_SHORT); + assert(zetas_start, "nttMult_zetas is null"); + Node* kyberNttMult = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberNttMult_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + result_start, ntta_start, nttb_start, + zetas_start); + + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberNttMult, TypeFunc::Parms)); + set_result(retvalue); + + return true; +} + +//------------------------------inline_kyberAddPoly_2 +bool LibraryCallKit::inline_kyberAddPoly_2() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 3, "kyberAddPoly_2 has 3 parameters"); + + stubAddr = StubRoutines::kyberAddPoly_2(); + stubName = "kyberAddPoly_2"; + if (!stubAddr) return false; + + Node* result = argument(0); + Node* a = argument(1); + Node* b = argument(2); + + result = must_be_not_null(result, true); + a = must_be_not_null(a, true); + b = must_be_not_null(b, true); + + Node* result_start = array_element_address(result, intcon(0), T_SHORT); + assert(result_start, "result is null"); + Node* a_start = array_element_address(a, intcon(0), T_SHORT); + assert(a_start, "a is null"); + Node* b_start = array_element_address(b, intcon(0), T_SHORT); + assert(b_start, "b is null"); + Node* kyberAddPoly_2 = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberAddPoly_2_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + result_start, a_start, b_start); + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberAddPoly_2, TypeFunc::Parms)); + set_result(retvalue); + return true; +} + +//------------------------------inline_kyberAddPoly_3 +bool LibraryCallKit::inline_kyberAddPoly_3() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 4, "kyberAddPoly_3 has 4 parameters"); + + stubAddr = StubRoutines::kyberAddPoly_3(); + stubName = "kyberAddPoly_3"; + if (!stubAddr) return false; + + Node* result = argument(0); + Node* a = argument(1); + Node* b = argument(2); + Node* c = argument(3); + + result = must_be_not_null(result, true); + a = must_be_not_null(a, true); + b = must_be_not_null(b, true); + c = must_be_not_null(c, true); + + Node* result_start = array_element_address(result, intcon(0), T_SHORT); + assert(result_start, "result is null"); + Node* a_start = array_element_address(a, intcon(0), T_SHORT); + assert(a_start, "a is null"); + Node* b_start = array_element_address(b, intcon(0), T_SHORT); + assert(b_start, "b is null"); + Node* c_start = array_element_address(c, intcon(0), T_SHORT); + assert(c_start, "c is null"); + Node* kyberAddPoly_3 = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberAddPoly_3_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + result_start, a_start, b_start, c_start); + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberAddPoly_3, TypeFunc::Parms)); + set_result(retvalue); + return true; +} + +//------------------------------inline_kyber12To16 +bool LibraryCallKit::inline_kyber12To16() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 4, "kyber12To16 has 4 parameters"); + + stubAddr = StubRoutines::kyber12To16(); + stubName = "kyber12To16"; + if (!stubAddr) return false; + + Node* condensed = argument(0); + Node* condensedOffs = argument(1); + Node* parsed = argument(2); + Node* parsedLength = argument(3); + + condensed = must_be_not_null(condensed, true); + parsed = must_be_not_null(parsed, true); + + Node* condensed_start = array_element_address(condensed, intcon(0), T_BYTE); + assert(condensed_start, "condensed is null"); + Node* parsed_start = array_element_address(parsed, intcon(0), T_SHORT); + assert(parsed_start, "parsed is null"); + Node* kyber12To16 = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyber12To16_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + condensed_start, condensedOffs, parsed_start, parsedLength); + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyber12To16, TypeFunc::Parms)); + set_result(retvalue); + return true; + +} + +//------------------------------inline_kyberBarrettReduce +bool LibraryCallKit::inline_kyberBarrettReduce() { + address stubAddr; + const char *stubName; + assert(UseKyberIntrinsics, "need Kyber intrinsics support"); + assert(callee()->signature()->size() == 1, "kyberBarrettReduce has 1 parameters"); + + stubAddr = StubRoutines::kyberBarrettReduce(); + stubName = "kyberBarrettReduce"; + if (!stubAddr) return false; + + Node* coeffs = argument(0); + + coeffs = must_be_not_null(coeffs, true); + + Node* coeffs_start = array_element_address(coeffs, intcon(0), T_SHORT); + assert(coeffs_start, "coeffs is null"); + Node* kyberBarrettReduce = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::kyberBarrettReduce_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + coeffs_start); + // return an int + Node* retvalue = _gvn.transform(new ProjNode(kyberBarrettReduce, TypeFunc::Parms)); + set_result(retvalue); + return true; +} + //------------------------------inline_dilithiumAlmostNtt bool LibraryCallKit::inline_dilithiumAlmostNtt() { address stubAddr; @@ -7326,7 +7579,6 @@ bool LibraryCallKit::inline_dilithiumAlmostInverseNtt() { OptoRuntime::dilithiumAlmostInverseNtt_Type(), stubAddr, stubName, TypePtr::BOTTOM, coeffs_start, zetas_start); - // return an int Node* retvalue = _gvn.transform(new ProjNode(dilithiumAlmostInverseNtt, TypeFunc::Parms)); set_result(retvalue); @@ -7347,10 +7599,12 @@ bool LibraryCallKit::inline_dilithiumNttMult() { Node* result = argument(0); Node* ntta = argument(1); Node* nttb = argument(2); + Node* zetas = argument(3); result = must_be_not_null(result, true); ntta = must_be_not_null(ntta, true); nttb = must_be_not_null(nttb, true); + zetas = must_be_not_null(zetas, true); Node* result_start = array_element_address(result, intcon(0), T_INT); assert(result_start, "result is null"); diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 5449d2f6d590..3a6d70734965 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -301,6 +301,13 @@ class LibraryCallKit : public GraphKit { Node* get_key_start_from_aescrypt_object(Node* aescrypt_object); bool inline_ghash_processBlocks(); bool inline_chacha20Block(); + bool inline_kyberNtt(); + bool inline_kyberInverseNtt(); + bool inline_kyberNttMult(); + bool inline_kyberAddPoly_2(); + bool inline_kyberAddPoly_3(); + bool inline_kyber12To16(); + bool inline_kyberBarrettReduce(); bool inline_dilithiumAlmostNtt(); bool inline_dilithiumAlmostInverseNtt(); bool inline_dilithiumNttMult(); diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp index 66e7e28db99c..5af4ebcf911d 100644 --- a/src/hotspot/share/opto/runtime.cpp +++ b/src/hotspot/share/opto/runtime.cpp @@ -1289,6 +1289,146 @@ const TypeFunc* OptoRuntime::chacha20Block_Type() { return TypeFunc::make(domain, range); } +// Kyber NTT function +const TypeFunc* OptoRuntime::kyberNtt_Type() { + int argcnt = 2; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // coeffs + fields[argp++] = TypePtr::NOTNULL; // NTT zetas + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + +// Kyber inverse NTT function +const TypeFunc* OptoRuntime::kyberInverseNtt_Type() { + int argcnt = 2; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // coeffs + fields[argp++] = TypePtr::NOTNULL; // inverse NTT zetas + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + +// Kyber NTT multiply function +const TypeFunc* OptoRuntime::kyberNttMult_Type() { + int argcnt = 4; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // result + fields[argp++] = TypePtr::NOTNULL; // ntta + fields[argp++] = TypePtr::NOTNULL; // nttb + fields[argp++] = TypePtr::NOTNULL; // NTT multiply zetas + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} +// Kyber add 2 polynomials function +const TypeFunc* OptoRuntime::kyberAddPoly_2_Type() { + int argcnt = 3; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // result + fields[argp++] = TypePtr::NOTNULL; // a + fields[argp++] = TypePtr::NOTNULL; // b + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + + +// Kyber add 3 polynomials function +const TypeFunc* OptoRuntime::kyberAddPoly_3_Type() { + int argcnt = 4; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // result + fields[argp++] = TypePtr::NOTNULL; // a + fields[argp++] = TypePtr::NOTNULL; // b + fields[argp++] = TypePtr::NOTNULL; // c + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + + +// Kyber XOF output parsing into polynomial coefficients candidates +// or decompress(12,...) function +const TypeFunc* OptoRuntime::kyber12To16_Type() { + int argcnt = 4; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // condensed + fields[argp++] = TypeInt::INT; // condensedOffs + fields[argp++] = TypePtr::NOTNULL; // parsed + fields[argp++] = TypeInt::INT; // parsedLength + + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + +// Kyber Barrett reduce function +const TypeFunc* OptoRuntime::kyberBarrettReduce_Type() { + int argcnt = 1; + + const Type** fields = TypeTuple::fields(argcnt); + int argp = TypeFunc::Parms; + fields[argp++] = TypePtr::NOTNULL; // coeffs + assert(argp == TypeFunc::Parms + argcnt, "correct decoding"); + const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields); + + // result type needed + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms + 0] = TypeInt::INT; + const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields); + return TypeFunc::make(domain, range); +} + // Dilithium NTT function except for the final "normalization" to |coeff| < Q const TypeFunc* OptoRuntime::dilithiumAlmostNtt_Type() { int argcnt = 2; diff --git a/src/hotspot/share/opto/runtime.hpp b/src/hotspot/share/opto/runtime.hpp index c00f9228716b..ef945cb5535c 100644 --- a/src/hotspot/share/opto/runtime.hpp +++ b/src/hotspot/share/opto/runtime.hpp @@ -292,6 +292,13 @@ class OptoRuntime : public AllStatic { static const TypeFunc* ghash_processBlocks_Type(); static const TypeFunc* chacha20Block_Type(); + static const TypeFunc* kyberNtt_Type(); + static const TypeFunc* kyberInverseNtt_Type(); + static const TypeFunc* kyberNttMult_Type(); + static const TypeFunc* kyberAddPoly_2_Type(); + static const TypeFunc* kyberAddPoly_3_Type(); + static const TypeFunc* kyber12To16_Type(); + static const TypeFunc* kyberBarrettReduce_Type(); static const TypeFunc* dilithiumAlmostNtt_Type(); static const TypeFunc* dilithiumAlmostInverseNtt_Type(); static const TypeFunc* dilithiumNttMult_Type(); diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index a56d12b6c79b..34cda5917baa 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -332,6 +332,8 @@ const int ObjectAlignmentInBytes = 8; product(bool, UseChaCha20Intrinsics, false, DIAGNOSTIC, \ "Use intrinsics for the vectorized version of ChaCha20") \ \ + product(bool, UseKyberIntrinsics, false, DIAGNOSTIC, \ + "Use intrinsics for the vectorized version of Kyber") \ product(bool, UseDilithiumIntrinsics, false, DIAGNOSTIC, \ "Use intrinsics for the vectorized version of Dilithium") \ \ diff --git a/src/hotspot/share/runtime/stubRoutines.cpp b/src/hotspot/share/runtime/stubRoutines.cpp index 4b80a34d3f4b..f782d240b5cc 100644 --- a/src/hotspot/share/runtime/stubRoutines.cpp +++ b/src/hotspot/share/runtime/stubRoutines.cpp @@ -127,6 +127,13 @@ address StubRoutines::_counterMode_AESCrypt = nullptr; address StubRoutines::_galoisCounterMode_AESCrypt = nullptr; address StubRoutines::_ghash_processBlocks = nullptr; address StubRoutines::_chacha20Block = nullptr; +address StubRoutines::_kyberNtt = nullptr; +address StubRoutines::_kyberInverseNtt = nullptr; +address StubRoutines::_kyberNttMult = nullptr; +address StubRoutines::_kyberAddPoly_2 = nullptr; +address StubRoutines::_kyberAddPoly_3 = nullptr; +address StubRoutines::_kyber12To16 = nullptr; +address StubRoutines::_kyberBarrettReduce = nullptr; address StubRoutines::_dilithiumAlmostNtt = nullptr; address StubRoutines::_dilithiumAlmostInverseNtt = nullptr; address StubRoutines::_dilithiumNttMult = nullptr; diff --git a/src/hotspot/share/runtime/stubRoutines.hpp b/src/hotspot/share/runtime/stubRoutines.hpp index 014aba74fbb4..503d8c2a2ba4 100644 --- a/src/hotspot/share/runtime/stubRoutines.hpp +++ b/src/hotspot/share/runtime/stubRoutines.hpp @@ -207,6 +207,13 @@ class StubRoutines: AllStatic { static address _galoisCounterMode_AESCrypt; static address _ghash_processBlocks; static address _chacha20Block; + static address _kyberNtt; + static address _kyberInverseNtt; + static address _kyberNttMult; + static address _kyberAddPoly_2; + static address _kyberAddPoly_3; + static address _kyber12To16; + static address _kyberBarrettReduce; static address _dilithiumAlmostNtt; static address _dilithiumAlmostInverseNtt; static address _dilithiumNttMult; @@ -402,6 +409,13 @@ class StubRoutines: AllStatic { static address counterMode_AESCrypt() { return _counterMode_AESCrypt; } static address ghash_processBlocks() { return _ghash_processBlocks; } static address chacha20Block() { return _chacha20Block; } + static address kyberNtt() { return _kyberNtt; } + static address kyberInverseNtt() { return _kyberInverseNtt; } + static address kyberNttMult() { return _kyberNttMult; } + static address kyberAddPoly_2() { return _kyberAddPoly_2; } + static address kyberAddPoly_3() { return _kyberAddPoly_3; } + static address kyber12To16() { return _kyber12To16; } + static address kyberBarrettReduce() { return _kyberBarrettReduce; } static address dilithiumAlmostNtt() { return _dilithiumAlmostNtt; } static address dilithiumAlmostInverseNtt() { return _dilithiumAlmostInverseNtt; } static address dilithiumNttMult() { return _dilithiumNttMult; } diff --git a/src/hotspot/share/runtime/vmStructs.cpp b/src/hotspot/share/runtime/vmStructs.cpp index 9d8034561647..51182b40d3e8 100644 --- a/src/hotspot/share/runtime/vmStructs.cpp +++ b/src/hotspot/share/runtime/vmStructs.cpp @@ -540,6 +540,13 @@ static_field(StubRoutines, _galoisCounterMode_AESCrypt, address) \ static_field(StubRoutines, _ghash_processBlocks, address) \ static_field(StubRoutines, _chacha20Block, address) \ + static_field(StubRoutines, _kyberNtt, address) \ + static_field(StubRoutines, _kyberInverseNtt, address) \ + static_field(StubRoutines, _kyberNttMult, address) \ + static_field(StubRoutines, _kyberAddPoly_2, address) \ + static_field(StubRoutines, _kyberAddPoly_3, address) \ + static_field(StubRoutines, _kyber12To16, address) \ + static_field(StubRoutines, _kyberBarrettReduce, address) \ static_field(StubRoutines, _dilithiumAlmostNtt, address) \ static_field(StubRoutines, _dilithiumAlmostInverseNtt, address) \ static_field(StubRoutines, _dilithiumNttMult, address) \ diff --git a/src/java.base/share/classes/com/sun/crypto/provider/ML_KEM.java b/src/java.base/share/classes/com/sun/crypto/provider/ML_KEM.java index 9808a0133032..b45b655e1f3e 100644 --- a/src/java.base/share/classes/com/sun/crypto/provider/ML_KEM.java +++ b/src/java.base/share/classes/com/sun/crypto/provider/ML_KEM.java @@ -28,6 +28,7 @@ import java.security.*; import java.util.Arrays; import javax.crypto.DecapsulateException; +import jdk.internal.vm.annotation.IntrinsicCandidate; import sun.security.provider.SHA3.SHAKE256; import sun.security.provider.SHA3Parallel.Shake128Parallel; @@ -71,6 +72,268 @@ public final class ML_KEM { -1599, -709, -789, -1317, -57, 1049, -584 }; + private static final short[] montZetasForVectorNttArr = new short[]{ + // level 0 + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + -758, -758, -758, -758, -758, -758, -758, -758, + // level 1 + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -359, -359, -359, -359, -359, -359, -359, -359, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + -1517, -1517, -1517, -1517, -1517, -1517, -1517, -1517, + // level 2 + 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, + 1493, 1493, 1493, 1493, 1493, 1493, 1493, 1493, + 1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, + 1422, 1422, 1422, 1422, 1422, 1422, 1422, 1422, + 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, + 202, 202, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 202, + // level 3 + -171, -171, -171, -171, -171, -171, -171, -171, + -171, -171, -171, -171, -171, -171, -171, -171, + 622, 622, 622, 622, 622, 622, 622, 622, + 622, 622, 622, 622, 622, 622, 622, 622, + 1577, 1577, 1577, 1577, 1577, 1577, 1577, 1577, + 1577, 1577, 1577, 1577, 1577, 1577, 1577, 1577, + 182, 182, 182, 182, 182, 182, 182, 182, + 182, 182, 182, 182, 182, 182, 182, 182, + 962, 962, 962, 962, 962, 962, 962, 962, + 962, 962, 962, 962, 962, 962, 962, 962, + -1202, -1202, -1202, -1202, -1202, -1202, -1202, -1202, + -1202, -1202, -1202, -1202, -1202, -1202, -1202, -1202, + -1474, -1474, -1474, -1474, -1474, -1474, -1474, -1474, + -1474, -1474, -1474, -1474, -1474, -1474, -1474, -1474, + 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, + 1468, 1468, 1468, 1468, 1468, 1468, 1468, 1468, + // level 4 + 573, 573, 573, 573, 573, 573, 573, 573, + -1325, -1325, -1325, -1325, -1325, -1325, -1325, -1325, + 264, 264, 264, 264, 264, 264, 264, 264, + 383, 383, 383, 383, 383, 383, 383, 383, + -829, -829, -829, -829, -829, -829, -829, -829, + 1458, 1458, 1458, 1458, 1458, 1458, 1458, 1458, + -1602, -1602, -1602, -1602, -1602, -1602, -1602, -1602, + -130, -130, -130, -130, -130, -130, -130, -130, + -681, -681, -681, -681, -681, -681, -681, -681, + 1017, 1017, 1017, 1017, 1017, 1017, 1017, 1017, + 732, 732, 732, 732, 732, 732, 732, 732, + 608, 608, 608, 608, 608, 608, 608, 608, + -1542, -1542, -1542, -1542, -1542, -1542, -1542, -1542, + 411, 411, 411, 411, 411, 411, 411, 411, + -205, -205, -205, -205, -205, -205, -205, -205, + -1571, -1571, -1571, -1571, -1571, -1571, -1571, -1571, + // level 5 + 1223, 1223, 1223, 1223, 652, 652, 652, 652, + -552, -552, -552, -552, 1015, 1015, 1015, 1015, + -1293, -1293, -1293, -1293, 1491, 1491, 1491, 1491, + -282, -282, -282, -282, -1544, -1544, -1544, -1544, + 516, 516, 516, 516, -8, -8, -8, -8, + -320, -320, -320, -320, -666, -666, -666, -666, + 1711, 1711, 1711, 1711, -1162, -1162, -1162, -1162, + 126, 126, 126, 126, 1469, 1469, 1469, 1469, + -853, -853, -853, -853, -90, -90, -90, -90, + -271, -271, -271, -271, 830, 830, 830, 830, + 107, 107, 107, 107, -1421, -1421, -1421, -1421, + -247, -247, -247, -247, -951, -951, -951, -951, + -398, -398, -398, -398, 961, 961, 961, 961, + -1508, -1508, -1508, -1508, -725, -725, -725, -725, + 448, 448, 448, 448, -1065, -1065, -1065, -1065, + 677, 677, 677, 677, -1275, -1275, -1275, -1275, + // level 6 + -1103, -1103, 430, 430, 555, 555, 843, 843, + -1251, -1251, 871, 871, 1550, 1550, 105, 105, + 422, 422, 587, 587, 177, 177, -235, -235, + -291, -291, -460, -460, 1574, 1574, 1653, 1653, + -246, -246, 778, 778, 1159, 1159, -147, -147, + -777, -777, 1483, 1483, -602, -602, 1119, 1119, + -1590, -1590, 644, 644, -872, -872, 349, 349, + 418, 418, 329, 329, -156, -156, -75, -75, + 817, 817, 1097, 1097, 603, 603, 610, 610, + 1322, 1322, -1285, -1285, -1465, -1465, 384, 384, + -1215, -1215, -136, -136, 1218, 1218, -1335, -1335, + -874, -874, 220, 220, -1187, -1187, 1670, 1670, + -1185, -1185, -1530, -1530, -1278, -1278, 794, 794, + -1510, -1510, -854, -854, -870, -870, 478, 478, + -108, -108, -308, -308, 996, 996, 991, 991, + 958, 958, -1460, -1460, 1522, 1522, 1628, 1628 + }; + private static final int[] MONT_ZETAS_FOR_INVERSE_NTT = new int[]{ + 584, -1049, 57, 1317, 789, 709, 1599, -1601, + -990, 604, 348, 857, 612, 474, 1177, -1014, + -88, -982, -191, 668, 1386, 486, -1153, -534, + 514, 137, 586, -1178, 227, 339, -907, 244, + 1200, -833, 1394, -30, 1074, 636, -317, -1192, + -1259, -355, -425, -884, -977, 1430, 868, 607, + 184, 1448, 702, 1327, 431, 497, 595, -94, + 1649, -1497, -620, 42, -172, 1107, -222, 1003, + 426, -845, 395, -510, 1613, 825, 1269, -290, + -1429, 623, -567, 1617, 36, 1007, 1440, 332, + -201, 1313, -1382, -744, 669, -1538, 128, -1598, + 1401, 1183, -553, 714, 405, -1155, -445, 406, + -1496, -49, 82, 1369, 259, 1604, 373, 909, + -1249, -1000, -25, -52, 530, -895, 1226, 819, + -185, 281, -742, 1253, 417, 1400, 35, -593, + 97, -1263, 551, -585, 969, -914, -1188 + }; + + private static final short[] montZetasForVectorInverseNttArr = new short[]{ + // level 0 + -1628, -1628, -1522, -1522, 1460, 1460, -958, -958, + -991, -991, -996, -996, 308, 308, 108, 108, + -478, -478, 870, 870, 854, 854, 1510, 1510, + -794, -794, 1278, 1278, 1530, 1530, 1185, 1185, + 1659, 1659, 1187, 1187, -220, -220, 874, 874, + 1335, 1335, -1218, -1218, 136, 136, 1215, 1215, + -384, -384, 1465, 1465, 1285, 1285, -1322, -1322, + -610, -610, -603, -603, -1097, -1097, -817, -817, + 75, 75, 156, 156, -329, -329, -418, -418, + -349, -349, 872, 872, -644, -644, 1590, 1590, + -1119, -1119, 602, 602, -1483, -1483, 777, 777, + 147, 147, -1159, -1159, -778, -778, 246, 246, + -1653, -1653, -1574, -1574, 460, 460, 291, 291, + 235, 235, -177, -177, -587, -587, -422, -422, + -105, -105, -1550, -1550, -871, -871, 1251, 1251, + -843, -843, -555, -555, -430, -430, 1103, 1103, + // level 1 + 1275, 1275, 1275, 1275, -677, -677, -677, -677, + 1065, 1065, 1065, 1065, -448, -448, -448, -448, + 725, 725, 725, 725, 1508, 1508, 1508, 1508, + -961, -961, -961, -961, 398, 398, 398, 398, + 951, 951, 951, 951, 247, 247, 247, 247, + 1421, 1421, 1421, 1421, -107, -107, -107, -107, + -830, -830, -830, -830, 271, 271, 271, 271, + 90, 90, 90, 90, 853, 853, 853, 853, + -1469, -1469, -1469, -1469, -126, -126, -126, -126, + 1162, 1162, 1162, 1162, 1618, 1618, 1618, 1618, + 666, 666, 666, 666, 320, 320, 320, 320, + 8, 8, 8, 8, -516, -516, -516, -516, + 1544, 1544, 1544, 1544, 282, 282, 282, 282, + -1491, -1491, -1491, -1491, 1293, 1293, 1293, 1293, + -1015, -1015, -1015, -1015, 552, 552, 552, 552, + -652, -652, -652, -652, -1223, -1223, -1223, -1223, + // level 2 + 1571, 1571, 1571, 1571, 1571, 1571, 1571, 1571, + 205, 205, 205, 205, 205, 205, 205, 205, + -411, -411, -411, -411, -411, -411, -411, -411, + 1542, 1542, 1542, 1542, 1542, 1542, 1542, 1542, + -608, -608, -608, -608, -608, -608, -608, -608, + -732, -732, -732, -732, -732, -732, -732, -732, + -1017, -1017, -1017, -1017, -1017, -1017, -1017, -1017, + 681, 681, 681, 681, 681, 681, 681, 681, + 130, 130, 130, 130, 130, 130, 130, 130, + 1602, 1602, 1602, 1602, 1602, 1602, 1602, 1602, + -1458, -1458, -1458, -1458, -1458, -1458, -1458, -1458, + 829, 829, 829, 829, 829, 829, 829, 829, + -383, -383, -383, -383, -383, -383, -383, -383, + -264, -264, -264, -264, -264, -264, -264, -264, + 1325, 1325, 1325, 1325, 1325, 1325, 1325, 1325, + -573, -573, -573, -573, -573, -573, -573, -573, + // level 3 + -1468, -1468, -1468, -1468, -1468, -1468, -1468, -1468, + -1468, -1468, -1468, -1468, -1468, -1468, -1468, -1468, + 1474, 1474, 1474, 1474, 1474, 1474, 1474, 1474, + 1474, 1474, 1474, 1474, 1474, 1474, 1474, 1474, + 1202, 1202, 1202, 1202, 1202, 1202, 1202, 1202, + 1202, 1202, 1202, 1202, 1202, 1202, 1202, 1202, + -962, -962, -962, -962, -962, -962, -962, -962, + -962, -962, -962, -962, -962, -962, -962, -962, + -182, -182, -182, -182, -182, -182, -182, -182, + -182, -182, -182, -182, -182, -182, -182, -182, + -1577, -1577, -1577, -1577, -1577, -1577, -1577, -1577, + -1577, -1577, -1577, -1577, -1577, -1577, -1577, -1577, + -622, -622, -622, -622, -622, -622, -622, -622, + -622, -622, -622, -622, -622, -622, -622, -622, + 171, 171, 171, 171, 171, 171, 171, 171, + 171, 171, 171, 171, 171, 171, 171, 171, + // level 4 + -202, -202, -202, -202, -202, -202, -202, -202, + -202, -202, -202, -202, -202, -202, -202, -202, + -202, -202, -202, -202, -202, -202, -202, -202, + -202, -202, -202, -202, -202, -202, -202, -202, + -287, -287, -287, -287, -287, -287, -287, -287, + -287, -287, -287, -287, -287, -287, -287, -287, + -287, -287, -287, -287, -287, -287, -287, -287, + -287, -287, -287, -287, -287, -287, -287, -287, + -1422, -1422, -1422, -1422, -1422, -1422, -1422, -1422, + -1422, -1422, -1422, -1422, -1422, -1422, -1422, -1422, + -1422, -1422, -1422, -1422, -1422, -1422, -1422, -1422, + -1422, -1422, -1422, -1422, -1422, -1422, -1422, -1422, + -1493, -1493, -1493, -1493, -1493, -1493, -1493, -1493, + -1493, -1493, -1493, -1493, -1493, -1493, -1493, -1493, + -1493, -1493, -1493, -1493, -1493, -1493, -1493, -1493, + -1493, -1493, -1493, -1493, -1493, -1493, -1493, -1493, + // level 5 + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 1517, 1517, 1517, 1517, 1517, 1517, 1517, 1517, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, + // level 6 + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758, + 758, 758, 758, 758, 758, 758, 758, 758 + }; + private static final int[] MONT_ZETAS_FOR_NTT_MULT = new int[]{ -1003, 1003, 222, -222, -1107, 1107, 172, -172, -42, 42, 620, -620, 1497, -1497, -1649, 1649, @@ -89,6 +352,24 @@ public final class ML_KEM { 1601, -1601, -1599, 1599, -709, 709, -789, 789, -1317, 1317, -57, 57, 1049, -1049, -584, 584 }; + private static final short[] montZetasForVectorNttMultArr = new short[]{ + -1103, 1103, 430, -430, 555, -555, 843, -843, + -1251, 1251, 871, -871, 1550, -1550, 105, -105, + 422, -422, 587, -587, 177, -177, -235, 235, + -291, 291, -460, 460, 1574, -1574, 1653, -1653, + -246, 246, 778, -778, 1159, -1159, -147, 147, + -777, 777, 1483, -1483, -602, 602, 1119, -1119, + -1590, 1590, 644, -644, -872, 872, 349, -349, + 418, -418, 329, -329, -156, 156, -75, 75, + 817, -817, 1097, -1097, 603, -603, 610, -610, + 1322, -1322, -1285, 1285, -1465, 1465, 384, -384, + -1215, 1215, -136, 136, 1218, -1218, -1335, 1335, + -874, 874, 220, -220, -1187, 1187, 1670, 1659, + -1185, 1185, -1530, 1530, -1278, 1278, 794, -794, + -1510, 1510, -854, 854, -870, 870, 478, -478, + -108, 108, -308, 308, 996, -996, 991, -991, + 958, -958, -1460, 1460, 1522, -1522, 1628, -1628 + }; private final int mlKem_k; private final int mlKem_eta1; @@ -261,7 +542,7 @@ protected ML_KEM_EncapsulateResult encapsulate( try { mlKemH = MessageDigest.getInstance(HASH_H_NAME); mlKemG = MessageDigest.getInstance(HASH_G_NAME); - } catch (NoSuchAlgorithmException e){ + } catch (NoSuchAlgorithmException e) { // This should never happen. throw new RuntimeException(e); } @@ -527,7 +808,7 @@ private short[][][] generateA(byte[] rho, Boolean transposed) { for (int i = 0; i < mlKem_k; i++) { for (int j = 0; j < mlKem_k; j++) { - xofBufArr[parInd] = seedBuf.clone(); + System.arraycopy(seedBuf, 0, xofBufArr[parInd], 0, seedBuf.length); if (transposed) { xofBufArr[parInd][rhoLen] = (byte) i; xofBufArr[parInd][rhoLen + 1] = (byte) j; @@ -707,9 +988,13 @@ private short[][] mlKemVectorInverseNTT(short[][] vector) { return vector; } - // The elements of poly should be in the range [-ML_KEM_Q, ML_KEM_Q] - // The elements of poly at return will be in the range of [0, ML_KEM_Q] - private void mlKemNTT(short[] poly) { + @IntrinsicCandidate + static int implKyberNtt(short[] poly, short[] ntt_zetas) { + implKyberNttJava(poly); + return 1; + } + + static void implKyberNttJava(short[] poly) { int[] coeffs = new int[ML_KEM_N]; for (int m = 0; m < ML_KEM_N; m++) { coeffs[m] = poly[m]; @@ -718,12 +1003,23 @@ private void mlKemNTT(short[] poly) { for (int m = 0; m < ML_KEM_N; m++) { poly[m] = (short) coeffs[m]; } + } + + // The elements of poly should be in the range [-mlKem_q, mlKem_q] + // The elements of poly at return will be in the range of [0, mlKem_q] + private void mlKemNTT(short[] poly) { + assert poly.length == ML_KEM_N; + implKyberNtt(poly, montZetasForVectorNttArr); mlKemBarrettReduce(poly); } - // Works in place, but also returns its (modified) input so that it can - // be used in expressions - private short[] mlKemInverseNTT(short[] poly) { + @IntrinsicCandidate + static int implKyberInverseNtt(short[] poly, short[] zetas) { + implKyberInverseNttJava(poly); + return 1; + } + + static void implKyberInverseNttJava(short[] poly) { int[] coeffs = new int[ML_KEM_N]; for (int m = 0; m < ML_KEM_N; m++) { coeffs[m] = poly[m]; @@ -732,6 +1028,13 @@ private short[] mlKemInverseNTT(short[] poly) { for (int m = 0; m < ML_KEM_N; m++) { poly[m] = (short) coeffs[m]; } + } + + // Works in place, but also returns its (modified) input so that it can + // be used in expressions + private short[] mlKemInverseNTT(short[] poly) { + assert poly.length == ML_KEM_N; + implKyberInverseNtt(poly, montZetasForVectorInverseNttArr); return poly; } @@ -822,11 +1125,16 @@ private short[] mlKemVectorScalarMult(short[][] a, short[][] b) { return result; } - // Multiplies two polynomials represented in the NTT domain. - // The result is a representation of the product still in the NTT domain. - // The coefficients in the result are in the range (-ML_KEM_Q, ML_KEM_Q). - private void nttMult(short[] result, short[] ntta, short[] nttb) { + @IntrinsicCandidate + static int implKyberNttMult(short[] result, short[] ntta, short[] nttb, + short[] zetas) { + implKyberNttMultJava(result, ntta, nttb); + return 1; + } + + static void implKyberNttMultJava(short[] result, short[] ntta, short[] nttb) { for (int m = 0; m < ML_KEM_N / 2; m++) { + int a0 = ntta[2 * m]; int a1 = ntta[2 * m + 1]; int b0 = nttb[2 * m]; @@ -839,6 +1147,15 @@ private void nttMult(short[] result, short[] ntta, short[] nttb) { } } + // Multiplies two polynomials represented in the NTT domain. + // The result is a representation of the product still in the NTT domain. + // The coefficients in the result are in the range (-mlKem_q, mlKem_q). + private void nttMult(short[] result, short[] ntta, short[] nttb) { + assert (result.length == ML_KEM_N) && (ntta.length == ML_KEM_N) && + (nttb.length == ML_KEM_N); + implKyberNttMult(result, ntta, nttb, montZetasForVectorNttMultArr); + } + // Adds the vector of polynomials b to a in place, i.e. a will hold // the result. It also returns (the modified) a so that it can be used // in an expression. @@ -853,15 +1170,41 @@ private short[][] mlKemAddVec(short[][] a, short[][] b) { return a; } + @IntrinsicCandidate + static int implKyberAddPoly(short[] result, short[] a, short[] b) { + implKyberAddPolyJava(result, a, b); + return 1; + } + + static void implKyberAddPolyJava(short[] result, short[] a, short[] b) { + for (int m = 0; m < ML_KEM_N; m++) { + int r = a[m] + b[m] + ML_KEM_Q; // This makes r > - ML_KEM_Q + a[m] = (short) r; + } + mlKemBarrettReduce(a); + } + // Adds the polynomial b to a in place, i.e. (the modified) a will hold // the result. // The coefficients are supposed be greater than -ML_KEM_Q in a and // greater than -ML_KEM_Q and less than ML_KEM_Q in b. // The coefficients in the result are greater than -ML_KEM_Q. - private void mlKemAddPoly(short[] a, short[] b) { + private short[] mlKemAddPoly(short[] a, short[] b) { + assert (a.length == ML_KEM_N) && (b.length == ML_KEM_N); + implKyberAddPoly(a, a, b); + return a; + } + + @IntrinsicCandidate + static int implKyberAddPoly(short[] result, short[] a, short[] b, short[] c) { + implKyberAddPolyJava(result, a, b, c); + return 1; + } + + static void implKyberAddPolyJava(short[] result, short[] a, short[] b, short[] c) { for (int m = 0; m < ML_KEM_N; m++) { - int r = a[m] + b[m] + ML_KEM_Q; // This makes r > -ML_KEM_Q - a[m] = (short) r; + int r = a[m] + b[m] + c[m] + 2 * ML_KEM_Q; // This makes r > - ML_KEM_Q + result[m] = (short) r; } } @@ -871,10 +1214,9 @@ private void mlKemAddPoly(short[] a, short[] b) { // greater than -ML_KEM_Q and less than ML_KEM_Q. // The coefficients in the result are nonnegative and less than ML_KEM_Q. private short[] mlKemAddPoly(short[] a, short[] b, short[] c) { - for (int m = 0; m < ML_KEM_N; m++) { - int r = a[m] + b[m] + c[m] + 2 * ML_KEM_Q; // This makes r > - ML_KEM_Q - a[m] = (short) r; - } + assert (a.length == ML_KEM_N) && (b.length == ML_KEM_N) && + (c.length == ML_KEM_N); + implKyberAddPoly(a, a, b, c); mlKemBarrettReduce(a); return a; } @@ -997,15 +1339,13 @@ private short[][] decodeVector(int l, byte[] encodedVector) { return result; } - // The intrinsic implementations assume that the input and output buffers - // are such that condensed can be read in 192-byte chunks and - // parsed can be written in 128 shorts chunks. In other words, - // if (i - 1) * 128 < parsedLengths <= i * 128 then - // parsed.size should be at least i * 128 and - // condensed.size should be at least index + i * 192 - private void twelve2Sixteen(byte[] condensed, int index, - short[] parsed, int parsedLength) { + @IntrinsicCandidate + private static int implKyber12To16(byte[] condensed, int index, short[] parsed, int parsedLength) { + implKyber12To16Java(condensed, index, parsed, parsedLength); + return 1; + } + private static void implKyber12To16Java(byte[] condensed, int index, short[] parsed, int parsedLength) { for (int i = 0; i < parsedLength * 3 / 2; i += 3) { parsed[(i / 3) * 2] = (short) ((condensed[i + index] & 0xff) + 256 * (condensed[i + index + 1] & 0xf)); @@ -1014,6 +1354,25 @@ private void twelve2Sixteen(byte[] condensed, int index, } } + // The intrinsic implementations assume that the input and output buffers + // are such that condensed can be read in 96-byte chunks and + // parsed can be written in 64 shorts chunks except for the last chunk + // that can be either 48 or 64 shorts. In other words, + // if (i - 1) * 64 < parsedLengths <= i * 64 then + // parsed.length should be either i * 64 or (i-1) * 64 + 48 and + // condensed.length should be at least index + i * 96. + private void twelve2Sixteen(byte[] condensed, int index, + short[] parsed, int parsedLength) { + int i = parsedLength / 64; + int remainder = parsedLength - i * 64; + if (remainder != 0) { + i++; + } + assert ((remainder == 0) || (remainder == 48)) && + (index + i * 96 <= condensed.length); + implKyber12To16(condensed, index, parsed, parsedLength); + } + private static void decodePoly5(byte[] condensed, int index, short[] parsed) { int j = index; for (int i = 0; i < ML_KEM_N; i += 8) { @@ -1152,6 +1511,19 @@ private static short[] decompressDecode(byte[] input) { return result; } + @IntrinsicCandidate + static int implKyberBarrettReduce(short[] coeffs) { + implKyberBarrettReduceJava(coeffs); + return 1; + } + + static void implKyberBarrettReduceJava(short[] poly) { + for (int m = 0; m < ML_KEM_N; m++) { + int tmp = ((int) poly[m] * BARRETT_MULTIPLIER) >> BARRETT_SHIFT; + poly[m] = (short) (poly[m] - tmp * ML_KEM_Q); + } + } + // The input elements can have any short value. // Modifies poly such that upon return poly[i] will be // in the range [0, ML_KEM_Q] and will be congruent with the original @@ -1161,11 +1533,9 @@ private static short[] decompressDecode(byte[] input) { // That means that if the original poly[i] > -ML_KEM_Q then at return it // will be in the range [0, ML_KEM_Q), i.e. it will be the canonical // representative of its residue class. - private void mlKemBarrettReduce(short[] poly) { - for (int m = 0; m < ML_KEM_N; m++) { - int tmp = ((int) poly[m] * BARRETT_MULTIPLIER) >> BARRETT_SHIFT; - poly[m] = (short) (poly[m] - tmp * ML_KEM_Q); - } + private static void mlKemBarrettReduce(short[] poly) { + assert poly.length == ML_KEM_N; + implKyberBarrettReduce(poly); } // Precondition: -(2^MONT_R_BITS -1) * MONT_Q <= b * c < (2^MONT_R_BITS - 1) * MONT_Q diff --git a/src/java.base/share/classes/sun/security/provider/ML_DSA.java b/src/java.base/share/classes/sun/security/provider/ML_DSA.java index 969b8fffa39f..ff25eb527efd 100644 --- a/src/java.base/share/classes/sun/security/provider/ML_DSA.java +++ b/src/java.base/share/classes/sun/security/provider/ML_DSA.java @@ -26,7 +26,6 @@ package sun.security.provider; import jdk.internal.vm.annotation.IntrinsicCandidate; -import sun.security.provider.SHA3.SHAKE128; import sun.security.provider.SHA3.SHAKE256; import sun.security.provider.SHA3Parallel.Shake128Parallel; @@ -1317,6 +1316,7 @@ private int[][] useHint(boolean[][] h, int[][] r) { */ public static void mlDsaNtt(int[] coeffs) { + assert coeffs.length == ML_DSA_N; implDilithiumAlmostNtt(coeffs, MONT_ZETAS_FOR_VECTOR_NTT); implDilithiumMontMulByConstant(coeffs, MONT_R_MOD_Q); } @@ -1343,6 +1343,7 @@ static void implDilithiumAlmostNttJava(int[] coeffs) { } public static void mlDsaInverseNtt(int[] coeffs) { + assert coeffs.length == ML_DSA_N; implDilithiumAlmostInverseNtt(coeffs, MONT_ZETAS_FOR_VECTOR_INVERSE_NTT); implDilithiumMontMulByConstant(coeffs, MONT_DIM_INVERSE); } @@ -1382,6 +1383,7 @@ void mlDsaVectorInverseNtt(int[][] vector) { } public static void mlDsaNttMultiply(int[] product, int[] coeffs1, int[] coeffs2) { + assert (coeffs1.length == ML_DSA_N) && (coeffs2.length == ML_DSA_N); implDilithiumNttMult(product, coeffs1, coeffs2); } @@ -1412,6 +1414,8 @@ static void implDilithiumMontMulByConstantJava(int[] coeffs, int constant) { public static void mlDsaDecomposePoly(int[] input, int[] lowPart, int[] highPart, int twoGamma2, int multiplier) { + assert (input.length == ML_DSA_N) && (lowPart.length == ML_DSA_N) + && (highPart.length == ML_DSA_N); implDilithiumDecomposePoly(input, lowPart, highPart,twoGamma2, multiplier); } @@ -1550,7 +1554,7 @@ boolean vectorNormBound(int[][] vec, int bound) { // precondition: -2^31 * MONT_Q <= a, b < 2^31, -2^31 < a * b < 2^31 * MONT_Q // computes a * b * 2^-32 mod MONT_Q // the result is greater than -MONT_Q and less than MONT_Q - // see e.g. Algorithm 3 in https://eprint.iacr.org/2018/039.pdf + // See e.g. Algorithm 3 in https://eprint.iacr.org/2018/039.pdf private static int montMul(int b, int c) { long a = (long) b * (long) c; int aHigh = (int) (a >> MONT_R_BITS); diff --git a/src/java.base/share/classes/sun/security/provider/SHA3.java b/src/java.base/share/classes/sun/security/provider/SHA3.java index 5f974bc6ea65..a096cac5f504 100644 --- a/src/java.base/share/classes/sun/security/provider/SHA3.java +++ b/src/java.base/share/classes/sun/security/provider/SHA3.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -476,9 +476,28 @@ public byte[] squeeze(int numBytes) { public void reset() { engineReset(); + // engineReset (final in DigestBase) skips implReset if there's + // no update. This works for MessageDigest, since digest() always + // resets. But for XOF, squeeze() may be called without update, + // and still modifies state. So we always call implReset here + // to ensure correct behavior. + implReset(); } } + public static final class SHAKE128Hash extends SHA3 { + public SHAKE128Hash() { + super("SHAKE128-256", 32, (byte) 0x1F, 32); + } + } + + public static final class SHAKE256Hash extends SHA3 { + public SHAKE256Hash() { + super("SHAKE256-512", 64, (byte) 0x1F, 64); + } + } + + /* * The SHAKE128 extendable output function. */ diff --git a/src/java.base/share/classes/sun/security/provider/SunEntries.java b/src/java.base/share/classes/sun/security/provider/SunEntries.java index 1487e521c547..769e5bfc6511 100644 --- a/src/java.base/share/classes/sun/security/provider/SunEntries.java +++ b/src/java.base/share/classes/sun/security/provider/SunEntries.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1996, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1996, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -269,6 +269,10 @@ public final class SunEntries { "sun.security.provider.SHA3$SHA384", attrs); addWithAlias(p, "MessageDigest", "SHA3-512", "sun.security.provider.SHA3$SHA512", attrs); + addWithAlias(p, "MessageDigest", "SHAKE128-256", + "sun.security.provider.SHA3$SHAKE128Hash", attrs); + addWithAlias(p, "MessageDigest", "SHAKE256-512", + "sun.security.provider.SHA3$SHAKE256Hash", attrs); /* * Certificates diff --git a/src/java.base/share/classes/sun/security/util/KnownOIDs.java b/src/java.base/share/classes/sun/security/util/KnownOIDs.java index bca4ee72b394..cf58d55b7fba 100644 --- a/src/java.base/share/classes/sun/security/util/KnownOIDs.java +++ b/src/java.base/share/classes/sun/security/util/KnownOIDs.java @@ -152,8 +152,8 @@ public enum KnownOIDs { SHA3_256("2.16.840.1.101.3.4.2.8", "SHA3-256"), SHA3_384("2.16.840.1.101.3.4.2.9", "SHA3-384"), SHA3_512("2.16.840.1.101.3.4.2.10", "SHA3-512"), - SHAKE128("2.16.840.1.101.3.4.2.11"), - SHAKE256("2.16.840.1.101.3.4.2.12"), + SHAKE128_256("2.16.840.1.101.3.4.2.11", "SHAKE128-256", "SHAKE128"), + SHAKE256_512("2.16.840.1.101.3.4.2.12", "SHAKE256-512", "SHAKE256"), HmacSHA3_224("2.16.840.1.101.3.4.2.13", "HmacSHA3-224"), HmacSHA3_256("2.16.840.1.101.3.4.2.14", "HmacSHA3-256"), HmacSHA3_384("2.16.840.1.101.3.4.2.15", "HmacSHA3-384"), diff --git a/src/java.base/share/classes/sun/security/util/SignatureUtil.java b/src/java.base/share/classes/sun/security/util/SignatureUtil.java index 36b65b28c206..73fe6a167062 100644 --- a/src/java.base/share/classes/sun/security/util/SignatureUtil.java +++ b/src/java.base/share/classes/sun/security/util/SignatureUtil.java @@ -199,7 +199,7 @@ public static class EdDSADigestAlgHolder { static { try { sha512 = new AlgorithmId(ObjectIdentifier.of(KnownOIDs.SHA_512)); - shake256 = new AlgorithmId(ObjectIdentifier.of(KnownOIDs.SHAKE256)); + shake256 = new AlgorithmId(ObjectIdentifier.of(KnownOIDs.SHAKE256_512)); shake256$512 = new AlgorithmId( ObjectIdentifier.of(KnownOIDs.SHAKE256_LEN), new DerValue((byte) 2, new byte[]{2, 0})); // int 512 diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.m index 8951ae8e110d..a143d6854607 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.m @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,19 @@ extern void MTLGC_DestroyMTLGraphicsConfig(jlong pConfigInfo); +/** + * Triggers the display link for the current destination surface. + */ +static void MTLSD_Flush() { + if (dstOps != NULL) { + MTLSDOps *dstMTLOps = (MTLSDOps *)dstOps->privOps; + MTLLayer *layer = (MTLLayer*)dstMTLOps->layer; + if (layer != NULL) { + [layer startDisplayLink]; + } + } +} + void MTLRenderQueue_CheckPreviousOp(jint op) { if (mtlPreviousOp == op) { @@ -575,6 +588,7 @@ void MTLRenderQueue_CheckPreviousOp(jint op) { [cbwrapper release]; }]; [commandbuf commit]; + MTLSD_Flush(); } mtlc = [MTLContext setSurfacesEnv:env src:pSrc dst:pDst]; dstOps = (BMTLSDOps *)jlong_to_ptr(pDst); @@ -602,6 +616,7 @@ void MTLRenderQueue_CheckPreviousOp(jint op) { [cbwrapper release]; }]; [commandbuf commit]; + MTLSD_Flush(); } mtlc = newMtlc; dstOps = NULL; @@ -871,14 +886,7 @@ void MTLRenderQueue_CheckPreviousOp(jint op) { [cbwrapper release]; }]; [commandbuf commit]; - BMTLSDOps *dstOps = MTLRenderQueue_GetCurrentDestination(); - if (dstOps != NULL) { - MTLSDOps *dstMTLOps = (MTLSDOps *)dstOps->privOps; - MTLLayer *layer = (MTLLayer*)dstMTLOps->layer; - if (layer != NULL) { - [layer startDisplayLink]; - } - } + MTLSD_Flush(); } RESET_PREVIOUS_OP(); } diff --git a/src/java.desktop/share/native/common/java2d/opengl/OGLRenderQueue.c b/src/java.desktop/share/native/common/java2d/opengl/OGLRenderQueue.c index 6328707a3e2a..42acf70a58f5 100644 --- a/src/java.desktop/share/native/common/java2d/opengl/OGLRenderQueue.c +++ b/src/java.desktop/share/native/common/java2d/opengl/OGLRenderQueue.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -433,6 +433,7 @@ Java_sun_java2d_opengl_OGLRenderQueue_flushBuffer jlong pDst = NEXT_LONG(b); if (oglc != NULL) { RESET_PREVIOUS_OP(); + OGLSD_Flush(env); } oglc = OGLContext_SetSurfaces(env, pSrc, pDst); dstOps = (OGLSDOps *)jlong_to_ptr(pDst); @@ -443,6 +444,7 @@ Java_sun_java2d_opengl_OGLRenderQueue_flushBuffer jlong pConfigInfo = NEXT_LONG(b); if (oglc != NULL) { RESET_PREVIOUS_OP(); + OGLSD_Flush(env); } oglc = OGLSD_SetScratchSurface(env, pConfigInfo); dstOps = NULL; diff --git a/test/jdk/ProblemList.txt b/test/jdk/ProblemList.txt index a72f9aecf8fe..0b610816bd45 100644 --- a/test/jdk/ProblemList.txt +++ b/test/jdk/ProblemList.txt @@ -248,7 +248,6 @@ sun/awt/datatransfer/SuplementaryCharactersTransferTest.java 8011371 generic-all sun/awt/shell/ShellFolderMemoryLeak.java 8197794 windows-all sun/java2d/DirectX/OverriddenInsetsTest/OverriddenInsetsTest.java 8196102 generic-all sun/java2d/DirectX/RenderingToCachedGraphicsTest/RenderingToCachedGraphicsTest.java 8196180 windows-all,macosx-all -sun/java2d/OpenGL/MultiWindowFillTest.java 8378506 macosx-all sun/java2d/OpenGL/OpaqueDest.java#id1 8367574 macosx-all sun/java2d/OpenGL/ScaleParamsOOB.java#id0 8377908 linux-all sun/java2d/SunGraphics2D/EmptyClipRenderingTest.java 8144029 macosx-all,linux-all @@ -647,18 +646,9 @@ security/infra/java/security/cert/CertPathValidator/certification/CAInterop.java ############################################################################ # jdk_sound -javax/sound/sampled/DirectAudio/bug6372428.java 8055097 generic-all -javax/sound/sampled/Clip/bug5070081.java 8055097 generic-all -javax/sound/sampled/DataLine/LongFramePosition.java 8055097 generic-all javax/sound/sampled/Clip/Drain/ClipDrain.java 7062792 generic-all -javax/sound/sampled/Mixers/DisabledAssertionCrash.java 7067310 generic-all - -javax/sound/midi/Sequencer/Recording.java 8167580,8265485 linux-all,macosx-aarch64 -javax/sound/midi/Sequencer/Looping.java 8136897 generic-all -javax/sound/sampled/Clip/ClipIsRunningAfterStop.java 8307574 linux-x64 - ############################################################################ # jdk_imageio diff --git a/test/jdk/java/io/Serializable/records/ProhibitedMethods.java b/test/jdk/java/io/Serializable/records/ProhibitedMethods.java index 7465c80e4533..48b4dad78827 100644 --- a/test/jdk/java/io/Serializable/records/ProhibitedMethods.java +++ b/test/jdk/java/io/Serializable/records/ProhibitedMethods.java @@ -295,7 +295,7 @@ public void visitEnd() { MethodVisitor mv = cv.visitMethod(ACC_PRIVATE, READ_OBJECT_NAME, READ_OBJECT_DESC, null, null); mv.visitCode(); mv.visitLdcInsn(READ_OBJECT_NAME + " should not be invoked"); - mv.visitMethodInsn(INVOKESTATIC, "org/testng/Assert", "fail", "(Ljava/lang/String;)V", false); + mv.visitMethodInsn(INVOKESTATIC, "org/junit/jupiter/api/Assertions", "fail", "(Ljava/lang/String;)Ljava/lang/Object;", false); mv.visitInsn(RETURN); mv.visitMaxs(0, 0); mv.visitEnd(); @@ -314,7 +314,7 @@ public void visitEnd() { MethodVisitor mv = cv.visitMethod(ACC_PRIVATE, READ_OBJECT_NO_DATA_NAME, READ_OBJECT_NO_DATA_DESC, null, null); mv.visitCode(); mv.visitLdcInsn(READ_OBJECT_NO_DATA_NAME + " should not be invoked"); - mv.visitMethodInsn(INVOKESTATIC, "org/testng/Assert", "fail", "(Ljava/lang/String;)V", false); + mv.visitMethodInsn(INVOKESTATIC, "org/junit/jupiter/api/Assertions", "fail", "(Ljava/lang/String;)Ljava/lang/Object;", false); mv.visitInsn(RETURN); mv.visitMaxs(0, 0); mv.visitEnd(); @@ -371,4 +371,5 @@ public void wellFormedGeneratedClasses() throws Exception { } } } -} \ No newline at end of file +} + diff --git a/test/jdk/sun/java2d/OpenGL/MultiWindowFillTest.java b/test/jdk/sun/java2d/OpenGL/MultiWindowFillTest.java index 59c58d944d79..302bb43e24a5 100644 --- a/test/jdk/sun/java2d/OpenGL/MultiWindowFillTest.java +++ b/test/jdk/sun/java2d/OpenGL/MultiWindowFillTest.java @@ -34,7 +34,7 @@ /** * @test - * @bug 8378201 + * @bug 8378201 8378506 * @key headful * @summary Verifies that window content survives a GL context switch to another * window and back diff --git a/test/jdk/sun/security/provider/MessageDigest/SHAKEhash.java b/test/jdk/sun/security/provider/MessageDigest/SHAKEhash.java new file mode 100644 index 000000000000..1f4cbbf034a1 --- /dev/null +++ b/test/jdk/sun/security/provider/MessageDigest/SHAKEhash.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8354305 + * @summary Ensure SHAKE message digest algorithms behave the same + * as correspondent XOF of the same output size + * @library /test/lib + * @modules java.base/sun.security.provider + */ + +import jdk.test.lib.Asserts; +import jdk.test.lib.security.SeededSecureRandom; +import sun.security.provider.SHA3; + +import java.security.MessageDigest; + +public class SHAKEhash { + public static void main(String[] args) throws Exception { + var random = SeededSecureRandom.one(); + var s1 = new SHA3.SHAKE128(); + var m1 = MessageDigest.getInstance("SHAKE128-256"); // use standard name + var s2 = new SHA3.SHAKE256(); + var m2 = MessageDigest.getInstance("SHAKE256"); // use alias + for (var i = 0; i < 1_000_000; i++) { + var msg = random.nBytes(random.nextInt(100)); + s1.update(msg); + m1.update(msg); + Asserts.assertEqualsByteArray(s1.squeeze(32), m1.digest()); + s2.update(msg); + m2.update(msg); + Asserts.assertEqualsByteArray(s2.squeeze(64), m2.digest()); + s1.reset(); + s2.reset(); + } + } +} diff --git a/test/jdk/sun/security/provider/MessageDigest/SHAKEsqueeze.java b/test/jdk/sun/security/provider/MessageDigest/SHAKEsqueeze.java index 5cdcce00e357..30edb9dbf134 100644 --- a/test/jdk/sun/security/provider/MessageDigest/SHAKEsqueeze.java +++ b/test/jdk/sun/security/provider/MessageDigest/SHAKEsqueeze.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,18 @@ public class SHAKEsqueeze { public static void main(String[] args) throws Exception { + resetFix(); + random(); + } + + static void resetFix() throws Exception { + var s = new SHA3.SHAKE256(); + var d1 = s.squeeze(10); + s.reset(); + Asserts.assertEqualsByteArray(d1, s.squeeze(10)); + } + + static void random() throws Exception { var r = SeededSecureRandom.one(); var atlast = 0; // Random test on SHAKE diff --git a/test/jdk/sun/security/provider/acvp/Launcher.java b/test/jdk/sun/security/provider/acvp/Launcher.java index 1405847de5d1..0b63e92429c4 100644 --- a/test/jdk/sun/security/provider/acvp/Launcher.java +++ b/test/jdk/sun/security/provider/acvp/Launcher.java @@ -39,6 +39,16 @@ * @bug 8342442 8345057 * @library /test/lib * @modules java.base/sun.security.provider + * @run main Launcher + */ + +/* + * @test + * @summary Test verifying the intrinsic implementation. + * @bug 8342442 8345057 + * @library /test/lib + * @modules java.base/sun.security.provider + * @run main/othervm -Xcomp Launcher */ /// This test runs on `internalProjection.json`-style files generated by NIST's