From 9fb50a831062f700df7d2886e93a8ee6edfd5905 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 13:04:00 +0200 Subject: [PATCH 1/6] Arm64: opportunistically light up LSE atomics for NativeAOT NativeAOT's arm64 baseline is armv8-a, so `Interlocked` operations were stuck emitting `ldaxr`/`stlxr` retry loops even though virtually all real hardware implements the Armv8.1 LSE atomics. The runtime JIT already picks the single instruction forms because it knows the exact CPU it is running on; AOT could not, because the check has to happen at runtime. Rather than teach codegen to branch on a CPU feature word, the check is expressed in managed code. A new internal `System.Runtime.Intrinsics.Arm.Lse` hardware intrinsic class exposes the five atomic primitives, and the `Interlocked` bodies call them behind `if (Lse.IsSupported)`. On NativeAOT `IsSupported` already lowers to a cached CPU feature query, so the existing opportunistic light-up machinery does all the work and no new JIT/EE API is needed. To make that check observable, the importer declines to intrinsify `Interlocked` exactly when the atomics are opportunistically available but not part of the baseline - the "Dynamic" state, which in practice only NativeAOT ever reaches. The call is then imported normally, the body is inlined, and the check lights up. Recursion is bounded by `mustExpand`: the fallback arm calls the same API from within its own body, and recursive intrinsic calls must expand, so it always bottoms out in the ldaxr/stlxr sequence. Every other configuration keeps the state it had before - baseline LSE stays baseline, no-LSE stays no-LSE - so this is a strict no-op for the runtime JIT, R2R, and NativeAOT on Windows/macOS. SuperPMI asmdiffs over the linux-arm64 benchmarks.run collection report zero diffs across 142,660 contexts. Covers CompareExchange, Exchange, ExchangeAdd (Add/Increment/Decrement), And, and Or, for both int and long. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- .../System/Threading/Interlocked.CoreCLR.cs | 45 ++++++++++++ src/coreclr/jit/codegenarm64.cpp | 7 +- src/coreclr/jit/compiler.cpp | 62 ++++++++++++++++ src/coreclr/jit/compiler.h | 41 +++++++++++ src/coreclr/jit/gentree.cpp | 9 +++ src/coreclr/jit/gentree.h | 4 ++ src/coreclr/jit/hwintrinsic.cpp | 2 +- src/coreclr/jit/hwintrinsicarm64.cpp | 70 ++++++++++++++++++- src/coreclr/jit/hwintrinsiclistarm64.h | 16 +++++ src/coreclr/jit/importercalls.cpp | 67 ++++++++++++++++++ src/coreclr/jit/jitconfigvalues.h | 8 +++ src/coreclr/jit/lowerarmarch.cpp | 7 +- src/coreclr/jit/lsraarm64.cpp | 12 ++-- .../src/System/Threading/Interlocked.cs | 39 +++++++++++ .../JitInterface/CorInfoInstructionSet.cs | 15 +++- .../ThunkGenerator/InstructionSetDesc.txt | 2 +- .../System.Private.CoreLib.Shared.projitems | 1 + .../src/System/Runtime/Intrinsics/Arm/Lse.cs | 49 +++++++++++++ .../src/System/Threading/Interlocked.cs | 27 +++++++ 19 files changed, 470 insertions(+), 13 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs index 5fb05077081bbb..946d1974d899d8 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -52,6 +55,13 @@ public static long Decrement(ref long location) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Exchange(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -73,6 +83,13 @@ public static int Exchange(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Exchange(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -116,6 +133,13 @@ public static long Exchange(ref long location1, long value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareExchange(ref int location1, int value, int comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -158,6 +182,13 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long CompareExchange(ref long location1, long value, long comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -212,6 +243,13 @@ public static long Add(ref long location1, long value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ExchangeAdd(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return ExchangeAdd(ref location1, value); // Must expand intrinsic #else @@ -228,6 +266,13 @@ private static int ExchangeAdd(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static long ExchangeAdd(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return ExchangeAdd(ref location1, value); // Must expand intrinsic #else diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 85bebf191cc2ec..666d2f36fc1cb3 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -3810,7 +3810,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) emitAttr dataSize = emitActualTypeSize(data); - if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (m_compiler->compGetAtomicsImplForNode(treeNode) == Compiler::AtomicsImpl::Lse) { assert(!data->isContainedIntOrIImmed()); @@ -3988,8 +3988,11 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) emitAttr dataSize = emitActualTypeSize(data); - if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (m_compiler->compGetAtomicsImplForNode(treeNode) == Compiler::AtomicsImpl::Lse) { + // 'casal' has no immediate form, so lowering must not have contained the comparand. + assert(!comparand->isContained()); + // casal use the comparand as the target reg GetEmitter()->emitIns_Mov(INS_mov, dataSize, targetReg, comparandReg, /* canSkip */ true); diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index dbc56f2344e513..caee14b53cd896 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2017,6 +2017,68 @@ bool Compiler::notifyInstructionSetUsage(CORINFO_InstructionSet isa, bool suppor return info.compCompHnd->notifyInstructionSetUsage(isa, supported); } +#ifdef TARGET_ARM64 +//------------------------------------------------------------------------ +// compGetAtomicsImpl: Decide how atomic operations are to be expanded in this method. +// +// Return Value: +// AtomicsImpl::Lse - the Armv8.1 atomics are known to be available, use them directly. +// AtomicsImpl::LlSc - they are known not to be available, the ldaxr/stlxr retry loops +// have to be used. +// AtomicsImpl::Dynamic - they are not part of the baseline instruction set, but they may +// still be available on the machine that ends up running this code. +// The Interlocked APIs are left unexpanded so that the managed +// `if (Lse.IsSupported)` check in their bodies gets inlined instead. +// +// Notes: +// The result is computed on demand and cached: asking the EE about instruction set support has +// side effects (for ReadyToRun it records a hard requirement on the method), so we must not do +// it for methods that contain no atomic operations at all. +// +// 'Dynamic' is what makes NativeAOT interesting here: its baseline is typically armv8-a, so +// compExactlyDependsOn(InstructionSet_Atomics) is false even though the vast majority of the +// hardware in the wild does implement LSE. Note that this state cannot occur for the runtime +// JIT, where the opportunistic and the exact answer always agree - see JitStressAtomicsLightUp +// for a way to force it for testing. +// +Compiler::AtomicsImpl Compiler::compGetAtomicsImpl() +{ + if (m_atomicsImpl != AtomicsImpl::Uninitialized) + { + return m_atomicsImpl; + } + + AtomicsImpl impl; + if (!opts.compSupportsISA.HasInstructionSet(InstructionSet_Atomics)) + { + impl = AtomicsImpl::LlSc; + } +#ifdef DEBUG + else if (JitConfig.JitStressAtomicsLightUp() != 0) + { + // Pretend the atomics are only opportunistically available so that we exercise the + // managed light-up path. Note we deliberately do not report the ISA as used to the EE. + impl = AtomicsImpl::Dynamic; + } +#endif + else if (compExactlyDependsOn(InstructionSet_Atomics)) + { + impl = AtomicsImpl::Lse; + } + else + { + impl = AtomicsImpl::Dynamic; + } + + JITDUMP("Atomics will be expanded as %s\n", (impl == AtomicsImpl::Lse) ? "LSE" + : (impl == AtomicsImpl::LlSc) ? "ldaxr/stlxr loops" + : "managed light-up"); + + m_atomicsImpl = impl; + return impl; +} +#endif // TARGET_ARM64 + #ifdef PROFILING_SUPPORTED // A Dummy routine to receive Enter/Leave/Tailcall profiler callbacks. // These are used when DOTNET_JitEltHookEnabled=1 diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 03fa22ec8c5463..436ee32af6ca72 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5418,6 +5418,7 @@ class Compiler R2RARG(CORINFO_CONST_LOOKUP* entryPoint), NamedIntrinsic* pIntrinsicName, bool* isSpecialIntrinsic = nullptr); + bool impIsAtomicLightUpCandidate(NamedIntrinsic ni, CORINFO_SIG_INFO* sig, bool mustExpand); GenTree* impEstimateIntrinsic(CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, CorInfoType callJitType, @@ -10936,6 +10937,46 @@ class Compiler return opts.compSupportsISA.HasInstructionSet(isa); } +#ifdef TARGET_ARM64 + // How the Interlocked/atomic operations (GT_CMPXCHG, GT_XADD, GT_XCHG, GT_XORR, GT_XAND) + // are to be expanded by codegen. + enum class AtomicsImpl : uint8_t + { + Uninitialized, + LlSc, // ldaxr/stlxr retry loop + Lse, // Armv8.1 single instruction atomics + Dynamic, // not in the baseline, but may still be available at run time + }; + + AtomicsImpl compGetAtomicsImpl(); + + //------------------------------------------------------------------------ + // compGetAtomicsImplForNode: How a specific atomic node is to be expanded. + // + // The importer only ever creates an atomic node when it knows which sequence to use: in the + // AtomicsImpl::Dynamic case the Interlocked APIs are left unexpanded and the managed + // `if (Lse.IsSupported)` check in their bodies picks the arm, marking the node it creates + // with GTF_ATOMIC_LSE. So lowering, LSRA and codegen only ever see a concrete answer. + // + AtomicsImpl compGetAtomicsImplForNode(GenTree* node) + { + assert(node->OperIsAtomicOp()); + + if ((node->gtFlags & GTF_ATOMIC_LSE) != 0) + { + return AtomicsImpl::Lse; + } + + AtomicsImpl impl = compGetAtomicsImpl(); + return (impl == AtomicsImpl::Lse) ? AtomicsImpl::Lse : AtomicsImpl::LlSc; + } + +private: + AtomicsImpl m_atomicsImpl = AtomicsImpl::Uninitialized; + +public: +#endif // TARGET_ARM64 + private: #ifdef DEBUG //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index f09808f88d6f56..fe3b81c0779053 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2770,6 +2770,15 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) return false; } +#ifdef TARGET_ARM64 + // The two arms of an opportunistic atomics check hold otherwise identical nodes that must not + // be merged back together - the flag is what tells codegen which sequence to emit. + if (OperIsAtomicOp(oper) && (((op1->gtFlags ^ op2->gtFlags) & GTF_ATOMIC_LSE) != 0)) + { + return false; + } +#endif // TARGET_ARM64 + /* Sensible flags must be equal */ if (op1->IsUnsigned() != op2->IsUnsigned()) { diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 7033e6cb81cd42..7907f09217d8ba 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -487,6 +487,10 @@ enum GenTreeFlags : unsigned GTF_IND_INITCLASS = 0x00200000, // OperIsIndir() -- the indirection requires preceding static cctor GTF_IND_ALLOW_NON_ATOMIC = 0x00100000, // GT_IND -- this memory access does not need to be atomic + // Set by the importer on arm64 for the atomic nodes created from the Lse intrinsics. Note this + // bit is not used by any of the GTF_IND_* flags, which the atomic opers otherwise share. + GTF_ATOMIC_LSE = 0x04000000, // GT_XADD/GT_XAND/GT_XORR/GT_XCHG/GT_CMPXCHG -- use the Armv8.1 atomics + // Represents flags that an indirection based on another indirection must preserve GTF_IND_MUST_PRESERVE_FLAGS = GTF_IND_VOLATILE | GTF_IND_UNALIGNED | GTF_IND_INITCLASS, diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index a1058fbffb0ccf..9a471fdbd6d4e5 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -983,7 +983,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Fp16, LAST_NI_Fp16 }, // Fp16 { FIRST_NI_Sha1, LAST_NI_Sha1 }, // Sha1 { FIRST_NI_Sha256, LAST_NI_Sha256 }, // Sha256 - { NI_Illegal, NI_Illegal }, // Atomics + { FIRST_NI_Atomics, LAST_NI_Atomics }, // Atomics { FIRST_NI_Vector, LAST_NI_Vector }, // Vector64 { FIRST_NI_Vector, LAST_NI_Vector }, // Vector128 { NI_Illegal, NI_Illegal }, // VectorT diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 91a3458999f833..7c3dcbecd816c0 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -94,6 +94,13 @@ CORINFO_InstructionSet Compiler::lookupInstructionSet(const char* className) return InstructionSet_Dp; } } + else if (className[0] == 'L') + { + if (strcmp(className, "Lse") == 0) + { + return InstructionSet_Atomics; + } + } else if (className[0] == 'R') { if (strcmp(className, "Rdm") == 0) @@ -703,7 +710,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, const int numArgs = sig->numArgs; // The vast majority of "special" intrinsics are Vector64/Vector128 methods. - // The only exception is ArmBase.Yield which should be treated differently. + // The only exceptions are ArmBase.Yield and the Lse atomics, which are treated differently. if (intrinsic == NI_ArmBase_Yield) { assert(sig->numArgs == 0); @@ -713,6 +720,67 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, return gtNewScalarHWIntrinsicNode(TYP_VOID, intrinsic); } + if (isa == InstructionSet_Atomics) + { + // The Armv8.1 atomics reuse the nodes that back the Interlocked APIs; GTF_ATOMIC_LSE is + // what tells lowering and codegen to use the single instruction forms unconditionally. + // + // Note we must use simdBaseType rather than retType here: the latter has been widened to + // TYP_INT for the byte and halfword forms, which would give us an access of the wrong size. + assert(simdSize == 0); + assert(varTypeIsIntegral(simdBaseType) && (genTypeSize(simdBaseType) <= TARGET_POINTER_SIZE)); + + genTreeOps oper; + switch (intrinsic) + { + case NI_Atomics_CompareAndSwap: + oper = GT_CMPXCHG; + break; + case NI_Atomics_LoadAdd: + oper = GT_XADD; + break; + case NI_Atomics_LoadClear: + oper = GT_XAND; + break; + case NI_Atomics_LoadSet: + oper = GT_XORR; + break; + case NI_Atomics_Swap: + oper = GT_XCHG; + break; + default: + unreached(); + } + + // Only "cas" and "swp" have byte and halfword forms. + assert(!varTypeIsSmall(simdBaseType) || (oper == GT_CMPXCHG) || (oper == GT_XCHG)); + + GenTree* comparand = nullptr; + if (oper == GT_CMPXCHG) + { + assert(sig->numArgs == 3); + comparand = impPopStack().val; + + if (varTypeIsSmall(simdBaseType)) + { + // Small types need the comparand to have its upper bits zeroed. + comparand = gtNewCastNode(genActualType(simdBaseType), comparand, /* uns */ false, + varTypeToUnsigned(simdBaseType)); + } + } + else + { + assert(sig->numArgs == 2); + } + + GenTree* value = impPopStack().val; + GenTree* addr = impPopStack().val; + + GenTree* node = gtNewAtomicNode(oper, simdBaseType, addr, value, comparand); + node->gtFlags |= GTF_ATOMIC_LSE; + return node; + } + bool isScalar = (category == HW_Category_Scalar); assert(numArgs >= 0); diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 161ac83c1a2510..5ed4dd9e3048f2 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -630,6 +630,22 @@ HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha256 NI_Sha256_ScheduleUpdate1 +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// Lse Intrinsics (Armv8.1 atomics) +// +// These import into the same GT_CMPXCHG/GT_XADD/GT_XCHG/GT_XORR/GT_XAND nodes as the Interlocked +// APIs they implement, marked with GTF_ATOMIC_LSE. See Compiler::impSpecialIntrinsic. +#define FIRST_NI_Atomics NI_Atomics_CompareAndSwap +HARDWARE_INTRINSIC(Atomics, CompareAndSwap, 0, 3, INS_casalb, INS_casalb, INS_casalh, INS_casalh, INS_casal, INS_casal, INS_casal, INS_casal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadAdd, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldaddal, INS_ldaddal, INS_ldaddal, INS_ldaddal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadClear, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldclral, INS_ldclral, INS_ldclral, INS_ldclral, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadSet, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldsetal, INS_ldsetal, INS_ldsetal, INS_ldsetal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, Swap, 0, 2, INS_swpalb, INS_swpalb, INS_swpalh, INS_swpalh, INS_swpal, INS_swpal, INS_swpal, INS_swpal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +#define LAST_NI_Atomics NI_Atomics_Swap + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 19b001e5026bb2..5309a63a6295b2 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -3266,6 +3266,64 @@ GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) return impCreateLocalNode(spanTempNum DEBUGARG(0)); } +//------------------------------------------------------------------------ +// impIsAtomicLightUpCandidate: Should the given Interlocked API be left unexpanded so that the +// managed `if (Lse.IsSupported)` check in its body can light up the Armv8.1 atomics? +// +// Arguments: +// ni - The named intrinsic +// sig - Signature of the call +// mustExpand - Whether the JIT is required to expand this call +// +// Return Value: +// True if the call should be imported as a regular (inlineable) call. +// +// Notes: +// This is only ever true when the Armv8.1 atomics are not part of our baseline instruction set +// but may still be present on the machine that ends up running the code - in practice that means +// NativeAOT, whose baseline is typically armv8-a. In every other configuration the answer is +// "no" and these APIs keep expanding exactly like they always have. +// +// 'mustExpand' is what stops the light-up from recursing forever: the fallback arm of the check +// calls the very same API from within its own body, and recursive intrinsic calls must expand. +// +bool Compiler::impIsAtomicLightUpCandidate(NamedIntrinsic ni, CORINFO_SIG_INFO* sig, bool mustExpand) +{ +#ifdef TARGET_ARM64 + switch (ni) + { + case NI_System_Threading_Interlocked_And: + case NI_System_Threading_Interlocked_Or: + case NI_System_Threading_Interlocked_CompareExchange: + case NI_System_Threading_Interlocked_Exchange: + case NI_System_Threading_Interlocked_ExchangeAdd: + break; + + default: + return false; + } + + if (mustExpand) + { + return false; + } + + // Only the overloads that actually carry the managed check qualify: the ones operating on a + // machine word sized integer. The object overloads use a write barrier helper instead, and the + // byte and halfword ones funnel through a masking loop, so leaving those unexpanded would just + // pessimize them. + var_types retType = JITtype2varType(sig->retType); + if (!varTypeIsIntegral(retType) || (genTypeSize(retType) < 4) || (genTypeSize(retType) > TARGET_POINTER_SIZE)) + { + return false; + } + + return compGetAtomicsImpl() == AtomicsImpl::Dynamic; +#else + return false; +#endif // TARGET_ARM64 +} + //------------------------------------------------------------------------ // impIntrinsic: possibly expand intrinsic call into alternate IR sequence // @@ -3548,6 +3606,15 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, *pIntrinsicName = ni; + if (impIsAtomicLightUpCandidate(ni, sig, mustExpand)) + { + // The Armv8.1 atomics may be available on the machine running this code even though they + // are not part of our baseline instruction set. Leave the call alone so that the managed + // `if (Lse.IsSupported)` check in the API's body gets inlined and lights them up. + JITDUMP("Not expanding %s to allow the managed Armv8.1 atomics light-up\n", eeGetMethodFullName(method)); + return nullptr; + } + if (ni == NI_System_StubHelpers_GetStubContext) { // must be done regardless of DbgCode and MinOpts diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 1e95c26b1e31cf..dbd797f334d6b6 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -868,6 +868,14 @@ RELEASE_CONFIG_INTEGER(JitEnableStrengthReduction, "JitEnableStrengthReduction", // Enable IV optimizations RELEASE_CONFIG_INTEGER(JitEnableInductionVariableOpts, "JitEnableInductionVariableOpts", 1) +#if defined(TARGET_ARM64) +// Stress the opportunistic light-up of the Armv8.1 LSE atomics: pretend they are not part of the +// baseline instruction set even when they are, so that the managed `if (Lse.IsSupported)` checks in +// the Interlocked APIs are used. This state occurs naturally for NativeAOT, but never for the +// runtime JIT, so this is the only way to cover it outside of AOT compilation. +CONFIG_INTEGER(JitStressAtomicsLightUp, "JitStressAtomicsLightUp", 0) +#endif // TARGET_ARM64 + // JitFunctionFile: Name of a file that contains a list of functions. If the currently compiled function is in the // file, certain other JIT config variables will be active. If the currently compiled function is not in the file, // the specific JIT config variables will not be active. diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d3955c7861ae1d..7050412fe55137 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -102,9 +102,10 @@ bool Lowering::IsContainableImmed(GenTree* parentNode, GenTree* childNode) const case GT_XORR: case GT_XAND: case GT_XADD: - return m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics) - ? false - : emitter::emitIns_valid_imm_for_add(immVal, size); + // The LSE forms of these have no immediate encoding, so only the ldaxr/stlxr + // sequences can make use of a contained immediate. + return (m_compiler->compGetAtomicsImplForNode(parentNode) == Compiler::AtomicsImpl::LlSc) && + emitter::emitIns_valid_imm_for_add(immVal, size); #elif defined(TARGET_ARM) return emitter::emitIns_valid_imm_for_add(immVal, flags); #endif diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..ba7aaf373dbbdf 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1133,7 +1133,9 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = cmpXchgNode->Comparand()->isContained() ? 2 : 3; assert(dstCount == 1); - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + const bool useLse = (m_compiler->compGetAtomicsImplForNode(tree) == Compiler::AtomicsImpl::Lse); + + if (!useLse) { // For ARMv8 exclusives requires a single internal register buildInternalIntRegisterDefForNode(tree); @@ -1155,7 +1157,7 @@ int LinearScan::BuildNode(GenTree* tree) // For ARMv8 exclusives the lifetime of the comparand must be extended because // it may be used used multiple during retries - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (!useLse) { setDelayFree(comparandUse); } @@ -1177,7 +1179,9 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == (tree->TypeIs(TYP_VOID) ? 0 : 1)); srcCount = tree->gtGetOp2()->isContained() ? 1 : 2; - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + const bool useLse = (m_compiler->compGetAtomicsImplForNode(tree) == Compiler::AtomicsImpl::Lse); + + if (!useLse) { // GT_XCHG requires a single internal register; the others require two. buildInternalIntRegisterDefForNode(tree); @@ -1202,7 +1206,7 @@ int LinearScan::BuildNode(GenTree* tree) // For ARMv8 exclusives the lifetime of the addr and data must be extended because // it may be used used multiple during retries - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (!useLse) { // Internals may not collide with target if (dstCount == 1) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs index c8a9c739831fcc..31f7a427878c73 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime; using System.Runtime.CompilerServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -15,6 +18,12 @@ public static partial class Interlocked [Intrinsic] public static int CompareExchange(ref int location1, int value, int comparand) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -45,6 +54,12 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long CompareExchange(ref long location1, long value, long comparand) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -71,6 +86,12 @@ public static long CompareExchange(ref long location1, long value, long comparan [Intrinsic] public static int Exchange(ref int location1, int value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -88,6 +109,12 @@ public static int Exchange(ref int location1, int value) [Intrinsic] public static long Exchange(ref long location1, long value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -164,6 +191,12 @@ public static long Add(ref long location1, long value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ExchangeAdd(ref int location1, int value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif int oldValue; do @@ -178,6 +211,12 @@ private static int ExchangeAdd(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static long ExchangeAdd(ref long location1, long value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif long oldValue; do diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 870544e2156425..4293c8a79d159e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -1164,7 +1164,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("fp16", "", InstructionSet.ARM64_Fp16, true); yield return new InstructionSetInfo("sha1", "Sha1", InstructionSet.ARM64_Sha1, true); yield return new InstructionSetInfo("sha2", "Sha256", InstructionSet.ARM64_Sha256, true); - yield return new InstructionSetInfo("lse", "", InstructionSet.ARM64_Atomics, true); + yield return new InstructionSetInfo("lse", "Lse", InstructionSet.ARM64_Atomics, true); yield return new InstructionSetInfo("Vector64", "", InstructionSet.ARM64_Vector64, false); yield return new InstructionSetInfo("Vector128", "", InstructionSet.ARM64_Vector128, false); yield return new InstructionSetInfo("VectorT", "", InstructionSet.ARM64_VectorT, false); @@ -1603,6 +1603,9 @@ public static InstructionSet LookupPlatformIntrinsicInstructionSet(TargetArchite else return InstructionSet.ARM64_Sha256; + case "Lse": + return InstructionSet.ARM64_Atomics; + case "Sve": if (nestedTypeName == "Arm64") return InstructionSet.ARM64_Sve_Arm64; @@ -2216,6 +2219,16 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } break; + case (InstructionSet.ARM64_Atomics, TargetArchitecture.ARM64): + { + var type = context.SystemModule.GetType("System.Runtime.Intrinsics.Arm"u8, "Lse"u8, false); + if (type != null) + { + yield return type; + } + } + break; + case (InstructionSet.ARM64_Sve, TargetArchitecture.ARM64): case (InstructionSet.ARM64_Sve_Arm64, TargetArchitecture.ARM64): { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index a70cd7d9991687..c7a57e87c51de6 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -218,7 +218,7 @@ instructionset ,ARM64 ,Rdm , ,24 ,Rdm instructionset ,ARM64 , ,Fp16 ,95 ,Fp16 ,fp16 instructionset ,ARM64 ,Sha1 , ,19 ,Sha1 ,sha1 instructionset ,ARM64 ,Sha256 , ,20 ,Sha256 ,sha2 -instructionset ,ARM64 , ,Atomics ,21 ,Atomics ,lse +instructionset ,ARM64 ,Lse ,Atomics ,21 ,Atomics ,lse instructionset ,ARM64 , , , ,Vector64 , instructionset ,ARM64 , , , ,Vector128 , instructionset ,ARM64 , , , ,VectorT , diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index bd846a6aed09b1..6c9a990525c416 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2880,6 +2880,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs new file mode 100644 index 00000000000000..bc25da8951a3dd --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Runtime.Intrinsics.Arm +{ + /// + /// Provides access to the ARM Large System Extensions (the Armv8.1 atomic instructions). + /// + /// + /// This type is internal because it exists so that can + /// light the instructions up opportunistically: folds to a constant + /// whenever LSE is (or is not) part of the compilation's baseline instruction set, and becomes a + /// runtime check against the CPU features word otherwise - which is the common case for NativeAOT, + /// whose baseline is typically armv8-a. + /// + /// Unlike the Interlocked APIs these never fall back to an ldaxr/stlxr retry loop, so they may + /// only be called when is true. The type argument must be a primitive + /// integer type; sizes of 1 and 2 bytes are only supported by + /// and . + /// + [Intrinsic] + internal abstract class Lse : ArmBase + { + internal Lse() { } + + public static new bool IsSupported { get => IsSupported; } + + /// Compare and swap: casal. + public static T CompareAndSwap(ref T location, T value, T comparand) => + CompareAndSwap(ref location, value, comparand); + + /// Atomic add, returning the original value: ldaddal. + public static T LoadAdd(ref T location, T value) => LoadAdd(ref location, value); + + /// Atomically clears the bits of that are not set in + /// - that is, an atomic "and" - and returns the original value. + /// Lowers to mvn followed by ldclral, since ldclral itself clears the + /// bits that are set in its operand. + public static T LoadClear(ref T location, T value) => LoadClear(ref location, value); + + /// Atomic bit set, returning the original value: ldsetal. + public static T LoadSet(ref T location, T value) => LoadSet(ref location, value); + + /// Atomic exchange: swpal. + public static T Swap(ref T location, T value) => Swap(ref location, value); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs index dfa92e8b9381e1..49286606479143 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -580,6 +583,12 @@ public static ulong Read(ref readonly ulong location) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int And(ref int location1, int value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadClear(ref location1, value); + } +#endif int current = location1; while (true) { @@ -612,6 +621,12 @@ public static uint And(ref uint location1, uint value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long And(ref long location1, long value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadClear(ref location1, value); + } +#endif long current = location1; while (true) { @@ -723,6 +738,12 @@ ref Unsafe.As(ref location1), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Or(ref int location1, int value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadSet(ref location1, value); + } +#endif int current = location1; while (true) { @@ -755,6 +776,12 @@ public static uint Or(ref uint location1, uint value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Or(ref long location1, long value) { +#if TARGET_ARM64 + if (Lse.IsSupported) + { + return Lse.LoadSet(ref location1, value); + } +#endif long current = location1; while (true) { From 2033af021a0c7635cae75704719aaa8457adfee5 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 14:25:34 +0200 Subject: [PATCH 2/6] Address review feedback: comments, and harden the Lse type contract - Add the "outlined on AOT" comment to the other two Interlocked files; it was only in Interlocked.CoreCLR.cs. - Promote the two base type checks in the Lse importer to noway_assert. These APIs are generic, so an unsupported type argument would otherwise silently produce an atomic access of the wrong width in a release JIT; now it fails the compilation instead. Verified: Lse.LoadAdd fails importation rather than emitting a bogus ldaddal, and Lse.Swap correctly emits a 32-bit swpal (the JIT sees enums as their underlying primitive). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- src/coreclr/jit/hwintrinsicarm64.cpp | 10 ++++++++-- .../src/System/Threading/Interlocked.cs | 6 ++++++ .../src/System/Runtime/Intrinsics/Arm/Lse.cs | 6 +++--- .../src/System/Threading/Interlocked.cs | 4 ++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 7c3dcbecd816c0..c1d6ecc9babc1f 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -728,7 +728,13 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, // Note we must use simdBaseType rather than retType here: the latter has been widened to // TYP_INT for the byte and halfword forms, which would give us an access of the wrong size. assert(simdSize == 0); - assert(varTypeIsIntegral(simdBaseType) && (genTypeSize(simdBaseType) <= TARGET_POINTER_SIZE)); + + // These are internal helpers with a contract the callers in Interlocked have to honor (see + // Lse.cs). Because they are generic, an unsupported type argument would otherwise silently + // produce an atomic access of the wrong width, so fail the compilation instead. Note this + // already rejects floating point, object references and anything larger than a pointer, + // none of which are integral. + noway_assert(varTypeIsIntegral(simdBaseType) && (genTypeSize(simdBaseType) <= TARGET_POINTER_SIZE)); genTreeOps oper; switch (intrinsic) @@ -753,7 +759,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, } // Only "cas" and "swp" have byte and halfword forms. - assert(!varTypeIsSmall(simdBaseType) || (oper == GT_CMPXCHG) || (oper == GT_XCHG)); + noway_assert(!varTypeIsSmall(simdBaseType) || (oper == GT_CMPXCHG) || (oper == GT_XCHG)); GenTree* comparand = nullptr; if (oper == GT_CMPXCHG) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs index 31f7a427878c73..495d444807d563 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -19,6 +19,7 @@ public static partial class Interlocked public static int CompareExchange(ref int location1, int value, int comparand) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.CompareAndSwap(ref location1, value, comparand); @@ -55,6 +56,7 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar public static long CompareExchange(ref long location1, long value, long comparand) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.CompareAndSwap(ref location1, value, comparand); @@ -87,6 +89,7 @@ public static long CompareExchange(ref long location1, long value, long comparan public static int Exchange(ref int location1, int value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.Swap(ref location1, value); @@ -110,6 +113,7 @@ public static int Exchange(ref int location1, int value) public static long Exchange(ref long location1, long value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.Swap(ref location1, value); @@ -192,6 +196,7 @@ public static long Add(ref long location1, long value) private static int ExchangeAdd(ref int location1, int value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadAdd(ref location1, value); @@ -212,6 +217,7 @@ private static int ExchangeAdd(ref int location1, int value) private static long ExchangeAdd(ref long location1, long value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadAdd(ref location1, value); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs index bc25da8951a3dd..f24bbb3e9cfeb0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs @@ -16,9 +16,9 @@ namespace System.Runtime.Intrinsics.Arm /// whose baseline is typically armv8-a. /// /// Unlike the Interlocked APIs these never fall back to an ldaxr/stlxr retry loop, so they may - /// only be called when is true. The type argument must be a primitive - /// integer type; sizes of 1 and 2 bytes are only supported by - /// and . + /// only be called when is true. The type argument must be an integer + /// (or an enum over one) no wider than a pointer; sizes of 1 and 2 bytes are only supported by + /// and . Anything else fails to compile. /// [Intrinsic] internal abstract class Lse : ArmBase diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs index 49286606479143..b7f09a14dcc985 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -584,6 +584,7 @@ public static ulong Read(ref readonly ulong location) => public static int And(ref int location1, int value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadClear(ref location1, value); @@ -622,6 +623,7 @@ public static uint And(ref uint location1, uint value) => public static long And(ref long location1, long value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadClear(ref location1, value); @@ -739,6 +741,7 @@ ref Unsafe.As(ref location1), public static int Or(ref int location1, int value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadSet(ref location1, value); @@ -777,6 +780,7 @@ public static uint Or(ref uint location1, uint value) => public static long Or(ref long location1, long value) { #if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. if (Lse.IsSupported) { return Lse.LoadSet(ref location1, value); From 97e467b8759bb89058c757623a0a2c6c1dfa018e Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 14:44:37 +0200 Subject: [PATCH 3/6] Throw from the Lse importer instead of failing the compilation Replace the noway_asserts guarding the Lse type argument with the standard impUnsupportedNamedIntrinsic path, so an unsupported instantiation compiles into a throw helper call rather than killing the compilation. The check is made before any argument is popped, and asks for a throwing expansion unconditionally because the managed bodies are recursive stubs - returning nullptr would fall back to a real call that recurses forever. Verified on arm64: Lse.LoadAdd now emits a call to CORINFO_HELP_THROW_PLATFORM_NOT_SUPPORTED (AOT remaps TypeNotSupported to PNSE) instead of tripping an assert, and Lse.Swap still emits swpal. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- src/coreclr/jit/hwintrinsicarm64.cpp | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index c1d6ecc9babc1f..a68eacc4e22a29 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -729,13 +729,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, // TYP_INT for the byte and halfword forms, which would give us an access of the wrong size. assert(simdSize == 0); - // These are internal helpers with a contract the callers in Interlocked have to honor (see - // Lse.cs). Because they are generic, an unsupported type argument would otherwise silently - // produce an atomic access of the wrong width, so fail the compilation instead. Note this - // already rejects floating point, object references and anything larger than a pointer, - // none of which are integral. - noway_assert(varTypeIsIntegral(simdBaseType) && (genTypeSize(simdBaseType) <= TARGET_POINTER_SIZE)); - genTreeOps oper; switch (intrinsic) { @@ -759,7 +752,21 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, } // Only "cas" and "swp" have byte and halfword forms. - noway_assert(!varTypeIsSmall(simdBaseType) || (oper == GT_CMPXCHG) || (oper == GT_XCHG)); + const bool hasSmallForms = (oper == GT_CMPXCHG) || (oper == GT_XCHG); + + // These are generic, so the type argument has to be checked rather than assumed: only an + // integer no wider than a pointer can be encoded. Anything else (floating point, an object + // reference, a struct) throws instead of silently accessing memory at the wrong width. + // + // Note the call is made before any argument is popped, and asks for a throwing expansion + // unconditionally: the managed bodies are recursive stubs, so falling back to a real call + // would simply recurse forever. + if (!varTypeIsIntegral(simdBaseType) || (genTypeSize(simdBaseType) > TARGET_POINTER_SIZE) || + (varTypeIsSmall(simdBaseType) && !hasSmallForms)) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_TYPE_NOT_SUPPORTED, method, sig, + /* mustExpand */ true); + } GenTree* comparand = nullptr; if (oper == GT_CMPXCHG) From 724dc0a4a71fdddd4f09642c49aa5c376800cde4 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 14:58:25 +0200 Subject: [PATCH 4/6] Keep Interlocked inlineable on NativeAOT arm64 Validated the real ILC path for the first time (x64-hosted ILC, --targetarch arm64 --targetos linux) rather than the JitStressAtomicsLightUp knob, and found that Exchange(ref int), Exchange(ref long) and CompareExchange(ref int) were left as out-of-line calls: bl System.Threading.Interlocked:Exchange(byref,int):int Those three are the only ones of the group that were not already marked AggressiveInlining. It never mattered before because they were always intrinsified at the call site, but now that the JIT declines to intrinsify them the inliner has to carry them, and its cost model rejects the larger body. With the attribute they inline as intended on linux-arm64: ldr w2, [x2] ; cached CPU features tbnz w2, #6, LSE ; bit 6 == Arm64IntrinsicConstants.Atomics ldaxr/stlxr ... ; armv8-a fallback LSE: swpal w0, w3, [x1] win-arm64 is unchanged (LSE is in its baseline): no feature check and no ldaxr/stlxr anywhere, just casal/swpal/ldaddal/ldsetal/ldclral. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- .../System.Private.CoreLib/src/System/Threading/Interlocked.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs index 495d444807d563..fbf0d5bccd9b98 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -16,6 +16,7 @@ public static partial class Interlocked #region CompareExchange [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareExchange(ref int location1, int value, int comparand) { #if TARGET_ARM64 @@ -86,6 +87,7 @@ public static long CompareExchange(ref long location1, long value, long comparan #region Exchange [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Exchange(ref int location1, int value) { #if TARGET_ARM64 @@ -110,6 +112,7 @@ public static int Exchange(ref int location1, int value) } [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Exchange(ref long location1, long value) { #if TARGET_ARM64 From 2e5c0f51930e15c06a4d9a1da6afad3af55a1ee4 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 15:54:55 +0200 Subject: [PATCH 5/6] Fix crossgen of the Lse generic definitions on Apple arm64 crossgen-corelib for maccatalyst-arm64 failed: Code generation failed for method '[S.P.CoreLib]System.Runtime.Intrinsics.Arm.Lse.CompareAndSwap(!!0&,!!0,!!0)' ---> System.InvalidCastException: Unable to cast object of type 'Internal.TypeSystem.SignatureMethodVariable' to type 'Internal.TypeSystem.DefType' R2R compiles the uninstantiated definition of a generic method. The usual recursive intrinsic stub then presents the importer with a self-call whose type argument is still a signature method variable, and trying to intrinsify that falls over. Apple arm64 is where it shows up because its baseline is apple-m1, so Lse.IsSupported folds to true and the call is live; everywhere else the ISA is not in the baseline, IsSupported folds to false and the body is dead code. The recursive stub is a convention for non-generic intrinsics and buys nothing here, so throw instead. All call sites are expanded at the call site, and a caller that ignores the contract now gets an exception rather than unbounded recursion. Verified by reproducing the exact crossgen2 command line locally (composite, mibc, macho, --targetos:maccatalyst) before and after. NativeAOT linux-arm64 light-up is unchanged (tbnz #6 plus swpal/casal/ldaddal/ldsetal/ldclral, no out-of-line calls), maccatalyst still emits baseline LSE with no feature checks, SPMI asmdiffs remain at 0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- .../src/System/Runtime/Intrinsics/Arm/Lse.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs index f24bbb3e9cfeb0..28a4f70de6d413 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs @@ -27,23 +27,28 @@ internal Lse() { } public static new bool IsSupported { get => IsSupported; } + // Note these are deliberately not the usual recursive intrinsic stubs. They are generic, and + // R2R compiles the uninstantiated definition of a generic method, which would then try to + // intrinsify a self-call whose type argument is still a signature method variable. Throwing + // keeps that body trivial, and is also what a caller that ignores the contract deserves. + /// Compare and swap: casal. public static T CompareAndSwap(ref T location, T value, T comparand) => - CompareAndSwap(ref location, value, comparand); + throw new PlatformNotSupportedException(); /// Atomic add, returning the original value: ldaddal. - public static T LoadAdd(ref T location, T value) => LoadAdd(ref location, value); + public static T LoadAdd(ref T location, T value) => throw new PlatformNotSupportedException(); /// Atomically clears the bits of that are not set in /// - that is, an atomic "and" - and returns the original value. /// Lowers to mvn followed by ldclral, since ldclral itself clears the /// bits that are set in its operand. - public static T LoadClear(ref T location, T value) => LoadClear(ref location, value); + public static T LoadClear(ref T location, T value) => throw new PlatformNotSupportedException(); /// Atomic bit set, returning the original value: ldsetal. - public static T LoadSet(ref T location, T value) => LoadSet(ref location, value); + public static T LoadSet(ref T location, T value) => throw new PlatformNotSupportedException(); /// Atomic exchange: swpal. - public static T Swap(ref T location, T value) => Swap(ref location, value); + public static T Swap(ref T location, T value) => throw new PlatformNotSupportedException(); } } From b9565d8762820c6d51935c95f305e811f96ccdfb Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Sat, 15 Aug 2026 16:58:28 +0200 Subject: [PATCH 6/6] Do not root generic methods in ReadyToRunHardwareIntrinsicRootProvider Reverts the previous commit's workaround and fixes the actual problem, which jkotas correctly pointed out my comment had described nonsensically: an uninstantiated generic method definition is never supposed to be compiled. ReadyToRunHardwareIntrinsicRootProvider roots every method on an intrinsic class whose ISA is in the supported set. AddCompilationRoot canonicalizes with GetCanonMethodTarget(CanonicalFormKind.Specific), which only maps reference types to __Canon - a SignatureMethodVariable passes through unchanged, so the typical definition gets rooted and queued, and compilation then fails as soon as the body resolves a token mentioning !!0: Unable to cast object of type 'Internal.TypeSystem.SignatureMethodVariable' to type 'Internal.TypeSystem.DefType' No existing hardware intrinsic class has generic methods, so this had never been exercised. Skip them; instantiations that are actually used are rooted through their callers. This lets Lse.cs go back to the conventional recursive intrinsic stubs. Verified: composite crossgen of CoreLib with the CI command line (mibc, --embed-pgo-data) now succeeds for arm64 on maccatalyst, osx, linux and windows, with the recursive stubs restored. NativeAOT linux-arm64 light-up is unchanged (tbnz #6 plus swpal/casal/ldaddal/ldsetal/ldclral, no out-of-line calls), an unsupported type argument still compiles to a throw helper rather than recursing, and SPMI asmdiffs remain at 0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ec0dae32-314b-42fd-a971-95921ea8ebb4 --- .../ReadyToRunHardwareIntrinsicRootProvider.cs | 9 +++++++++ .../src/System/Runtime/Intrinsics/Arm/Lse.cs | 18 +++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs index 972eb2686e2614..7e1bf1fa24e062 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs @@ -25,6 +25,15 @@ public void AddCompilationRoots(IRootingServiceProvider rootProvider) { foreach (MethodDesc method in hardwareIntrinsicType.GetMethods()) { + // A generic method has no code of its own to compile - rooting the typical + // definition would queue a method whose signature and body still refer to + // its own type variables. Instantiations that are actually used get rooted + // through the callers that use them. + if (method.HasInstantiation) + { + continue; + } + rootProvider.AddCompilationRoot(method, rootMinimalDependencies: false, "Supported hardware intrinsic method"); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs index 28a4f70de6d413..83c5dd6083cd1e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs @@ -18,7 +18,8 @@ namespace System.Runtime.Intrinsics.Arm /// Unlike the Interlocked APIs these never fall back to an ldaxr/stlxr retry loop, so they may /// only be called when is true. The type argument must be an integer /// (or an enum over one) no wider than a pointer; sizes of 1 and 2 bytes are only supported by - /// and . Anything else fails to compile. + /// and . Any other type argument compiles + /// into a throw. /// [Intrinsic] internal abstract class Lse : ArmBase @@ -27,28 +28,23 @@ internal Lse() { } public static new bool IsSupported { get => IsSupported; } - // Note these are deliberately not the usual recursive intrinsic stubs. They are generic, and - // R2R compiles the uninstantiated definition of a generic method, which would then try to - // intrinsify a self-call whose type argument is still a signature method variable. Throwing - // keeps that body trivial, and is also what a caller that ignores the contract deserves. - /// Compare and swap: casal. public static T CompareAndSwap(ref T location, T value, T comparand) => - throw new PlatformNotSupportedException(); + CompareAndSwap(ref location, value, comparand); /// Atomic add, returning the original value: ldaddal. - public static T LoadAdd(ref T location, T value) => throw new PlatformNotSupportedException(); + public static T LoadAdd(ref T location, T value) => LoadAdd(ref location, value); /// Atomically clears the bits of that are not set in /// - that is, an atomic "and" - and returns the original value. /// Lowers to mvn followed by ldclral, since ldclral itself clears the /// bits that are set in its operand. - public static T LoadClear(ref T location, T value) => throw new PlatformNotSupportedException(); + public static T LoadClear(ref T location, T value) => LoadClear(ref location, value); /// Atomic bit set, returning the original value: ldsetal. - public static T LoadSet(ref T location, T value) => throw new PlatformNotSupportedException(); + public static T LoadSet(ref T location, T value) => LoadSet(ref location, value); /// Atomic exchange: swpal. - public static T Swap(ref T location, T value) => throw new PlatformNotSupportedException(); + public static T Swap(ref T location, T value) => Swap(ref location, value); } }