diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs index 5fb05077081bbb..946d1974d899d8 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -52,6 +55,13 @@ public static long Decrement(ref long location) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Exchange(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -73,6 +83,13 @@ public static int Exchange(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Exchange(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -116,6 +133,13 @@ public static long Exchange(ref long location1, long value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareExchange(ref int location1, int value, int comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -158,6 +182,13 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long CompareExchange(ref long location1, long value, long comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -212,6 +243,13 @@ public static long Add(ref long location1, long value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ExchangeAdd(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return ExchangeAdd(ref location1, value); // Must expand intrinsic #else @@ -228,6 +266,13 @@ private static int ExchangeAdd(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static long ExchangeAdd(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return ExchangeAdd(ref location1, value); // Must expand intrinsic #else diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 85bebf191cc2ec..666d2f36fc1cb3 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -3810,7 +3810,7 @@ void CodeGen::genLockedInstructions(GenTreeOp* treeNode) emitAttr dataSize = emitActualTypeSize(data); - if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (m_compiler->compGetAtomicsImplForNode(treeNode) == Compiler::AtomicsImpl::Lse) { assert(!data->isContainedIntOrIImmed()); @@ -3988,8 +3988,11 @@ void CodeGen::genCodeForCmpXchg(GenTreeCmpXchg* treeNode) emitAttr dataSize = emitActualTypeSize(data); - if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (m_compiler->compGetAtomicsImplForNode(treeNode) == Compiler::AtomicsImpl::Lse) { + // 'casal' has no immediate form, so lowering must not have contained the comparand. + assert(!comparand->isContained()); + // casal use the comparand as the target reg GetEmitter()->emitIns_Mov(INS_mov, dataSize, targetReg, comparandReg, /* canSkip */ true); diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index dbc56f2344e513..caee14b53cd896 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -2017,6 +2017,68 @@ bool Compiler::notifyInstructionSetUsage(CORINFO_InstructionSet isa, bool suppor return info.compCompHnd->notifyInstructionSetUsage(isa, supported); } +#ifdef TARGET_ARM64 +//------------------------------------------------------------------------ +// compGetAtomicsImpl: Decide how atomic operations are to be expanded in this method. +// +// Return Value: +// AtomicsImpl::Lse - the Armv8.1 atomics are known to be available, use them directly. +// AtomicsImpl::LlSc - they are known not to be available, the ldaxr/stlxr retry loops +// have to be used. +// AtomicsImpl::Dynamic - they are not part of the baseline instruction set, but they may +// still be available on the machine that ends up running this code. +// The Interlocked APIs are left unexpanded so that the managed +// `if (Lse.IsSupported)` check in their bodies gets inlined instead. +// +// Notes: +// The result is computed on demand and cached: asking the EE about instruction set support has +// side effects (for ReadyToRun it records a hard requirement on the method), so we must not do +// it for methods that contain no atomic operations at all. +// +// 'Dynamic' is what makes NativeAOT interesting here: its baseline is typically armv8-a, so +// compExactlyDependsOn(InstructionSet_Atomics) is false even though the vast majority of the +// hardware in the wild does implement LSE. Note that this state cannot occur for the runtime +// JIT, where the opportunistic and the exact answer always agree - see JitStressAtomicsLightUp +// for a way to force it for testing. +// +Compiler::AtomicsImpl Compiler::compGetAtomicsImpl() +{ + if (m_atomicsImpl != AtomicsImpl::Uninitialized) + { + return m_atomicsImpl; + } + + AtomicsImpl impl; + if (!opts.compSupportsISA.HasInstructionSet(InstructionSet_Atomics)) + { + impl = AtomicsImpl::LlSc; + } +#ifdef DEBUG + else if (JitConfig.JitStressAtomicsLightUp() != 0) + { + // Pretend the atomics are only opportunistically available so that we exercise the + // managed light-up path. Note we deliberately do not report the ISA as used to the EE. + impl = AtomicsImpl::Dynamic; + } +#endif + else if (compExactlyDependsOn(InstructionSet_Atomics)) + { + impl = AtomicsImpl::Lse; + } + else + { + impl = AtomicsImpl::Dynamic; + } + + JITDUMP("Atomics will be expanded as %s\n", (impl == AtomicsImpl::Lse) ? "LSE" + : (impl == AtomicsImpl::LlSc) ? "ldaxr/stlxr loops" + : "managed light-up"); + + m_atomicsImpl = impl; + return impl; +} +#endif // TARGET_ARM64 + #ifdef PROFILING_SUPPORTED // A Dummy routine to receive Enter/Leave/Tailcall profiler callbacks. // These are used when DOTNET_JitEltHookEnabled=1 diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 03fa22ec8c5463..436ee32af6ca72 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -5418,6 +5418,7 @@ class Compiler R2RARG(CORINFO_CONST_LOOKUP* entryPoint), NamedIntrinsic* pIntrinsicName, bool* isSpecialIntrinsic = nullptr); + bool impIsAtomicLightUpCandidate(NamedIntrinsic ni, CORINFO_SIG_INFO* sig, bool mustExpand); GenTree* impEstimateIntrinsic(CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig, CorInfoType callJitType, @@ -10936,6 +10937,46 @@ class Compiler return opts.compSupportsISA.HasInstructionSet(isa); } +#ifdef TARGET_ARM64 + // How the Interlocked/atomic operations (GT_CMPXCHG, GT_XADD, GT_XCHG, GT_XORR, GT_XAND) + // are to be expanded by codegen. + enum class AtomicsImpl : uint8_t + { + Uninitialized, + LlSc, // ldaxr/stlxr retry loop + Lse, // Armv8.1 single instruction atomics + Dynamic, // not in the baseline, but may still be available at run time + }; + + AtomicsImpl compGetAtomicsImpl(); + + //------------------------------------------------------------------------ + // compGetAtomicsImplForNode: How a specific atomic node is to be expanded. + // + // The importer only ever creates an atomic node when it knows which sequence to use: in the + // AtomicsImpl::Dynamic case the Interlocked APIs are left unexpanded and the managed + // `if (Lse.IsSupported)` check in their bodies picks the arm, marking the node it creates + // with GTF_ATOMIC_LSE. So lowering, LSRA and codegen only ever see a concrete answer. + // + AtomicsImpl compGetAtomicsImplForNode(GenTree* node) + { + assert(node->OperIsAtomicOp()); + + if ((node->gtFlags & GTF_ATOMIC_LSE) != 0) + { + return AtomicsImpl::Lse; + } + + AtomicsImpl impl = compGetAtomicsImpl(); + return (impl == AtomicsImpl::Lse) ? AtomicsImpl::Lse : AtomicsImpl::LlSc; + } + +private: + AtomicsImpl m_atomicsImpl = AtomicsImpl::Uninitialized; + +public: +#endif // TARGET_ARM64 + private: #ifdef DEBUG //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index f09808f88d6f56..fe3b81c0779053 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2770,6 +2770,15 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) return false; } +#ifdef TARGET_ARM64 + // The two arms of an opportunistic atomics check hold otherwise identical nodes that must not + // be merged back together - the flag is what tells codegen which sequence to emit. + if (OperIsAtomicOp(oper) && (((op1->gtFlags ^ op2->gtFlags) & GTF_ATOMIC_LSE) != 0)) + { + return false; + } +#endif // TARGET_ARM64 + /* Sensible flags must be equal */ if (op1->IsUnsigned() != op2->IsUnsigned()) { diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 7033e6cb81cd42..7907f09217d8ba 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -487,6 +487,10 @@ enum GenTreeFlags : unsigned GTF_IND_INITCLASS = 0x00200000, // OperIsIndir() -- the indirection requires preceding static cctor GTF_IND_ALLOW_NON_ATOMIC = 0x00100000, // GT_IND -- this memory access does not need to be atomic + // Set by the importer on arm64 for the atomic nodes created from the Lse intrinsics. Note this + // bit is not used by any of the GTF_IND_* flags, which the atomic opers otherwise share. + GTF_ATOMIC_LSE = 0x04000000, // GT_XADD/GT_XAND/GT_XORR/GT_XCHG/GT_CMPXCHG -- use the Armv8.1 atomics + // Represents flags that an indirection based on another indirection must preserve GTF_IND_MUST_PRESERVE_FLAGS = GTF_IND_VOLATILE | GTF_IND_UNALIGNED | GTF_IND_INITCLASS, diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index a1058fbffb0ccf..9a471fdbd6d4e5 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -983,7 +983,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = { { FIRST_NI_Fp16, LAST_NI_Fp16 }, // Fp16 { FIRST_NI_Sha1, LAST_NI_Sha1 }, // Sha1 { FIRST_NI_Sha256, LAST_NI_Sha256 }, // Sha256 - { NI_Illegal, NI_Illegal }, // Atomics + { FIRST_NI_Atomics, LAST_NI_Atomics }, // Atomics { FIRST_NI_Vector, LAST_NI_Vector }, // Vector64 { FIRST_NI_Vector, LAST_NI_Vector }, // Vector128 { NI_Illegal, NI_Illegal }, // VectorT diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 91a3458999f833..a68eacc4e22a29 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -94,6 +94,13 @@ CORINFO_InstructionSet Compiler::lookupInstructionSet(const char* className) return InstructionSet_Dp; } } + else if (className[0] == 'L') + { + if (strcmp(className, "Lse") == 0) + { + return InstructionSet_Atomics; + } + } else if (className[0] == 'R') { if (strcmp(className, "Rdm") == 0) @@ -703,7 +710,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, const int numArgs = sig->numArgs; // The vast majority of "special" intrinsics are Vector64/Vector128 methods. - // The only exception is ArmBase.Yield which should be treated differently. + // The only exceptions are ArmBase.Yield and the Lse atomics, which are treated differently. if (intrinsic == NI_ArmBase_Yield) { assert(sig->numArgs == 0); @@ -713,6 +720,80 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, return gtNewScalarHWIntrinsicNode(TYP_VOID, intrinsic); } + if (isa == InstructionSet_Atomics) + { + // The Armv8.1 atomics reuse the nodes that back the Interlocked APIs; GTF_ATOMIC_LSE is + // what tells lowering and codegen to use the single instruction forms unconditionally. + // + // Note we must use simdBaseType rather than retType here: the latter has been widened to + // TYP_INT for the byte and halfword forms, which would give us an access of the wrong size. + assert(simdSize == 0); + + genTreeOps oper; + switch (intrinsic) + { + case NI_Atomics_CompareAndSwap: + oper = GT_CMPXCHG; + break; + case NI_Atomics_LoadAdd: + oper = GT_XADD; + break; + case NI_Atomics_LoadClear: + oper = GT_XAND; + break; + case NI_Atomics_LoadSet: + oper = GT_XORR; + break; + case NI_Atomics_Swap: + oper = GT_XCHG; + break; + default: + unreached(); + } + + // Only "cas" and "swp" have byte and halfword forms. + const bool hasSmallForms = (oper == GT_CMPXCHG) || (oper == GT_XCHG); + + // These are generic, so the type argument has to be checked rather than assumed: only an + // integer no wider than a pointer can be encoded. Anything else (floating point, an object + // reference, a struct) throws instead of silently accessing memory at the wrong width. + // + // Note the call is made before any argument is popped, and asks for a throwing expansion + // unconditionally: the managed bodies are recursive stubs, so falling back to a real call + // would simply recurse forever. + if (!varTypeIsIntegral(simdBaseType) || (genTypeSize(simdBaseType) > TARGET_POINTER_SIZE) || + (varTypeIsSmall(simdBaseType) && !hasSmallForms)) + { + return impUnsupportedNamedIntrinsic(CORINFO_HELP_THROW_TYPE_NOT_SUPPORTED, method, sig, + /* mustExpand */ true); + } + + GenTree* comparand = nullptr; + if (oper == GT_CMPXCHG) + { + assert(sig->numArgs == 3); + comparand = impPopStack().val; + + if (varTypeIsSmall(simdBaseType)) + { + // Small types need the comparand to have its upper bits zeroed. + comparand = gtNewCastNode(genActualType(simdBaseType), comparand, /* uns */ false, + varTypeToUnsigned(simdBaseType)); + } + } + else + { + assert(sig->numArgs == 2); + } + + GenTree* value = impPopStack().val; + GenTree* addr = impPopStack().val; + + GenTree* node = gtNewAtomicNode(oper, simdBaseType, addr, value, comparand); + node->gtFlags |= GTF_ATOMIC_LSE; + return node; + } + bool isScalar = (category == HW_Category_Scalar); assert(numArgs >= 0); diff --git a/src/coreclr/jit/hwintrinsiclistarm64.h b/src/coreclr/jit/hwintrinsiclistarm64.h index 161ac83c1a2510..5ed4dd9e3048f2 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64.h +++ b/src/coreclr/jit/hwintrinsiclistarm64.h @@ -630,6 +630,22 @@ HARDWARE_INTRINSIC(Sha256, ScheduleUpdate0, HARDWARE_INTRINSIC(Sha256, ScheduleUpdate1, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sha256su1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_HasRMWSemantics) #define LAST_NI_Sha256 NI_Sha256_ScheduleUpdate1 +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// Lse Intrinsics (Armv8.1 atomics) +// +// These import into the same GT_CMPXCHG/GT_XADD/GT_XCHG/GT_XORR/GT_XAND nodes as the Interlocked +// APIs they implement, marked with GTF_ATOMIC_LSE. See Compiler::impSpecialIntrinsic. +#define FIRST_NI_Atomics NI_Atomics_CompareAndSwap +HARDWARE_INTRINSIC(Atomics, CompareAndSwap, 0, 3, INS_casalb, INS_casalb, INS_casalh, INS_casalh, INS_casal, INS_casal, INS_casal, INS_casal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadAdd, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldaddal, INS_ldaddal, INS_ldaddal, INS_ldaddal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadClear, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldclral, INS_ldclral, INS_ldclral, INS_ldclral, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, LoadSet, 0, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ldsetal, INS_ldsetal, INS_ldsetal, INS_ldsetal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +HARDWARE_INTRINSIC(Atomics, Swap, 0, 2, INS_swpalb, INS_swpalb, INS_swpalh, INS_swpalh, INS_swpal, INS_swpal, INS_swpal, INS_swpal, INS_invalid, INS_invalid, -1, -1, HW_Category_Special, HW_Flag_NoFloatingPointUsed|HW_Flag_SpecialImport|HW_Flag_SpecialSideEffect_Other) +#define LAST_NI_Atomics NI_Atomics_Swap + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 19b001e5026bb2..5309a63a6295b2 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -3266,6 +3266,64 @@ GenTree* Compiler::impCreateSpanIntrinsic(CORINFO_SIG_INFO* sig) return impCreateLocalNode(spanTempNum DEBUGARG(0)); } +//------------------------------------------------------------------------ +// impIsAtomicLightUpCandidate: Should the given Interlocked API be left unexpanded so that the +// managed `if (Lse.IsSupported)` check in its body can light up the Armv8.1 atomics? +// +// Arguments: +// ni - The named intrinsic +// sig - Signature of the call +// mustExpand - Whether the JIT is required to expand this call +// +// Return Value: +// True if the call should be imported as a regular (inlineable) call. +// +// Notes: +// This is only ever true when the Armv8.1 atomics are not part of our baseline instruction set +// but may still be present on the machine that ends up running the code - in practice that means +// NativeAOT, whose baseline is typically armv8-a. In every other configuration the answer is +// "no" and these APIs keep expanding exactly like they always have. +// +// 'mustExpand' is what stops the light-up from recursing forever: the fallback arm of the check +// calls the very same API from within its own body, and recursive intrinsic calls must expand. +// +bool Compiler::impIsAtomicLightUpCandidate(NamedIntrinsic ni, CORINFO_SIG_INFO* sig, bool mustExpand) +{ +#ifdef TARGET_ARM64 + switch (ni) + { + case NI_System_Threading_Interlocked_And: + case NI_System_Threading_Interlocked_Or: + case NI_System_Threading_Interlocked_CompareExchange: + case NI_System_Threading_Interlocked_Exchange: + case NI_System_Threading_Interlocked_ExchangeAdd: + break; + + default: + return false; + } + + if (mustExpand) + { + return false; + } + + // Only the overloads that actually carry the managed check qualify: the ones operating on a + // machine word sized integer. The object overloads use a write barrier helper instead, and the + // byte and halfword ones funnel through a masking loop, so leaving those unexpanded would just + // pessimize them. + var_types retType = JITtype2varType(sig->retType); + if (!varTypeIsIntegral(retType) || (genTypeSize(retType) < 4) || (genTypeSize(retType) > TARGET_POINTER_SIZE)) + { + return false; + } + + return compGetAtomicsImpl() == AtomicsImpl::Dynamic; +#else + return false; +#endif // TARGET_ARM64 +} + //------------------------------------------------------------------------ // impIntrinsic: possibly expand intrinsic call into alternate IR sequence // @@ -3548,6 +3606,15 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, *pIntrinsicName = ni; + if (impIsAtomicLightUpCandidate(ni, sig, mustExpand)) + { + // The Armv8.1 atomics may be available on the machine running this code even though they + // are not part of our baseline instruction set. Leave the call alone so that the managed + // `if (Lse.IsSupported)` check in the API's body gets inlined and lights them up. + JITDUMP("Not expanding %s to allow the managed Armv8.1 atomics light-up\n", eeGetMethodFullName(method)); + return nullptr; + } + if (ni == NI_System_StubHelpers_GetStubContext) { // must be done regardless of DbgCode and MinOpts diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 1e95c26b1e31cf..dbd797f334d6b6 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -868,6 +868,14 @@ RELEASE_CONFIG_INTEGER(JitEnableStrengthReduction, "JitEnableStrengthReduction", // Enable IV optimizations RELEASE_CONFIG_INTEGER(JitEnableInductionVariableOpts, "JitEnableInductionVariableOpts", 1) +#if defined(TARGET_ARM64) +// Stress the opportunistic light-up of the Armv8.1 LSE atomics: pretend they are not part of the +// baseline instruction set even when they are, so that the managed `if (Lse.IsSupported)` checks in +// the Interlocked APIs are used. This state occurs naturally for NativeAOT, but never for the +// runtime JIT, so this is the only way to cover it outside of AOT compilation. +CONFIG_INTEGER(JitStressAtomicsLightUp, "JitStressAtomicsLightUp", 0) +#endif // TARGET_ARM64 + // JitFunctionFile: Name of a file that contains a list of functions. If the currently compiled function is in the // file, certain other JIT config variables will be active. If the currently compiled function is not in the file, // the specific JIT config variables will not be active. diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d3955c7861ae1d..7050412fe55137 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -102,9 +102,10 @@ bool Lowering::IsContainableImmed(GenTree* parentNode, GenTree* childNode) const case GT_XORR: case GT_XAND: case GT_XADD: - return m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics) - ? false - : emitter::emitIns_valid_imm_for_add(immVal, size); + // The LSE forms of these have no immediate encoding, so only the ldaxr/stlxr + // sequences can make use of a contained immediate. + return (m_compiler->compGetAtomicsImplForNode(parentNode) == Compiler::AtomicsImpl::LlSc) && + emitter::emitIns_valid_imm_for_add(immVal, size); #elif defined(TARGET_ARM) return emitter::emitIns_valid_imm_for_add(immVal, flags); #endif diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..ba7aaf373dbbdf 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1133,7 +1133,9 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = cmpXchgNode->Comparand()->isContained() ? 2 : 3; assert(dstCount == 1); - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + const bool useLse = (m_compiler->compGetAtomicsImplForNode(tree) == Compiler::AtomicsImpl::Lse); + + if (!useLse) { // For ARMv8 exclusives requires a single internal register buildInternalIntRegisterDefForNode(tree); @@ -1155,7 +1157,7 @@ int LinearScan::BuildNode(GenTree* tree) // For ARMv8 exclusives the lifetime of the comparand must be extended because // it may be used used multiple during retries - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (!useLse) { setDelayFree(comparandUse); } @@ -1177,7 +1179,9 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == (tree->TypeIs(TYP_VOID) ? 0 : 1)); srcCount = tree->gtGetOp2()->isContained() ? 1 : 2; - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + const bool useLse = (m_compiler->compGetAtomicsImplForNode(tree) == Compiler::AtomicsImpl::Lse); + + if (!useLse) { // GT_XCHG requires a single internal register; the others require two. buildInternalIntRegisterDefForNode(tree); @@ -1202,7 +1206,7 @@ int LinearScan::BuildNode(GenTree* tree) // For ARMv8 exclusives the lifetime of the addr and data must be extended because // it may be used used multiple during retries - if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Atomics)) + if (!useLse) { // Internals may not collide with target if (dstCount == 1) diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs index c8a9c739831fcc..fbf0d5bccd9b98 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime; using System.Runtime.CompilerServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -13,8 +16,16 @@ public static partial class Interlocked #region CompareExchange [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int CompareExchange(ref int location1, int value, int comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -45,6 +56,13 @@ internal static unsafe int CompareExchange(int* location1, int value, int compar [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long CompareExchange(ref long location1, long value, long comparand) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.CompareAndSwap(ref location1, value, comparand); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return CompareExchange(ref location1, value, comparand); // Must expand intrinsic #else @@ -69,8 +87,16 @@ public static long CompareExchange(ref long location1, long value, long comparan #region Exchange [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Exchange(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_X86 || TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -86,8 +112,16 @@ public static int Exchange(ref int location1, int value) } [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Exchange(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.Swap(ref location1, value); + } +#endif #if TARGET_AMD64 || TARGET_ARM64 || TARGET_RISCV64 return Exchange(ref location1, value); // Must expand intrinsic #else @@ -164,6 +198,13 @@ public static long Add(ref long location1, long value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int ExchangeAdd(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif int oldValue; do @@ -178,6 +219,13 @@ private static int ExchangeAdd(ref int location1, int value) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static long ExchangeAdd(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadAdd(ref location1, value); + } +#endif long oldValue; do diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 870544e2156425..4293c8a79d159e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -1164,7 +1164,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("fp16", "", InstructionSet.ARM64_Fp16, true); yield return new InstructionSetInfo("sha1", "Sha1", InstructionSet.ARM64_Sha1, true); yield return new InstructionSetInfo("sha2", "Sha256", InstructionSet.ARM64_Sha256, true); - yield return new InstructionSetInfo("lse", "", InstructionSet.ARM64_Atomics, true); + yield return new InstructionSetInfo("lse", "Lse", InstructionSet.ARM64_Atomics, true); yield return new InstructionSetInfo("Vector64", "", InstructionSet.ARM64_Vector64, false); yield return new InstructionSetInfo("Vector128", "", InstructionSet.ARM64_Vector128, false); yield return new InstructionSetInfo("VectorT", "", InstructionSet.ARM64_VectorT, false); @@ -1603,6 +1603,9 @@ public static InstructionSet LookupPlatformIntrinsicInstructionSet(TargetArchite else return InstructionSet.ARM64_Sha256; + case "Lse": + return InstructionSet.ARM64_Atomics; + case "Sve": if (nestedTypeName == "Arm64") return InstructionSet.ARM64_Sve_Arm64; @@ -2216,6 +2219,16 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } break; + case (InstructionSet.ARM64_Atomics, TargetArchitecture.ARM64): + { + var type = context.SystemModule.GetType("System.Runtime.Intrinsics.Arm"u8, "Lse"u8, false); + if (type != null) + { + yield return type; + } + } + break; + case (InstructionSet.ARM64_Sve, TargetArchitecture.ARM64): case (InstructionSet.ARM64_Sve_Arm64, TargetArchitecture.ARM64): { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index a70cd7d9991687..c7a57e87c51de6 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -218,7 +218,7 @@ instructionset ,ARM64 ,Rdm , ,24 ,Rdm instructionset ,ARM64 , ,Fp16 ,95 ,Fp16 ,fp16 instructionset ,ARM64 ,Sha1 , ,19 ,Sha1 ,sha1 instructionset ,ARM64 ,Sha256 , ,20 ,Sha256 ,sha2 -instructionset ,ARM64 , ,Atomics ,21 ,Atomics ,lse +instructionset ,ARM64 ,Lse ,Atomics ,21 ,Atomics ,lse instructionset ,ARM64 , , , ,Vector64 , instructionset ,ARM64 , , , ,Vector128 , instructionset ,ARM64 , , , ,VectorT , diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs index 972eb2686e2614..7e1bf1fa24e062 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunHardwareIntrinsicRootProvider.cs @@ -25,6 +25,15 @@ public void AddCompilationRoots(IRootingServiceProvider rootProvider) { foreach (MethodDesc method in hardwareIntrinsicType.GetMethods()) { + // A generic method has no code of its own to compile - rooting the typical + // definition would queue a method whose signature and body still refer to + // its own type variables. Instantiations that are actually used get rooted + // through the callers that use them. + if (method.HasInstantiation) + { + continue; + } + rootProvider.AddCompilationRoot(method, rootMinimalDependencies: false, "Supported hardware intrinsic method"); } } diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index bd846a6aed09b1..6c9a990525c416 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2880,6 +2880,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs new file mode 100644 index 00000000000000..83c5dd6083cd1e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Lse.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Runtime.Intrinsics.Arm +{ + /// + /// Provides access to the ARM Large System Extensions (the Armv8.1 atomic instructions). + /// + /// + /// This type is internal because it exists so that can + /// light the instructions up opportunistically: folds to a constant + /// whenever LSE is (or is not) part of the compilation's baseline instruction set, and becomes a + /// runtime check against the CPU features word otherwise - which is the common case for NativeAOT, + /// whose baseline is typically armv8-a. + /// + /// Unlike the Interlocked APIs these never fall back to an ldaxr/stlxr retry loop, so they may + /// only be called when is true. The type argument must be an integer + /// (or an enum over one) no wider than a pointer; sizes of 1 and 2 bytes are only supported by + /// and . Any other type argument compiles + /// into a throw. + /// + [Intrinsic] + internal abstract class Lse : ArmBase + { + internal Lse() { } + + public static new bool IsSupported { get => IsSupported; } + + /// Compare and swap: casal. + public static T CompareAndSwap(ref T location, T value, T comparand) => + CompareAndSwap(ref location, value, comparand); + + /// Atomic add, returning the original value: ldaddal. + public static T LoadAdd(ref T location, T value) => LoadAdd(ref location, value); + + /// Atomically clears the bits of that are not set in + /// - that is, an atomic "and" - and returns the original value. + /// Lowers to mvn followed by ldclral, since ldclral itself clears the + /// bits that are set in its operand. + public static T LoadClear(ref T location, T value) => LoadClear(ref location, value); + + /// Atomic bit set, returning the original value: ldsetal. + public static T LoadSet(ref T location, T value) => LoadSet(ref location, value); + + /// Atomic exchange: swpal. + public static T Swap(ref T location, T value) => Swap(ref location, value); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs index dfa92e8b9381e1..b7f09a14dcc985 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Interlocked.cs @@ -5,6 +5,9 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if TARGET_ARM64 +using System.Runtime.Intrinsics.Arm; +#endif namespace System.Threading { @@ -580,6 +583,13 @@ public static ulong Read(ref readonly ulong location) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int And(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadClear(ref location1, value); + } +#endif int current = location1; while (true) { @@ -612,6 +622,13 @@ public static uint And(ref uint location1, uint value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long And(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadClear(ref location1, value); + } +#endif long current = location1; while (true) { @@ -723,6 +740,13 @@ ref Unsafe.As(ref location1), [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Or(ref int location1, int value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadSet(ref location1, value); + } +#endif int current = location1; while (true) { @@ -755,6 +779,13 @@ public static uint Or(ref uint location1, uint value) => [MethodImpl(MethodImplOptions.AggressiveInlining)] public static long Or(ref long location1, long value) { +#if TARGET_ARM64 + // Outlined on AOT, where LSE may not be in the baseline instruction set. + if (Lse.IsSupported) + { + return Lse.LoadSet(ref location1, value); + } +#endif long current = location1; while (true) {