Updates

scripts/benchmarks_local.py

@@ -399,13 +399,12 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type:
         os.environ['RestoreAdditionalProjectSources'] = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle")
 
     elif specific_run_type == RunType.WasmAOT:
-        benchmark_ci_args += ['--wasm', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")]
+        benchmark_ci_args += ['--wasm', '--wasm-runtime-flavor', 'MonoAOT', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")]
         bdn_args_unescaped += [
             '--anyCategories', 'Libraries', 'Runtime',
             '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono',
             '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet", "dotnet"),
             '--wasmEngine', parsed_args.wasm_engine_path,
-            '--aotcompilermode', 'wasm',
             '--logBuildOutput',
             '--generateBinLog'
         ]

scripts/micro_benchmarks.py

@@ -145,8 +145,8 @@ def __get_bdn_arguments(user_input: str) -> list[str]:
         dest='wasm_runtime_flavor',
         required=False,
         default='Mono',
-        choices=['Mono', 'CoreCLR'],
-        help='Runtime flavor for WASM benchmarks: Mono (default) or CoreCLR'
+        choices=['Mono', 'MonoAOT', 'CoreCLR'],
+        help='Runtime flavor for WASM benchmarks: Mono interpreter (default), MonoAOT, or CoreCLR'
     )
 
     parser.add_argument(
@@ -268,22 +268,22 @@ def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]:
     if framework.startswith("nativeaot"):
         run_args += ['--runtimes', framework]
     if args.wasm:
-        if framework == "net6.0":
-            run_args += ['--runtimes', 'wasm']
-        elif framework == "net7.0":
-            run_args += ['--runtimes', 'wasmnet70']
-        elif framework == "net8.0":
-            run_args += ['--runtimes', 'wasmnet80']
-        elif framework == "net9.0":
-            run_args += ['--runtimes', 'wasmnet90']
-        elif framework == "net10.0":
-            run_args += ['--runtimes', 'wasmnet10_0']
-        elif framework == "net11.0":
-            run_args += ['--runtimes', 'wasmnet11_0']
-        else:
+        wasm_moniker_prefix = {
+            'Mono': 'monowasm',
+            'MonoAOT': 'monowasmaot',
+            'CoreCLR': 'corewasm',
+        }[args.wasm_runtime_flavor]
+        wasm_version = {
+            'net6.0': '6.0',
+            'net7.0': '7.0',
+            'net8.0': '8.0',
+            'net9.0': '9.0',
+            'net10.0': '10.0',
+            'net11.0': '11.0',
+        }.get(framework)
+        if wasm_version is None:
             raise ArgumentTypeError('Framework {} is not supported for wasm'.format(framework))
-        if args.wasm_runtime_flavor != 'Mono':
-            run_args += ['--wasmRuntimeFlavor', args.wasm_runtime_flavor]
+        run_args += ['--runtimes', wasm_moniker_prefix + wasm_version]
 
     # Increase default 2 min build timeout to accommodate slow (or even very slow) hardware
     if not args.bdn_arguments or '--buildTimeout' not in args.bdn_arguments:

scripts/run_performance_job.py

@@ -501,7 +501,7 @@ def get_bdn_arguments(
             bdn_arguments += [
                 "--runtimes", "monoaotllvm",
                 "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross",
-                "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", 
+                "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack",
                 "--aotcompilermode", "llvm",
             ]
         else:
@@ -530,10 +530,7 @@ def get_bdn_arguments(
             bdn_arguments += ["\\\"--wasmArgs=--experimental-wasm-exnref\\\""]
 
         if is_aot:
-            bdn_arguments += [
-                "--aotcompilermode", "wasm",
-                "--buildTimeout", "3600"
-            ]
+            bdn_arguments += ["--buildTimeout", "3600"]
 
     if runtime_type == "wasm_coreclr":
         category_exclusions += ["NoWASM", "NoWasmCoreCLR", "NoMono"]
@@ -555,13 +552,13 @@ def get_bdn_arguments(
     if runtime_type == "coreclr_r2r_interpreter":
         if os_group == "windows":
             bdn_arguments += [
-                "--runtimes", "r2r11_0",
+                "--runtimes", "r2r11.0",
                 "--customruntimepack", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\runtimepack",
                 "--aotcompilerpath", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\crossgen2",
             ]
         else:
             bdn_arguments += [
-                "--runtimes", "r2r11_0",
+                "--runtimes", "r2r11.0",
                 "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/runtimepack",
                 "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/crossgen2",
             ]
@@ -684,7 +681,7 @@ def get_run_configurations(
 
     return configurations
 
-def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, only_sanity_check: bool = False):
+def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str, wasm_coreclr: bool = False, wasm_aot: bool = False, only_sanity_check: bool = False):
     if os_group == "windows":
         work_item_command = [
             "python",
@@ -714,6 +711,8 @@ def get_work_item_command(os_group: str, target_csproj: str, architecture: str,
         work_item_command += ["--run-isolated", "--wasm", "--dotnet-path", "$HELIX_CORRELATION_PAYLOAD/dotnet/"]
         if wasm_coreclr:
             work_item_command += ["--wasm-runtime-flavor", "CoreCLR"]
+        elif wasm_aot:
+            work_item_command += ["--wasm-runtime-flavor", "MonoAOT"]
 
     work_item_command += ["--bdn-artifacts", bdn_artifacts_dir]
 
@@ -1360,7 +1359,7 @@ def get_bdn_args_for_coreroot_dir(coreroot_dir: Optional[str]):
 
     def get_work_item_command_for_artifact_dir(artifact_dir: str):
         assert args.target_csproj is not None
-        return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir, wasm_coreclr, args.only_sanity_check)
+        return get_work_item_command(args.os_group, args.target_csproj, args.architecture, perf_lab_framework, args.internal, wasm, artifact_dir, wasm_coreclr, wasm_aot, args.only_sanity_check)
 
     work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory)
     baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir)

src/benchmarks/micro/Program.cs

@@ -23,6 +23,7 @@ static async Task<int> Main(string[] args)
             List<string> exclusionFilterValue;
             List<string> categoryExclusionFilterValue;
             bool getDiffableDisasm;
+            MonoAotLLVMToolChain monoAotToolchain = null;
 
             // Parse and remove any additional parameters that we need that aren't part of BDN
             try
@@ -33,6 +34,69 @@ static async Task<int> Main(string[] args)
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue);
                 CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
 
+                // Extract monoaotllvm args not recognized by BDN and build the toolchain.
+                argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--runtimes", out var runtimesValues);
+                if (runtimesValues.Remove("monoaotllvm"))
+                {
+                    if (runtimesValues.Count > 0)
+                    {
+                        argsList.Add("--runtimes");
+                        argsList.AddRange(runtimesValues);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--cli", out var cliPathValue);
+                    if (cliPathValue.Count > 0)
+                    {
+                        argsList.Add("--cli");
+                        argsList.AddRange(cliPathValue);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--packages", out var packagesPathValue);
+                    if (packagesPathValue.Count > 0)
+                    {
+                        argsList.Add("--packages");
+                        argsList.AddRange(packagesPathValue);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilerpath", out var aotCompilerPathValue);
+                    string aotCompilerPath;
+                    if (aotCompilerPathValue.Count > 0)
+                    {
+                        aotCompilerPath = aotCompilerPathValue[0];
+                        argsList.Add("--aotcompilerpath");
+                        argsList.AddRange(aotCompilerPathValue);
+                    }
+                    else
+                    {
+                        aotCompilerPath = "";
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--customruntimepack", out var customRuntimePackValue);
+                    string customRuntimePack;
+                    if (customRuntimePackValue.Count > 0)
+                    {
+                        customRuntimePack = customRuntimePackValue[0];
+                        argsList.Add("--customruntimepack");
+                        argsList.AddRange(customRuntimePackValue);
+
+                    }
+                    else
+                    {
+                        customRuntimePack = "";
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilermode", out var aotCompilerModeValue);
+                    monoAotToolchain = new MonoAotLLVMToolChain(new MonoAotLLVMRuntime(Environment.Version), new()
+                    {
+                        CliPath = cliPathValue.Count > 0 ? new(cliPathValue[0]) : null,
+                        PackagesPath = packagesPathValue.Count > 0 ? new(packagesPathValue[0]) : null,
+                        TargetFrameworkMoniker = $"net{Environment.Version.Major}.0",
+                        CustomRuntimePack = customRuntimePack,
+                        AotCompilerPath = aotCompilerPath,
+                        AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse<MonoAotCompilerMode>(aotCompilerModeValue[0], ignoreCase: true) : 0
+                    });
+                }
+                else
+                {
+                    argsList.Add("--runtimes");
+                    argsList.AddRange(runtimesValues);
+                }
+
                 CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
             }
             catch (ArgumentException e)
@@ -56,7 +120,8 @@ static async Task<int> Main(string[] args)
                         partitionIndex: partitionIndex,
                         exclusionFilterValue: exclusionFilterValue,
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
-                        getDiffableDisasm: getDiffableDisasm)
+                        getDiffableDisasm: getDiffableDisasm,
+                        toolchain: monoAotToolchain)
                     .AddValidator(new NoWasmValidator(Categories.NoWASM)))
                 .ConfigureAwait(false);
 

src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs

@@ -99,7 +99,6 @@ public void SetupBase64IsValid()
     // To make the results stable the Iteration needs to last at least 100ms, this is why we are using bigger value for NumberOfBytes
     // Due to limitation of BDN, where Params have no Target and are applied to entire class the benchmarks live in a separate class.
     [BenchmarkCategory(Categories.Libraries, Categories.NoMono)]
-    [WarmupCount(30)] // make sure it's promoted to Tier 1
     public class Base64EncodeDecodeInPlaceTests
     {
         [Params(1000 * 1000 * 200)] // allows for stable iteration around 200ms

src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs

@@ -2,29 +2,18 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Attributes.Filters;
-using BenchmarkDotNet.Configs;
-using BenchmarkDotNet.Filters;
 using MicroBenchmarks;
 
 namespace System.Collections.Concurrent
 {
-    internal class AddRemoveFromDifferentThreadsDisabledConfig : ManualConfig
-    {
-        public AddRemoveFromDifferentThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false));
-    }
-
     [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)]
     [GenericTypeArguments(typeof(int))] // value type
     [GenericTypeArguments(typeof(string))] // reference type
-    [MinWarmupCount(6, forceAutoWarmup: true)]
-    [MaxWarmupCount(10, forceAutoWarmup: true)]
     [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")]
-    [Config(typeof(AddRemoveFromDifferentThreadsDisabledConfig))]
     public class AddRemoveFromDifferentThreads<T>
     {
         const int NumThreads = 2;

src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromSameThreads.cs

@@ -6,23 +6,13 @@
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
-using BenchmarkDotNet.Configs;
-using BenchmarkDotNet.Filters;
 using MicroBenchmarks;
 
 namespace System.Collections.Concurrent
 {
-    internal class AddRemoveFromSameThreadsDisabledConfig : ManualConfig
-    {
-        public AddRemoveFromSameThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false));
-    }
-
     [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)]
     [GenericTypeArguments(typeof(int))] // value type
     [GenericTypeArguments(typeof(string))] // reference type
-    [MinWarmupCount(6, forceAutoWarmup: true)]
-    [MaxWarmupCount(10, forceAutoWarmup: true)]
-    [Config(typeof(AddRemoveFromSameThreadsDisabledConfig))]
     public class AddRemoveFromSameThreads<T>
     {
         const int NumThreads = 2;

src/benchmarks/micro/libraries/System.Collections/Sort.cs

@@ -17,7 +17,6 @@ namespace System.Collections
     [GenericTypeArguments(typeof(IntClass))] // custom reference type, sort in managed code
     [GenericTypeArguments(typeof(BigStruct))] // custom value type, sort in managed code
     [InvocationCount(InvocationsPerIteration)]
-    [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups
     public class Sort<T> where T : IComparable<T>
     {
         private static readonly ComparableComparerClass _comparableComparerClass = new ComparableComparerClass();

src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs

@@ -11,8 +11,6 @@
 namespace System.Threading.Tasks
 {
     [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)]
-    [MinWarmupCount(2, forceAutoWarmup: true)] // these benchmarks require more warmups than in our default config
-    [MaxWarmupCount(10, forceAutoWarmup: true)]
     public class ValueTaskPerfTest
     {
         private Task<int> _completedTask = Task.FromResult(42);

src/benchmarks/micro/runtime/Span/Sorting.cs

@@ -13,7 +13,6 @@ namespace Span
 {
     [BenchmarkCategory(Categories.Runtime)]
     [InvocationCount(InvocationsPerIteration)]
-    [MinWarmupCount(6, forceAutoWarmup: true)] // when InvocationCount is set, BDN does not run Pilot Stage, so to get the code promoted to Tier 1 before Actual Workload, we enforce more Warmups
     public class Sorting
     {
         private const int InvocationsPerIteration = 10_000;

src/harness/BenchmarkDotNet.Extensions/AotFilterAttribute.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Filters;
+
+namespace BenchmarkDotNet.Attributes.Filters;
+
+public class AotFilterAttribute : FilterConfigBaseAttribute
+{
+    public AotFilterAttribute(string? reason = null)
+        : base(new SimpleFilter(benchmark => benchmark.GetRuntime() is not (NativeAotRuntime or MonoAotLLVMRuntime)))
+    {
+    }
+}

src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -23,4 +23,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\tools\Reporting\Reporting\Reporting.csproj" />
   </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="MonoAotLLVM\MonoAotLLVMCsProj.txt" />
+  </ItemGroup>
 </Project>

src/harness/BenchmarkDotNet.Extensions/MonoAotLLVM/MonoAotCompilerMode.cs

@@ -0,0 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace BenchmarkDotNet.Extensions;
+
+public enum MonoAotCompilerMode
+{
+    mini,
+    llvm
+}