From 875f53450871c2cdb8f78dacfa50aff9ed8826c2 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 3 Feb 2026 19:06:30 -0600 Subject: [PATCH 1/2] [tests] remove "flaky" MSBuild perf tests (#10756) These two commonly fail (but also randomly pass!): * `Build_From_Clean_DontIncludeRestore` * `Build_JLO_Change` The general goal of the MSBuild performance tests is to catch large regressions in incremental builds. Let's remove these two as they are lower value: * Flaky, due to the long time they take and the variability of build machine performance. * Perf regressions in initial build time are less important than incremental build time regressions. It would be higher value for PRs to be greener than run these, I think. `Build_No_Changes` and `Build_CSharp_Change` are more valuable to keep as they are less flaky and more likely to catch real-world regressions. --- .../Tests/PerformanceTest.cs | 38 ------------------- .../MSBuildDeviceIntegration.csv | 2 - 2 files changed, 40 deletions(-) diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index abcb1c8d67f..154b7a04d92 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -140,21 +140,6 @@ XamarinAndroidApplicationProject CreateApplicationProject () return proj; } - [Test] - [Retry (Retry)] - public void Build_From_Clean_DontIncludeRestore () - { - AssertCommercialBuild (); // If runs, this test will fail without Fast Deployment - - var proj = CreateApplicationProject (); - using (var builder = CreateBuilderWithoutLogFile ()) { - builder.AutomaticNuGetRestore = false; - builder.Target = "Build"; - builder.Restore (proj); - Profile (builder, b => b.Build (proj)); - } - } - [Test] [Retry (Retry)] public void Build_No_Changes () @@ -255,29 +240,6 @@ public void Build_AndroidAsset_Change () } } - [Test] - [Retry (Retry)] - public void Build_JLO_Change () - { - AssertCommercialBuild (); // If runs, this test will fail without Fast Deployment - - var className = "Foo"; - var proj = CreateApplicationProject (); - proj.Sources.Add (new BuildItem.Source ("Foo.cs") { - TextContent = () => $"class {className} : Java.Lang.Object {{}}" - }); - using (var builder = CreateBuilderWithoutLogFile ()) { - builder.Target = "Build"; - builder.Build (proj); - builder.AutomaticNuGetRestore = false; - - // Profile Java.Lang.Object rename - className = "Foo2"; - proj.Touch ("Foo.cs"); - Profile (builder, b => b.Build (proj)); - } - } - [Test] [Retry (Retry)] public void Build_AndroidManifest_Change () diff --git a/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv b/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv index f367c961422..a5bce9027ae 100644 --- a/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv +++ b/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv @@ -2,13 +2,11 @@ # First non-comment row is human description of columns Test Name,Time in ms (int) # Data -Build_From_Clean_DontIncludeRestore,13000 Build_No_Changes,2250 Build_CSharp_Change,3500 Build_AndroidResource_Change,2250 Build_AndroidAsset_Change,6500 Build_AndroidManifest_Change,4500 -Build_JLO_Change,4500 Build_XAML_Change,3000 Install_CSharp_Change,4000 Install_XAML_Change,3750 From ff0c535d05d060f0e943cf3ea2dc74da958c8f38 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 1 May 2026 02:46:43 -0500 Subject: [PATCH 2/2] [tests] Skip PerformanceTest on slow CI machines using evaluation time (#11249) * [tests] Skip PerformanceTest on slow CI machines using evaluation time When a performance test exceeds its expected time, check the MSBuild project evaluation duration from the binlog. If evaluation alone exceeds 500ms (normal is ~200-350ms), the CI machine is too slow for reliable performance measurements. Use Assert.Inconclusive() instead of Assert.Fail() so the test is marked as skipped rather than failed. Analysis of 10 recent CI failures showed evaluation times of 558-854ms on slow machines, while normal machines should be well under 500ms. Also refactored binlog reading so BinaryLog.ReadBuild() is called once per iteration, with the Build object passed to helper methods. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix CS0118: use alias to disambiguate Build type from namespace Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/PerformanceTest.cs | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index 154b7a04d92..e819212d479 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using Microsoft.Build.Framework; using Microsoft.Build.Logging.StructuredLogger; +using StructuredBuild = Microsoft.Build.Logging.StructuredLogger.Build; using NUnit.Framework; using Xamarin.ProjectTools; @@ -15,6 +16,7 @@ namespace Xamarin.Android.Build.Tests public class PerformanceTest : DeviceTest { const int Retry = 2; + const int MaxEvaluationTimeInMs = 500; static readonly Dictionary csv_values = new Dictionary (); [OneTimeSetUp] @@ -57,9 +59,11 @@ void Profile (ProjectBuilder builder, int iterations, Action act } double total = 0; + StructuredBuild build = null; for (int i=0; i < iterations; i++) { action (builder); - var actual = GetDurationFromBinLog (builder); + build = ReadBinLog (builder); + var actual = GetDuration (build, builder); TestContext.Out.WriteLine ($"run {i} took: {actual}ms"); total += actual; if (afterRun is not null) @@ -68,6 +72,7 @@ void Profile (ProjectBuilder builder, int iterations, Action act total /= iterations; TestContext.Out.WriteLine ($"expected: {expected}ms, actual: {total}ms"); if (total > expected) { + AssertSlowMachine (build, expected, total); Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms"); } } @@ -78,47 +83,63 @@ void ProfileTask (ProjectBuilder builder, string task, int iterations, Action expected) { + AssertSlowMachine (build, expected, total); Assert.Fail ($"Exceeded expected time of {expected}ms, actual {total}ms"); } } - double GetTaskDurationFromBinLog (ProjectBuilder builder, string task) + void AssertSlowMachine (StructuredBuild build, int expected, double actual) + { + var evaluations = build.FindChildrenRecursive (); + var maxEval = evaluations.Any () ? evaluations.Max (e => e.Duration.TotalMilliseconds) : 0; + TestContext.Out.WriteLine ($"max evaluation time: {maxEval}ms (threshold: {MaxEvaluationTimeInMs}ms)"); + if (maxEval > MaxEvaluationTimeInMs) { + Assert.Inconclusive ($"Exceeded expected time of {expected}ms, actual {actual}ms, but evaluation time was {maxEval:F0}ms (threshold: {MaxEvaluationTimeInMs}ms), indicating a slow CI machine."); + } + } + + StructuredBuild ReadBinLog (ProjectBuilder builder) { var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); FileAssert.Exists (binlog); + return BinaryLog.ReadBuild (binlog); + } - var build = BinaryLog.ReadBuild (binlog); + double GetTaskDuration (StructuredBuild build, ProjectBuilder builder, string task) + { var duration = build .FindChildrenRecursive (t => t.Name == task) .Aggregate (TimeSpan.Zero, (duration, target) => duration + target.Duration); - if (duration == TimeSpan.Zero) + if (duration == TimeSpan.Zero) { + var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); throw new InvalidDataException ($"No task build duration found in {binlog}"); + } return duration.TotalMilliseconds; } - double GetDurationFromBinLog (ProjectBuilder builder) + double GetDuration (StructuredBuild build, ProjectBuilder builder) { - var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); - FileAssert.Exists (binlog); - - var build = BinaryLog.ReadBuild (binlog); var duration = build .FindChildrenRecursive () .Aggregate (TimeSpan.Zero, (duration, project) => duration + project.Duration); - if (duration == TimeSpan.Zero) + if (duration == TimeSpan.Zero) { + var binlog = Path.Combine (Root, builder.ProjectDirectory, $"{Path.GetFileNameWithoutExtension (builder.BuildLogFile)}.binlog"); throw new InvalidDataException ($"No project build duration found in {binlog}"); + } return duration.TotalMilliseconds; }