Repair the Linux benchmark and CI contracts the first complete runs exposed

Benchmarks run 40046956549 was the first to get past the compiled-bundle
import; every remaining failure was a Linux- or CI-only contract defect:

- KeyLoad cells: the workload client re-verified each server process by an
  exact Process.StartTime match against the bootstrap's observation. Linux
  .NET caches a boot-time estimate once per observing process, so two
  processes disagree by the coarse clock granularity. Identity now pairs the
  PID, executable path and compiled-file hashes with a start time inside a
  configurable observer window (TestExecutionOptions.ProcessStartTimeObserverWindow,
  one second by default); a focused unit test exercises both sides.
- Neo4j cells: the runtime-import session-id pattern rejected ids with digits.
- PostgreSQL, Redis, Qdrant, RabbitMQ and KurrentDB finalization: the
  resource-evidence validator expected runtimeMaxOutputBytes while the C#
  observation policy writes maxOutputBytes; the validator follows the producer.
- Redis three nodes: replicas name the primary by its container-network
  address, so the client-visible primary endpoint can never match. Both
  replica proofs now follow the primary's replication id.
- SurrealDB: the image runs as nonroot and its declared /data volume root is
  root-owned; the session volume now lives under the image user's home.
- OpenSearch cells: a synchronous _update_by_query over one million documents
  returns one TASK_RESOURCE_USAGE response header per scroll batch (about
  230 KB), above the .NET 64 KB default; the document HTTP client declares
  DocumentClientMaxResponseHeadersKilobytes (1024) as recorded policy.
- MongoDB: ubuntu-latest moved to Ubuntu 26.04 with a 7.0 kernel that the
  pinned server refuses; Benchmarks jobs declare ubuntu-24.04 like Build and
  Tests, and the serialization job identity follows that label.

Build and Tests ran the unit suite with unlimited parallelism on four
processors (234 of 3718 failed, about 105 of them load cancellations) and
stopped at the first failing suite. Each suite is now its own step that runs
after a failing one, unit parallelism is bounded to the runner's processors,
and the job timeout covers all four suites.

Docs, the closed unit test inventory and status.json record the current
contracts; no figures are published.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: KSemenenko

.github/workflows/benchmarks.yml

@@ -19,7 +19,7 @@ jobs:
   comparison-runtime:
     name: Prepare comparison runtime
     if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 150
     outputs:
       databases: ${{ steps.plan.outputs.database_matrices }}
@@ -109,7 +109,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -156,7 +156,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -167,7 +167,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -178,7 +178,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -189,7 +189,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -200,7 +200,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -211,7 +211,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -222,7 +222,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -233,7 +233,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -244,7 +244,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -255,7 +255,7 @@ jobs:
     name: ${{ matrix.jobName }}
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -266,7 +266,7 @@ jobs:
     name: KeyLoad functional CLUSTER million-record ingestion
     if: (github.event_name == 'push' || inputs.mode == 'full')
     needs: [comparison-runtime]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 180
     steps:
       - name: Download source code
@@ -292,7 +292,7 @@ jobs:
     name: Combine benchmark results
     needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, keyload-functional-heavy-load]
     if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-runtime.result == 'success' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 150
     env:
       GH_TOKEN: ${{ github.token }}
@@ -333,7 +333,7 @@ jobs:
       !contains(needs.*.result, 'skipped') &&
       github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 &&
       github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     permissions: {actions: write}
     env:

.github/workflows/build-and-tests.yml

@@ -9,7 +9,7 @@ permissions:
 jobs:
   build-and-test:
     runs-on: ubuntu-24.04
-    timeout-minutes: 120
+    timeout-minutes: 180
     steps:
       - name: Download source code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -26,9 +26,14 @@ jobs:
       - name: Install administration browser
         shell: pwsh
         run: tests/KeyLoad.IntegrationTests/bin/Debug/net10.0/playwright.ps1 install --with-deps chromium
-      - name: Test
-        run: |
-          dotnet test --project tests/KeyLoad.Analyzers.Tests/KeyLoad.Analyzers.Tests.csproj --no-build --output Detailed
-          dotnet test --project tests/KeyLoad.UnitTests/KeyLoad.UnitTests.csproj --no-build --output Detailed
-          dotnet test --project tests/KeyLoad.RecoveryTests/KeyLoad.RecoveryTests.csproj --no-build --output Detailed
-          dotnet test --project tests/KeyLoad.IntegrationTests/KeyLoad.IntegrationTests.csproj --no-build --output Detailed --treenode-filter '/*/*/*/*[Category!=HeavyLoad]'
+      - name: Test code analyzers
+        run: dotnet test --project tests/KeyLoad.Analyzers.Tests/KeyLoad.Analyzers.Tests.csproj --no-build --output Detailed
+      - name: Test units
+        if: ${{ !cancelled() }}
+        run: dotnet test --project tests/KeyLoad.UnitTests/KeyLoad.UnitTests.csproj --no-build --output Detailed --maximum-parallel-tests "$(nproc)"
+      - name: Test process recovery
+        if: ${{ !cancelled() }}
+        run: dotnet test --project tests/KeyLoad.RecoveryTests/KeyLoad.RecoveryTests.csproj --no-build --output Detailed
+      - name: Test Aspire topology
+        if: ${{ !cancelled() }}
+        run: dotnet test --project tests/KeyLoad.IntegrationTests/KeyLoad.IntegrationTests.csproj --no-build --output Detailed --treenode-filter '/*/*/*/*[Category!=HeavyLoad]'

benchmarks/KeyLoad.ComparisonHost/Configuration/execution.json

@@ -14,6 +14,7 @@
     "OpenSearchHealthWaitTimeoutSeconds": 60,
     "CleanupTimeout": "00:00:30",
     "DocumentClientMaxConnections": 1,
+    "DocumentClientMaxResponseHeadersKilobytes": 1024,
     "DocumentVerificationConnectionReserve": 1,
     "DocumentCleanupTimeout": "00:20:00",
     "IndexBuildTimeout": "00:20:00",

benchmarks/KeyLoad.ComparisonHost/Session/Configuration/BenchmarkSessionProcessEvidence.cs

@@ -1,5 +1,6 @@
 using System.Collections.Immutable;
 using System.Text.Json;
+using KeyLoad.AppHost.Hosting;
 using KeyLoad.AppHost.TestInfrastructure;
 using KeyLoad.Comparisons;
 using KeyLoad.Diagnostics.ResourceExecution;
@@ -42,9 +43,10 @@ internal static BenchmarkSessionProcesses Read(IConfiguration configuration, Ben
         {
             throw new InvalidDataException(ServerTopologyMismatch);
         }
+        var execution = AppHostOptionsRegistration.BindTestExecution(configuration);
         foreach (var server in servers)
         {
-            ProjectProcessEvidence.Verify(server);
+            ProjectProcessEvidence.Verify(server, execution);
         }
         return new(client, servers);
     }

benchmarks/KeyLoad.Comparisons/Configuration/ComparisonExecutionOptions.cs

@@ -3,12 +3,15 @@ namespace KeyLoad.Comparisons;
 public sealed partial class ComparisonExecutionOptions
 {
     internal const int RequiredDocumentClientConnections = 1;
+    internal const int RequiredDocumentClientResponseHeadersKilobytes = 1024;
     private const int RequiredDocumentCleanupMinutes = 20;
     internal static readonly TimeSpan RequiredDocumentCleanupTimeout = TimeSpan.FromMinutes(RequiredDocumentCleanupMinutes);
     /// <summary>Additional execution primary connection reserved for untimed final verification.</summary>
     public int DocumentVerificationConnectionReserve { get; set; } = RequiredDocumentClientConnections;
     /// <summary>One actual connection per independently admitted HTTP document client.</summary>
     public int DocumentClientMaxConnections { get; set; } = RequiredDocumentClientConnections;
+    /// <summary>Response header bound of the document HTTP client; OpenSearch answers a synchronous update-by-query with one TASK_RESOURCE_USAGE header per scroll batch, about 230 KB for one million documents.</summary>
+    public int DocumentClientMaxResponseHeadersKilobytes { get; set; } = RequiredDocumentClientResponseHeadersKilobytes;
     /// <summary>Joined cleanup bound for a complete million-record execution document namespace.</summary>
     public TimeSpan DocumentCleanupTimeout { get; set; } = RequiredDocumentCleanupTimeout;
 }

benchmarks/KeyLoad.Comparisons/Configuration/DocumentPolicy.cs

@@ -9,6 +9,7 @@ internal static class DocumentPolicy
     internal static void Validate(ComparisonExecutionOptions options)
     {
         if (options.DocumentClientMaxConnections != ComparisonExecutionOptions.RequiredDocumentClientConnections
+            || options.DocumentClientMaxResponseHeadersKilobytes != ComparisonExecutionOptions.RequiredDocumentClientResponseHeadersKilobytes
             || options.DocumentVerificationConnectionReserve != ComparisonExecutionOptions.RequiredDocumentClientConnections
             || options.DocumentCleanupTimeout != ComparisonExecutionOptions.RequiredDocumentCleanupTimeout)
         {
@@ -18,6 +19,7 @@ internal static void Validate(ComparisonExecutionOptions options)
     internal static void Record(ComparisonExecutionOptions options, IDictionary<string, string> parameters)
     {
         parameters[nameof(options.DocumentClientMaxConnections)] = options.DocumentClientMaxConnections.ToString(CultureInfo.InvariantCulture);
+        parameters[nameof(options.DocumentClientMaxResponseHeadersKilobytes)] = options.DocumentClientMaxResponseHeadersKilobytes.ToString(CultureInfo.InvariantCulture);
         parameters[nameof(options.DocumentVerificationConnectionReserve)] = options.DocumentVerificationConnectionReserve.ToString(CultureInfo.InvariantCulture);
         parameters[nameof(options.DocumentCleanupTimeout)] = options.DocumentCleanupTimeout.ToString(DurationFormat, CultureInfo.InvariantCulture);
     }

benchmarks/KeyLoad.Comparisons/Documents/Lifetimes/DocumentHttpClient.cs

@@ -11,9 +11,11 @@ internal HttpClient Client
     }
     internal DocumentHttpClient(HttpClient source, IOptions<ComparisonExecutionOptions> options)
     {
+        var settings = ComparisonExecutionOptions.Require(options).Value;
         handler = new()
         {
-            MaxConnectionsPerServer = ComparisonExecutionOptions.Require(options).Value.DocumentClientMaxConnections
+            MaxConnectionsPerServer = settings.DocumentClientMaxConnections,
+            MaxResponseHeadersLength = settings.DocumentClientMaxResponseHeadersKilobytes
         };
         try
         {

benchmarks/KeyLoad.Comparisons/Documents/Validation/RedisDocumentCopyProof.cs

@@ -36,12 +36,12 @@ internal static async Task<ClusterEvidence> VerifyAsync(ConnectionMultiplexer pr
         }
 
         await RequireAofAsync(primaryServer, CommandFlags.DemandMaster, deadline.Token).ConfigureAwait(false);
-        await JoinAsync(RedisReplicaProof.VerifyWorkerPrimaryAsync(primary, topology, CancellationToken.None), deadline.Token).ConfigureAwait(false);
+        var replicationId = await JoinAsync(RedisReplicaProof.VerifyWorkerPrimaryAsync(primary, topology, CancellationToken.None), deadline.Token).ConfigureAwait(false);
         var database = primary.GetDatabase();
         var fenced = false;
         try
         {
-            var identities = await ObserveReplicasAsync(primaryEndpoint, primaryIdentity, endpoints, replicas, deadline.Token).ConfigureAwait(false);
+            var identities = await ObserveReplicasAsync(primaryEndpoint, replicationId, primaryIdentity, endpoints, replicas, deadline.Token).ConfigureAwait(false);
             RedisNodeIdentity.RequireUniqueVersionedSet(identities, primaryIdentity, copies);
             var evidence = replicas.Count == NoReplicas ? RedisNodeIdentity.SingleEvidence(primaryIdentity, SingleState)
                 : RedisNodeIdentity.ReplicatedEvidence(primaryIdentity, identities.Skip(PrimaryIdentityCount).ToArray(), ReplicatedState);
@@ -57,9 +57,8 @@ internal static async Task<ClusterEvidence> VerifyAsync(ConnectionMultiplexer pr
             fenced = true;
             await RequireAllFsyncAsync(database, copies - PrimaryCopyCount, clientId, lifecycleOptions.Value.RedisReceiptTimeout, deadline.Token).ConfigureAwait(false);
             var observations = evidence.Observations.ToList();
-            await VerifyReplicaReadbacksAsync(replicas, identities, primaryEndpoint, schedule, readReplica, observations, token).ConfigureAwait(false);
-            await JoinAsync(RedisReplicaProof.VerifyWorkerPrimaryAsync(primary, topology, CancellationToken.None), token).ConfigureAwait(false);
-            RedisNodeIdentity.RequireUnchanged(await JoinAsync(RedisNodeIdentity.ReadAsync(primaryServer, CommandFlags.DemandMaster, CancellationToken.None), token).ConfigureAwait(false), primaryIdentity);
+            await VerifyReplicaReadbacksAsync(replicas, identities, replicationId, schedule, readReplica, observations, token).ConfigureAwait(false);
+            await RequireFinalPrimaryAsync(primary, primaryServer, topology, replicationId, primaryIdentity, token).ConfigureAwait(false);
             return evidence with
             {
                 Observations = observations.Append(string.Format(System.Globalization.CultureInfo.InvariantCulture, FinalFsyncObservationFormat, schedule.Final, copies - PrimaryCopyCount)).ToImmutableArray()
@@ -83,7 +82,16 @@ internal static async Task<ClusterEvidence> VerifyAsync(ConnectionMultiplexer pr
             }
         }
     }
-    private static async Task<List<RedisNodeIdentity>> ObserveReplicasAsync(System.Net.EndPoint primaryEndpoint,
+    private static async Task RequireFinalPrimaryAsync(ConnectionMultiplexer primary, IServer primaryServer, ComparisonTopology topology,
+        string replicationId, RedisNodeIdentity primaryIdentity, CancellationToken token)
+    {
+        if (await JoinAsync(RedisReplicaProof.VerifyWorkerPrimaryAsync(primary, topology, CancellationToken.None), token).ConfigureAwait(false) != replicationId)
+        {
+            throw new ComparisonFailureException(Failure);
+        }
+        RedisNodeIdentity.RequireUnchanged(await JoinAsync(RedisNodeIdentity.ReadAsync(primaryServer, CommandFlags.DemandMaster, CancellationToken.None), token).ConfigureAwait(false), primaryIdentity);
+    }
+    private static async Task<List<RedisNodeIdentity>> ObserveReplicasAsync(System.Net.EndPoint primaryEndpoint, string replicationId,
         RedisNodeIdentity primaryIdentity, string[] endpoints, List<ConnectionMultiplexer> replicas, CancellationToken token)
     {
         var identities = new List<RedisNodeIdentity>(endpoints.Length + PrimaryIdentityCount) { primaryIdentity };
@@ -101,14 +109,14 @@ private static async Task<List<RedisNodeIdentity>> ObserveReplicasAsync(System.N
             }
 
             var server = replica.GetServer(actualEndpoint);
-            await RequireReplicaAsync(server, primaryEndpoint, token).ConfigureAwait(false);
+            await RequireReplicaAsync(server, replicationId, token).ConfigureAwait(false);
             await RequireAofAsync(server, CommandFlags.DemandReplica, token).ConfigureAwait(false);
             identities.Add(await JoinAsync(RedisNodeIdentity.ReadAsync(server, CommandFlags.DemandReplica, CancellationToken.None), token).ConfigureAwait(false));
         }
         return identities;
     }
     private static async Task VerifyReplicaReadbacksAsync(List<ConnectionMultiplexer> replicas, List<RedisNodeIdentity> identities,
-        System.Net.EndPoint primaryEndpoint, DocumentComparisonSchedule schedule,
+        string replicationId, DocumentComparisonSchedule schedule,
         Func<ConnectionMultiplexer, CancellationToken, IAsyncEnumerable<FoundDocument>> readReplica,
         List<string> observations, CancellationToken token)
     {
@@ -117,7 +125,7 @@ private static async Task VerifyReplicaReadbacksAsync(List<ConnectionMultiplexer
             var readback = await DocumentComparisonOracle.VerifyAsync(readReplica(replicas[index], token), schedule.FinalDocuments(), token).ConfigureAwait(false);
             observations.Add(string.Format(System.Globalization.CultureInfo.InvariantCulture, FinalReplicaObservationFormat, identities[index + PrimaryIdentityCount].RunId, readback.Records, readback.ActualSha256));
             var server = replicas[index].GetServer(RedisProtocol.ConfiguredEndpoint(replicas[index], Failure));
-            await RequireReplicaAsync(server, primaryEndpoint, token).ConfigureAwait(false);
+            await RequireReplicaAsync(server, replicationId, token).ConfigureAwait(false);
             RedisNodeIdentity.RequireUnchanged(await JoinAsync(RedisNodeIdentity.ReadAsync(server, CommandFlags.DemandReplica, CancellationToken.None), token).ConfigureAwait(false), identities[index + PrimaryIdentityCount]);
         }
     }
@@ -144,15 +152,17 @@ private static async Task RequireAofAsync(IServer server, CommandFlags flags, Ca
             throw new ComparisonFailureException(Failure);
         }
     }
-    private static async Task RequireReplicaAsync(IServer server, System.Net.EndPoint primary, CancellationToken token)
+    // Replicas name the primary by its container-network address; the primary's replication id is the follow oracle.
+    private static async Task RequireReplicaAsync(IServer server, string replicationId, CancellationToken token)
     {
         var info = await JoinAsync(RedisProtocol.ReadInfoAsync(server, RedisProtocol.ReplicationSection, CommandFlags.DemandReplica, CancellationToken.None), token).ConfigureAwait(false);
         var role = await JoinAsync(RedisProtocol.ReadRoleAsync(server, CommandFlags.DemandReplica, CancellationToken.None), token).ConfigureAwait(false);
         var host = info.GetValueOrDefault(RedisProtocol.MasterHostField);
         var port = RedisProtocol.ParseInteger(info, RedisProtocol.MasterPortField);
         if (info.GetValueOrDefault(RedisProtocol.RoleField) != RedisProtocol.ReplicaRole
             || info.GetValueOrDefault(RedisProtocol.LinkField) != RedisProtocol.LinkUp || host is null
-            || !RedisProtocol.EndpointMatches(primary, host, port) || role.Length < RedisProtocol.RoleReplicaResponseLength
+            || !string.Equals(info.GetValueOrDefault(RedisProtocol.ReplicationIdField), replicationId, StringComparison.Ordinal)
+            || role.Length < RedisProtocol.RoleReplicaResponseLength
             || role[RedisProtocol.RoleNameIndex].ToString() != RedisProtocol.ReplicaRole
             || !StringComparer.OrdinalIgnoreCase.Equals(role[RedisProtocol.RoleHostIndex].ToString(), host) || role[RedisProtocol.RolePortIndex].ToString() != port.ToString(System.Globalization.CultureInfo.InvariantCulture))
         {
@@ -182,11 +192,7 @@ internal static void RequireFsyncReceipt(long[] counts, int replicas)
             throw new ComparisonFailureException(Failure);
         }
     }
-    private static async Task JoinAsync(Task original, CancellationToken token)
-    {
-        await original.ConfigureAwait(false);
-        token.ThrowIfCancellationRequested();
-    }
+
     private static async Task<T> JoinAsync<T>(Task<T> original, CancellationToken token)
     {
         var result = await original.ConfigureAwait(false);