benchmarks.yml

name: Benchmarks
on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      mode:
        description: Run harness contracts or the full database suite
        type: choice
        options: [contracts, full]
        default: full
permissions: {contents: read, actions: read}
concurrency:
  group: keyload-benchmarks-${{ github.ref }}
  cancel-in-progress: false
env:
  NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages
jobs:
  comparison-runtime:
    name: Prepare comparison runtime
    if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-24.04
    timeout-minutes: 150
    permissions: {contents: read, actions: read, packages: read}
    outputs:
      databases: ${{ steps.plan.outputs.database_matrices }}
      runtime-artifact-id: ${{ steps.runtime-bundle.outputs.artifact-id }}
      keyload-image: ${{ steps.keyload-image.outputs.reference }}
    env:
      GH_TOKEN: ${{ github.token }}
      Configuration: Release
      KEYLOAD_COMPARISON_JOB_NAME: Prepare comparison runtime
    steps:
      - name: Download source code
        uses: actions/checkout@v7
      - name: Set up .NET
        uses: actions/setup-dotnet@v6
        with:
          global-json-file: global.json
      # KeyLoad cells run the released server image; this job compiles only the comparison harness.
      - name: Restore the comparison harness packages
        run: dotnet restore benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj
      - name: Prepare the pinned Aspire runtime inside the archive inputs
        shell: bash
        run: |
          aspire_version=$(jq -er '."msbuild-sdks"."Aspire.AppHost.Sdk"' global.json)
          [[ "$aspire_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
          aspire_rid="linux-${RUNNER_ARCH,,}"
          [[ "$aspire_rid" == linux-x64 || "$aspire_rid" == linux-arm64 ]]
          aspire_cli="$NUGET_PACKAGES/aspire.cli.$aspire_rid/$aspire_version/tools/net10.0/$aspire_rid"
          dotnet dnx --yes "aspire.cli@$aspire_version" -- setup --install-path "$aspire_cli" --non-interactive
          test -x "$aspire_cli/bundle/dcp/dcp"
          test -x "$aspire_cli/bundle/dashboard/Aspire.Dashboard"
          printf 'AspireCliBundlePath=%s/bundle\n' "$aspire_cli" >> "$GITHUB_ENV"
      - name: Build the comparison harness
        run: dotnet build benchmarks/KeyLoad.ComparisonHost/KeyLoad.ComparisonHost.csproj --no-restore --configuration Release -p:AspireCliInvocationMode=DnxPinned -warnaserror
      - name: Check repository rules
        run: node scripts/Features/RepositoryGovernance/verify.mjs
      - name: Plan the shared database sessions
        id: plan
        run: node scripts/Features/BenchmarkComparisons/session-plan.mjs --output="$RUNNER_TEMP/session-plan.json" --github-output="$GITHUB_OUTPUT"
      - name: Save the canonical session plan
        uses: actions/upload-artifact@v7
        with:
          name: comparison-plan
          path: ${{ runner.temp }}/session-plan.json
          if-no-files-found: error
          retention-days: 90
      - name: Check Docker
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: docker version
      - name: Resolve the latest released KeyLoad server image to one digest
        id: keyload-image
        if: (github.event_name == 'push' || inputs.mode == 'full')
        env:
          REGISTRY_TOKEN: ${{ github.token }}
          KEYLOAD_IMAGE_REPOSITORY: ghcr.io/managedcode/keyload
        run: |
          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin
          digest=$(docker buildx imagetools inspect "$KEYLOAD_IMAGE_REPOSITORY:latest" --format '{{json .Manifest}}' | jq -er '.digest')
          [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]
          reference="$KEYLOAD_IMAGE_REPOSITORY:latest@$digest"
          printf 'reference=%s\n' "$reference" >> "$GITHUB_OUTPUT"
          printf 'KeyLoad cells use %s\n' "$reference" >> "$GITHUB_STEP_SUMMARY"
          docker logout ghcr.io
      - name: Pack the compiled benchmark runtime
        if: (github.event_name == 'push' || inputs.mode == 'full')
        run: node scripts/Features/BenchmarkComparisons/compiled-test-bundle.mjs --pack
      - name: Save compiled runtime identities
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: actions/upload-artifact@v7
        with:
          name: comparison-compiled-runtime-identity
          path: |
            artifacts/benchmark-runtime-bundle/benchmark-runtime.json
            artifacts/benchmark-runtime-bundle/benchmark-runtime-files.json
          if-no-files-found: error
          retention-days: 90
      - name: Save the compiled benchmark runtime
        id: runtime-bundle
        if: (github.event_name == 'push' || inputs.mode == 'full')
        uses: actions/upload-artifact@v7
        with:
          name: comparison-compiled-test-bundle
          path: artifacts/benchmark-runtime-bundle/*
          if-no-files-found: error
          compression-level: 0
          retention-days: 90
      - name: Save original preparation diagnostics
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: comparison-runtime-qualification
          path: |
            TestResults/**
            artifacts/qualification/**
            artifacts/code-quality/**
            ${{ runner.temp }}/keyload-cell-github/**
          if-no-files-found: error
          retention-days: 90
  comparison-keyload:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    permissions: {contents: read, actions: read, packages: read}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).keyload }}
    env: &database-environment
      GH_TOKEN: ${{ github.token }}
      KEYLOAD_BENCHMARK_SESSION_ID: ${{ matrix.id }}
      KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }}
      Benchmarks__Target: ${{ matrix.target }}
      Benchmarks__NodeCount: ${{ matrix.nodeCount }}
      Benchmarks__Session: 'true'
      Benchmarks__SessionId: ${{ matrix.id }}
      Benchmarks__EvidenceProfile: session-v1
      Benchmarks__KeyLoadImage: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }}
    steps: &database-steps
      - name: Download source code
        uses: actions/checkout@v7
      - name: Prepare the execution database session
        id: setup
        uses: ./.github/workflows/Features/BenchmarkComparisons/SessionSetup
        with:
          runtime-artifact-id: ${{ needs.comparison-runtime.outputs.runtime-artifact-id }}
          keyload-image: ${{ matrix.target == 'KeyLoad' && needs.comparison-runtime.outputs.keyload-image || '' }}
      - name: Test selected database adapter against real infrastructure
        if: matrix.nodeCount == 1 && contains(fromJSON('["TimescaleDB","SQL Server","MySQL","ClickHouse","InfluxDB"]'), matrix.target)
        run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/AdditionalDatabaseTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/AdapterTestResults" --report-trx --timeout 15m
      - name: Run database workload
        id: workload
        run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --github-reporter-style full --maximum-parallel-tests 1 --minimum-expected-tests 1 --treenode-filter '/*/*/BenchmarkSessionTests/*' --results-directory "$KEYLOAD_COMPARISON_EVIDENCE_DIRECTORY/TestResults" --report-trx --timeout 210m
      - name: Record benchmark availability
        if: ${{ !cancelled() && (steps.workload.outcome == 'success' || steps.workload.outcome == 'failure') }}
        env:
          KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }}
        run: node scripts/Features/BenchmarkComparisons/session-finalize.mjs
      - name: Save benchmark results
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v7
        with:
          name: ${{ matrix.artifactName }}
          path: artifacts/comparisons/sessions/workers/${{ matrix.id }}/**
          if-no-files-found: error
          retention-days: 90
      - name: End this job and retain execution diagnostics
        if: always()
        uses: ./.github/workflows/Features/BenchmarkComparisons/SessionTeardown
        with:
          artifact-name: comparison-case-qualification-session-${{ matrix.id }}
  comparison-postgresql:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).postgresql }}
    env: *database-environment
    steps: *database-steps
  comparison-qdrant:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).qdrant }}
    env: *database-environment
    steps: *database-steps
  comparison-rabbitmq:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).rabbitmq }}
    env: *database-environment
    steps: *database-steps
  comparison-redis:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).redis }}
    env: *database-environment
    steps: *database-steps
  comparison-neo4j:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).neo4j }}
    env: *database-environment
    steps: *database-steps
  comparison-mongodb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mongodb }}
    env: *database-environment
    steps: *database-steps
  comparison-opensearch:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).opensearch }}
    env: *database-environment
    steps: *database-steps
  comparison-kurrentdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).kurrentdb }}
    env: *database-environment
    steps: *database-steps
  comparison-surrealdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).surrealdb }}
    env: *database-environment
    steps: *database-steps
  comparison-helixdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).helixdb }}
    env: *database-environment
    steps: *database-steps
  comparison-timescaledb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).timescaledb }}
    env: *database-environment
    steps: *database-steps
  comparison-sqlserver:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).sqlserver }}
    env: *database-environment
    steps: *database-steps
  comparison-mysql:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).mysql }}
    env: *database-environment
    steps: *database-steps
  comparison-clickhouse:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).clickhouse }}
    env: *database-environment
    steps: *database-steps
  comparison-influxdb:
    name: ${{ matrix.jobName }}
    if: (github.event_name == 'push' || inputs.mode == 'full')
    needs: [comparison-runtime]
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.comparison-runtime.outputs.databases).influxdb }}
    env: *database-environment
    steps: *database-steps
  comparison-aggregate:
    name: Combine benchmark results
    needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb]
    if: ${{ always() && !cancelled() && (github.event_name == 'push' || inputs.mode == 'full') && needs.comparison-runtime.result == 'success' }}
    runs-on: ubuntu-24.04
    timeout-minutes: 150
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Download source code
        uses: actions/checkout@v7
      - name: Download original authenticated session results
        run: |
          mkdir -p artifacts/comparisons/sessions
          node scripts/Features/BenchmarkComparisons/session-github-collect.mjs --input="$PWD/artifacts/comparisons/sessions/provider"
      - name: Check complete session accounting
        run: node scripts/Features/BenchmarkComparisons/session-aggregate-cli.mjs --input="$PWD/artifacts/comparisons/sessions/provider" --output="$PWD/artifacts/comparisons/sessions/aggregate"
      - name: Save combined benchmark results
        uses: actions/upload-artifact@v7
        with:
          name: comparison-session-suite
          path: artifacts/comparisons/sessions/aggregate/**
          if-no-files-found: error
          retention-days: 90
      - name: Save GitHub result verification
        uses: actions/upload-artifact@v7
        with:
          name: comparison-session-provider-evidence
          path: |
            artifacts/comparisons/sessions/provider/github/**
            artifacts/comparisons/sessions/provider/proof.json
          if-no-files-found: error
          retention-days: 90
  website-trigger:
    name: Trigger Website
    needs: [comparison-runtime, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb, comparison-timescaledb, comparison-sqlserver, comparison-mysql, comparison-clickhouse, comparison-influxdb, comparison-aggregate]
    if: >-
      always() && !cancelled() &&
      (github.event_name == 'push' || inputs.mode == 'full') &&
      needs.comparison-aggregate.result == 'success' &&
      !contains(needs.*.result, 'failure') &&
      !contains(needs.*.result, 'cancelled') &&
      !contains(needs.*.result, 'skipped') &&
      github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 &&
      github.ref == 'refs/heads/main'
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    permissions: {actions: write}
    env:
      GH_TOKEN: ${{ github.token }}
      GH_REPO: ${{ github.repository }}
      BENCHMARK_RUN_ID: ${{ github.run_id }}
    steps:
      - name: Start the separate Website workflow
        run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID"