diff --git a/.github/workflows/docker-buildkit-windows.yml b/.github/workflows/docker-buildkit-windows.yml deleted file mode 100644 index b72c2057e..000000000 --- a/.github/workflows/docker-buildkit-windows.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Docker BuildKit on Windows - -on: - workflow_call: - -permissions: - contents: read - -jobs: - docker-buildkit: - name: Windows WSL SSH build - runs-on: windows-latest - timeout-minutes: 25 - defaults: - run: - shell: pwsh - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: go.mod - cache: false - - - name: build CI helper - run: go build -trimpath -o "$env:RUNNER_TEMP\devsy-ci.exe" ./e2e/cmd/devsy-ci - - - name: download Windows CLI - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: devsy-windows - path: ${{ runner.temp }}/devsy-buildkit-bin - - - name: download Linux agent - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: devsy-linux - path: ${{ runner.temp }}/devsy-buildkit-bin - - - name: download E2E test binary - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: e2e-test-windows - path: ./e2e - - - name: prepare CLI and agent binaries - run: | - New-Item -ItemType Directory -Force ./e2e/bin | Out-Null - Get-ChildItem "$env:RUNNER_TEMP\devsy-buildkit-bin" -Recurse -File -Filter 'devsy-*' | - Copy-Item -Destination ./e2e/bin - - - name: setup WSL2 Ubuntu - uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 - with: - distribution: Ubuntu-24.04 - use-cache: true - - - name: provision Docker over WSL SSH - timeout-minutes: 15 - run: >- - & "$env:RUNNER_TEMP\devsy-ci.exe" docker-wsl setup - --distribution Ubuntu-24.04 --timeout 4m - --state-file "$env:RUNNER_TEMP\devsy-ci\docker-wsl-state.json" - - - name: run BuildKit regression specs - working-directory: ./e2e - env: - GH_USERNAME: ${{ github.repository_owner }} - GH_ACCESS_TOKEN: ${{ github.token }} - GH_CREDENTIAL_USERNAME: x-access-token - DOCKER_HOST: npipe:////./pipe/devsy-ci-no-daemon - DOCKER_CONTEXT: "" - run: | - ./e2e.test.exe '-test.v' '-ginkgo.v' '-test.timeout=900s' ` - '-ginkgo.fail-on-empty' '-ginkgo.flake-attempts=1' ` - '-ginkgo.label-filter=docker-internal-buildkit' ` - '-ginkgo.json-report=docker-internal-buildkit-report.json' - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - & "$env:RUNNER_TEMP\devsy-ci.exe" check-report ` - --report docker-internal-buildkit-report.json ` - --label docker-internal-buildkit --expected-specs 6 - - - name: tear down Docker WSL SSH fixture - if: always() - timeout-minutes: 3 - run: | - if (Test-Path "$env:RUNNER_TEMP\devsy-ci.exe") { - & "$env:RUNNER_TEMP\devsy-ci.exe" docker-wsl cleanup ` - --state-file "$env:RUNNER_TEMP\devsy-ci\docker-wsl-state.json" --timeout 2m - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } - - - name: upload BuildKit report and setup or teardown diagnostics - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: e2e-diagnostics-docker-internal-buildkit - path: | - e2e/docker-internal-buildkit-report.json - ${{ runner.temp }}/devsy-ci/docker-wsl-diagnostics.log - if-no-files-found: warn diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index de115ac82..326bb1f30 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -1,5 +1,14 @@ name: Pull Request +# This workflow only wires GitHub Actions together. What tests exist, where they +# run and how they run is defined in hack/ci/suites.yaml and implemented by the +# Go CLI in hack/ci: +# - `plan` turns the manifest into the job matrices below. +# - unit tests run through `hack/ci unit`. +# - Ginkgo suites run through `hack/ci setup` and `hack/ci e2e`. +# - `ci-success` evaluates the needed jobs with `hack/ci success`. +# Add or change a test in the manifest, not in this file. + on: push: branches: [main] @@ -31,14 +40,39 @@ jobs: - 'go.sum' - '.goreleaser.yml' - 'e2e/**' + - 'hack/ci/**' - 'hack/microsandbox/**' - 'hack/provider_update_ci/**' - '.github/workflows/pr-ci.yml' - - '.github/workflows/docker-buildkit-windows.yml' - 'hack/licenses/**' - 'hack/podman_ci/**' - 'THIRD_PARTY_LICENSES.md' + plan: + name: Plan + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + unit: ${{ steps.matrix.outputs.unit }} + unprivileged: ${{ steps.matrix.outputs.unprivileged }} + integration: ${{ steps.matrix.outputs.integration }} + microsandbox: ${{ steps.matrix.outputs.microsandbox }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + + # Suites that need the GitHub App secret are left out of the matrices when + # the secret is unavailable (for example on pull requests from forks). + - name: build job matrices from hack/ci/suites.yaml + id: matrix + env: + SECRET_AVAILABLE: ${{ secrets.DEVSY_GITHUB_APP_ID != '' }} + run: go run ./hack/ci matrix --secret-available="$SECRET_AVAILABLE" >> "$GITHUB_OUTPUT" + precommit: name: Pre-commit runs-on: ubuntu-latest @@ -87,76 +121,8 @@ jobs: version: v2.13.2 only-new-issues: true - unit-tests: - name: Unit Tests (${{ matrix.runner }}) - needs: [changes] - if: needs.changes.outputs.go == 'true' - strategy: - fail-fast: false - matrix: - runner: - - ubuntu-latest - - macos-latest - runs-on: ${{ matrix.runner }} - # Leave room for race compilation and the existing crypto tests on macOS. - timeout-minutes: ${{ matrix.runner == 'macos-latest' && 45 || 30 }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: setup Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - with: - go-version-file: go.mod - - - name: run unit tests with race detector - shell: bash - run: | - set -euo pipefail - - packages="$(go run ./hack/unit_test_packages)" - - if [[ -z "$packages" ]]; then - echo "No unit-test packages found" - exit 1 - fi - - printf '%s\n' "$packages" | xargs go test -race -short - - - name: test agent bootstrap with BusyBox sh - if: runner.os == 'Linux' - shell: bash - run: | - set -euo pipefail - sudo apt-get update -qq - sudo apt-get install -y busybox-static - original_sh="$(readlink /bin/sh)" - trap 'sudo ln -sf "$original_sh" /bin/sh' EXIT - sudo ln -sf /bin/busybox /bin/sh - go test -race -count=1 -timeout 90s ./pkg/inject -run '^(TestBootstrapSuite|TestInjectScript_RealShell.*)$' - - - name: test E2E runtime helpers - run: go test -race -count=1 ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/cmd/devsy-ci - - - name: test Compose UID expectation matrix - shell: bash - run: | - set -euo pipefail - go test ./e2e/tests/up-docker-compose -list '^TestExpectedUIDMapping$' \ - | grep -Fx 'TestExpectedUIDMapping' - go test -race -short -count=1 -v ./e2e/tests/up-docker-compose \ - -run '^TestExpectedUIDMapping$' - - - name: test workspace fixture helpers - run: go test -race -short -count=1 -timeout 90s ./e2e/tests/up - - - name: test Linux registry fixture - if: runner.os == 'Linux' - run: go test -race -count=1 -timeout 90s ./e2e/framework/registryfixture - - - name: test Linux GitHub API oracle - if: runner.os == 'Linux' - run: go test -race -count=1 -timeout 90s ./e2e/framework/ghfixture - + # Native host tests: they execute the real `gh` CLI and, on macOS, the native + # keyring, so they run per host OS outside the unit tier's package list. github-credential-host-tests: name: GitHub Credential Host Tests (${{ matrix.runner }}) needs: [changes] @@ -192,26 +158,28 @@ jobs: go test -race -count=1 -v -timeout 2m ./pkg/tunnel ./pkg/credentials -run '^(TestCredentials.*Host.*|TestOptionalGitHubService.*|TestServicesStartup.*)$' - windows-transport-tests: - name: Windows Transport Tests - needs: [changes] + # Unit tier: every Go test that needs no running workspace. The steps per + # runner are defined under `unit` in hack/ci/suites.yaml. + unit-tests: + name: Unit Tests (${{ matrix.runner }}) + needs: [changes, plan] if: needs.changes.outputs.go == 'true' - runs-on: windows-latest - timeout-minutes: 15 - permissions: - contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.unit) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ matrix.timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + + - name: setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod - cache: false - - name: test native Windows transport - run: >- - go test -race -count=1 -v -timeout 5m ./pkg/docker - -run '^(TestInvocationSuite|TestProviderDaemonOverrides.*|TestStdioTransportSuite|TestLaunch.*|TestEnvBlock.*)$' - - name: test E2E runtime helpers - run: go test -race -count=1 -timeout 5m ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/cmd/devsy-ci + + - name: run unit tests + shell: bash + run: go run ./hack/ci unit provider-update-windows: name: Provider Update Tests (windows-latest) @@ -228,9 +196,6 @@ jobs: with: go-version-file: go.mod - - name: test Go CI helpers on Windows - run: go test -race -count=1 -timeout 10m ./hack/microsandbox ./hack/provider_update_ci - - name: run provider update tests and require Windows coverage shell: pwsh run: go run ./hack/provider_update_ci --artifacts-dir "$env:RUNNER_TEMP" @@ -245,21 +210,6 @@ jobs: ${{ runner.temp }}/provider-update-windows.exitcode if-no-files-found: ignore - can-read-secret: - name: Can Read Secret - runs-on: ubuntu-latest - outputs: - secret-set: ${{ steps.check-secret.outputs.secret-set }} - steps: - - name: Check secret - id: check-secret - run: | - if [ -z "${{ secrets.DEVSY_GITHUB_APP_ID }}" ]; then - echo "secret-set=false" >> "$GITHUB_OUTPUT" - else - echo "secret-set=true" >> "$GITHUB_OUTPUT" - fi - build-cli: name: Build CLI Binary on ${{ matrix.runner }} needs: [changes, precommit, lint] @@ -292,24 +242,21 @@ jobs: fi echo "runner_os=$OS" >> "$GITHUB_OUTPUT" - - name: test Podman CI helper - run: go test ./hack/podman_ci - - - name: build Podman CI helper + - name: build CI helpers if: runner.os == 'Linux' || runner.os == 'Windows' shell: bash env: CGO_ENABLED: 0 run: | OUT_DIR="${{ runner.temp }}" + SUFFIX="" if [ "${{ runner.os }}" = "Windows" ]; then OUT_DIR="${OUT_DIR//\\//}" + SUFFIX=".exe" fi - OUT="$OUT_DIR/devsy-ci-podman" - if [ "${{ runner.os }}" = "Windows" ]; then - OUT="${OUT}.exe" - fi - go build -trimpath -o "$OUT" ./hack/podman_ci + go build -trimpath -o "$OUT_DIR/devsy-ci-podman$SUFFIX" ./hack/podman_ci + go build -trimpath -o "$OUT_DIR/devsy-ci-tool$SUFFIX" ./hack/ci + go build -trimpath -o "$OUT_DIR/devsy-ci-e2e$SUFFIX" ./e2e/cmd/devsy-ci - name: upload Podman CI helper if: runner.os == 'Linux' || runner.os == 'Windows' @@ -318,6 +265,20 @@ jobs: name: devsy-ci-podman-${{ steps.os.outputs.runner_os }} path: ${{ runner.temp }}/devsy-ci-podman* + - name: upload E2E runtime helper + if: runner.os == 'Windows' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: devsy-ci-e2e-${{ steps.os.outputs.runner_os }} + path: ${{ runner.temp }}/devsy-ci-e2e* + + - name: upload CI tool + if: runner.os == 'Linux' || runner.os == 'Windows' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: devsy-ci-tool-${{ steps.os.outputs.runner_os }} + path: ${{ runner.temp }}/devsy-ci-tool* + - name: disk usage before Linux snapshot if: runner.os == 'Linux' run: | @@ -408,7 +369,6 @@ jobs: name: e2e-test-${{ steps.os.outputs.runner_os }} path: ./e2e/e2e.test* - licenses: name: Third-party licenses needs: [changes, precommit, lint] @@ -424,152 +384,109 @@ jobs: - name: Check license policy and attribution run: go run ./hack/licenses --check + # E2E tier, group "unprivileged": runs as the runner user. integration-tests-unprivileged: - name: Test ${{ matrix.label }} on ${{ matrix.runner }} - needs: [changes, build-cli] + name: ${{ matrix.title }} + needs: [changes, plan, build-cli] if: needs.changes.outputs.go == 'true' strategy: fail-fast: false - matrix: - include: - - label: context - runner: ubuntu-latest - - - label: extends - runner: ubuntu-latest - - - label: outdated - runner: ubuntu-latest - - - label: machine - runner: ubuntu-latest - - - label: self-update - runner: ubuntu-latest - - - label: upgrade - runner: ubuntu-latest - + matrix: ${{ fromJSON(needs.plan.outputs.unprivileged) }} runs-on: ${{ matrix.runner }} - timeout-minutes: 45 + timeout-minutes: ${{ matrix.job_timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: get operating system lowercase - id: os + - name: download CI tool + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: devsy-ci-tool-${{ matrix.os }} + path: ${{ runner.temp }}/devsy-ci-tool/ + + - name: set up CI tool shell: bash run: | - OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') - echo "runner_os=$OS" >> "$GITHUB_OUTPUT" + tool="${RUNNER_TEMP//\\//}/devsy-ci-tool/devsy-ci-tool" + chmod +x "$tool" + echo "CI_TOOL=$tool" >> "$GITHUB_ENV" - name: download CLI artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: devsy-linux + pattern: devsy-${{ matrix.os }} path: ${{ runner.temp }}/devsy-bin/ merge-multiple: true - name: download e2e test binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: e2e-test-linux + name: e2e-test-${{ matrix.os }} path: ./e2e/ - name: setup executable shell: bash - working-directory: ./e2e - run: | - TEMP_DIR="${{ runner.temp }}" - ls -R "$TEMP_DIR/devsy-bin/" - mkdir -p ./bin/ - find "$TEMP_DIR/devsy-bin/" -type f -name "devsy-*" -exec cp {} ./bin \; - find ./bin -name "devsy-*" -exec chmod +x {} \; - chmod +x ./e2e.test - ls -R ./bin/ + run: '"$CI_TOOL" e2e prepare --bin-src "$RUNNER_TEMP/devsy-bin" --e2e-dir ./e2e' - name: run test shell: bash - working-directory: ./e2e env: GH_USERNAME: ${{ github.repository_owner }} GH_ACCESS_TOKEN: ${{ github.token }} GH_CREDENTIAL_USERNAME: x-access-token - run: | - ./e2e.test -test.v -ginkgo.v -test.timeout 1500s -ginkgo.fail-on-empty --ginkgo.label-filter="${{ matrix.label }}" + run: '"$CI_TOOL" e2e run --label "${{ matrix.name }}" --runner "${{ matrix.runner }}" --e2e-dir ./e2e' + # E2E tier, group "microsandbox": the MicroSandbox CI helper owns the run and + # its report checks. integration-tests-microsandbox: - name: Test ${{ matrix.label }} on ubuntu-latest - needs: [changes, build-cli] + name: ${{ matrix.title }} + needs: [changes, plan, build-cli] if: needs.changes.outputs.go == 'true' strategy: fail-fast: false - matrix: - include: - - label: up-provider-microsandbox - expected-spec-count: 2 - - label: up-provider-microsandbox-images - expected-spec-count: 6 - - label: up-provider-microsandbox-mounts - expected-spec-count: 10 - - label: up-provider-microsandbox-resources - expected-spec-count: 12 - - label: up-provider-microsandbox-storage - expected-spec-count: 6 - - label: up-provider-microsandbox-ephemeral - expected-spec-count: 6 - - label: up-provider-microsandbox-egress - expected-spec-count: 4 - - label: up-provider-microsandbox-prebuilds - expected-spec-count: 4 - - label: up-provider-microsandbox-dockerless - expected-spec-count: 8 - - label: up-provider-microsandbox-recreate - expected-spec-count: 4 - - label: up-provider-microsandbox-managed-recreate - expected-spec-count: 4 - - label: up-provider-microsandbox-logs-cancel - expected-spec-count: 6 - - label: up-provider-microsandbox-compatibility - expected-spec-count: 2 - - label: up-provider-microsandbox-registry-consumer - expected-spec-count: 3 - provider-mode: external - - label: up-provider-microsandbox-remote-agent-binaries - expected-spec-count: 1 - provider-mode: external - - label: up-provider-microsandbox-source-migration - expected-spec-count: 2 - provider-mode: external - runs-on: ubuntu-latest - timeout-minutes: 25 + matrix: ${{ fromJSON(needs.plan.outputs.microsandbox) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: ${{ matrix.job_timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: install gnupg2 run: sudo apt-get install -y gnupg2 + - name: download CI tool + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: devsy-ci-tool-${{ matrix.os }} + path: ${{ runner.temp }}/devsy-ci-tool/ + + - name: set up CI tool + shell: bash + run: | + tool="${RUNNER_TEMP//\\//}/devsy-ci-tool/devsy-ci-tool" + chmod +x "$tool" + echo "CI_TOOL=$tool" >> "$GITHUB_ENV" + - name: download CLI artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: devsy-linux + pattern: devsy-${{ matrix.os }} path: ${{ runner.temp }}/devsy-bin/ merge-multiple: true - name: download e2e test binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: e2e-test-linux + name: e2e-test-${{ matrix.os }} path: ./e2e/ - name: download runtime performance tools - if: matrix.label == 'up-provider-microsandbox' + if: matrix.performance_capture uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runtime-performance-tools-linux path: ${{ runner.temp }}/runtime-performance-tools/ - name: prepare runtime performance capture - if: matrix.label == 'up-provider-microsandbox' + if: matrix.performance_capture shell: bash run: | chmod +x "${RUNNER_TEMP}/runtime-performance-tools/devsy-runtime-spawn-probe" \ @@ -594,14 +511,7 @@ jobs: - name: setup executable shell: bash - working-directory: ./e2e - run: | - ls -R "${RUNNER_TEMP}/devsy-bin/" - mkdir -p ./bin/ - find "${RUNNER_TEMP}/devsy-bin/" -type f -name "devsy-*" -exec cp {} ./bin \; - find ./bin -name "devsy-*" -exec chmod +x {} \; - chmod +x ./e2e.test - ls -R ./bin/ + run: '"$CI_TOOL" e2e prepare --bin-src "$RUNNER_TEMP/devsy-bin" --e2e-dir ./e2e' - name: cache microsandbox release bundle (Linux) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -619,23 +529,23 @@ jobs: GH_USERNAME: ${{ github.repository_owner }} GH_ACCESS_TOKEN: ${{ github.token }} GH_CREDENTIAL_USERNAME: x-access-token - TEST_TIMEOUT: 1200s + TEST_TIMEOUT: ${{ matrix.test_timeout }} DEVSY_REQUIRE_MICROSANDBOX: "true" - run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.label }}" --expected-count "${{ matrix.expected-spec-count }}" --provider-mode "${{ matrix.provider-mode || ''parity'' }}"' + run: '"${RUNNER_TEMP}/devsy-ci-microsandbox" run --label "${{ matrix.name }}" --expected-count "${{ matrix.expected_spec_count }}" --provider-mode "${{ matrix.provider_mode }}"' - name: upload MicroSandbox test evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: microsandbox-${{ matrix.label }}-evidence + name: microsandbox-${{ matrix.name }}-evidence path: | - ${{ runner.temp }}/microsandbox-${{ matrix.label }}.json - ${{ runner.temp }}/microsandbox-${{ matrix.label }}.log + ${{ runner.temp }}/microsandbox-${{ matrix.name }}.json + ${{ runner.temp }}/microsandbox-${{ matrix.name }}.log if-no-files-found: warn retention-days: 14 - name: upload runtime performance evidence - if: always() && matrix.label == 'up-provider-microsandbox' + if: always() && matrix.performance_capture uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: microsandbox-runtime-performance @@ -643,452 +553,67 @@ jobs: if-no-files-found: error retention-days: 14 + # E2E tier, group "integration": runtimes (Docker, Podman, kind) are + # provisioned per suite from the matrix flags computed by `hack/ci matrix`. integration-tests: - name: Test ${{ matrix.label }}${{ (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && format(' ({0})', matrix.install-podman) || '' }} on ${{ matrix.runner }} - needs: [changes, can-read-secret, build-cli] + name: ${{ matrix.title }} + needs: [changes, plan, build-cli] if: needs.changes.outputs.go == 'true' strategy: fail-fast: false - matrix: - include: - - label: docker-context - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - test-timeout: 300s - job-timeout-minutes: 10 - - - label: ide - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: machine-provider - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: integration - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: tunnel - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - flake-attempts: 2 - - - label: mcp - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: provider - runner: ubuntu-latest - free-disk-space: true - install-kind: true - requires-secret: false - flake-attempts: 2 - - - label: ssh - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: ssh-agent-forward - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: ssh-ports-attributes - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: ssh-tunnel-mode - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: task-cancel - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: ssh-credentials-server-race - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: build - runner: ubuntu-latest - free-disk-space: false - install-kind: true - requires-secret: true - - - label: docker-install - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: exec - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: ci - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: feature - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: extends-up - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: down - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: rename - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: delivery - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: logs - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: templates - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: config-apply - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: run-user-commands - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: config-read - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - # Up tests - - - label: up-workspaces - runner: ubuntu-latest - free-disk-space: true - install-kind: true - requires-secret: false - - - label: up-behaviors - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - - label: up-handle-errors - runner: ubuntu-latest - free-disk-space: true - install-kind: true - requires-secret: false - - - label: up-private-token - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: true - - - label: up-provider-kubernetes - runner: ubuntu-latest - free-disk-space: true - install-kind: true - requires-secret: false - - - label: up-provider-kubernetes-restricted-scc - runner: ubuntu-latest - free-disk-space: true - install-kind: true - requires-secret: false - - - label: up-provider-podman-rootless-basic - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootless - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootless-exec - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootless - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootless-lifecycle - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootless - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootless-config - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootless - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootless-features - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootless - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootful-basic - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootful - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootful-lifecycle - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootful - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootful-lifecycle-2 - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootful - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootful-config - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootful - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-podman-rootful-features - runner: ubuntu-24.04 - free-disk-space: false - install-kind: false - requires-secret: false - install-podman: rootful - test-timeout: 600s - job-timeout-minutes: 20 - flake-attempts: 1 - - - label: up-provider-docker - runner: ubuntu-latest - free-disk-space: false - install-kind: true - requires-secret: false - - # Snapshot tests - - - label: snapshot - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - configure-insecure-registry: true - - # Up Docker Compose tests - - - label: up-docker-compose - runner: ubuntu-latest - free-disk-space: false - install-kind: false - requires-secret: false - - # Up Docker Feature tests - - - label: up-features - runner: ubuntu-latest - free-disk-space: false - install-kind: true - requires-secret: false - - # Windows tests - - - label: provider - runner: windows-latest - free-disk-space: false - install-kind: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - - label: up-docker-wsl - runner: windows-latest - free-disk-space: false - install-kind: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - - label: up-features - runner: windows-latest - free-disk-space: false - install-kind: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - label: ssh-proxy-command - runner: windows-latest - free-disk-space: false - install-kind: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - label: ssh-tunnel-mode - runner: windows-latest - free-disk-space: false - install-kind: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - - label: task-cancel - runner: windows-latest - free-disk-space: false - install-kind: false - install-podman: true - requires-secret: false - test-timeout: 900s - job-timeout-minutes: 20 - - + matrix: ${{ fromJSON(needs.plan.outputs.integration) }} runs-on: ${{ matrix.runner }} - timeout-minutes: ${{ matrix.job-timeout-minutes || 45 }} + timeout-minutes: ${{ matrix.job_timeout_minutes }} steps: - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # v2.0.0 background: true - if: matrix.free-disk-space == true && runner.os == 'Linux' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.free_disk_space && matrix.os == 'linux' with: docker-images: false swap-storage: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' - name: install gnupg2 - if: runner.os == 'Linux' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.os == 'linux' run: sudo apt-get install -y gnupg2 - - name: get operating system lowercase - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' - id: os + - name: download CI tool + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: devsy-ci-tool-${{ matrix.os }} + path: ${{ runner.temp }}/devsy-ci-tool/ + + - name: set up CI tool shell: bash run: | - OS=$(echo "${{ runner.os }}" | tr '[:upper:]' '[:lower:]') - if [ "$OS" == "macos" ]; then - OS="darwin" - fi - echo "runner_os=$OS" >> "$GITHUB_OUTPUT" + tool="${RUNNER_TEMP//\\//}/devsy-ci-tool/devsy-ci-tool" + chmod +x "$tool" + echo "CI_TOOL=$tool" >> "$GITHUB_ENV" - name: download Podman CI helper - if: > - ( - matrix.install-kind == true || - matrix.install-podman == true || - matrix.install-podman == 'rootless' || - matrix.install-podman == 'rootful' - ) && - (runner.os == 'Windows' || runner.os == 'Linux') && - (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.needs_podman_helper uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: devsy-ci-podman-${{ steps.os.outputs.runner_os }} + name: devsy-ci-podman-${{ matrix.os }} path: ${{ runner.temp }}/devsy-ci-podman/ + - name: download E2E runtime helper + if: matrix.setup_docker_wsl + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: devsy-ci-e2e-${{ matrix.os }} + path: ${{ runner.temp }}/devsy-ci-e2e/ + - name: download CLI artifacts - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: devsy-${{ steps.os.outputs.runner_os }} + pattern: devsy-${{ matrix.os }} path: ${{ runner.temp }}/devsy-bin/ merge-multiple: true - name: download Linux CLI artifacts for Windows - if: runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.os == 'windows' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: devsy-linux @@ -1096,40 +621,24 @@ jobs: merge-multiple: true - name: download e2e test binary - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: e2e-test-${{ steps.os.outputs.runner_os }} + name: e2e-test-${{ matrix.os }} path: ./e2e/ - name: download GitHub credential fixture - if: matrix.label == 'ssh' && runner.os == 'Linux' + if: matrix.ghfixture_linux uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ghfixture-linux path: ./e2e/bin/ - name: setup executable - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' shell: bash - working-directory: ./e2e - run: | - TEMP_DIR="${{ runner.temp }}" - if [ "${{ runner.os }}" == "Windows" ]; then - TEMP_DIR="${TEMP_DIR//\\//}" - fi - - ls -R "$TEMP_DIR/devsy-bin/" - mkdir -p ./bin/ - find "$TEMP_DIR/devsy-bin/" -type f -name "devsy-*" -exec cp {} ./bin \; - find ./bin -name "devsy-*" -exec chmod +x {} \; - if [ "${{ runner.os }}" == "Linux" ]; then - chmod +x ./e2e.test - fi - ls -R ./bin/ + run: '"$CI_TOOL" e2e prepare --bin-src "$RUNNER_TEMP/devsy-bin" --e2e-dir ./e2e' - name: generate uuid - if: matrix.install-kind == true && runner.os != 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_kind_linux id: uuid shell: bash run: | @@ -1137,7 +646,7 @@ jobs: echo "result=$UUID" >> "$GITHUB_OUTPUT" - name: cache podman installer (Windows) - if: (matrix.install-kind == true || matrix.install-podman == true) && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_podman_windows id: podman-cache-windows uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -1145,7 +654,7 @@ jobs: key: ${{ runner.os }}-${{ runner.arch }}-podman-v5.8.2 - name: install podman (Windows) - if: (matrix.install-kind == true || matrix.install-podman == true) && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_podman_windows shell: pwsh timeout-minutes: 5 run: | @@ -1155,7 +664,7 @@ jobs: --install-dir "C:\Program Files\RedHat\Podman" --log-path "$env:RUNNER_TEMP\podman-install.log" - name: start podman (Windows) - if: (matrix.install-kind == true || matrix.install-podman == true) && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_podman_windows shell: pwsh timeout-minutes: 5 run: | @@ -1165,14 +674,14 @@ jobs: --bootstrap-timeout 270s - name: cache kind.exe (Windows) - if: matrix.install-kind == true && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_kind_windows uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/kind.exe key: ${{ runner.os }}-${{ runner.arch }}-kind-v0.33.0 - name: setup kind (Windows) - if: matrix.install-kind == true && runner.os == 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_kind_windows shell: pwsh env: KIND_EXPERIMENTAL_PROVIDER: podman @@ -1186,7 +695,7 @@ jobs: # NOTE: skevetter/setup-kind does not work on Windows runners - name: setup kind - if: matrix.install-kind == true && runner.os != 'Windows' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') + if: matrix.setup_kind_linux uses: skevetter/setup-kind@7febac2ed35df332069b3bb687c6b1fa00fc0883 # v1 with: name: ${{ steps.uuid.outputs.result }} @@ -1195,67 +704,58 @@ jobs: skipClusterLogsExport: true - name: cache podman installer (Linux) - if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' - id: podman-cache-linux + if: matrix.setup_podman_linux uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/podman-linux-amd64.tar.gz key: ${{ runner.os }}-${{ runner.arch }}-podman-static-v5.8.4-sha256-v2 - name: install podman (Linux) - if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' + if: matrix.setup_podman_linux timeout-minutes: 5 run: | - archive="${{ runner.temp }}/podman-linux-amd64.tar.gz" - if [ "${{ steps.podman-cache-linux.outputs.cache-hit }}" != "true" ]; then - curl -fsSL -o "$archive" \ - https://github.com/mgoltzsche/podman-static/releases/download/v5.8.4/podman-linux-amd64.tar.gz - fi - - expectedHash="a58765fe8be6ab3fb79f892f1a027b4ce4a7e8eb589df1ef960c167cbde08d69" - actualHash=$(sha256sum "$archive" | cut -d' ' -f1) - if [ "$actualHash" != "$expectedHash" ]; then - echo "::error::SHA256 mismatch for podman-static archive! Expected: $expectedHash, Got: $actualHash" - exit 1 - fi - - tar -xzf "$archive" -C "${{ runner.temp }}" - sudo cp -r "${{ runner.temp }}/podman-linux-amd64/usr" "${{ runner.temp }}/podman-linux-amd64/etc" / - - command -v newuidmap >/dev/null || sudo apt-get install -y uidmap + "$CI_TOOL" setup podman-linux \ + --archive "${RUNNER_TEMP}/podman-linux-amd64.tar.gz" \ + --url https://github.com/mgoltzsche/podman-static/releases/download/v5.8.4/podman-linux-amd64.tar.gz \ + --sha256 a58765fe8be6ab3fb79f892f1a027b4ce4a7e8eb589df1ef960c167cbde08d69 \ + --extract-dir "${RUNNER_TEMP}" - name: configure and verify podman (Linux) - if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' + if: matrix.setup_podman_linux shell: bash timeout-minutes: 5 run: | chmod +x "${RUNNER_TEMP}/devsy-ci-podman/devsy-ci-podman" "${RUNNER_TEMP}/devsy-ci-podman/devsy-ci-podman" setup \ - --mode "${{ matrix.install-podman }}" \ + --mode "${{ matrix.podman_mode }}" \ --podman-path /usr/local/bin/podman \ --crun-path /usr/local/bin/crun \ --bootstrap-timeout 270s - - name: remove docker - if: matrix.label == 'docker-install' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') - run: | - sudo systemctl stop docker.socket docker.service - sudo apt-get remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - sudo rm -rf /var/lib/docker /var/lib/containerd /usr/bin/docker /usr/bin/dockerd + - name: setup WSL2 Ubuntu + if: matrix.setup_docker_wsl + uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 + with: + distribution: Ubuntu-24.04 + use-cache: true - - name: configure insecure registry for snapshot tests (Linux) - if: matrix.configure-insecure-registry == true && runner.os == 'Linux' - run: | - echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts - sudo mkdir -p /etc/docker - existing="$(sudo cat /etc/docker/daemon.json 2>/dev/null || echo '{}')" - echo "$existing" | jq '.["insecure-registries"] = ((.["insecure-registries"] // []) + ["host.docker.internal:15500"] | unique)' \ - | sudo tee /etc/docker/daemon.json > /dev/null - sudo systemctl restart docker - timeout 30 bash -c 'until docker info >/dev/null 2>&1; do sleep 1; done' + - name: provision Docker over WSL SSH + if: matrix.setup_docker_wsl + shell: pwsh + timeout-minutes: 15 + run: >- + & "$env:RUNNER_TEMP\devsy-ci-e2e\devsy-ci-e2e.exe" docker-wsl setup + --distribution Ubuntu-24.04 --timeout 4m + --state-file "$env:RUNNER_TEMP\devsy-ci\docker-wsl-state.json" + + # Runs the suite's `host_setup` actions from hack/ci/suites.yaml + # (for example removing Docker or trusting a local registry). + - name: host setup + shell: bash + run: '"$CI_TOOL" setup host --label "${{ matrix.name }}" --runner "${{ matrix.runner }}"' - name: generate GitHub App token - if: matrix.requires-secret == true && needs.can-read-secret.outputs.secret-set == 'true' + if: matrix.requires_secret id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 with: @@ -1263,167 +763,68 @@ jobs: private-key: ${{ secrets.DEVSY_GITHUB_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - - name: test Compose UID mapping as existing non-root runner - if: runner.os == 'Linux' && matrix.label == 'up-docker-compose' - shell: bash - working-directory: ./e2e - env: - TEST_TIMEOUT: ${{ matrix.test-timeout || '1500s' }} - run: | - set -euo pipefail - id - test "$(id -u)" = 1001 - test "$(id -g)" = 1001 - getent passwd "$(id -u)" - getent group "$(id -g)" - test -d "$HOME" && test -w "$HOME" - test "$(stat -c %u "$HOME")" = "$(id -u)" - test -d "$RUNNER_TEMP" && test -w "$RUNNER_TEMP" - docker info - docker compose version - printf 'checkout=%s\n' "$(git rev-parse HEAD)" - sha256sum ./e2e.test ./bin/devsy-linux-amd64 - ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" \ - -ginkgo.fail-on-empty -ginkgo.flake-attempts=1 \ - --ginkgo.label-filter=up-docker-compose \ - --ginkgo.focus='implements updateRemoteUserUID with root container user$' \ - --ginkgo.json-report="$RUNNER_TEMP/compose-uid-nonroot.json" - python3 - "$RUNNER_TEMP/compose-uid-nonroot.json" <<'PY' - import json - import sys - - with open(sys.argv[1], encoding="utf-8") as report: - suites = json.load(report) - assert isinstance(suites, list) and len(suites) == 1, "unexpected suite report shape" - suite = suites[0] - assert suite["SuiteConfig"]["DryRun"] is False, "dry-run is not execution" - assert suite["SuiteSucceeded"] is True, "suite failed" - selected = [ - spec for spec in suite["SpecReports"] - if spec.get("LeafNodeType") == "It" and spec.get("State") != "skipped" - ] - assert len(selected) == 1, f"expected one executed leaf, found {len(selected)}" - spec = selected[0] - assert spec["LeafNodeText"] == "implements updateRemoteUserUID with root container user", "wrong UID spec" - assert spec["State"] == "passed", "UID spec did not pass" - assert spec["NumAttempts"] == 1, "unexpected retries" - print("one www-data UID spec passed in one attempt") - PY - + # Focused checks, privilege (sudo), runtime environment and post checks + # all come from the suite definition. - name: run test - if: matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true' shell: bash - working-directory: ./e2e env: # NOTE: GitHub credentials are required for tests using private repos GH_USERNAME: ${{ github.repository_owner }} GH_ACCESS_TOKEN: ${{ steps.app-token.outputs.token || github.token }} GH_CREDENTIAL_USERNAME: x-access-token - TEST_TIMEOUT: ${{ matrix.test-timeout || '1500s' }} - FLAKE_ATTEMPTS: ${{ matrix.flake-attempts || '1' }} - run: | - if [ "${{ runner.os }}" == "Linux" ]; then - if [ "${{ matrix.install-podman || '' }}" = "rootless" ]; then - echo "Podman rootless test identity:" - id - printf 'HOME=%s\n' "${HOME}" - printf 'XDG_RUNTIME_DIR=%s\n' "${XDG_RUNTIME_DIR:-}" - podman info - GH_USERNAME="${GH_USERNAME}" \ - GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ - GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ - KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ - PATH="${PATH}" \ - ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" - else - sudo \ - GH_USERNAME="${GH_USERNAME}" \ - GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ - GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ - KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ - ${DOCKER_HOST:+DOCKER_HOST="${DOCKER_HOST}"} \ - PATH="${PATH}" \ - ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" - fi - else - GH_USERNAME="${GH_USERNAME}" \ - GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ - GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ - KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" \ - PATH="${PATH}" \ - DOCKER_HOST="npipe:////./pipe/podman-machine-default" \ - ./e2e.test.exe -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" - fi + run: '"$CI_TOOL" e2e run --label "${{ matrix.name }}" --runner "${{ matrix.runner }}" --e2e-dir ./e2e' - - name: verify docker is installed - if: matrix.label == 'docker-install' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') - run: docker --version && docker ps + - name: tear down Docker WSL SSH fixture + if: always() && matrix.setup_docker_wsl + shell: pwsh + timeout-minutes: 3 + run: | + if (Test-Path "$env:RUNNER_TEMP\devsy-ci-e2e\devsy-ci-e2e.exe") { + & "$env:RUNNER_TEMP\devsy-ci-e2e\devsy-ci-e2e.exe" docker-wsl cleanup ` + --state-file "$env:RUNNER_TEMP\devsy-ci\docker-wsl-state.json" --timeout 2m + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } - docker-buildkit-windows: - needs: [changes, build-cli, windows-transport-tests] - if: needs.changes.outputs.go == 'true' - uses: ./.github/workflows/docker-buildkit-windows.yml + - name: upload report and setup or teardown diagnostics + if: always() && matrix.setup_docker_wsl + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: e2e-diagnostics-${{ matrix.name }} + path: | + e2e/${{ matrix.name }}-report.json + ${{ runner.temp }}/devsy-ci/docker-wsl-diagnostics.log + if-no-files-found: warn ci-success: name: CI Success runs-on: ubuntu-latest if: always() + permissions: + contents: read needs: - changes + - plan - precommit - lint - agent-skill - unit-tests - github-credential-host-tests - - windows-transport-tests - provider-update-windows - licenses - build-cli - integration-tests-unprivileged - integration-tests - integration-tests-microsandbox - - docker-buildkit-windows steps: - - name: Check required jobs - run: | - go_changes="${{ needs.changes.outputs.go }}" - - check_result() { - local name="$1" - local result="$2" - local skip_when_no_go_changes="$3" - - case "$result" in - success) - ;; - skipped) - if [[ "$skip_when_no_go_changes" != "true" || "$go_changes" == "true" ]]; then - echo "::error::$name ended with result=skipped unexpectedly" - return 1 - fi - ;; - *) - echo "::error::$name ended with result=$result" - return 1 - ;; - esac - } + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - status=0 - check_result "changes" "${{ needs.changes.result }}" false || status=1 - check_result "precommit" "${{ needs.precommit.result }}" false || status=1 - check_result "lint" "${{ needs.lint.result }}" false || status=1 - check_result "agent-skill" "${{ needs.agent-skill.result }}" false || status=1 - check_result "unit-tests" "${{ needs.unit-tests.result }}" true || status=1 - check_result "github-credential-host-tests" "${{ needs.github-credential-host-tests.result }}" true || status=1 - check_result "windows-transport-tests" "${{ needs.windows-transport-tests.result }}" true || status=1 - check_result "provider-update-windows" "${{ needs.provider-update-windows.result }}" true || status=1 - check_result "licenses" "${{ needs.licenses.result }}" true || status=1 - check_result "build-cli" "${{ needs.build-cli.result }}" true || status=1 - check_result "integration-tests-unprivileged" "${{ needs.integration-tests-unprivileged.result }}" true || status=1 - check_result "integration-tests" "${{ needs.integration-tests.result }}" true || status=1 - check_result "integration-tests-microsandbox" "${{ needs.integration-tests-microsandbox.result }}" true || status=1 - - check_result "docker-buildkit-windows" "${{ needs.docker-buildkit-windows.result }}" true || status=1 - - exit "$status" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + + # Jobs gated on Go changes may be skipped only when no Go-relevant file + # changed; the jobs listed in --always-run must never be skipped. + - name: Check required jobs + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: go run ./hack/ci success --always-run changes,plan,precommit,lint,agent-skill diff --git a/ci b/ci new file mode 100755 index 000000000..e6b4a38c4 Binary files /dev/null and b/ci differ diff --git a/e2e/README.md b/e2e/README.md index 1ed95b059..23d4a89a1 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -49,14 +49,21 @@ Remove the cluster afterwards: mise run cli:test:e2e:kind:teardown ``` +## CI test tiers + +`hack/ci/suites.yaml` defines every test tier that `.github/workflows/pr-ci.yml` runs; +see `hack/ci/README.md`. To make a new Ginkgo label run in CI, add it to +`e2e.suites` there. `go test ./hack/ci` fails while a label in this directory has no +entry. + #### CI helpers -The `Windows Transport Tests` job runs native transport and runtime-helper unit -tests independently of CLI artifact builds. After that job and the builds pass, -the reusable `docker-buildkit-windows.yml` workflow runs the -`docker-internal-buildkit` regression specs. It provisions Docker Engine in WSL2, -reached over SSH, with `devsy-ci docker-wsl setup` (`e2e/cmd/devsy-ci`). The runtime -logic lives in `e2e/framework`: +Native Windows transport and runtime-helper unit tests run in the `Unit Tests +(windows-latest)` job, independently of CLI artifact builds. After the CLI builds +pass, the `docker-internal-buildkit` suite of the integration group (runtime +`docker-wsl` in `hack/ci/suites.yaml`) runs the BuildKit regression specs. Its job +provisions Docker Engine in WSL2, reached over SSH, with `devsy-ci docker-wsl setup` +(`e2e/cmd/devsy-ci`). The runtime logic lives in `e2e/framework`: | Package | Responsibility | | --- | --- | @@ -66,10 +73,10 @@ logic lives in `e2e/framework`: Their unit tests need no Docker, WSL or network: ```bash -go test ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/cmd/devsy-ci +go test ./e2e/framework/runenv/... ./e2e/framework/dockerwsl ./e2e/framework/ginkgoreport ./e2e/cmd/devsy-ci ``` -Ginkgo executes the specs directly. `devsy-ci check-report` checks its JSON -report: all six BuildKit specs must pass on their first attempt, with no +Ginkgo executes the specs directly. `ci e2e run` (or `devsy-ci check-report`) checks the JSON +report through `e2e/framework/ginkgoreport`: all six BuildKit specs must pass on their first attempt, with no selected skips, dry run or programmatic focus. Teardown runs on success and failure, before the report and non-secret diagnostics are uploaded. diff --git a/e2e/cmd/devsy-ci/report.go b/e2e/cmd/devsy-ci/report.go index 8419b9404..5696e7b22 100644 --- a/e2e/cmd/devsy-ci/report.go +++ b/e2e/cmd/devsy-ci/report.go @@ -1,13 +1,10 @@ package main import ( - "encoding/json" - "errors" "fmt" - "slices" + "github.com/devsy-org/devsy/e2e/framework/ginkgoreport" "github.com/devsy-org/devsy/e2e/framework/runenv" - "github.com/onsi/ginkgo/v2/types" "github.com/spf13/cobra" ) @@ -26,7 +23,7 @@ func newReportCommand(deps *runenv.Dependencies) *cobra.Command { return runenv.NewStageError(runenv.CodeGinkgoReportInvalid, reportCommandName, fmt.Errorf("read Ginkgo report: %w", err)) } - if err := checkReport(data, label, expected); err != nil { + if err := ginkgoreport.Check(data, label, expected); err != nil { return runenv.NewStageError(runenv.CodeGinkgoReportInvalid, reportCommandName, err) } return nil @@ -37,49 +34,3 @@ func newReportCommand(deps *runenv.Dependencies) *cobra.Command { cmd.Flags().IntVar(&expected, "expected-specs", 0, "required number of selected specs") return cmd } - -func checkReport(data []byte, label string, expected int) error { - if label == "" || expected <= 0 { - return errors.New("report validation needs a label and a positive expected-specs count") - } - report, err := readReport(data) - if err != nil { - return err - } - return checkSelectedSpecs(report.SpecReports, label, expected) -} - -func readReport(data []byte) (types.Report, error) { - var reports []types.Report - if err := json.Unmarshal(data, &reports); err != nil { - return types.Report{}, fmt.Errorf("decode Ginkgo report: %w", err) - } - if len(reports) != 1 { - return types.Report{}, errors.New("expected exactly one Ginkgo suite report") - } - report := reports[0] - if !report.SuiteSucceeded || report.SuiteConfig.DryRun || report.SuiteHasProgrammaticFocus { - return types.Report{}, errors.New( - "report must prove a successful run without dry-run or programmatic focus", - ) - } - return report, nil -} - -func checkSelectedSpecs(specs types.SpecReports, label string, expected int) error { - selected := 0 - for _, spec := range specs { - if spec.LeafNodeType != types.NodeTypeIt || !slices.Contains(spec.Labels(), label) { - continue - } - selected++ - if spec.State != types.SpecStatePassed || spec.NumAttempts != 1 { - return fmt.Errorf("spec %q: state=%s attempts=%d, expected passed on the first attempt", - spec.FullText(), spec.State, spec.NumAttempts) - } - } - if selected != expected { - return fmt.Errorf("label %q: expected %d specs, found %d", label, expected, selected) - } - return nil -} diff --git a/e2e/cmd/devsy-ci/report_test.go b/e2e/cmd/devsy-ci/report_test.go index 9d06b7a96..53da3e696 100644 --- a/e2e/cmd/devsy-ci/report_test.go +++ b/e2e/cmd/devsy-ci/report_test.go @@ -33,32 +33,6 @@ func passedReport() types.Report { } } -func (s *ReportSuite) TestRequiresEverySelectedSpecToPassOnFirstAttempt() { - for _, tc := range []struct { - name string - mutate func(*types.Report) - }{ - {"failed suite", func(r *types.Report) { r.SuiteSucceeded = false }}, - {"dry run", func(r *types.Report) { r.SuiteConfig.DryRun = true }}, - {"focused suite", func(r *types.Report) { r.SuiteHasProgrammaticFocus = true }}, - {"skipped selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStateSkipped }}, - {"pending selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStatePending }}, - {"failed selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStateFailed }}, - {"retried spec", func(r *types.Report) { r.SpecReports[0].NumAttempts = 2 }}, - {"not executed", func(r *types.Report) { r.SpecReports[0].NumAttempts = 0 }}, - {"wrong label", func(r *types.Report) { r.SpecReports[0].ContainerHierarchyLabels = nil }}, - {"extra selected spec", func(r *types.Report) { r.SpecReports = append(r.SpecReports, r.SpecReports[0]) }}, - } { - s.Run(tc.name, func() { - r := passedReport() - tc.mutate(&r) - data, err := json.Marshal([]types.Report{r}) - s.Require().NoError(err) - s.Error(checkReport(data, reportLabel, 1)) - }) - } -} - func (s *ReportSuite) TestReportCommandChecksTheRequestedFile() { data, err := json.Marshal([]types.Report{passedReport()}) s.Require().NoError(err) @@ -79,13 +53,7 @@ func (s *ReportSuite) TestReportCommandChecksTheRequestedFile() { s.NoError(cmd.Execute()) } -func (s *ReportSuite) TestRejectsMissingOrMalformedReportsAndInvalidExpectations() { - for _, data := range []string{"not JSON", "null", "[]", "[{},{}]", "[{}]"} { - s.Error(checkReport([]byte(data), reportLabel, 1), data) - } - s.Error(checkReport(nil, "", 1)) - s.Error(checkReport(nil, reportLabel, 0)) - s.Error(checkReport(nil, reportLabel, -1)) +func (s *ReportSuite) TestRejectsMissingReportFile() { cmd := newRootCommand(runenvtest.NewRig(runenv.GoosLinux, nil).Dependencies) cmd.SetArgs( []string{ diff --git a/e2e/framework/ginkgoreport/report.go b/e2e/framework/ginkgoreport/report.go new file mode 100644 index 000000000..caf2e9096 --- /dev/null +++ b/e2e/framework/ginkgoreport/report.go @@ -0,0 +1,59 @@ +// Package ginkgoreport validates Ginkgo JSON reports produced by E2E runs. +package ginkgoreport + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/onsi/ginkgo/v2/types" +) + +// Check requires every spec carrying label to pass on its first attempt, exactly expected specs +// to be selected, and the report to prove a real, successful, unfocused run. +func Check(data []byte, label string, expected int) error { + if label == "" || expected <= 0 { + return errors.New("report validation needs a label and a positive expected-specs count") + } + report, err := readReport(data) + if err != nil { + return err + } + return checkSelectedSpecs(report.SpecReports, label, expected) +} + +func readReport(data []byte) (types.Report, error) { + var reports []types.Report + if err := json.Unmarshal(data, &reports); err != nil { + return types.Report{}, fmt.Errorf("decode Ginkgo report: %w", err) + } + if len(reports) != 1 { + return types.Report{}, errors.New("expected exactly one Ginkgo suite report") + } + report := reports[0] + if !report.SuiteSucceeded || report.SuiteConfig.DryRun || report.SuiteHasProgrammaticFocus { + return types.Report{}, errors.New( + "report must prove a successful run without dry-run or programmatic focus", + ) + } + return report, nil +} + +func checkSelectedSpecs(specs types.SpecReports, label string, expected int) error { + selected := 0 + for _, spec := range specs { + if spec.LeafNodeType != types.NodeTypeIt || !slices.Contains(spec.Labels(), label) { + continue + } + selected++ + if spec.State != types.SpecStatePassed || spec.NumAttempts != 1 { + return fmt.Errorf("spec %q: state=%s attempts=%d, expected passed on the first attempt", + spec.FullText(), spec.State, spec.NumAttempts) + } + } + if selected != expected { + return fmt.Errorf("label %q: expected %d specs, found %d", label, expected, selected) + } + return nil +} diff --git a/e2e/framework/ginkgoreport/report_test.go b/e2e/framework/ginkgoreport/report_test.go new file mode 100644 index 000000000..e9e76e003 --- /dev/null +++ b/e2e/framework/ginkgoreport/report_test.go @@ -0,0 +1,67 @@ +package ginkgoreport + +import ( + "encoding/json" + "testing" + + "github.com/onsi/ginkgo/v2/types" + "github.com/stretchr/testify/suite" +) + +const reportLabel = "docker-internal-buildkit" + +type ReportSuite struct{ suite.Suite } + +func TestReportSuite(t *testing.T) { suite.Run(t, new(ReportSuite)) } + +func passedReport() types.Report { + return types.Report{ + SuiteSucceeded: true, + SpecReports: types.SpecReports{ + { + LeafNodeType: types.NodeTypeIt, LeafNodeText: "builds on the selected daemon", + ContainerHierarchyLabels: [][]string{{reportLabel}}, + State: types.SpecStatePassed, NumAttempts: 1, + }, + { + LeafNodeType: types.NodeTypeIt, LeafNodeText: "unrelated spec", + LeafNodeLabels: []string{"other-label"}, State: types.SpecStateSkipped, + }, + }, + } +} + +func (s *ReportSuite) TestRequiresEverySelectedSpecToPassOnFirstAttempt() { + for _, tc := range []struct { + name string + mutate func(*types.Report) + }{ + {"failed suite", func(r *types.Report) { r.SuiteSucceeded = false }}, + {"dry run", func(r *types.Report) { r.SuiteConfig.DryRun = true }}, + {"focused suite", func(r *types.Report) { r.SuiteHasProgrammaticFocus = true }}, + {"skipped selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStateSkipped }}, + {"pending selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStatePending }}, + {"failed selected spec", func(r *types.Report) { r.SpecReports[0].State = types.SpecStateFailed }}, + {"retried spec", func(r *types.Report) { r.SpecReports[0].NumAttempts = 2 }}, + {"not executed", func(r *types.Report) { r.SpecReports[0].NumAttempts = 0 }}, + {"wrong label", func(r *types.Report) { r.SpecReports[0].ContainerHierarchyLabels = nil }}, + {"extra selected spec", func(r *types.Report) { r.SpecReports = append(r.SpecReports, r.SpecReports[0]) }}, + } { + s.Run(tc.name, func() { + r := passedReport() + tc.mutate(&r) + data, err := json.Marshal([]types.Report{r}) + s.Require().NoError(err) + s.Error(Check(data, reportLabel, 1)) + }) + } +} + +func (s *ReportSuite) TestRejectsMissingOrMalformedReportsAndInvalidExpectations() { + for _, data := range []string{"not JSON", "null", "[]", "[{},{}]", "[{}]"} { + s.Error(Check([]byte(data), reportLabel, 1), data) + } + s.Error(Check(nil, "", 1)) + s.Error(Check(nil, reportLabel, 0)) + s.Error(Check(nil, reportLabel, -1)) +} diff --git a/hack/ci/README.md b/hack/ci/README.md new file mode 100644 index 000000000..d4e5bc922 --- /dev/null +++ b/hack/ci/README.md @@ -0,0 +1,54 @@ +# hack/ci + +Go implementation of the scripting behind `.github/workflows/pr-ci.yml`. The +workflow only wires GitHub Actions together (checkout, caches, artifacts). What +tests exist, where they run and how they run is defined here. + +## Test tiers + +[`suites.yaml`](suites.yaml) is the single manifest. + +| Tier | Contents | Runs | Defined by | +| --- | --- | --- | --- | +| `unit` | Go tests that need no running workspace | `go run ./hack/ci unit` (also `mise run cli:test`) | `unit.steps` | +| `e2e`, group `unprivileged` | Ginkgo suites run as the runner user | `ci e2e run` | `e2e.suites` | +| `e2e`, group `integration` | Ginkgo suites run with sudo on Linux, on Podman-backed Windows, or on rootless/rootful Podman | `ci e2e run` | `e2e.suites` | +| `e2e`, group `microsandbox` | MicroSandbox suites run through `devsy-ci-microsandbox` | `devsy-ci-microsandbox run` | `e2e.suites` | + +All unit tests run together: `ci unit` runs every step of the manifest that +applies to the current OS, and the `Unit Tests` job runs it on each runner under +`unit.runners`. All integration suites run as one block of jobs generated from +`e2e.suites`. + +## Adding tests + +- **Go unit test in an ordinary package**: nothing to do. The `packages` step + discovers every package outside `test/` and `e2e/` directories, on Linux and + macOS only. +- **Go test under `test/` or `e2e/` that must run as a unit test**: add the + package to a `unit.steps` entry (or to `unit.excluded` with a reason). +- **Go test that must run on Windows**: discovery does not run there, so list + its packages in a `unit.steps` entry with `os: [windows]`. +- **Ginkgo spec**: add its label to `e2e.suites`, choosing a `group`. Set only + what differs from the group defaults (`runner`, `test_timeout`, + `install_kind`, `install_podman`, `requires_secret`, `host_setup`, ...). Labels + that another workflow runs go to `e2e.ungated` with a reason. + +`go test ./hack/ci` enforces both rules: it fails when a Ginkgo label or a Go test +package is not assigned to a tier, and when the manifest names something that no +longer exists. The workflow needs no edits for new tests. + +## Commands + +| Command | Purpose | +| --- | --- | +| `ci matrix [--group G] [--secret-available]` | Print the job matrices as GitHub outputs. Suites that need the GitHub App secret are dropped when it is unavailable. | +| `ci unit [--os GOOS] [--step NAME] [--list] [--coverprofile P]` | Run the unit tier on this machine. | +| `ci e2e prepare --bin-src DIR --e2e-dir DIR` | Stage the CLI binaries next to `e2e.test`. | +| `ci e2e run --label L --runner R` | Run a suite: focused checks, sudo and environment handling, post checks. | +| `ci setup host --label L --runner R` | Run the suite's `host_setup` actions (`remove-docker`, `insecure-registry`). | +| `ci setup podman-linux ...` | Download, verify and install the pinned static Podman build. | +| `ci success --always-run JOBS` | Fold the `needs` results into the required `CI Success` check. | + +The Podman and MicroSandbox runtime helpers stay in `hack/podman_ci` and +`hack/microsandbox`. diff --git a/hack/ci/coverage_test.go b/hack/ci/coverage_test.go new file mode 100644 index 000000000..006998233 --- /dev/null +++ b/hack/ci/coverage_test.go @@ -0,0 +1,343 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +// coverageRepoRoot is the repository root relative to this package directory. +const coverageRepoRoot = "../.." + +const coverageGoListFormat = "{{.ImportPath}}|{{.Dir}}|{{len .TestGoFiles}}|{{len .XTestGoFiles}}" + +var ( + // coverageLabelCall matches `Label(...)` and `ginkgo.Label(...)`. + coverageLabelCall = regexp.MustCompile(`\bLabel\(([^)]*)\)`) + coverageStringLit = regexp.MustCompile(`"([^"]+)"`) + // coverageStringConst matches a bare `name = "value"` assignment (not a + // field such as `fixture.label = ...`), used to resolve labels passed as + // named constants such as Label(suiteLabel). + coverageStringConst = regexp.MustCompile(`(?m)(?:^|[^.\w])([A-Za-z_]\w*)\s*=\s*"([^"]+)"`) + coverageSelector = regexp.MustCompile(`\.[A-Za-z_]\w*`) + coverageIdent = regexp.MustCompile(`[A-Za-z_]\w*`) + // coverageNonUnitDir matches directories whose tests are not unit tests. + // It mirrors the rule the unit tier uses to discover packages. + coverageNonUnitDir = regexp.MustCompile(`(^|/)(test|e2e)(/|$)`) +) + +// coverageSuite guards that every Ginkgo label, Go test package and manifest +// entry is assigned to a CI tier, so no test is silently orphaned. +type coverageSuite struct { + suite.Suite + root string + manifest *Manifest + labels map[string]bool +} + +func TestCoverageSuite(t *testing.T) { + suite.Run(t, new(coverageSuite)) +} + +func (s *coverageSuite) SetupSuite() { + root, err := filepath.Abs(coverageRepoRoot) + s.Require().NoError(err) + s.root = root + + m, err := LoadManifest() + s.Require().NoError(err) + s.manifest = m + + labels, err := coverageScanLabels(filepath.Join(root, "e2e")) + s.Require().NoError(err) + s.labels = labels +} + +// Footgun: an e2e spec gets a new Label that no suite selects, so the spec +// never runs in PR CI and nobody notices. +func (s *coverageSuite) TestEveryGinkgoLabelIsAssigned() { + assigned := s.assignedLabels() + + var missing []string + for _, label := range sortedKeys(s.labels) { + if !assigned[label] { + missing = append(missing, fmt.Sprintf( + "Ginkgo label %q is used in e2e/ but not assigned to a tier; "+ + "add it to hack/ci/suites.yaml (e2e.suites, or e2e.ungated with a reason)", + label, + )) + } + } + if len(missing) > 0 { + s.Failf("unassigned Ginkgo labels", "%s", strings.Join(missing, "\n")) + } +} + +// Footgun: a label is renamed or removed in e2e/ but its manifest entry stays, +// so the CI job selects nothing and reports a green result. +func (s *coverageSuite) TestEveryManifestLabelExistsInE2E() { + var stale []string + for _, label := range s.manifestLabels() { + if !s.labels[label] { + stale = append(stale, fmt.Sprintf( + "label %q is in hack/ci/suites.yaml but no Label(...) in e2e/ uses it; remove the stale entry or restore the label", + label, + )) + } + } + if len(stale) > 0 { + s.Failf("stale manifest labels", "%s", strings.Join(stale, "\n")) + } +} + +// Footgun: a new Go package with tests lands outside every unit step and +// exclusion, so `go test` never runs it in CI. +func (s *coverageSuite) TestEveryTestPackageIsAccounted() { + if _, err := exec.LookPath("go"); err != nil { + s.T().Skip("go tool not available") + } + dirs, err := coverageTestPackageDirs(s.root) + s.Require().NoError(err) + + patterns := s.unitPatterns() + var unaccounted []string + for _, dir := range dirs { + pkg := coverageDotPath(dir) + if coverageNonUnitDir.MatchString(dir) && !coverageAnyMatch(patterns, pkg) { + unaccounted = append(unaccounted, pkg) + } + } + if len(unaccounted) > 0 { + s.Failf( + "packages with tests not assigned to a tier", + "add these packages to a unit step in hack/ci/suites.yaml "+ + "(unit.steps packages, or unit.excluded with a reason):\n%s", + strings.Join(unaccounted, "\n"), + ) + } +} + +// Footgun: a unit step or excluded entry names a package that was moved or +// deleted, so the step quietly tests nothing. +func (s *coverageSuite) TestUnitEntriesExistOnDisk() { + var missing []string + for _, pkg := range s.literalUnitPackages() { + dir := filepath.Join(s.root, filepath.FromSlash(strings.TrimPrefix(pkg, "./"))) + if !coverageIsDir(dir) { + missing = append(missing, pkg) + } + } + if len(missing) > 0 { + s.Failf( + "unit entries point at missing packages", + "remove or fix these entries in hack/ci/suites.yaml:\n%s", + strings.Join(missing, "\n"), + ) + } +} + +func (s *coverageSuite) assignedLabels() map[string]bool { + assigned := map[string]bool{} + for _, label := range s.manifestLabels() { + assigned[label] = true + } + return assigned +} + +func (s *coverageSuite) manifestLabels() []string { + var labels []string + for _, entry := range s.manifest.E2E.Suites { + labels = append(labels, entry.Label) + } + for _, ungated := range s.manifest.E2E.Ungated { + labels = append(labels, ungated.Label) + } + return labels +} + +// unitPatterns returns every package pattern that a unit step or exclusion names. +func (s *coverageSuite) unitPatterns() []string { + var patterns []string + for _, step := range s.manifest.Unit.Steps { + patterns = append(patterns, step.Packages...) + } + for _, excluded := range s.manifest.Unit.Excluded { + patterns = append(patterns, excluded.Package) + } + return patterns +} + +// literalUnitPackages returns the step packages and exclusions without a +// `/...` suffix, which must name a real directory. +func (s *coverageSuite) literalUnitPackages() []string { + var pkgs []string + for _, pattern := range s.unitPatterns() { + if !strings.HasSuffix(pattern, "/...") { + pkgs = append(pkgs, pattern) + } + } + return pkgs +} + +// coverageScanLabels collects every label passed to a Label(...) call under +// dir. Literal strings count, and so do identifiers that name a string +// constant somewhere under dir. Dot-imported and qualified calls both match. +func coverageScanLabels(dir string) (map[string]bool, error) { + sources, err := coverageReadGoFiles(dir) + if err != nil { + return nil, err + } + consts := map[string]string{} + for _, data := range sources { + for _, m := range coverageStringConst.FindAllSubmatch(data, -1) { + consts[string(m[1])] = string(m[2]) + } + } + labels := map[string]bool{} + for _, data := range sources { + coverageCollectLabels(data, consts, labels) + } + return labels, nil +} + +func coverageReadGoFiles(dir string) ([][]byte, error) { + var sources [][]byte + err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || filepath.Ext(path) != ".go" { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + sources = append(sources, data) + return nil + }) + if err != nil { + return nil, fmt.Errorf("scan e2e sources: %w", err) + } + return sources, nil +} + +func coverageCollectLabels(data []byte, consts map[string]string, labels map[string]bool) { + for _, call := range coverageLabelCall.FindAllSubmatch(data, -1) { + args := call[1] + for _, lit := range coverageStringLit.FindAllSubmatch(args, -1) { + labels[string(lit[1])] = true + } + // Selectors (provider.label) are struct fields, not package constants. + bare := coverageSelector.ReplaceAll(coverageStringLit.ReplaceAll(args, nil), nil) + for _, ident := range coverageIdent.FindAll(bare, -1) { + if value, ok := consts[string(ident)]; ok { + labels[value] = true + } + } + } +} + +// coverageTestPackageDirs returns the root-relative directories of packages +// that have _test.go files, as reported by `go list`. +func coverageTestPackageDirs(root string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, "go", "list", "-f", coverageGoListFormat, "./...") + cmd.Dir = root + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("go list ./...: %w\n%s", err, stderr.String()) + } + return coverageParseGoList(string(out), root) +} + +// coverageParseGoList keeps the root-relative directories of packages that +// have test files. +func coverageParseGoList(output, root string) ([]string, error) { + var dirs []string + for line := range strings.SplitSeq(strings.TrimSpace(output), "\n") { + if line == "" { + continue + } + dir, hasTests, err := coverageParseGoListLine(line, root) + if err != nil { + return nil, err + } + if hasTests { + dirs = append(dirs, dir) + } + } + return dirs, nil +} + +// coverageParseGoListLine parses one `importpath|dir|tests|xtests` line. +func coverageParseGoListLine(line, root string) (string, bool, error) { + fields := strings.Split(line, "|") + if len(fields) != 4 { + return "", false, fmt.Errorf("unexpected go list line %q", line) + } + testFiles, errTest := strconv.Atoi(fields[2]) + xTestFiles, errX := strconv.Atoi(fields[3]) + if errTest != nil || errX != nil { + return "", false, fmt.Errorf("unexpected go list counts in %q", line) + } + rel, err := filepath.Rel(root, fields[1]) + if err != nil { + return "", false, fmt.Errorf("relative path of %s: %w", fields[1], err) + } + return filepath.ToSlash(rel), testFiles+xTestFiles > 0, nil +} + +// coverageDotPath converts a root-relative directory into the `./dir` form +// used by manifest package entries. +func coverageDotPath(rel string) string { + if rel == "." { + return "." + } + return "./" + rel +} + +func coverageAnyMatch(patterns []string, pkg string) bool { + return slices.ContainsFunc(patterns, func(pattern string) bool { + return coverageMatches(pattern, pkg) + }) +} + +// coverageMatches compares a manifest entry with a `./dir` package path. A +// `/...` suffix matches the directory and everything below it. +func coverageMatches(pattern, pkg string) bool { + if prefix, ok := strings.CutSuffix(pattern, "/..."); ok { + return pkg == prefix || strings.HasPrefix(pkg, prefix+"/") + } + return pattern == pkg +} + +func coverageIsDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +func sortedKeys(set map[string]bool) []string { + keys := make([]string, 0, len(set)) + for key := range set { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} diff --git a/hack/ci/e2e.go b/hack/ci/e2e.go new file mode 100644 index 000000000..0ec881697 --- /dev/null +++ b/hack/ci/e2e.go @@ -0,0 +1,427 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + + "github.com/devsy-org/devsy/e2e/framework/ginkgoreport" +) + +const ( + defaultE2EDir = "./e2e" + e2eBinDirName = "bin" + e2eTestBinary = "e2e.test" + e2eTestBinaryWindows = "e2e.test.exe" + devsyBinaryPrefix = "devsy-" + executableMode = fs.FileMode(0o755) + windowsDockerHost = "npipe:////./pipe/podman-machine-default" + windowsDockerWSLHost = "npipe:////./pipe/devsy-ci-no-daemon" + reportNameSuffix = "-report.json" + + postCheckDockerInstalled = "docker-installed" + dockerBinary = "docker" +) + +var ( + errNoDevsyBinaries = errors.New("no devsy-* binaries found") + errMicrosandboxSuite = errors.New("microsandbox suites are run by devsy-ci-microsandbox") + errUnknownPostCheck = errors.New("unknown post check") + + // postCheckCommands maps a suite post check to the commands it runs, in order. + postCheckCommands = map[string][][]string{ + postCheckDockerInstalled: {{dockerBinary, "--version"}, {dockerBinary, "ps"}}, + } +) + +// focusedChecksFunc matches runFocusedChecks. Tests substitute a fake. +type focusedChecksFunc func(context.Context, commandRunner, Suite, e2eOptions) error + +// e2eRunner executes one Ginkgo suite through the e2e binary. +type e2eRunner struct { + runner commandRunner + getenv func(string) string + readFile func(string) ([]byte, error) + focused focusedChecksFunc + opts e2eOptions +} + +// e2eCLI implements `ci e2e prepare` and `ci e2e run`. +func e2eCLI(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + _, _ = fmt.Fprintln(stderr, "usage: ci e2e [flags]") + return 2 + } + switch args[0] { + case "prepare": + return e2ePrepareCLI(args[1:], stdout, stderr) + case "run": + return e2eRunCLI(args[1:], stdout, stderr) + default: + _, _ = fmt.Fprintf(stderr, "unknown e2e subcommand: %s\n", args[0]) + return 2 + } +} + +func e2ePrepareCLI(args []string, stdout, stderr io.Writer) int { + fl := newE2EFlagSet("ci e2e prepare", stderr) + binSrc := fl.String("bin-src", "", "directory the devsy binaries were downloaded to") + e2eDir := fl.String("e2e-dir", defaultE2EDir, "directory containing e2e.test") + if code, ok := parseE2EFlags(fl, args); !ok { + return code + } + if *binSrc == "" { + _, _ = fmt.Fprintln(stderr, "ci e2e prepare: --bin-src is required") + return 2 + } + if err := prepareE2E(stdout, runtime.GOOS, *binSrc, *e2eDir); err != nil { + _, _ = fmt.Fprintf(stderr, "ci e2e prepare: %v\n", err) + return 1 + } + return 0 +} + +func e2eRunCLI(args []string, stdout, stderr io.Writer) int { + fl := newE2EFlagSet("ci e2e run", stderr) + label := fl.String("label", "", "Ginkgo label of the suite to run") + runnerName := fl.String("runner", "", "runner of the suite; needed when the label is ambiguous") + e2eDir := fl.String("e2e-dir", defaultE2EDir, "directory containing e2e.test and bin/") + if code, ok := parseE2EFlags(fl, args); !ok { + return code + } + if *label == "" { + _, _ = fmt.Fprintln(stderr, "ci e2e run: --label is required") + return 2 + } + suite, code := e2eLoadSuite(*label, *runnerName, stderr) + if code != 0 { + return code + } + r := e2eRunner{ + runner: execRunner{}, + getenv: os.Getenv, + readFile: os.ReadFile, + focused: runFocusedChecks, + opts: e2eOptions{ + E2EDir: *e2eDir, + TempDir: os.Getenv("RUNNER_TEMP"), + GOOS: runtime.GOOS, + Stdout: stdout, + Stderr: stderr, + }, + } + if err := r.run(context.Background(), suite); err != nil { + _, _ = fmt.Fprintf(stderr, "ci e2e run %s: %v\n", *label, err) + return 1 + } + return 0 +} + +// e2eLoadSuite resolves the suite from the manifest and refuses the groups owned by other commands. +func e2eLoadSuite(label, runnerName string, stderr io.Writer) (Suite, int) { + manifest, err := LoadManifest() + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci e2e run: %v\n", err) + return Suite{}, 1 + } + suite, err := manifest.FindSuite(label, runnerName) + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci e2e run: %v\n", err) + return Suite{}, 2 + } + if suite.Group == groupMicrosandbox { + _, _ = fmt.Fprintf(stderr, "ci e2e run: %v (label %q)\n", errMicrosandboxSuite, label) + return Suite{}, 2 + } + return suite, 0 +} + +func newE2EFlagSet(name string, output io.Writer) *flag.FlagSet { + fl := flag.NewFlagSet(name, flag.ContinueOnError) + fl.SetOutput(output) + return fl +} + +// parseE2EFlags parses the flags of one subcommand. It returns false together with the exit code +// when the subcommand must stop. +func parseE2EFlags(fl *flag.FlagSet, args []string) (int, bool) { + if err := fl.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0, false + } + return 2, false + } + if fl.NArg() > 0 { + _, _ = fmt.Fprintf(fl.Output(), "unexpected arguments: %v\n", fl.Args()) + return 2, false + } + return 0, true +} + +// prepareE2E copies the devsy-* regular files found under binSrc flat into e2eDir/bin and makes +// them executable. It also marks e2e.test executable when it exists and the platform has exec bits. +func prepareE2E(stdout io.Writer, goos, binSrc, e2eDir string) error { + found, err := e2eFindBinaries(stdout, binSrc) + if err != nil { + return fmt.Errorf("list %s: %w", binSrc, err) + } + if len(found) == 0 { + return fmt.Errorf("%w under %s", errNoDevsyBinaries, binSrc) + } + binDir := filepath.Join(e2eDir, e2eBinDirName) + if err := os.MkdirAll(binDir, executableMode); err != nil { + return fmt.Errorf("create %s: %w", binDir, err) + } + for _, src := range found { + dst := filepath.Join(binDir, filepath.Base(src)) + if err := e2eCopyExecutable(src, dst); err != nil { + return err + } + _, _ = fmt.Fprintf(stdout, "copied %s\n", dst) + } + return e2eMarkTestBinary(goos, e2eDir) +} + +// e2eFindBinaries prints every entry under root, like `ls -R`, and returns the devsy-* regular files. +func e2eFindBinaries(stdout io.Writer, root string) ([]string, error) { + var found []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("relative path of %s: %w", path, err) + } + _, _ = fmt.Fprintln(stdout, rel) + if d.Type().IsRegular() && strings.HasPrefix(d.Name(), devsyBinaryPrefix) { + found = append(found, path) + } + return nil + }) + return found, err +} + +func e2eCopyExecutable(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("open %s: %w", src, err) + } + defer func() { _ = in.Close() }() + + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, executableMode) + if err != nil { + return fmt.Errorf("create %s: %w", dst, err) + } + _, copyErr := io.Copy(out, in) + if err := errors.Join(copyErr, out.Close()); err != nil { + return fmt.Errorf("copy %s: %w", src, err) + } + if err := os.Chmod(dst, executableMode); err != nil { + return fmt.Errorf("chmod %s: %w", dst, err) + } + return nil +} + +// e2eMarkTestBinary adds the execute bits to e2e.test when it exists. Windows has no execute bits. +func e2eMarkTestBinary(goos, e2eDir string) error { + if goos == osWindows { + return nil + } + path := filepath.Join(e2eDir, e2eTestBinary) + info, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("stat %s: %w", path, err) + } + if err := os.Chmod(path, info.Mode().Perm()|0o111); err != nil { + return fmt.Errorf("chmod %s: %w", path, err) + } + return nil +} + +// run executes one suite: focused checks, optional rootless diagnostics, the e2e binary, the +// report check and then the suite's post checks. +func (r e2eRunner) run(ctx context.Context, suite Suite) error { + r.printBanner(suite) + if err := r.focused(ctx, r.runner, suite, r.opts); err != nil { + return fmt.Errorf("focused checks: %w", err) + } + if suite.InstallPodman == podmanRootless { + if err := r.printRootlessDiagnostics(ctx); err != nil { + return fmt.Errorf("rootless diagnostics: %w", err) + } + } + if err := r.runner.Run(ctx, r.binarySpec(suite)); err != nil { + return fmt.Errorf("e2e run: %w", err) + } + if err := r.checkReport(suite); err != nil { + return err + } + for _, check := range suite.PostChecks { + if err := r.postCheck(ctx, check); err != nil { + return fmt.Errorf("post check %s: %w", check, err) + } + } + return nil +} + +// checkReport verifies the Ginkgo JSON report of a suite that pins its spec count. It runs after +// the binary succeeded and before the post checks. +func (r e2eRunner) checkReport(suite Suite) error { + if !reportEnabled(suite) { + return nil + } + name := reportFileName(suite.Label) + data, err := r.readFile(filepath.Join(r.opts.E2EDir, name)) + if err != nil { + return fmt.Errorf("read report %s: %w", name, err) + } + if err := ginkgoreport.Check(data, suite.Label, suite.ExpectedSpecCount); err != nil { + return fmt.Errorf("verify report %s: %w", name, err) + } + _, _ = fmt.Fprintf(r.opts.Stdout, "report check passed: %d specs\n", suite.ExpectedSpecCount) + return nil +} + +// reportEnabled reports whether the suite's selected specs are verified from a Ginkgo JSON report. +func reportEnabled(suite Suite) bool { + return suite.ExpectedSpecCount > 0 && suite.Group != groupMicrosandbox +} + +// reportFileName is the JSON report name, relative to the e2e directory the binary runs in. +func reportFileName(label string) string { + return label + reportNameSuffix +} + +func (r e2eRunner) printBanner(suite Suite) { + _, _ = fmt.Fprintf( + r.opts.Stdout, + "running suite %s on %s (privileged=%t)\n", + suite.Label, + suite.Runner, + suite.Resolved, + ) +} + +func (r e2eRunner) printRootlessDiagnostics(ctx context.Context) error { + _, _ = fmt.Fprintln(r.opts.Stdout, "Podman rootless test identity:") + if err := r.runner.Run(ctx, r.hostCommand("id")); err != nil { + return err + } + _, _ = fmt.Fprintf(r.opts.Stdout, "HOME=%s\n", r.getenv("HOME")) + _, _ = fmt.Fprintf(r.opts.Stdout, "XDG_RUNTIME_DIR=%s\n", r.getenv("XDG_RUNTIME_DIR")) + return r.runner.Run(ctx, r.hostCommand("podman", "info")) +} + +// binarySpec returns the command that runs the e2e binary for the suite. On Linux a privileged +// suite runs through sudo, with the environment passed as K=V arguments. +func (r e2eRunner) binarySpec(suite Suite) commandSpec { + name := "./" + e2eTestBinary + if r.opts.GOOS == osWindows { + name = "./" + e2eTestBinaryWindows + } + args := e2eTestArgs(suite) + env := r.e2eEnv(suite) + spec := commandSpec{ + Name: name, + Args: args, + Dir: r.opts.E2EDir, + Stdout: r.opts.Stdout, + Stderr: r.opts.Stderr, + } + if r.opts.GOOS != osLinux || !suite.Resolved { + spec.Env = env + return spec + } + spec.Name = "sudo" + spec.Args = slices.Concat(env, []string{name}, args) + return spec +} + +// e2eEnv returns the variables the e2e binary needs, as K=V pairs in a fixed order. +func (r e2eRunner) e2eEnv(suite Suite) []string { + env := []string{ + "GH_USERNAME=" + r.getenv("GH_USERNAME"), + "GH_ACCESS_TOKEN=" + r.getenv("GH_ACCESS_TOKEN"), + "GH_CREDENTIAL_USERNAME=" + r.getenv("GH_CREDENTIAL_USERNAME"), + "KUBECONFIG=" + r.kubeconfig(), + "PATH=" + r.getenv("PATH"), + } + return append(env, r.dockerEnv(suite)...) +} + +// dockerEnv returns the Docker client variables of the suite. A docker-wsl suite talks to the +// WSL2 fixture and clears DOCKER_CONTEXT so that the explicit host is used. +func (r e2eRunner) dockerEnv(suite Suite) []string { + if suite.Runtime == runtimeDockerWSL { + return []string{"DOCKER_HOST=" + windowsDockerWSLHost, "DOCKER_CONTEXT="} + } + if dockerHost := r.dockerHost(); dockerHost != "" { + return []string{"DOCKER_HOST=" + dockerHost} + } + return nil +} + +func (r e2eRunner) kubeconfig() string { + if kubeconfig := r.getenv("KUBECONFIG"); kubeconfig != "" { + return kubeconfig + } + home := r.getenv("HOME") + if home == "" && r.opts.GOOS == osWindows { + home = r.getenv("USERPROFILE") + } + return filepath.Join(home, ".kube", "config") +} + +func (r e2eRunner) dockerHost() string { + if r.opts.GOOS == osWindows { + return windowsDockerHost + } + return r.getenv("DOCKER_HOST") +} + +// e2eTestArgs returns the Go test and Ginkgo flags for one suite. A suite with a pinned spec count +// also writes its JSON report next to the binary. +func e2eTestArgs(suite Suite) []string { + args := []string{ + "-test.v", + "-ginkgo.v", + "-test.timeout", suite.TestTimeout, + "-ginkgo.fail-on-empty", + fmt.Sprintf("-ginkgo.flake-attempts=%d", suite.FlakeAttempts), + "--ginkgo.label-filter=" + suite.Label, + } + if reportEnabled(suite) { + args = append(args, "-ginkgo.json-report="+reportFileName(suite.Label)) + } + return args +} + +func (r e2eRunner) postCheck(ctx context.Context, name string) error { + commands, ok := postCheckCommands[name] + if !ok { + return fmt.Errorf("%w %q", errUnknownPostCheck, name) + } + for _, argv := range commands { + if err := r.runner.Run(ctx, r.hostCommand(argv[0], argv[1:]...)); err != nil { + return err + } + } + return nil +} + +// hostCommand returns a spec that runs in the current directory and writes to the run's output. +func (r e2eRunner) hostCommand(name string, args ...string) commandSpec { + return commandSpec{Name: name, Args: args, Stdout: r.opts.Stdout, Stderr: r.opts.Stderr} +} diff --git a/hack/ci/e2e_check.go b/hack/ci/e2e_check.go new file mode 100644 index 000000000..21d58e23f --- /dev/null +++ b/hack/ci/e2e_check.go @@ -0,0 +1,356 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/user" + "path/filepath" + "strconv" + "strings" +) + +// e2eOptions is the environment shared by `ci e2e run` and the focused checks. +type e2eOptions struct { + E2EDir string // directory containing e2e.test and bin/ + TempDir string // RUNNER_TEMP; reports are written here + GOOS string + Stdout io.Writer + Stderr io.Writer +} + +var ( + errFocusedUID = errors.New("focused check uid mismatch") + errFocusedGID = errors.New("focused check gid mismatch") + errFocusedAccount = errors.New("current user or group does not resolve") + errFocusedDir = errors.New("directory is missing or not writable") + errFocusedOwner = errors.New("directory is not owned by the current user") + + errReportRead = errors.New("read ginkgo json report") + errReportShape = errors.New("unexpected suite report shape") + errReportDryRun = errors.New("dry-run is not execution") + errReportSuite = errors.New("suite failed") + errReportLeaves = errors.New("expected one executed leaf") + errReportLeafText = errors.New("wrong focused spec") + errReportState = errors.New("focused spec did not pass") + errReportRetried = errors.New("unexpected retries") +) + +// focusedChecksumFiles are the binaries whose digests are printed before each focused run. +var focusedChecksumFiles = []string{"e2e.test", "bin/devsy-linux-amd64"} + +// focusedHost is the process identity and account database the preflight inspects. +type focusedHost struct { + uid int + gid int + home string + lookupUser func(id string) (*user.User, error) + lookupGroup func(id string) (*user.Group, error) +} + +// focusedHostProvider returns the host focused checks run under. Tests replace it. +var focusedHostProvider = currentFocusedHost + +func currentFocusedHost() focusedHost { + return focusedHost{ + uid: os.Getuid(), + gid: os.Getgid(), + home: os.Getenv("HOME"), + lookupUser: user.LookupId, + lookupGroup: user.LookupGroupId, + } +} + +// focusedRun carries what every focused check of one suite needs. +type focusedRun struct { + runner commandRunner + suite Suite + opts e2eOptions +} + +// runFocusedChecks runs every focused check of the suite, in order. +func runFocusedChecks( + ctx context.Context, + runner commandRunner, + suite Suite, + opts e2eOptions, +) error { + run := focusedRun{runner: runner, suite: suite, opts: opts} + for _, check := range suite.FocusedChecks { + if err := run.runOne(ctx, check); err != nil { + return fmt.Errorf("focused check %s: %w", check.Name, err) + } + _, _ = fmt.Fprintf(opts.Stdout, "focused check %s passed in one attempt\n", check.Name) + } + return nil +} + +func (r focusedRun) runOne(ctx context.Context, check FocusedCheck) error { + if r.opts.GOOS == osLinux { + if err := focusedHostProvider().preflight(check, r.opts); err != nil { + return fmt.Errorf("preflight: %w", err) + } + } + if err := r.diagnostics(ctx); err != nil { + return err + } + reportPath, err := filepath.Abs(filepath.Join(r.opts.TempDir, check.Name+".json")) + if err != nil { + return fmt.Errorf("resolve report path: %w", err) + } + if err := r.runner.Run(ctx, r.e2eCommand(check, reportPath)); err != nil { + return fmt.Errorf("run e2e binary: %w", err) + } + return verifyFocusedReport(reportPath, check) +} + +// diagnostics logs the Docker, checkout and binary identity the CI log needs. +func (r focusedRun) diagnostics(ctx context.Context) error { + if err := r.exec(ctx, "docker", "info"); err != nil { + return err + } + if err := r.exec(ctx, "docker", "compose", "version"); err != nil { + return err + } + sha, err := r.checkout(ctx) + if err != nil { + return err + } + _, _ = fmt.Fprintf(r.opts.Stdout, "checkout=%s\n", sha) + return printChecksums(r.opts) +} + +// exec runs a command in the e2e directory, streaming its output. +func (r focusedRun) exec(ctx context.Context, name string, args ...string) error { + spec := commandSpec{ + Name: name, + Args: args, + Dir: r.opts.E2EDir, + Stdout: r.opts.Stdout, + Stderr: r.opts.Stderr, + } + if err := r.runner.Run(ctx, spec); err != nil { + return fmt.Errorf("%s: %w", strings.Join(append([]string{name}, args...), " "), err) + } + return nil +} + +// checkout returns the commit the e2e directory is checked out at. +func (r focusedRun) checkout(ctx context.Context) (string, error) { + var out bytes.Buffer + spec := commandSpec{ + Name: "git", + Args: []string{"rev-parse", "HEAD"}, + Dir: r.opts.E2EDir, + Stdout: &out, + Stderr: r.opts.Stderr, + } + if err := r.runner.Run(ctx, spec); err != nil { + return "", fmt.Errorf("git rev-parse HEAD: %w", err) + } + return strings.TrimSpace(out.String()), nil +} + +// e2eCommand is the direct, unprivileged run of the e2e binary for one check. +func (r focusedRun) e2eCommand(check FocusedCheck, reportPath string) commandSpec { + return commandSpec{ + Name: "./e2e.test", + Args: []string{ + "-test.v", + "-ginkgo.v", + "-test.timeout", r.suite.TestTimeout, + "-ginkgo.fail-on-empty", + "-ginkgo.flake-attempts=1", + "--ginkgo.label-filter=" + r.suite.Label, + "--ginkgo.focus=" + check.Focus, + "--ginkgo.json-report=" + reportPath, + }, + Dir: r.opts.E2EDir, + Stdout: r.opts.Stdout, + Stderr: r.opts.Stderr, + } +} + +// preflight verifies the identity and directories a focused check depends on. +func (h focusedHost) preflight(check FocusedCheck, opts e2eOptions) error { + _, _ = fmt.Fprintf(opts.Stdout, "uid=%d gid=%d\n", h.uid, h.gid) + if err := h.checkRequiredIDs(check); err != nil { + return err + } + if err := h.checkAccount(); err != nil { + return err + } + if err := h.checkHome(); err != nil { + return err + } + if opts.TempDir == "" { + return nil + } + return requireWritableDir(opts.TempDir) +} + +func (h focusedHost) checkRequiredIDs(check FocusedCheck) error { + if check.RequireUID != 0 && h.uid != check.RequireUID { + return fmt.Errorf("%w: uid %d, want %d", errFocusedUID, h.uid, check.RequireUID) + } + if check.RequireGID != 0 && h.gid != check.RequireGID { + return fmt.Errorf("%w: gid %d, want %d", errFocusedGID, h.gid, check.RequireGID) + } + return nil +} + +func (h focusedHost) checkAccount() error { + if _, err := h.lookupUser(strconv.Itoa(h.uid)); err != nil { + return fmt.Errorf("%w: uid %d: %w", errFocusedAccount, h.uid, err) + } + if _, err := h.lookupGroup(strconv.Itoa(h.gid)); err != nil { + return fmt.Errorf("%w: gid %d: %w", errFocusedAccount, h.gid, err) + } + return nil +} + +func (h focusedHost) checkHome() error { + if err := requireWritableDir(h.home); err != nil { + return err + } + return requireOwnedBy(h.home, h.uid) +} + +// requireWritableDir fails unless path is an existing directory the process can write to. +func requireWritableDir(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("%w: %s: %w", errFocusedDir, path, err) + } + if !info.IsDir() { + return fmt.Errorf("%w: %s is not a directory", errFocusedDir, path) + } + return probeWritable(path) +} + +// probeWritable creates and removes a temporary file in dir. +func probeWritable(dir string) error { + f, err := os.CreateTemp(dir, ".focused-check-*") + if err != nil { + return fmt.Errorf("%w: %s: %w", errFocusedDir, dir, err) + } + closeErr := f.Close() + removeErr := os.Remove(f.Name()) + if joined := errors.Join(closeErr, removeErr); joined != nil { + return fmt.Errorf("%w: %s: %w", errFocusedDir, dir, joined) + } + return nil +} + +// printChecksums prints ` ./` for each binary that exists. +func printChecksums(opts e2eOptions) error { + for _, rel := range focusedChecksumFiles { + sum, found, err := fileSHA256(filepath.Join(opts.E2EDir, filepath.FromSlash(rel))) + if err != nil { + return err + } + if found { + _, _ = fmt.Fprintf(opts.Stdout, "%s ./%s\n", sum, rel) + } + } + return nil +} + +// fileSHA256 hashes a file; found is false when the file does not exist. +func fileSHA256(path string) (string, bool, error) { + f, err := os.Open(path) + if errors.Is(err, fs.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("open %s: %w", path, err) + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", false, fmt.Errorf("hash %s: %w", path, err) + } + return hex.EncodeToString(h.Sum(nil)), true, nil +} + +// ginkgoJSONSuite is the subset of a Ginkgo JSON suite report the check reads. +type ginkgoJSONSuite struct { + SuiteConfig *ginkgoJSONConfig `json:"SuiteConfig"` + SuiteSucceeded *bool `json:"SuiteSucceeded"` + SpecReports []ginkgoJSONSpec `json:"SpecReports"` +} + +type ginkgoJSONConfig struct { + DryRun *bool `json:"DryRun"` +} + +type ginkgoJSONSpec struct { + LeafNodeType string `json:"LeafNodeType"` + LeafNodeText string `json:"LeafNodeText"` + State string `json:"State"` + NumAttempts int `json:"NumAttempts"` +} + +// verifyFocusedReport validates a Ginkgo JSON report against a check. +func verifyFocusedReport(path string, check FocusedCheck) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("%w: %w", errReportRead, err) + } + var suites []ginkgoJSONSuite + if err := json.Unmarshal(data, &suites); err != nil { + return fmt.Errorf("%w: %w", errReportShape, err) + } + if len(suites) != 1 { + return fmt.Errorf("%w: found %d suite reports", errReportShape, len(suites)) + } + return checkSuiteReport(suites[0], check) +} + +func checkSuiteReport(s ginkgoJSONSuite, check FocusedCheck) error { + if s.SuiteConfig == nil || s.SuiteConfig.DryRun == nil || s.SuiteSucceeded == nil { + return fmt.Errorf("%w: missing SuiteConfig.DryRun or SuiteSucceeded", errReportShape) + } + if *s.SuiteConfig.DryRun { + return errReportDryRun + } + if !*s.SuiteSucceeded { + return errReportSuite + } + return checkFocusedSpec(executedLeaves(s.SpecReports), check) +} + +// executedLeaves returns the It nodes that were not skipped. +func executedLeaves(specs []ginkgoJSONSpec) []ginkgoJSONSpec { + var leaves []ginkgoJSONSpec + for _, spec := range specs { + if spec.LeafNodeType == "It" && spec.State != "skipped" { + leaves = append(leaves, spec) + } + } + return leaves +} + +func checkFocusedSpec(leaves []ginkgoJSONSpec, check FocusedCheck) error { + if len(leaves) != 1 { + return fmt.Errorf("%w: found %d", errReportLeaves, len(leaves)) + } + spec := leaves[0] + if spec.LeafNodeText != check.ExpectLeaf { + return fmt.Errorf("%w: %q, want %q", errReportLeafText, spec.LeafNodeText, check.ExpectLeaf) + } + if spec.State != "passed" { + return fmt.Errorf("%w: state %q", errReportState, spec.State) + } + if spec.NumAttempts != 1 { + return fmt.Errorf("%w: %d attempts", errReportRetried, spec.NumAttempts) + } + return nil +} diff --git a/hack/ci/e2e_check_test.go b/hack/ci/e2e_check_test.go new file mode 100644 index 000000000..a5b7caa95 --- /dev/null +++ b/hack/ci/e2e_check_test.go @@ -0,0 +1,554 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/user" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +const ( + focusedTestName = "compose-uid-nonroot" + focusedTestLabel = "up-docker-compose" + focusedTestLeaf = "implements updateRemoteUserUID with root container user" + focusedTestFocus = focusedTestLeaf + "$" + focusedTestTimeout = "1500s" + focusedTestBinary = "test binary" + focusedTestAgent = "agent" + + focusedE2EBinary = "./e2e.test" + focusedGitBinary = "git" + focusedCompose = "compose" + focusedVersion = "version" + focusedInfo = "info" + focusedRevParse = "rev-parse" + focusedHEAD = "HEAD" + focusedTestV = "-test.v" + focusedGinkgoV = "-ginkgo.v" + focusedTestTO = "-test.timeout" + focusedFailOnEmpt = "-ginkgo.fail-on-empty" + focusedFlakeOne = "-ginkgo.flake-attempts=1" + focusedLabelArg = "--ginkgo.label-filter=" + focusedFocusArg = "--ginkgo.focus=" + focusedReportArgP = "--ginkgo.json-report=" + focusedMalformed = "malformed JSON" + + focusedLeafPassed = `{"LeafNodeType":"It","LeafNodeText":"` + focusedTestLeaf + `","State":"passed","NumAttempts":1}` + focusedLeafSkipped = `{"LeafNodeType":"It","LeafNodeText":"` + focusedTestLeaf + `","State":"skipped","NumAttempts":0}` + focusedBeforeSuite = `{"LeafNodeType":"BeforeSuite","LeafNodeText":"","State":"failed","NumAttempts":0}` +) + +// focusedCall is one command seen by focusedRecorder. +type focusedCall struct { + name string + args []string + dir string +} + +// focusedRecorder is a commandRunner that records calls. The e2e binary writes +// report to the --ginkgo.json-report path it is given, then returns runErr. +type focusedRecorder struct { + calls []focusedCall + report string + runErr error +} + +func newFocusedRecorder() *focusedRecorder { + return &focusedRecorder{report: focusedReport(false, true, focusedLeafPassed)} +} + +func (f *focusedRecorder) Run(_ context.Context, spec commandSpec) error { + f.calls = append(f.calls, focusedCall{name: spec.Name, args: spec.Args, dir: spec.Dir}) + switch spec.Name { + case focusedGitBinary: + _, _ = io.WriteString(spec.Stdout, "abc123\n") + case focusedE2EBinary: + return f.runE2E(spec.Args) + } + return nil +} + +func (f *focusedRecorder) runE2E(args []string) error { + path, ok := focusedReportArg(args) + if !ok { + return errors.New("missing --ginkgo.json-report argument") + } + if err := os.WriteFile(path, []byte(f.report), 0o600); err != nil { + return fmt.Errorf("write fake report: %w", err) + } + return f.runErr +} + +func focusedReportArg(args []string) (string, bool) { + for _, arg := range args { + if path, ok := strings.CutPrefix(arg, focusedReportArgP); ok { + return path, true + } + } + return "", false +} + +// focusedReport builds a one-suite Ginkgo JSON report from raw spec objects. +func focusedReport(dryRun, succeeded bool, specs ...string) string { + return "[" + focusedSuiteObject(dryRun, succeeded, specs...) + "]" +} + +// focusedSuiteObject builds one suite object of a Ginkgo JSON report. +func focusedSuiteObject(dryRun, succeeded bool, specs ...string) string { + return fmt.Sprintf(`{"SuiteConfig":{"DryRun":%t},"SuiteSucceeded":%t,"SpecReports":[%s]}`, + dryRun, succeeded, strings.Join(specs, ",")) +} + +func focusedTestCheck() FocusedCheck { + return FocusedCheck{Name: focusedTestName, Focus: focusedTestFocus, ExpectLeaf: focusedTestLeaf} +} + +func focusedTestSuite(check FocusedCheck) Suite { + return Suite{ + Label: focusedTestLabel, + TestTimeout: focusedTestTimeout, + FocusedChecks: []FocusedCheck{check}, + } +} + +func focusedSum(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} + +func focusedLookupUserFails(_ string) (*user.User, error) { + return nil, errors.New("no such user") +} + +func focusedLookupGroupFails(_ string) (*user.Group, error) { + return nil, errors.New("no such group") +} + +type focusedCheckSuite struct { + suite.Suite + check FocusedCheck +} + +type focusedPreflightCase struct { + name string + mutate func(h *focusedHost, c *FocusedCheck, o *e2eOptions) + wantErr error +} + +type focusedReportCase struct { + name string + report string + wantErr error +} + +func TestFocusedCheckSuite(t *testing.T) { + suite.Run(t, new(focusedCheckSuite)) +} + +func (s *focusedCheckSuite) SetupSuite() { + s.check = focusedTestCheck() +} + +func (s *focusedCheckSuite) TestRunsDiagnosticsThenE2EWithoutSudo() { + opts, stdout := s.options("linux") + host := s.passingHost() + s.installHost(host) + recorder := newFocusedRecorder() + + s.Require(). + NoError(runFocusedChecks(context.Background(), recorder, focusedTestSuite(s.checkFor(host)), opts)) + + reportPath := filepath.Join(opts.TempDir, focusedTestName+".json") + s.Equal([]focusedCall{ + {name: dockerBinary, args: []string{focusedInfo}, dir: opts.E2EDir}, + {name: dockerBinary, args: []string{focusedCompose, focusedVersion}, dir: opts.E2EDir}, + {name: focusedGitBinary, args: []string{focusedRevParse, focusedHEAD}, dir: opts.E2EDir}, + { + name: focusedE2EBinary, + args: []string{ + focusedTestV, + focusedGinkgoV, + focusedTestTO, focusedTestTimeout, + focusedFailOnEmpt, + focusedFlakeOne, + focusedLabelArg + focusedTestLabel, + focusedFocusArg + focusedTestFocus, + focusedReportArgP + reportPath, + }, + dir: opts.E2EDir, + }, + }, recorder.calls) + for _, call := range recorder.calls { + s.NotEqual("sudo", call.name) + } + + out := stdout.String() + s.Contains(out, fmt.Sprintf("uid=%d gid=%d\n", host.uid, host.gid)) + s.Contains(out, "checkout=abc123\n") + s.Contains(out, focusedSum(focusedTestBinary)+" ./e2e.test\n") + s.Contains(out, focusedSum(focusedTestAgent)+" ./bin/devsy-linux-amd64\n") + s.Contains(out, "focused check "+focusedTestName+" passed in one attempt\n") +} + +func (s *focusedCheckSuite) TestWrongUIDAbortsBeforeAnyCommand() { + opts, _ := s.options("linux") + host := s.passingHost() + host.uid = 1000 + check := s.check + check.RequireUID = 1001 + s.installHost(host) + recorder := newFocusedRecorder() + + err := runFocusedChecks(context.Background(), recorder, focusedTestSuite(check), opts) + + s.ErrorIs(err, errFocusedUID) + s.Empty(recorder.calls) +} + +func (s *focusedCheckSuite) TestReportFailurePropagates() { + opts, _ := s.options("linux") + host := s.passingHost() + s.installHost(host) + recorder := newFocusedRecorder() + recorder.report = focusedReport( + false, + true, + `{"LeafNodeType":"It","LeafNodeText":"`+focusedTestLeaf+`","State":"passed","NumAttempts":2}`, + ) + + err := runFocusedChecks( + context.Background(), + recorder, + focusedTestSuite(s.checkFor(host)), + opts, + ) + + s.ErrorIs(err, errReportRetried) + s.Len(recorder.calls, 4) +} + +func (s *focusedCheckSuite) TestE2ERunFailurePropagates() { + opts, _ := s.options("linux") + host := s.passingHost() + s.installHost(host) + recorder := newFocusedRecorder() + recorder.runErr = errors.New("e2e exploded") + + err := runFocusedChecks( + context.Background(), + recorder, + focusedTestSuite(s.checkFor(host)), + opts, + ) + + s.ErrorContains(err, "e2e exploded") + s.Len(recorder.calls, 4) +} + +func (s *focusedCheckSuite) TestNonLinuxSkipsPreflight() { + opts, _ := s.options("darwin") + host := s.passingHost() + host.uid = 1000 + host.lookupUser = focusedLookupUserFails + host.lookupGroup = focusedLookupGroupFails + s.installHost(host) + check := s.check + check.RequireUID = 1001 + recorder := newFocusedRecorder() + + s.Require(). + NoError(runFocusedChecks(context.Background(), recorder, focusedTestSuite(check), opts)) + s.Len(recorder.calls, 4) +} + +func (s *focusedCheckSuite) TestPreflight() { + for _, tc := range s.preflightCases() { + s.Run(tc.name, func() { + host := s.passingHost() + check := s.checkFor(host) + opts, stdout := s.options("linux") + if tc.mutate != nil { + tc.mutate(&host, &check, &opts) + } + + err := host.preflight(check, opts) + + if tc.wantErr == nil { + s.Require().NoError(err) + s.Contains(stdout.String(), fmt.Sprintf("uid=%d gid=%d\n", host.uid, host.gid)) + return + } + s.ErrorIs(err, tc.wantErr) + }) + } +} + +func (s *focusedCheckSuite) TestRequireWritableDirLeavesNoFiles() { + dir := s.T().TempDir() + + s.Require().NoError(requireWritableDir(dir)) + + entries, err := os.ReadDir(dir) + s.Require().NoError(err) + s.Empty(entries) +} + +func (s *focusedCheckSuite) TestRequireOwnedBy() { + dir := s.T().TempDir() + s.Require().NoError(requireOwnedBy(dir, os.Getuid())) + + if runtime.GOOS == osWindows { + s.Require().NoError(requireOwnedBy(dir, 1), "ownership is not checked on Windows") + return + } + s.ErrorIs(requireOwnedBy(dir, os.Getuid()+1), errFocusedOwner) +} + +func (s *focusedCheckSuite) TestPrintChecksumsSkipsMissingFiles() { + opts, stdout := s.options("linux") + s.Require().NoError(os.Remove(filepath.Join(opts.E2EDir, "bin", "devsy-linux-amd64"))) + + s.Require().NoError(printChecksums(opts)) + + s.Equal(focusedSum(focusedTestBinary)+" ./e2e.test\n", stdout.String()) +} + +func (s *focusedCheckSuite) TestVerifyFocusedReport() { + for _, tc := range s.reportCases() { + s.Run(tc.name, func() { + path := s.writeFile("report.json") + s.Require().NoError(os.WriteFile(path, []byte(tc.report), 0o600)) + + err := verifyFocusedReport(path, s.check) + + if tc.wantErr == nil { + s.Require().NoError(err) + return + } + s.ErrorIs(err, tc.wantErr) + }) + } +} + +func (s *focusedCheckSuite) TestVerifyFocusedReportMissingFile() { + err := verifyFocusedReport(filepath.Join(s.T().TempDir(), "absent.json"), s.check) + + s.ErrorIs(err, errReportRead) +} + +// preflightCases lists the host and directory conditions the preflight must reject or accept. +func (s *focusedCheckSuite) preflightCases() []focusedPreflightCase { + return []focusedPreflightCase{ + {name: "passes for the current identity"}, + { + name: "uid differs from the required uid", + mutate: func(h *focusedHost, c *FocusedCheck, _ *e2eOptions) { + h.uid = 1000 + c.RequireUID = 1001 + }, + wantErr: errFocusedUID, + }, + { + name: "gid differs from the required gid", + mutate: func(h *focusedHost, c *FocusedCheck, _ *e2eOptions) { + h.gid = 1000 + c.RequireGID = 1001 + }, + wantErr: errFocusedGID, + }, + { + name: "user does not resolve", + mutate: func(h *focusedHost, _ *FocusedCheck, _ *e2eOptions) { + h.lookupUser = focusedLookupUserFails + }, + wantErr: errFocusedAccount, + }, + { + name: "group does not resolve", + mutate: func(h *focusedHost, _ *FocusedCheck, _ *e2eOptions) { + h.lookupGroup = focusedLookupGroupFails + }, + wantErr: errFocusedAccount, + }, + { + name: "home is missing", + mutate: func(h *focusedHost, _ *FocusedCheck, _ *e2eOptions) { + h.home = filepath.Join(h.home, "missing") + }, + wantErr: errFocusedDir, + }, + { + name: "home is a file", + mutate: func(h *focusedHost, _ *FocusedCheck, _ *e2eOptions) { + h.home = s.writeFile("home-file") + }, + wantErr: errFocusedDir, + }, + { + name: "temp dir is missing", + mutate: func(_ *focusedHost, _ *FocusedCheck, o *e2eOptions) { + o.TempDir = filepath.Join(o.TempDir, "missing") + }, + wantErr: errFocusedDir, + }, + } +} + +// reportCases lists the Ginkgo JSON reports the verifier must accept or reject. +func (s *focusedCheckSuite) reportCases() []focusedReportCase { + return append(s.reportShapeCases(), s.reportLeafCases()...) +} + +// reportShapeCases lists the reports that are malformed or describe the wrong suite. +func (s *focusedCheckSuite) reportShapeCases() []focusedReportCase { + return []focusedReportCase{ + { + name: "passes with non-It and skipped nodes ignored", + report: focusedReport( + false, + true, + focusedBeforeSuite, + focusedLeafSkipped, + focusedLeafPassed, + ), + }, + {name: "not an array", report: `{}`, wantErr: errReportShape}, + {name: focusedMalformed, report: `[`, wantErr: errReportShape}, + {name: "empty array", report: `[]`, wantErr: errReportShape}, + { + name: "two suite reports", + report: "[" + focusedSuiteObject(false, true, focusedLeafPassed) + "," + + focusedSuiteObject(false, true, focusedLeafPassed) + "]", + wantErr: errReportShape, + }, + { + name: "missing SuiteConfig", + report: `[{"SuiteSucceeded":true,"SpecReports":[` + focusedLeafPassed + `]}]`, + wantErr: errReportShape, + }, + } +} + +// reportLeafCases lists the reports whose run status or selected spec is wrong. +func (s *focusedCheckSuite) reportLeafCases() []focusedReportCase { + return []focusedReportCase{ + { + name: "dry run", + report: focusedReport(true, true, focusedLeafPassed), + wantErr: errReportDryRun, + }, + { + name: "suite failed", + report: focusedReport(false, false, focusedLeafPassed), + wantErr: errReportSuite, + }, + { + name: "zero executed leaves", + report: focusedReport(false, true), + wantErr: errReportLeaves, + }, + { + name: "only skipped leaf", + report: focusedReport(false, true, focusedLeafSkipped), + wantErr: errReportLeaves, + }, + { + name: "two executed leaves", + report: focusedReport(false, true, focusedLeafPassed, focusedLeafPassed), + wantErr: errReportLeaves, + }, + { + name: "wrong spec text", + report: focusedReport( + false, + true, + `{"LeafNodeType":"It","LeafNodeText":"other spec","State":"passed","NumAttempts":1}`, + ), + wantErr: errReportLeafText, + }, + { + name: "spec did not pass", + report: focusedReport( + false, + true, + `{"LeafNodeType":"It","LeafNodeText":"`+focusedTestLeaf+`","State":"failed","NumAttempts":1}`, + ), + wantErr: errReportState, + }, + { + name: "spec retried", + report: focusedReport( + false, + true, + `{"LeafNodeType":"It","LeafNodeText":"`+focusedTestLeaf+`","State":"passed","NumAttempts":2}`, + ), + wantErr: errReportRetried, + }, + } +} + +// options returns e2e options backed by a temp directory holding e2e.test and bin/devsy-linux-amd64. +func (s *focusedCheckSuite) options(goos string) (e2eOptions, *bytes.Buffer) { + dir := s.T().TempDir() + s.Require().NoError(os.MkdirAll(filepath.Join(dir, "bin"), 0o750)) + s.Require(). + NoError(os.WriteFile(filepath.Join(dir, "e2e.test"), []byte(focusedTestBinary), 0o600)) + s.Require(). + NoError(os.WriteFile(filepath.Join(dir, "bin", "devsy-linux-amd64"), []byte(focusedTestAgent), 0o600)) + stdout := &bytes.Buffer{} + opts := e2eOptions{ + E2EDir: dir, + TempDir: s.T().TempDir(), + GOOS: goos, + Stdout: stdout, + Stderr: io.Discard, + } + return opts, stdout +} + +// passingHost is the current process identity with a temp HOME and successful account lookups. +func (s *focusedCheckSuite) passingHost() focusedHost { + return focusedHost{ + uid: os.Getuid(), + gid: os.Getgid(), + home: s.T().TempDir(), + lookupUser: func(id string) (*user.User, error) { + return &user.User{Uid: id}, nil + }, + lookupGroup: func(id string) (*user.Group, error) { + return &user.Group{Gid: id}, nil + }, + } +} + +// checkFor returns the suite's check requiring the identity of host. +func (s *focusedCheckSuite) checkFor(host focusedHost) FocusedCheck { + check := s.check + check.RequireUID = host.uid + check.RequireGID = host.gid + return check +} + +// installHost makes host the focused-check host until the test ends. +func (s *focusedCheckSuite) installHost(host focusedHost) { + previous := focusedHostProvider + focusedHostProvider = func() focusedHost { return host } + s.T().Cleanup(func() { focusedHostProvider = previous }) +} + +func (s *focusedCheckSuite) writeFile(name string) string { + path := filepath.Join(s.T().TempDir(), name) + s.Require().NoError(os.WriteFile(path, []byte("x"), 0o600)) + return path +} diff --git a/hack/ci/e2e_check_unix.go b/hack/ci/e2e_check_unix.go new file mode 100644 index 000000000..26ce18ed0 --- /dev/null +++ b/hack/ci/e2e_check_unix.go @@ -0,0 +1,31 @@ +//go:build !windows + +package main + +import ( + "fmt" + "os" + "syscall" +) + +// requireOwnedBy fails unless path is owned by uid. +func requireOwnedBy(path string, uid int) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("%w: %s: %w", errFocusedOwner, path, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("%w: %s has no owner information", errFocusedOwner, path) + } + if int(stat.Uid) != uid { + return fmt.Errorf( + "%w: %s is owned by uid %d, want %d", + errFocusedOwner, + path, + stat.Uid, + uid, + ) + } + return nil +} diff --git a/hack/ci/e2e_check_windows.go b/hack/ci/e2e_check_windows.go new file mode 100644 index 000000000..d43278d66 --- /dev/null +++ b/hack/ci/e2e_check_windows.go @@ -0,0 +1,6 @@ +//go:build windows + +package main + +// requireOwnedBy is a no-op on Windows, where Unix ownership does not apply. +func requireOwnedBy(_ string, _ int) error { return nil } diff --git a/hack/ci/e2e_test.go b/hack/ci/e2e_test.go new file mode 100644 index 000000000..3f84f6758 --- /dev/null +++ b/hack/ci/e2e_test.go @@ -0,0 +1,602 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/onsi/ginkgo/v2/types" + "github.com/stretchr/testify/suite" +) + +const ( + e2eTestToken = "tok-secret-value" + e2eTestDocker = "unix:///var/run/docker.sock" + e2eTestTarget = "integration-x" + e2eTestBinPath = "./e2e.test" + e2eTestTimeout = "1500s" + e2eCmdPodman = "podman" + e2eCmdPrepare = "prepare" + e2eCmdRun = "run" + e2eFlagLabel = "--label" +) + +var errE2EBoom = errors.New("boom") + +// e2eFakeRunner records every command and fails the names listed in errs. +type e2eFakeRunner struct { + specs []commandSpec + errs map[string]error +} + +func (f *e2eFakeRunner) Run(_ context.Context, spec commandSpec) error { + f.specs = append(f.specs, spec) + return f.errs[spec.Name] +} + +func (f *e2eFakeRunner) names() []string { + out := make([]string, 0, len(f.specs)) + for _, spec := range f.specs { + out = append(out, spec.Name) + } + return out +} + +func e2eNoFocusedChecks(context.Context, commandRunner, Suite, e2eOptions) error { + return nil +} + +func e2eEnvValue(env []string, key string) (string, bool) { + for _, kv := range env { + if k, v, ok := strings.Cut(kv, "="); ok && k == key { + return v, true + } + } + return "", false +} + +func e2eTargetSuite(privileged bool) Suite { + return Suite{ + Group: groupIntegration, + Label: e2eTestTarget, + Runner: "ubuntu-latest", + Resolved: privileged, + TestTimeout: e2eTestTimeout, + FlakeAttempts: 1, + } +} + +// e2eSuiteReport returns a Ginkgo report whose specs all carry the target label and passed on +// their first attempt. +func e2eSuiteReport(specs int) types.Report { + r := types.Report{SuiteSucceeded: true} + for range specs { + r.SpecReports = append(r.SpecReports, types.SpecReport{ + LeafNodeType: types.NodeTypeIt, + LeafNodeText: "spec", + ContainerHierarchyLabels: [][]string{{e2eTestTarget}}, + State: types.SpecStatePassed, + NumAttempts: 1, + }) + } + return r +} + +func e2eWantTestArgs(timeout string, flakes int) []string { + return []string{ + "-test.v", + "-ginkgo.v", + "-test.timeout", timeout, + "-ginkgo.fail-on-empty", + fmt.Sprintf("-ginkgo.flake-attempts=%d", flakes), + "--ginkgo.label-filter=" + e2eTestTarget, + } +} + +// E2ECommandSuite covers the `ci e2e` prepare and run subcommands. +type E2ECommandSuite struct { + suite.Suite + + dir string + fake *e2eFakeRunner + env map[string]string + report []byte + readErr error + reads []string + out bytes.Buffer + errOut bytes.Buffer +} + +func TestE2ECommandSuite(t *testing.T) { + t.Parallel() + suite.Run(t, new(E2ECommandSuite)) +} + +func (s *E2ECommandSuite) SetupTest() { + s.dir = s.T().TempDir() + s.fake = &e2eFakeRunner{} + s.env = map[string]string{ + "GH_USERNAME": "octo", + "GH_ACCESS_TOKEN": e2eTestToken, + "GH_CREDENTIAL_USERNAME": "x-access-token", + "HOME": "/home/runner", + "PATH": "/usr/bin:/bin", + "XDG_RUNTIME_DIR": "/run/user/1001", + } + s.report = nil + s.readErr = nil + s.reads = nil + s.out.Reset() + s.errOut.Reset() +} + +func (s *E2ECommandSuite) TestPrepareCopiesDevsyBinariesFlat() { + s.Require().NoError(prepareE2E(&s.out, osLinux, s.binSource(), s.dir)) + + entries, err := os.ReadDir(filepath.Join(s.dir, e2eBinDirName)) + s.Require().NoError(err) + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + s.Equal([]string{"devsy-darwin-arm64", "devsy-linux-amd64"}, names) + s.Contains(s.out.String(), filepath.Join("mac", "notes.txt")) +} + +func (s *E2ECommandSuite) TestPrepareSetsExecutableMode() { + if runtime.GOOS == osWindows { + s.T().Skip("windows has no unix mode bits") + } + s.Require().NoError(prepareE2E(&s.out, osLinux, s.binSource(), s.dir)) + + for _, name := range []string{"devsy-linux-amd64", "devsy-darwin-arm64"} { + info, err := os.Stat(filepath.Join(s.dir, e2eBinDirName, name)) + s.Require().NoError(err) + s.Equal(os.FileMode(0o755), info.Mode().Perm(), name) + } +} + +func (s *E2ECommandSuite) TestPrepareMarksTestBinaryExecutable() { + if runtime.GOOS == osWindows { + s.T().Skip("windows has no unix mode bits") + } + testBinary := filepath.Join(s.dir, e2eTestBinary) + s.writeFile(testBinary, 0o644) + + s.Require().NoError(prepareE2E(&s.out, osLinux, s.binSource(), s.dir)) + + info, err := os.Stat(testBinary) + s.Require().NoError(err) + s.Equal(os.FileMode(0o755), info.Mode().Perm()) +} + +func (s *E2ECommandSuite) TestPrepareLeavesTestBinaryAloneOnWindows() { + testBinary := filepath.Join(s.dir, e2eTestBinary) + s.writeFile(testBinary, 0o644) + before, err := os.Stat(testBinary) + s.Require().NoError(err) + + s.Require().NoError(prepareE2E(&s.out, osWindows, s.binSource(), s.dir)) + + after, err := os.Stat(testBinary) + s.Require().NoError(err) + s.Equal(before.Mode(), after.Mode()) +} + +func (s *E2ECommandSuite) TestPrepareWithoutTestBinary() { + s.Require().NoError(prepareE2E(&s.out, osLinux, s.binSource(), s.dir)) + s.NoFileExists(filepath.Join(s.dir, e2eTestBinary)) +} + +func (s *E2ECommandSuite) TestPrepareFailsWithoutDevsyBinaries() { + src := s.T().TempDir() + s.writeFile(filepath.Join(src, "other-tool"), 0o755) + + err := prepareE2E(&s.out, osLinux, src, s.dir) + + s.ErrorIs(err, errNoDevsyBinaries) + s.NoDirExists(filepath.Join(s.dir, e2eBinDirName)) +} + +func (s *E2ECommandSuite) TestCLIPrepareRequiresBinSrc() { + code := e2eCLI([]string{e2eCmdPrepare}, &s.out, &s.errOut) + + s.Equal(2, code) + s.Contains(s.errOut.String(), "--bin-src") +} + +func (s *E2ECommandSuite) TestCLIPrepareCopiesBinaries() { + code := e2eCLI( + []string{"prepare", "--bin-src", s.binSource(), "--e2e-dir", s.dir}, + &s.out, + &s.errOut, + ) + + s.Equal(0, code, s.errOut.String()) + s.FileExists(filepath.Join(s.dir, e2eBinDirName, "devsy-darwin-arm64")) +} + +func (s *E2ECommandSuite) TestRunLinuxPrivilegedBuildsSudoCommand() { + s.env["KUBECONFIG"] = "/etc/kube/config" + s.env["DOCKER_HOST"] = e2eTestDocker + + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(true))) + + s.Require().Len(s.fake.specs, 1) + spec := s.fake.specs[0] + s.Equal("sudo", spec.Name) + s.Empty(spec.Env) + s.Equal(s.dir, spec.Dir) + env := []string{ + "GH_USERNAME=octo", + "GH_ACCESS_TOKEN=" + e2eTestToken, + "GH_CREDENTIAL_USERNAME=x-access-token", + "KUBECONFIG=/etc/kube/config", + "PATH=/usr/bin:/bin", + "DOCKER_HOST=" + e2eTestDocker, + } + s.Equal( + slices.Concat( + env, + []string{e2eTestBinPath}, + e2eWantTestArgs(e2eTestTimeout, 1), + ), + spec.Args, + ) +} + +func (s *E2ECommandSuite) TestRunLinuxPrivilegedOmitsUnsetDockerHost() { + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(true))) + + spec := s.fake.specs[0] + _, ok := e2eEnvValue(spec.Args, "DOCKER_HOST") + s.False(ok) +} + +func (s *E2ECommandSuite) TestRunDockerWSLUsesWSLDaemonAndClearsContext() { + target := e2eTargetSuite(false) + target.Runner = "windows-latest" + target.Runtime = runtimeDockerWSL + target.ExpectedSpecCount = 1 + s.env["DOCKER_HOST"] = e2eTestDocker + s.setReport(e2eSuiteReport(1)) + + s.Require().NoError(s.runnerFor(osWindows).run(context.Background(), target)) + + spec := s.fake.specs[0] + host, ok := e2eEnvValue(spec.Env, "DOCKER_HOST") + s.True(ok) + s.Equal("npipe:////./pipe/devsy-ci-no-daemon", host) + s.NotEqual(windowsDockerHost, host) + dockerContext, ok := e2eEnvValue(spec.Env, "DOCKER_CONTEXT") + s.True(ok, "DOCKER_CONTEXT must be present so an inherited context is cleared") + s.Empty(dockerContext) +} + +func (s *E2ECommandSuite) TestRunWithoutDockerWSLLeavesDockerContextUnset() { + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(false))) + + _, ok := e2eEnvValue(s.fake.specs[0].Env, "DOCKER_CONTEXT") + s.False(ok) +} + +func (s *E2ECommandSuite) TestJSONReportArgOnlyWithPinnedSpecCount() { + reportArg := "-ginkgo.json-report=" + e2eTestTarget + "-report.json" + tests := map[string]struct { + group string + count int + want bool + }{ + "pinned in integration": {group: groupIntegration, count: 2, want: true}, + "unpinned integration": {group: groupIntegration, count: 0}, + "pinned in microsandbox": {group: groupMicrosandbox, count: 2}, + "unpinned in unprivileged": {group: groupUnprivileged}, + } + for name, tc := range tests { + target := e2eTargetSuite(false) + target.Group = tc.group + target.ExpectedSpecCount = tc.count + want := e2eWantTestArgs(e2eTestTimeout, 1) + if tc.want { + want = append(want, reportArg) + } + s.Equal(want, e2eTestArgs(target), name) + } +} + +func (s *E2ECommandSuite) TestRunChecksReportBeforePostChecks() { + target := e2eTargetSuite(false) + target.ExpectedSpecCount = 2 + target.PostChecks = []string{postCheckDockerInstalled} + s.setReport(e2eSuiteReport(2)) + + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), target)) + + s.Equal([]string{e2eTestBinPath, dockerBinary, dockerBinary}, s.fake.names()) + s.Equal([]string{filepath.Join(s.dir, e2eTestTarget+"-report.json")}, s.reads) + s.Contains(s.out.String(), "report check passed: 2 specs\n") +} + +func (s *E2ECommandSuite) TestRunRejectsReportThatFailsCheck() { + tests := map[string]func(*types.Report){ + "fewer specs than expected": func(r *types.Report) { r.SpecReports = r.SpecReports[:1] }, + "failed spec": func(r *types.Report) { r.SpecReports[0].State = types.SpecStateFailed }, + "suite not successful": func(r *types.Report) { r.SuiteSucceeded = false }, + } + for name, mutate := range tests { + s.Run(name, func() { + s.fake = &e2eFakeRunner{} + r := e2eSuiteReport(2) + mutate(&r) + s.setReport(r) + target := e2eTargetSuite(false) + target.ExpectedSpecCount = 2 + target.PostChecks = []string{postCheckDockerInstalled} + + err := s.runnerFor(osLinux).run(context.Background(), target) + + s.ErrorContains(err, "verify report") + s.Equal([]string{e2eTestBinPath}, s.fake.names(), "post checks must not run") + s.NotContains(s.out.String(), "report check passed") + }) + } +} + +func (s *E2ECommandSuite) TestRunFailsWhenReportIsUnreadable() { + target := e2eTargetSuite(false) + target.ExpectedSpecCount = 2 + target.PostChecks = []string{postCheckDockerInstalled} + s.readErr = errE2EBoom + + err := s.runnerFor(osLinux).run(context.Background(), target) + + s.ErrorIs(err, errE2EBoom) + s.Equal([]string{e2eTestBinPath}, s.fake.names()) +} + +func (s *E2ECommandSuite) TestRunSkipsReportWithoutPinnedSpecCount() { + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(false))) + + s.Empty(s.reads) + s.NotContains(s.out.String(), "report check") +} + +func (s *E2ECommandSuite) TestRunUnprivilegedRunsDirectly() { + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(false))) + + s.Require().Len(s.fake.specs, 1) + spec := s.fake.specs[0] + s.Equal(e2eTestBinPath, spec.Name) + s.Equal(s.dir, spec.Dir) + s.Equal(e2eWantTestArgs(e2eTestTimeout, 1), spec.Args) + token, ok := e2eEnvValue(spec.Env, "GH_ACCESS_TOKEN") + s.True(ok) + s.Equal(e2eTestToken, token) + kubeconfig, ok := e2eEnvValue(spec.Env, "KUBECONFIG") + s.True(ok) + s.Equal(filepath.Join("/home/runner", ".kube", "config"), kubeconfig) + _, ok = e2eEnvValue(spec.Env, "DOCKER_HOST") + s.False(ok) +} + +func (s *E2ECommandSuite) TestRunWindowsUsesExeWithoutSudo() { + s.env["DOCKER_HOST"] = "tcp://ignored:2375" + + s.Require().NoError(s.runnerFor(osWindows).run(context.Background(), e2eTargetSuite(true))) + + s.Require().Len(s.fake.specs, 1) + spec := s.fake.specs[0] + s.Equal("./e2e.test.exe", spec.Name) + s.NotEmpty(spec.Env) + dockerHost, ok := e2eEnvValue(spec.Env, "DOCKER_HOST") + s.True(ok) + s.Equal(windowsDockerHost, dockerHost) + s.Equal(e2eWantTestArgs(e2eTestTimeout, 1), spec.Args) +} + +func (s *E2ECommandSuite) TestRunWindowsKubeconfigFallsBackToUserProfile() { + s.env["HOME"] = "" + s.env["USERPROFILE"] = `C:\Users\runner` + + s.Require().NoError(s.runnerFor(osWindows).run(context.Background(), e2eTargetSuite(false))) + + s.Require().Len(s.fake.specs, 1) + kubeconfig, ok := e2eEnvValue(s.fake.specs[0].Env, "KUBECONFIG") + s.True(ok) + s.Equal(filepath.Join(`C:\Users\runner`, ".kube", "config"), kubeconfig) +} + +func (s *E2ECommandSuite) TestRunMacOSRunsDirectly() { + s.Require().NoError(s.runnerFor(osDarwin).run(context.Background(), e2eTargetSuite(true))) + + s.Require().Len(s.fake.specs, 1) + spec := s.fake.specs[0] + s.Equal(e2eTestBinPath, spec.Name) + token, ok := e2eEnvValue(spec.Env, "GH_ACCESS_TOKEN") + s.True(ok) + s.Equal(e2eTestToken, token) + _, ok = e2eEnvValue(spec.Env, "DOCKER_HOST") + s.False(ok) +} + +func (s *E2ECommandSuite) TestRunRootlessPrintsDiagnosticsBeforeRunning() { + target := e2eTargetSuite(false) + target.InstallPodman = podmanRootless + + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), target)) + + s.Equal([]string{"id", e2eCmdPodman, e2eTestBinPath}, s.fake.names()) + s.Equal([]string{"info"}, s.fake.specs[1].Args) + s.Contains(s.out.String(), "HOME=/home/runner\n") + s.Contains(s.out.String(), "XDG_RUNTIME_DIR=/run/user/1001\n") +} + +func (s *E2ECommandSuite) TestRunRootlessStopsWhenPodmanInfoFails() { + target := e2eTargetSuite(false) + target.InstallPodman = podmanRootless + s.fake.errs = map[string]error{e2eCmdPodman: errE2EBoom} + + err := s.runnerFor(osLinux).run(context.Background(), target) + + s.ErrorIs(err, errE2EBoom) + s.Equal([]string{"id", "podman"}, s.fake.names()) +} + +func (s *E2ECommandSuite) TestRunFocusedChecksRunFirst() { + r := s.runnerFor(osLinux) + r.focused = func(ctx context.Context, runner commandRunner, _ Suite, opts e2eOptions) error { + s.Equal(s.dir, opts.E2EDir) + return runner.Run(ctx, commandSpec{Name: "focus-check"}) + } + + s.Require().NoError(r.run(context.Background(), e2eTargetSuite(false))) + + s.Equal([]string{"focus-check", e2eTestBinPath}, s.fake.names()) +} + +func (s *E2ECommandSuite) TestRunAbortsWhenFocusedChecksFail() { + r := s.runnerFor(osLinux) + r.focused = func(context.Context, commandRunner, Suite, e2eOptions) error { + return errE2EBoom + } + + err := r.run(context.Background(), e2eTargetSuite(false)) + + s.ErrorIs(err, errE2EBoom) + s.Empty(s.fake.specs) +} + +func (s *E2ECommandSuite) TestRunDockerInstalledPostCheckAfterSuccess() { + target := e2eTargetSuite(false) + target.PostChecks = []string{postCheckDockerInstalled} + + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), target)) + + s.Equal([]string{e2eTestBinPath, dockerBinary, dockerBinary}, s.fake.names()) + s.Equal([]string{"--version"}, s.fake.specs[1].Args) + s.Equal([]string{"ps"}, s.fake.specs[2].Args) +} + +func (s *E2ECommandSuite) TestRunSkipsPostChecksWhenSuiteFails() { + target := e2eTargetSuite(false) + target.PostChecks = []string{postCheckDockerInstalled} + s.fake.errs = map[string]error{e2eTestBinPath: errE2EBoom} + + err := s.runnerFor(osLinux).run(context.Background(), target) + + s.ErrorIs(err, errE2EBoom) + s.Equal([]string{e2eTestBinPath}, s.fake.names()) +} + +func (s *E2ECommandSuite) TestRunRejectsUnknownPostCheck() { + target := e2eTargetSuite(false) + target.PostChecks = []string{"no-such-check"} + + err := s.runnerFor(osLinux).run(context.Background(), target) + + s.ErrorIs(err, errUnknownPostCheck) +} + +func (s *E2ECommandSuite) TestRunTakesTimeoutAndFlakeAttemptsFromSuite() { + target := e2eTargetSuite(false) + target.TestTimeout = "900s" + target.FlakeAttempts = 3 + + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), target)) + + s.Equal(e2eWantTestArgs("900s", 3), s.fake.specs[0].Args) +} + +func (s *E2ECommandSuite) TestRunPrintsBanner() { + s.Require().NoError(s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(true))) + + s.Contains(s.out.String(), "running suite integration-x on ubuntu-latest (privileged=true)\n") +} + +func (s *E2ECommandSuite) TestRunNeverPrintsToken() { + s.fake.errs = map[string]error{e2eTestBinPath: errE2EBoom} + + err := s.runnerFor(osLinux).run(context.Background(), e2eTargetSuite(false)) + + s.Error(err) + s.NotContains(s.out.String(), e2eTestToken) + s.NotContains(s.errOut.String(), e2eTestToken) +} + +func (s *E2ECommandSuite) TestCLIRunRefusesMicrosandbox() { + code := e2eCLI([]string{e2eCmdRun, e2eFlagLabel, "up-provider-microsandbox"}, &s.out, &s.errOut) + + s.Equal(2, code) + s.Contains(s.errOut.String(), "devsy-ci-microsandbox") +} + +func (s *E2ECommandSuite) TestCLIRejectsBadUsage() { + tests := map[string][]string{ + "no subcommand": {}, + "unknown subcommand": {"bogus"}, + "run without label": {e2eCmdRun}, + "unknown label": {e2eCmdRun, e2eFlagLabel, "no-such-suite"}, + "unknown flag": {e2eCmdRun, "--no-such-flag"}, + "stray argument": {e2eCmdPrepare, "--bin-src", s.dir, "extra"}, + } + for name, args := range tests { + s.Equal(2, e2eCLI(args, &s.out, &s.errOut), name) + } +} + +func (s *E2ECommandSuite) runnerFor(goos string) e2eRunner { + return e2eRunner{ + runner: s.fake, + getenv: func(key string) string { return s.env[key] }, + readFile: func(path string) ([]byte, error) { + s.reads = append(s.reads, path) + if s.readErr != nil { + return nil, s.readErr + } + return s.report, nil + }, + focused: e2eNoFocusedChecks, + opts: e2eOptions{ + E2EDir: s.dir, + TempDir: "/tmp/runner-temp", + GOOS: goos, + Stdout: &s.out, + Stderr: &s.errOut, + }, + } +} + +// setReport makes the fake report reader return r as a one-suite Ginkgo JSON report. +func (s *E2ECommandSuite) setReport(r types.Report) { + data, err := json.Marshal([]types.Report{r}) + s.Require().NoError(err) + s.report = data +} + +func (s *E2ECommandSuite) writeFile(path string, mode os.FileMode) { + s.Require().NoError(os.MkdirAll(filepath.Dir(path), 0o755)) + s.Require().NoError(os.WriteFile(path, []byte("content"), mode)) + s.Require().NoError(os.Chmod(path, mode)) +} + +// binSource builds a download tree with devsy binaries in nested directories and some decoys. +func (s *E2ECommandSuite) binSource() string { + src := filepath.Join(s.T().TempDir(), "devsy-bin") + s.writeFile(filepath.Join(src, "linux", "devsy-linux-amd64"), 0o700) + s.writeFile(filepath.Join(src, "mac", "devsy-darwin-arm64"), 0o600) + s.writeFile(filepath.Join(src, "mac", "notes.txt"), 0o644) + s.writeFile(filepath.Join(src, "devsy-dir", "inner"), 0o644) + if runtime.GOOS != osWindows { + s.Require(). + NoError(os.Symlink(filepath.Join("linux", "devsy-linux-amd64"), filepath.Join(src, "devsy-link"))) + } + return src +} diff --git a/hack/ci/exec.go b/hack/ci/exec.go new file mode 100644 index 000000000..43d2f4830 --- /dev/null +++ b/hack/ci/exec.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" +) + +// commandSpec describes one external command. +type commandSpec struct { + Name string + Args []string + Dir string + Env []string // appended to the current environment + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// commandRunner runs external commands. Tests substitute a fake so no real +// process, sudo or package manager is touched. +type commandRunner interface { + Run(ctx context.Context, spec commandSpec) error +} + +type execRunner struct{} + +func (execRunner) Run(ctx context.Context, spec commandSpec) error { + cmd := exec.CommandContext(ctx, spec.Name, spec.Args...) + cmd.Dir = spec.Dir + cmd.Env = append(os.Environ(), spec.Env...) + cmd.Stdin = spec.Stdin + cmd.Stdout = spec.Stdout + cmd.Stderr = spec.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s: %w", spec.Name, err) + } + return nil +} diff --git a/hack/ci/main.go b/hack/ci/main.go new file mode 100644 index 000000000..6e7ae3c7c --- /dev/null +++ b/hack/ci/main.go @@ -0,0 +1,44 @@ +// Command ci is the Go implementation of the PR workflow's scripting. +// +// The workflow (.github/workflows/pr-ci.yml) only wires GitHub Actions +// together. What tests exist, where they run and how they run lives in +// suites.yaml and in the subcommands here: +// +// matrix emit the job matrices for the workflow +// unit run the unit-test tier on this machine +// e2e prepare and run Ginkgo suites +// setup host setup for suites (Docker, Podman) +// success fold the needed job results into the required status check +package main + +import ( + "fmt" + "io" + "os" +) + +func main() { + os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr)) +} + +func runCLI(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + _, _ = fmt.Fprintln(stderr, "usage: ci [args]") + return 2 + } + switch args[0] { + case "matrix": + return matrixCLI(args[1:], stdout, stderr) + case "unit": + return unitCLI(args[1:], stdout, stderr) + case "e2e": + return e2eCLI(args[1:], stdout, stderr) + case "setup": + return setupCLI(args[1:], stdout, stderr) + case "success": + return successCLI(args[1:], stdout, stderr) + default: + _, _ = fmt.Fprintf(stderr, "unknown command: %s\n", args[0]) + return 2 + } +} diff --git a/hack/ci/manifest.go b/hack/ci/manifest.go new file mode 100644 index 000000000..3402b2ab1 --- /dev/null +++ b/hack/ci/manifest.go @@ -0,0 +1,359 @@ +package main + +import ( + _ "embed" + "errors" + "fmt" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +//go:embed suites.yaml +var embeddedSuites []byte + +const ( + groupUnprivileged = "unprivileged" + groupIntegration = "integration" + groupMicrosandbox = "microsandbox" + + osLinux = "linux" + osWindows = "windows" + osDarwin = "darwin" + + podmanRootless = "rootless" + podmanRootful = "rootful" + + defaultProviderMode = "parity" + + fixtureGHFixtureLinux = "ghfixture-linux" + + // runtimeDockerWSL is a Windows suite whose Docker daemon is a WSL2 fixture the workflow provisions. + runtimeDockerWSL = "docker-wsl" +) + +var ( + errUnknownGroup = errors.New("unknown group") + errDuplicateSuite = errors.New("duplicate suite") + errInvalidSuite = errors.New("invalid suite") + errInvalidStep = errors.New("invalid unit step") +) + +// Manifest is the parsed hack/ci/suites.yaml. +type Manifest struct { + Unit UnitTier `yaml:"unit"` + E2E E2ETier `yaml:"e2e"` +} + +// UnitTier describes how Go unit tests run. +type UnitTier struct { + Runners []UnitRunner `yaml:"runners"` + Steps []UnitStep `yaml:"steps"` + Excluded []Exclusion `yaml:"excluded"` +} + +// UnitRunner is one runner of the unit-test job matrix. +type UnitRunner struct { + Runner string `yaml:"runner" json:"runner"` + TimeoutMinutes int `yaml:"timeout_minutes" json:"timeout_minutes"` +} + +// UnitStep is one `go test` invocation. Steps run in manifest order. +type UnitStep struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + OS []string `yaml:"os"` + Discover bool `yaml:"discover"` + HostShell string `yaml:"host_shell"` + Packages []string `yaml:"packages"` + Run string `yaml:"run"` + RequireTests []string `yaml:"require_tests"` + Args []string `yaml:"args"` +} + +// Exclusion records a package with tests that is deliberately not a unit test. +type Exclusion struct { + Package string `yaml:"package"` + Reason string `yaml:"reason"` +} + +// E2ETier describes the Ginkgo suites. +type E2ETier struct { + Groups map[string]Group `yaml:"groups"` + Ungated []Ungated `yaml:"ungated"` + Suites []Suite `yaml:"suites"` +} + +// Group holds the defaults inherited by its suites. +type Group struct { + Description string `yaml:"description"` + Runner string `yaml:"runner"` + Privileged bool `yaml:"privileged"` + TestTimeout string `yaml:"test_timeout"` + JobTimeoutMinutes int `yaml:"job_timeout_minutes"` + FlakeAttempts int `yaml:"flake_attempts"` +} + +// Ungated is a Ginkgo label that PR CI intentionally does not select directly. +type Ungated struct { + Label string `yaml:"label"` + Reason string `yaml:"reason"` +} + +// FocusedCheck is a single-spec run that must pass exactly once before the +// suite's main run, optionally under a required numeric identity. +type FocusedCheck struct { + Name string `yaml:"name"` + Focus string `yaml:"focus"` + ExpectLeaf string `yaml:"expect_leaf"` + RequireUID int `yaml:"require_uid"` + RequireGID int `yaml:"require_gid"` +} + +// Suite is one Ginkgo label executed as one CI job. After Resolve, every +// inherited field is populated. +type Suite struct { + Group string `yaml:"group"` + Label string `yaml:"label"` + Runner string `yaml:"runner"` + // Privileged selects sudo for the e2e binary on Linux. Nil inherits the group. + Privileged *bool `yaml:"privileged"` + TestTimeout string `yaml:"test_timeout"` + JobTimeoutMinutes int `yaml:"job_timeout_minutes"` + FlakeAttempts int `yaml:"flake_attempts"` + FreeDiskSpace bool `yaml:"free_disk_space"` + InstallKind bool `yaml:"install_kind"` + InstallPodman string `yaml:"install_podman"` + RequiresSecret bool `yaml:"requires_secret"` + HostSetup []string `yaml:"host_setup"` + PostChecks []string `yaml:"post_checks"` + FocusedChecks []FocusedCheck `yaml:"focused_checks"` + + ExpectedSpecCount int `yaml:"expected_spec_count"` + ProviderMode string `yaml:"provider_mode"` + PerformanceCapture bool `yaml:"performance_capture"` + Runtime string `yaml:"runtime"` + // Fixtures names prebuilt artifacts the job downloads next to the e2e binary. + Fixtures []string `yaml:"fixtures"` + + // Resolved is the effective privilege after group inheritance. + Resolved bool `yaml:"-"` +} + +// LoadManifest parses and validates the embedded manifest. +func LoadManifest() (*Manifest, error) { + return ParseManifest(embeddedSuites) +} + +// ParseManifest parses, resolves and validates manifest bytes. +func ParseManifest(data []byte) (*Manifest, error) { + var m Manifest + dec := yaml.NewDecoder(strings.NewReader(string(data))) + dec.KnownFields(true) + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("parse suites manifest: %w", err) + } + if err := m.resolve(); err != nil { + return nil, err + } + if err := m.validate(); err != nil { + return nil, err + } + return &m, nil +} + +func validPodmanMode(mode string) bool { + return mode == "" || mode == podmanRootless || mode == podmanRootful +} + +// RunsOn reports whether the step applies to a GOOS value. +func (u UnitStep) RunsOn(goos string) bool { + return len(u.OS) == 0 || slices.Contains(u.OS, goos) +} + +func (u UnitStep) validate() error { + if u.Name == "" || (!u.Discover && len(u.Packages) == 0) { + return fmt.Errorf("%w: %q needs a name and packages or discover", errInvalidStep, u.Name) + } + for _, os := range u.OS { + if os != osLinux && os != osDarwin && os != osWindows { + return fmt.Errorf("%w: %q has os %q", errInvalidStep, u.Name, os) + } + } + return nil +} + +// OS returns the GOOS value implied by the suite's runner label. +func (s Suite) OS() string { + switch { + case strings.HasPrefix(s.Runner, "windows"): + return osWindows + case strings.HasPrefix(s.Runner, "macos"): + return osDarwin + default: + return osLinux + } +} + +// Title is the GitHub job display name. +func (s Suite) Title() string { + mode := "" + if s.InstallPodman != "" { + mode = " (" + s.InstallPodman + ")" + } + return fmt.Sprintf("Test %s%s on %s", s.Label, mode, s.Runner) +} + +func (s Suite) validate() error { + switch { + case s.Label == "": + return fmt.Errorf("%w: empty label", errInvalidSuite) + case !s.hasRequiredFields(): + return fmt.Errorf( + "%w: %s is missing runner, timeouts or flake attempts", + errInvalidSuite, + s.Label, + ) + case !validPodmanMode(s.InstallPodman): + return fmt.Errorf("%w: %s has install_podman %q", errInvalidSuite, s.Label, s.InstallPodman) + case s.requiresSpecCount() && s.ExpectedSpecCount <= 0: + return fmt.Errorf("%w: %s needs expected_spec_count", errInvalidSuite, s.Label) + } + if err := s.validateFixtures(); err != nil { + return err + } + return s.validateRuntime() +} + +func (s Suite) hasRequiredFields() bool { + return s.Runner != "" && s.TestTimeout != "" && s.JobTimeoutMinutes > 0 && s.FlakeAttempts > 0 +} + +// requiresSpecCount reports whether the suite must pin how many specs it selects. +func (s Suite) requiresSpecCount() bool { + return s.Group == groupMicrosandbox || s.Runtime == runtimeDockerWSL +} + +func (s Suite) validateRuntime() error { + switch { + case s.Runtime == "": + return nil + case s.Runtime != runtimeDockerWSL: + return fmt.Errorf("%w: %s has runtime %q", errInvalidSuite, s.Label, s.Runtime) + case s.OS() != osWindows: + return fmt.Errorf( + "%w: %s runtime %s needs a windows runner, not %s", + errInvalidSuite, + s.Label, + s.Runtime, + s.Runner, + ) + } + return nil +} + +// SuitesIn returns the resolved suites of one group in manifest order. +func (m *Manifest) SuitesIn(group string) []Suite { + var out []Suite + for _, s := range m.E2E.Suites { + if s.Group == group { + out = append(out, s) + } + } + return out +} + +// FindSuite returns the suite with a label on a runner. An empty runner +// matches only when the label is unique. +func (m *Manifest) FindSuite(label, runner string) (Suite, error) { + var found []Suite + for _, s := range m.E2E.Suites { + if s.Label == label && (runner == "" || s.Runner == runner) { + found = append(found, s) + } + } + if len(found) != 1 { + return Suite{}, fmt.Errorf( + "%w: %d suites match label %q runner %q", + errInvalidSuite, + len(found), + label, + runner, + ) + } + return found[0], nil +} + +func (m *Manifest) resolve() error { + for i := range m.E2E.Suites { + s := &m.E2E.Suites[i] + g, ok := m.E2E.Groups[s.Group] + if !ok { + return fmt.Errorf("%w %q for label %q", errUnknownGroup, s.Group, s.Label) + } + s.Runner = firstNonEmpty(s.Runner, g.Runner) + s.TestTimeout = firstNonEmpty(s.TestTimeout, g.TestTimeout) + if s.JobTimeoutMinutes == 0 { + s.JobTimeoutMinutes = g.JobTimeoutMinutes + } + if s.FlakeAttempts == 0 { + s.FlakeAttempts = g.FlakeAttempts + } + s.Resolved = g.Privileged + if s.Privileged != nil { + s.Resolved = *s.Privileged + } + if s.Group == groupMicrosandbox { + s.ProviderMode = firstNonEmpty(s.ProviderMode, defaultProviderMode) + } + } + return nil +} + +func (m *Manifest) validate() error { + seen := map[string]bool{} + for _, s := range m.E2E.Suites { + key := s.Runner + "/" + s.Label + if seen[key] { + return fmt.Errorf("%w: %s", errDuplicateSuite, key) + } + seen[key] = true + if err := s.validate(); err != nil { + return err + } + } + for _, step := range m.Unit.Steps { + if err := step.validate(); err != nil { + return err + } + } + return nil +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// validateFixtures rejects unknown fixture names and Linux fixtures on other runners. +func (s Suite) validateFixtures() error { + for _, name := range s.Fixtures { + if name != fixtureGHFixtureLinux { + return fmt.Errorf("%w: %s has unknown fixture %q", errInvalidSuite, s.Label, name) + } + if s.OS() != osLinux { + return fmt.Errorf( + "%w: %s needs a Linux runner for fixture %q", + errInvalidSuite, + s.Label, + name, + ) + } + } + return nil +} diff --git a/hack/ci/manifest_test.go b/hack/ci/manifest_test.go new file mode 100644 index 000000000..ce9f3f27d --- /dev/null +++ b/hack/ci/manifest_test.go @@ -0,0 +1,320 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +// testGroupsYAML is a minimal e2e section with the same group shapes as the +// embedded manifest. Invalid-manifest cases append their suites to it. +const testGroupsYAML = `e2e: + groups: + integration: + runner: ubuntu-latest + privileged: true + test_timeout: 1500s + job_timeout_minutes: 45 + flake_attempts: 1 + microsandbox: + runner: ubuntu-latest + privileged: false + test_timeout: 1200s + job_timeout_minutes: 25 + flake_attempts: 1 + suites: +` + +// manifestCase is one inline manifest that ParseManifest must reject. +type manifestCase struct { + name string + yaml string + want error +} + +// manifestSuite covers ParseManifest, LoadManifest and the derived Suite and +// UnitStep helpers. +type manifestSuite struct { + suite.Suite + m *Manifest +} + +func TestManifestSuite(t *testing.T) { + suite.Run(t, new(manifestSuite)) +} + +func (s *manifestSuite) SetupSuite() { + m, err := LoadManifest() + s.Require().NoError(err) + s.m = m +} + +// Footgun: the embedded suites.yaml stops parsing, which would fail every PR +// workflow job that reads the matrix. +func (s *manifestSuite) TestEmbeddedManifestLoads() { + s.NotEmpty(s.m.E2E.Groups) + s.NotEmpty(s.m.E2E.Suites) + s.NotEmpty(s.m.Unit.Steps) + s.NotEmpty(s.m.Unit.Runners) +} + +// Footgun: a suite that omits a field silently gets the wrong group default, +// for example an unprivileged suite running with sudo or the wrong timeout. +func (s *manifestSuite) TestSuitesInheritGroupDefaults() { + ctx := s.mustFind("context", "") + s.Equal("ubuntu-latest", ctx.Runner) + s.Equal("1500s", ctx.TestTimeout) + s.Equal(45, ctx.JobTimeoutMinutes) + s.Equal(1, ctx.FlakeAttempts) + s.False(ctx.Resolved) + + micro := s.mustFind("up-provider-microsandbox-images", "") + s.Equal("1200s", micro.TestTimeout) + s.Equal(25, micro.JobTimeoutMinutes) + s.Equal(1, micro.FlakeAttempts) + s.False(micro.Resolved) +} + +// Footgun: an explicit suite override is replaced by the group default, so +// the docker-context job gets the 1500s and 45 minute budget it was tuned away from. +func (s *manifestSuite) TestSuiteOverridesWinOverGroupDefaults() { + dc := s.mustFind("docker-context", "") + s.Equal("300s", dc.TestTimeout) + s.Equal(10, dc.JobTimeoutMinutes) + s.Equal(1, dc.FlakeAttempts) + s.True(dc.Resolved, "integration group is privileged and docker-context does not override it") + + tunnel := s.mustFind("tunnel", "") + s.Equal(2, tunnel.FlakeAttempts) + s.Equal("1500s", tunnel.TestTimeout, "tunnel inherits the integration timeout") +} + +// Footgun: a rootless Podman suite runs privileged, or a rootful one runs +// unprivileged, so the sudo decision in the CI helper is wrong. +func (s *manifestSuite) TestPodmanPrivilegeFollowsMode() { + rootless := s.mustFind("up-provider-podman-rootless-basic", "") + s.Equal("rootless", rootless.InstallPodman) + s.Equal("ubuntu-24.04", rootless.Runner) + s.Equal("600s", rootless.TestTimeout) + s.False(rootless.Resolved) + + rootful := s.mustFind("up-provider-podman-rootful-basic", "") + s.Equal("rootful", rootful.InstallPodman) + s.True(rootful.Resolved) +} + +// Footgun: the Windows provider suite loses its 900s override, or the +// Linux flake_attempts of 2 leaks into the Windows suite that does not set it. +func (s *manifestSuite) TestWindowsSuiteKeepsItsOverride() { + win := s.mustFind("provider", "windows-latest") + s.Equal("windows-latest", win.Runner) + s.Equal("900s", win.TestTimeout) + s.Equal(20, win.JobTimeoutMinutes) + s.Equal(1, win.FlakeAttempts) + s.True(win.Resolved) +} + +// Footgun: a microsandbox suite without provider_mode runs with an empty mode, +// and an explicit external mode is overwritten with the parity default. +func (s *manifestSuite) TestMicrosandboxProviderMode() { + s.Equal("parity", s.mustFind("up-provider-microsandbox-images", "").ProviderMode) + s.Equal("external", s.mustFind("up-provider-microsandbox-registry-consumer", "").ProviderMode) + s.Empty(s.mustFind("context", "").ProviderMode, "only microsandbox suites get a provider mode") +} + +// Footgun: the GitHub job name drops the podman mode, so rootless and rootful +// jobs of the same label show up as indistinguishable checks. +func (s *manifestSuite) TestSuiteTitle() { + rootless := s.mustFind("up-provider-podman-rootless-basic", "ubuntu-24.04") + s.Equal("Test up-provider-podman-rootless-basic (rootless) on ubuntu-24.04", rootless.Title()) + + ctx := s.mustFind("context", "") + s.Equal("Test context on ubuntu-latest", ctx.Title()) +} + +// Footgun: the runner in the GOOS mapping is wrong, so Windows-only steps run +// on Linux runners or the reverse. +func (s *manifestSuite) TestSuiteOS() { + tests := []struct { + runner string + want string + }{ + {runner: "windows-latest", want: osWindows}, + {runner: "macos-latest", want: osDarwin}, + {runner: "ubuntu-latest", want: osLinux}, + {runner: "ubuntu-24.04", want: osLinux}, + } + for _, tc := range tests { + s.Equal(tc.want, Suite{Runner: tc.runner}.OS(), "runner %s", tc.runner) + } +} + +// Footgun: an ambiguous label silently resolves to the wrong suite, or a +// runner that is set but matches nothing is accepted. +func (s *manifestSuite) TestFindSuite() { + _, err := s.m.FindSuite("provider", "") + s.ErrorIs(err, errInvalidSuite, "provider exists on ubuntu-latest and windows-latest") + + found, err := s.m.FindSuite("provider", "windows-latest") + s.Require().NoError(err) + s.Equal("windows-latest", found.Runner) + _, err = s.m.FindSuite("provider", "macos-latest") + s.ErrorIs(err, errInvalidSuite, "no provider suite runs on macos-latest") + + _, err = s.m.FindSuite("no-such-label", "") + s.ErrorIs(err, errInvalidSuite) +} + +// Footgun: a docker-wsl suite is parsed without the Windows runner or the spec count that its +// JSON report check depends on, or a plain integration suite is forced to pin a spec count. +func (s *manifestSuite) TestRuntimeDockerWSLSuite() { + m, err := ParseManifest([]byte(testGroupsYAML + + " - { group: integration, label: x, runner: windows-latest, runtime: docker-wsl, expected_spec_count: 2 }\n")) + s.Require().NoError(err) + wsl := s.mustFindIn(m, "x", "windows-latest") + s.Equal(runtimeDockerWSL, wsl.Runtime) + s.Equal(2, wsl.ExpectedSpecCount) + + _, err = ParseManifest([]byte(testGroupsYAML + " - { group: integration, label: y }\n")) + s.NoError(err, "integration suites without a runtime do not need expected_spec_count") +} + +// Footgun: a rejected manifest is accepted because a validation rule was lost +// in refactoring, letting a bad suite reach the CI matrix. +func (s *manifestSuite) TestParseRejectsInvalidEntries() { + for _, tc := range invalidManifestCases() { + s.Run(tc.name, func() { + _, err := ParseManifest([]byte(tc.yaml)) + s.ErrorIs(err, tc.want) + }) + } +} + +// Footgun: a misspelled key such as `flake_attempt` is silently ignored and the +// suite runs with the group default instead of the intended value. +func (s *manifestSuite) TestParseRejectsUnknownKey() { + _, err := ParseManifest( + []byte(testGroupsYAML + " - { group: integration, label: x, flake_attempt: 2 }\n"), + ) + s.Error(err) + s.ErrorContains(err, "flake_attempt") +} + +// Footgun: RunsOn disagrees with the os list, so a step runs on a platform it +// was meant to skip or is skipped on its own platform. +func (s *manifestSuite) TestUnitStepRunsOn() { + all := UnitStep{Name: "all", Packages: []string{"./pkg/x"}} + s.True(all.RunsOn(osWindows), "a step without os runs everywhere") + + posix := UnitStep{Name: "posix", Packages: []string{"./pkg/x"}, OS: []string{osLinux, osDarwin}} + s.True(posix.RunsOn(osLinux)) + s.True(posix.RunsOn(osDarwin)) + s.False(posix.RunsOn(osWindows)) +} + +// Footgun: a typo in a fixture name, or a Linux-only fixture on a Windows runner, makes the +// job download an artifact that cannot exist and fail late. +func (s *manifestSuite) TestFixtureValidation() { + m, err := ParseManifest([]byte(testGroupsYAML + + " - { group: integration, label: x, fixtures: [ghfixture-linux] }\n")) + s.Require().NoError(err) + s.Equal([]string{fixtureGHFixtureLinux}, s.mustFindIn(m, "x", "ubuntu-latest").Fixtures) + + for _, entry := range []string{ + " - { group: integration, label: x, fixtures: [nope] }\n", + " - { group: integration, label: x, runner: windows-latest, fixtures: [ghfixture-linux] }\n", + } { + _, err := ParseManifest([]byte(testGroupsYAML + entry)) + s.ErrorIs(err, errInvalidSuite, entry) + } +} + +func (s *manifestSuite) mustFind(label, runner string) Suite { + s.T().Helper() + return s.mustFindIn(s.m, label, runner) +} + +func (s *manifestSuite) mustFindIn(m *Manifest, label, runner string) Suite { + s.T().Helper() + found, err := m.FindSuite(label, runner) + s.Require().NoError(err) + return found +} + +func invalidManifestCases() []manifestCase { + return append(invalidSuiteCases(), invalidUnitStepCases()...) +} + +func invalidSuiteCases() []manifestCase { + return []manifestCase{ + { + name: "unknown group", + yaml: testGroupsYAML + " - { group: nope, label: x }\n", + want: errUnknownGroup, + }, + { + name: "empty label", + yaml: testGroupsYAML + " - { group: integration, label: '' }\n", + want: errInvalidSuite, + }, + { + name: "group without suite defaults", + yaml: "e2e:\n groups:\n bare: {}\n suites:\n - { group: bare, label: x }\n", + want: errInvalidSuite, + }, + { + name: "duplicate runner and label", + yaml: testGroupsYAML + + " - { group: integration, label: x }\n" + + " - { group: integration, label: x }\n", + want: errDuplicateSuite, + }, + { + name: "microsandbox without expected_spec_count", + yaml: testGroupsYAML + " - { group: microsandbox, label: up-provider-microsandbox }\n", + want: errInvalidSuite, + }, + { + name: "bad install_podman value", + yaml: testGroupsYAML + " - { group: integration, label: x, install_podman: sometimes }\n", + want: errInvalidSuite, + }, + { + name: "unknown runtime", + yaml: testGroupsYAML + " - { group: integration, label: x, runtime: hyperv }\n", + want: errInvalidSuite, + }, + { + name: "docker-wsl on a linux runner", + yaml: testGroupsYAML + + " - { group: integration, label: x, runtime: docker-wsl, expected_spec_count: 1 }\n", + want: errInvalidSuite, + }, + { + name: "docker-wsl without expected_spec_count", + yaml: testGroupsYAML + + " - { group: integration, label: x, runner: windows-latest, runtime: docker-wsl }\n", + want: errInvalidSuite, + }, + } +} + +func invalidUnitStepCases() []manifestCase { + return []manifestCase{ + { + name: "unit step without packages or discover", + yaml: "unit:\n steps:\n - name: bare\n", + want: errInvalidStep, + }, + { + name: "unit step without name", + yaml: "unit:\n steps:\n - packages: [./pkg/x]\n", + want: errInvalidStep, + }, + { + name: "unit step with unknown os", + yaml: "unit:\n steps:\n - name: s\n packages: [./pkg/x]\n os: [solaris]\n", + want: errInvalidStep, + }, + } +} diff --git a/hack/ci/matrix.go b/hack/ci/matrix.go new file mode 100644 index 000000000..22eb566a4 --- /dev/null +++ b/hack/ci/matrix.go @@ -0,0 +1,216 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "slices" + "strings" +) + +const matrixUnitGroup = "unit" + +// matrixGroupNames lists the job groups in the order `ci matrix` prints them. +var matrixGroupNames = []string{ + matrixUnitGroup, + groupUnprivileged, + groupIntegration, + groupMicrosandbox, +} + +var errUnknownMatrixGroup = errors.New("unknown matrix group") + +// matrixDoc is the GitHub Actions matrix object. +type matrixDoc struct { + Include any `json:"include"` +} + +// matrixUnitEntry is one unit-test runner with the GOOS its steps run as. +type matrixUnitEntry struct { + UnitRunner + OS string `json:"os"` +} + +// matrixEntry is one e2e suite as a job of the workflow matrix. +type matrixEntry struct { + Name string `json:"name"` + Title string `json:"title"` + Runner string `json:"runner"` + OS string `json:"os"` + Privileged bool `json:"privileged"` + TestTimeout string `json:"test_timeout"` + JobTimeoutMinutes int `json:"job_timeout_minutes"` + FlakeAttempts int `json:"flake_attempts"` + FreeDiskSpace bool `json:"free_disk_space"` + InstallKind bool `json:"install_kind"` + PodmanMode string `json:"podman_mode"` + RequiresSecret bool `json:"requires_secret"` + ExpectedSpecCount int `json:"expected_spec_count"` + ProviderMode string `json:"provider_mode"` + PerformanceCapture bool `json:"performance_capture"` + Runtime string `json:"runtime"` + SetupDockerWSL bool `json:"setup_docker_wsl"` + SetupKindLinux bool `json:"setup_kind_linux"` + SetupKindWindows bool `json:"setup_kind_windows"` + SetupPodmanLinux bool `json:"setup_podman_linux"` + SetupPodmanWindows bool `json:"setup_podman_windows"` + NeedsPodmanHelper bool `json:"needs_podman_helper"` + GHFixtureLinux bool `json:"ghfixture_linux"` +} + +// matrixCLI prints the job matrices for the PR workflow. +func matrixCLI(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("matrix", flag.ContinueOnError) + fs.SetOutput(stderr) + group := fs.String( + "group", + "", + "print only this group: unit, unprivileged, integration or microsandbox", + ) + secretAvailable := fs.Bool("secret-available", false, "include suites that require a secret") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + _, _ = fmt.Fprintf( + stderr, + "ci matrix: unexpected arguments: %s\n", + strings.Join(fs.Args(), " "), + ) + return 2 + } + m, err := LoadManifest() + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) + return 1 + } + out, err := matrixOutput(m, *group, *secretAvailable) + if err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: %v\n", err) + return matrixFailureCode(err) + } + if _, err := io.WriteString(stdout, out); err != nil { + _, _ = fmt.Fprintf(stderr, "ci matrix: write output: %v\n", err) + return 1 + } + return 0 +} + +// matrixFailureCode maps a matrixOutput error to an exit code. +func matrixFailureCode(err error) int { + if errors.Is(err, errUnknownMatrixGroup) { + return 2 + } + return 1 +} + +// matrixOutput renders the --group JSON line, or without a group one +// `name=` line per group, each terminated by a newline. +func matrixOutput(m *Manifest, group string, secretAvailable bool) (string, error) { + if group != "" { + line, err := matrixGroupJSON(m, group, secretAvailable) + if err != nil { + return "", err + } + return line + "\n", nil + } + lines := make([]string, 0, len(matrixGroupNames)) + for _, name := range matrixGroupNames { + line, err := matrixGroupJSON(m, name, secretAvailable) + if err != nil { + return "", err + } + lines = append(lines, name+"="+line) + } + return strings.Join(lines, "\n") + "\n", nil +} + +// matrixGroupJSON returns the compact single-line matrix object of a group. +func matrixGroupJSON(m *Manifest, group string, secretAvailable bool) (string, error) { + include, err := matrixInclude(m, group, secretAvailable) + if err != nil { + return "", err + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(matrixDoc{Include: include}); err != nil { + return "", fmt.Errorf("encode %s matrix: %w", group, err) + } + return strings.TrimSuffix(buf.String(), "\n"), nil +} + +// matrixInclude returns the entries of one group. Entry slices are never nil +// so an empty group encodes as [] rather than null. +func matrixInclude(m *Manifest, group string, secretAvailable bool) (any, error) { + switch group { + case matrixUnitGroup: + return unitMatrixEntries(m), nil + case groupUnprivileged, groupIntegration, groupMicrosandbox: + return e2eMatrixEntries(m, group, secretAvailable), nil + default: + return nil, fmt.Errorf("%w: %q", errUnknownMatrixGroup, group) + } +} + +func unitMatrixEntries(m *Manifest) []matrixUnitEntry { + entries := make([]matrixUnitEntry, 0, len(m.Unit.Runners)) + for _, r := range m.Unit.Runners { + entries = append(entries, matrixUnitEntry{UnitRunner: r, OS: Suite{Runner: r.Runner}.OS()}) + } + return entries +} + +func e2eMatrixEntries(m *Manifest, group string, secretAvailable bool) []matrixEntry { + suites := m.SuitesIn(group) + entries := make([]matrixEntry, 0, len(suites)) + for _, s := range suites { + if s.RequiresSecret && !secretAvailable { + continue + } + entries = append(entries, newMatrixEntry(s)) + } + return entries +} + +// newMatrixEntry maps a resolved suite to its matrix job, including the setup +// steps the workflow must run for it. +func newMatrixEntry(s Suite) matrixEntry { + osName := s.OS() + linux := osName == osLinux + windows := osName == osWindows + hasPodman := s.InstallPodman != "" + setupPodmanLinux := linux && hasPodman + setupPodmanWindows := windows && (s.InstallKind || hasPodman) + return matrixEntry{ + Name: s.Label, + Title: s.Title(), + Runner: s.Runner, + OS: osName, + Privileged: s.Resolved, + TestTimeout: s.TestTimeout, + JobTimeoutMinutes: s.JobTimeoutMinutes, + FlakeAttempts: s.FlakeAttempts, + FreeDiskSpace: s.FreeDiskSpace, + InstallKind: s.InstallKind, + PodmanMode: s.InstallPodman, + RequiresSecret: s.RequiresSecret, + ExpectedSpecCount: s.ExpectedSpecCount, + ProviderMode: s.ProviderMode, + PerformanceCapture: s.PerformanceCapture, + Runtime: s.Runtime, + SetupDockerWSL: s.Runtime == runtimeDockerWSL, + SetupKindLinux: linux && s.InstallKind, + SetupKindWindows: windows && s.InstallKind, + SetupPodmanLinux: setupPodmanLinux, + SetupPodmanWindows: setupPodmanWindows, + NeedsPodmanHelper: setupPodmanLinux || setupPodmanWindows, + GHFixtureLinux: linux && slices.Contains(s.Fixtures, fixtureGHFixtureLinux), + } +} diff --git a/hack/ci/matrix_test.go b/hack/ci/matrix_test.go new file mode 100644 index 000000000..26431b4de --- /dev/null +++ b/hack/ci/matrix_test.go @@ -0,0 +1,366 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +// matrixTestManifest is a small manifest with one suite per interesting case. +const matrixTestManifest = ` +unit: + runners: + - { runner: ubuntu-latest, timeout_minutes: 30 } + - { runner: windows-latest, timeout_minutes: 30 } +e2e: + groups: + unprivileged: + runner: ubuntu-latest + privileged: false + test_timeout: 1500s + job_timeout_minutes: 45 + flake_attempts: 1 + integration: + runner: ubuntu-latest + privileged: true + test_timeout: 1500s + job_timeout_minutes: 45 + flake_attempts: 1 + microsandbox: + runner: ubuntu-latest + test_timeout: 1200s + job_timeout_minutes: 25 + flake_attempts: 1 + suites: + - { group: unprivileged, label: context } + - { group: integration, label: ide } + - { group: integration, label: build, requires_secret: true } + - { group: microsandbox, label: up-provider-microsandbox, expected_spec_count: 2, performance_capture: true } + - { group: integration, label: "a 1 && slices.Contains(fields[1:], name) { + return true + } + } + return false +} + +// readOptionalFile returns nil when the file does not exist. +func readOptionalFile(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + return data, nil +} + +// mergeInsecureRegistry adds registry to the insecure-registries list of a +// daemon.json document. Other keys are kept; the list is sorted and unique. +func mergeInsecureRegistry(existing []byte, registry string) ([]byte, error) { + cfg, err := decodeDaemonConfig(existing) + if err != nil { + return nil, err + } + registries, err := insecureRegistriesOf(cfg) + if err != nil { + return nil, err + } + registries = append(registries, registry) + slices.Sort(registries) + cfg[insecureKey] = slices.Compact(registries) + return encodeDaemonConfig(cfg) +} + +// decodeDaemonConfig parses a daemon.json document; empty input is an empty object. +func decodeDaemonConfig(data []byte) (map[string]any, error) { + cfg := map[string]any{} + if len(bytes.TrimSpace(data)) == 0 { + return cfg, nil + } + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("%w: %w", errInvalidDaemon, err) + } + if cfg == nil { + cfg = map[string]any{} + } + return cfg, nil +} + +func insecureRegistriesOf(cfg map[string]any) ([]string, error) { + raw, ok := cfg[insecureKey] + if !ok || raw == nil { + return nil, nil + } + items, ok := raw.([]any) + if !ok { + return nil, fmt.Errorf("%w: %s is not a list", errInvalidDaemon, insecureKey) + } + registries := make([]string, 0, len(items)) + for _, item := range items { + name, ok := item.(string) + if !ok { + return nil, fmt.Errorf("%w: %s has a non-string entry", errInvalidDaemon, insecureKey) + } + registries = append(registries, name) + } + return registries, nil +} + +// encodeDaemonConfig writes indented JSON with sorted keys and a trailing newline. +func encodeDaemonConfig(cfg map[string]any) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(cfg); err != nil { + return nil, fmt.Errorf("encode docker daemon config: %w", err) + } + return buf.Bytes(), nil +} + +// podmanInstall installs the podman-static archive system-wide. +type podmanInstall struct { + archive string + url string + sha256Hex string + extractDir string + client *http.Client + runner commandRunner + lookPath func(string) (string, error) + stdout io.Writer + stderr io.Writer +} + +// checksumMismatchError reports an archive whose SHA-256 differs from the pinned value. +type checksumMismatchError struct { + expected string + actual string +} + +func (e *checksumMismatchError) Error() string { + return fmt.Sprintf( + "SHA256 mismatch for podman-static archive! Expected: %s, Got: %s", + e.expected, + e.actual, + ) +} + +func (p *podmanInstall) install(ctx context.Context) error { + if err := p.ensureArchive(ctx); err != nil { + return err + } + if err := verifySHA256(p.archive, p.sha256Hex); err != nil { + return err + } + if err := extractTarGz(p.archive, p.extractDir); err != nil { + return fmt.Errorf("extract %s: %w", p.archive, err) + } + src := filepath.Join(p.extractDir, podmanPackageDir) + if err := p.sudo( + ctx, + "cp", + "-r", + filepath.Join(src, "usr"), + filepath.Join(src, "etc"), + "/", + ); err != nil { + return fmt.Errorf("install podman files: %w", err) + } + return p.ensureUIDMap(ctx) +} + +func (p *podmanInstall) sudo(ctx context.Context, args ...string) error { + return p.runner.Run( + ctx, + commandSpec{Name: sudoCmd, Args: args, Stdout: p.stdout, Stderr: p.stderr}, + ) +} + +// ensureUIDMap installs uidmap only when newuidmap is missing. +func (p *podmanInstall) ensureUIDMap(ctx context.Context) error { + if _, err := p.lookPath(uidmapTool); err == nil { + return nil + } + return p.sudo(ctx, "apt-get", "install", "-y", "uidmap") +} + +func (p *podmanInstall) ensureArchive(ctx context.Context) error { + _, err := os.Stat(p.archive) + switch { + case err == nil: + return nil + case errors.Is(err, fs.ErrNotExist): + return p.download(ctx) + default: + return fmt.Errorf("stat %s: %w", p.archive, err) + } +} + +func (p *podmanInstall) download(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, podmanDownloadTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.url, nil) + if err != nil { + return fmt.Errorf("create request for %s: %w", p.url, err) + } + resp, err := p.client.Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", p.url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: unexpected status %s", p.url, resp.Status) + } + return saveDownload(p.archive, resp.Body) +} + +// saveDownload writes body to path through a partial file so a failed +// download never leaves a truncated archive behind. +func saveDownload(path string, body io.Reader) error { + if err := os.MkdirAll(filepath.Dir(path), setupDirPerm); err != nil { + return fmt.Errorf("create directory for %s: %w", path, err) + } + partial := path + partialSuffix + flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC + if err := writeFile(partial, flags, setupDownloadPerm, body); err != nil { + _ = os.Remove(partial) + return err + } + return os.Rename(partial, path) +} + +// writeFile copies body into the file at path, opened with flags. +func writeFile(path string, flags int, perm fs.FileMode, body io.Reader) (err error) { + f, err := os.OpenFile(path, flags, perm) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + defer func() { err = errors.Join(err, f.Close()) }() + if _, err := io.Copy(f, body); err != nil { + return fmt.Errorf("write %s: %w", path, err) + } + return nil +} + +func verifySHA256(path, expected string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return fmt.Errorf("hash %s: %w", path, err) + } + actual := hex.EncodeToString(h.Sum(nil)) + if !strings.EqualFold(actual, expected) { + return &checksumMismatchError{expected: expected, actual: actual} + } + return nil +} + +// extractTarGz unpacks a gzipped tar archive below dest. +func extractTarGz(archive, dest string) error { + f, err := os.Open(archive) + if err != nil { + return fmt.Errorf("open %s: %w", archive, err) + } + defer func() { _ = f.Close() }() + zr, err := gzip.NewReader(f) + if err != nil { + return fmt.Errorf("read gzip: %w", err) + } + defer func() { _ = zr.Close() }() + if err := os.MkdirAll(dest, setupDirPerm); err != nil { + return fmt.Errorf("create %s: %w", dest, err) + } + return extractTar(dest, tar.NewReader(zr)) +} + +func extractTar(dest string, tr *tar.Reader) error { + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("read tar: %w", err) + } + if err := extractEntry(dest, hdr, tr); err != nil { + return err + } + } +} + +func extractEntry(dest string, hdr *tar.Header, body io.Reader) error { + name := filepath.FromSlash(hdr.Name) + if !filepath.IsLocal(name) { + return fmt.Errorf("%w: %q", errEscapesDest, hdr.Name) + } + target := filepath.Join(dest, name) + if err := prepareParent(dest, target); err != nil { + return err + } + switch hdr.Typeflag { + case tar.TypeDir: + return makeDir(dest, target, hdr.FileInfo().Mode().Perm()) + case tar.TypeReg: + return extractRegular(target, hdr.FileInfo().Mode().Perm(), body) + case tar.TypeSymlink: + return extractSymlink(name, target, hdr.Linkname) + default: + return fmt.Errorf("%w: %q", errUnsupportedEntry, hdr.Name) + } +} + +// prepareParent creates the parent of target and checks that it resolves inside dest. +func prepareParent(dest, target string) error { + parent := filepath.Dir(target) + if err := os.MkdirAll(parent, setupDirPerm); err != nil { + return fmt.Errorf("create %s: %w", parent, err) + } + return ensureInside(dest, parent) +} + +func makeDir(dest, target string, perm fs.FileMode) error { + if err := os.MkdirAll(target, perm|setupOwnerDirBits); err != nil { + return fmt.Errorf("create %s: %w", target, err) + } + return ensureInside(dest, target) +} + +func extractRegular(target string, perm fs.FileMode, body io.Reader) error { + if err := removeExisting(target); err != nil { + return err + } + return writeFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm, body) +} + +// extractSymlink accepts only relative links that stay inside the archive. +func extractSymlink(name, target, linkname string) error { + if filepath.IsAbs(linkname) || !filepath.IsLocal(filepath.Join(filepath.Dir(name), linkname)) { + return fmt.Errorf("%w: symlink %q points to %q", errEscapesDest, name, linkname) + } + if err := removeExisting(target); err != nil { + return err + } + return os.Symlink(linkname, target) +} + +// removeExisting deletes whatever sits at path so that the next create never +// writes through a symlink. +func removeExisting(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("replace %s: %w", path, err) + } + return nil +} + +// ensureInside checks that path, with symlinks resolved, lies inside dest. +func ensureInside(dest, path string) error { + realDest, err := filepath.EvalSymlinks(dest) + if err != nil { + return fmt.Errorf("resolve %s: %w", dest, err) + } + realPath, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("resolve %s: %w", path, err) + } + rel, err := filepath.Rel(realDest, realPath) + if err != nil || !filepath.IsLocal(rel) { + return fmt.Errorf("%w: %s resolves outside %s", errEscapesDest, path, dest) + } + return nil +} + +// setupCLI implements `ci setup`. +func setupCLI(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + _, _ = fmt.Fprintln(stderr, "usage: ci setup [flags]") + return 2 + } + switch args[0] { + case "host": + return setupHostCLI(args[1:], stdout, stderr) + case "podman-linux": + return setupPodmanCLI(args[1:], stdout, stderr) + default: + _, _ = fmt.Fprintf(stderr, "unknown setup subcommand: %s\n", args[0]) + return 2 + } +} + +func setupHostCLI(args []string, stdout, stderr io.Writer) int { + flags := setupFlagSet("setup host", stderr) + label := flags.String("label", "", "suite label from suites.yaml") + runner := flags.String( + "runner", + "", + "runner label; required when the label runs on several runners", + ) + if err := flags.Parse(args); err != nil { + return setupFlagExit(err) + } + if !setupRequireFlags(stderr, flags, "label") { + return 2 + } + suite, err := setupFindSuite(*label, *runner) + if err != nil { + setupReport(stderr, err) + return 1 + } + if err := newHostSetup( + stdout, + stderr, + ).apply(context.Background(), suite.HostSetup); err != nil { + setupReport(stderr, err) + return 1 + } + return 0 +} + +func setupPodmanCLI(args []string, stdout, stderr io.Writer) int { + flags := setupFlagSet("setup podman-linux", stderr) + archive := flags.String( + "archive", + "", + "path of the podman-static archive; downloaded from --url when absent", + ) + url := flags.String("url", "", "URL to download the archive from") + sum := flags.String("sha256", "", "expected SHA-256 of the archive, hex encoded") + extractDir := flags.String("extract-dir", "", "directory to extract the archive into") + if err := flags.Parse(args); err != nil { + return setupFlagExit(err) + } + if !setupRequireFlags(stderr, flags, "archive", "url", "sha256", "extract-dir") { + return 2 + } + p := &podmanInstall{ + archive: *archive, + url: *url, + sha256Hex: *sum, + extractDir: *extractDir, + client: http.DefaultClient, + runner: execRunner{}, + lookPath: exec.LookPath, + stdout: stdout, + stderr: stderr, + } + if err := p.install(context.Background()); err != nil { + setupReport(stderr, err) + return 1 + } + return 0 +} + +func setupFlagSet(name string, stderr io.Writer) *flag.FlagSet { + flags := flag.NewFlagSet(name, flag.ContinueOnError) + flags.SetOutput(stderr) + return flags +} + +// setupFlagExit maps a flag parse error to an exit code; asking for help succeeds. +func setupFlagExit(err error) int { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 +} + +// setupRequireFlags reports the named flags that are empty and returns false when any is. +func setupRequireFlags(stderr io.Writer, flags *flag.FlagSet, names ...string) bool { + var missing []string + for _, name := range names { + if f := flags.Lookup(name); f == nil || f.Value.String() == "" { + missing = append(missing, "-"+name) + } + } + if len(missing) == 0 { + return true + } + _, _ = fmt.Fprintf(stderr, "missing required flags: %s\n", strings.Join(missing, ", ")) + return false +} + +func setupFindSuite(label, runner string) (Suite, error) { + m, err := LoadManifest() + if err != nil { + return Suite{}, err + } + return m.FindSuite(label, runner) +} + +// setupReport prints err to stderr. A checksum mismatch is also emitted as a +// GitHub Actions error annotation. +func setupReport(stderr io.Writer, err error) { + if _, ok := errors.AsType[*checksumMismatchError](err); ok { + _, _ = fmt.Fprintf(stderr, "::error::%s\n", err) + return + } + _, _ = fmt.Fprintf(stderr, "error: %s\n", err) +} diff --git a/hack/ci/setup_test.go b/hack/ci/setup_test.go new file mode 100644 index 000000000..7de059748 --- /dev/null +++ b/hack/ci/setup_test.go @@ -0,0 +1,590 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +const ( + setupWantDaemon = `{ + "insecure-registries": [ + "host.docker.internal:15500" + ], + "log-level": "warn" +} +` + setupWantHostsLine = "127.0.0.1 host.docker.internal\n" + setupEmptyDaemon = `{ + "insecure-registries": [ + "host.docker.internal:15500" + ] +} +` + setupSortedDaemon = `{ + "insecure-registries": [ + "alpha:5000", + "host.docker.internal:15500", + "zeta:5000" + ] +} +` + setupOtherKeysDaemon = `{ + "features": { + "buildkit": true + }, + "insecure-registries": [ + "host.docker.internal:15500" + ], + "max-concurrent-downloads": 3 +} +` + setupNullDoc = "null" + setupPodmanCmd = "podman-linux" + setupHostCmd = "host" + setupBogusFlag = "--bogus" + setupDockerDir = "docker" + setupLabelFlag = "--label" +) + +// setupFakeRunner records every command instead of running it. The first +// infoFails docker info calls fail. +type setupFakeRunner struct { + calls []setupCall + infoFails int + infoCalls int +} + +type setupCall struct { + cmd string + stdin string +} + +func (r *setupFakeRunner) Run(_ context.Context, spec commandSpec) error { + call := setupCall{cmd: strings.Join(append([]string{spec.Name}, spec.Args...), " ")} + if spec.Stdin != nil { + data, err := io.ReadAll(spec.Stdin) + if err != nil { + return err + } + call.stdin = string(data) + } + r.calls = append(r.calls, call) + if spec.Name != dockerBinary { + return nil + } + r.infoCalls++ + if r.infoCalls <= r.infoFails { + return errors.New("cannot connect to the docker daemon") + } + return nil +} + +func (r *setupFakeRunner) commands() []string { + out := make([]string, 0, len(r.calls)) + for _, call := range r.calls { + out = append(out, call.cmd) + } + return out +} + +// setupClock is a fake clock whose sleep advances time. +type setupClock struct { + t time.Time + sleeps int +} + +func (c *setupClock) now() time.Time { return c.t } + +func (c *setupClock) sleep(d time.Duration) { + c.t = c.t.Add(d) + c.sleeps++ +} + +// setupTarEntry describes one entry of a generated archive fixture. +type setupTarEntry struct { + name string + typeflag byte + mode int64 + body string + linkname string +} + +func buildSetupArchive(t testing.TB, entries []setupTarEntry) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, e := range entries { + hdr := &tar.Header{ + Name: e.name, + Typeflag: e.typeflag, + Mode: e.mode, + Linkname: e.linkname, + Size: int64(len(e.body)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write tar header: %v", err) + } + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatalf("write tar body: %v", err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +// podmanEntries is a podman-static like archive. Symlinks are left out on +// Windows, where creating them needs privileges. +func podmanEntries() []setupTarEntry { + entries := []setupTarEntry{ + {name: "podman-linux-amd64/", typeflag: tar.TypeDir, mode: 0o755}, + { + name: "podman-linux-amd64/usr/bin/podman", + typeflag: tar.TypeReg, + mode: 0o755, + body: "podman-binary", + }, + { + name: "podman-linux-amd64/etc/containers/policy.json", + typeflag: tar.TypeReg, + mode: 0o644, + body: "{}", + }, + } + if runtime.GOOS != osWindows { + entries = append(entries, setupTarEntry{ + name: "podman-linux-amd64/usr/bin/docker", + typeflag: tar.TypeSymlink, + mode: 0o777, + linkname: "podman", + }) + } + return entries +} + +func setupSHA(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +type setupSuite struct { + suite.Suite + hosts string + daemon string + archive string + extract string + runner *setupFakeRunner + clock *setupClock + start time.Time + h *hostSetup +} + +func (s *setupSuite) SetupTest() { + dir := s.T().TempDir() + s.hosts = filepath.Join(dir, "hosts") + s.daemon = filepath.Join(dir, setupDockerDir, "daemon.json") + s.archive = filepath.Join(dir, "podman-linux-amd64.tar.gz") + s.extract = filepath.Join(dir, "extract") + s.Require().NoError(os.WriteFile(s.hosts, []byte("127.0.0.1 localhost\n"), 0o600)) + s.runner = &setupFakeRunner{} + s.start = time.Unix(1_000_000, 0) + s.clock = &setupClock{t: s.start} + s.h = &hostSetup{ + runner: s.runner, + goos: osLinux, + hostsFile: s.hosts, + daemonFile: s.daemon, + stdout: io.Discard, + stderr: io.Discard, + now: s.clock.now, + sleep: s.clock.sleep, + } +} + +func (s *setupSuite) TestMergeInsecureRegistry() { + tests := []struct { + name string + existing string + want string + wantErr bool + }{ + {name: "empty input", existing: "", want: setupEmptyDaemon}, + {name: "whitespace only", existing: " \n\t", want: setupEmptyDaemon}, + {name: "null document", existing: setupNullDoc, want: setupEmptyDaemon}, + { + name: "missing key keeps other keys", + existing: `{"log-level":"warn"}`, + want: setupWantDaemon, + }, + { + name: "existing entries are sorted", + existing: `{"insecure-registries":["zeta:5000","alpha:5000"]}`, + want: setupSortedDaemon, + }, + { + name: "registry already present is not duplicated", + existing: `{"insecure-registries":["host.docker.internal:15500","host.docker.internal:15500"]}`, + want: setupEmptyDaemon, + }, + { + name: "other keys are preserved", + existing: `{"max-concurrent-downloads":3,"features":{"buildkit":true}}`, + want: setupOtherKeysDaemon, + }, + {name: "invalid JSON", existing: `{"log-level":`, wantErr: true}, + { + name: "insecure-registries not a list", + existing: `{"insecure-registries":"a:5000"}`, + wantErr: true, + }, + {name: "non-string entry", existing: `{"insecure-registries":[1]}`, wantErr: true}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + got, err := mergeInsecureRegistry([]byte(tt.existing), snapshotRegistry) + if tt.wantErr { + s.ErrorIs(err, errInvalidDaemon) + return + } + s.Require().NoError(err) + s.Equal(tt.want, string(got)) + }) + } +} + +func (s *setupSuite) TestRemoveDockerRunsCommandsInOrder() { + s.Require().NoError(s.h.apply(context.Background(), []string{actionRemoveDocker})) + s.Equal([]string{ + "sudo systemctl stop docker.socket docker.service", + "sudo apt-get remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin", + "sudo rm -rf /var/lib/docker /var/lib/containerd /usr/bin/docker /usr/bin/dockerd", + }, s.runner.commands()) +} + +func (s *setupSuite) TestInsecureRegistryFullSequence() { + s.Require().NoError(os.MkdirAll(filepath.Dir(s.daemon), 0o755)) + s.Require().NoError(os.WriteFile(s.daemon, []byte(`{"log-level":"warn"}`), 0o600)) + + s.Require().NoError(s.h.apply(context.Background(), []string{actionInsecureRegistry})) + + s.Equal([]string{ + "sudo tee -a " + s.hosts, + "sudo mkdir -p " + filepath.Dir(s.daemon), + "sudo tee " + s.daemon, + "sudo systemctl restart docker", + "docker info", + }, s.runner.commands()) + s.Equal(setupWantHostsLine, s.runner.calls[0].stdin) + s.Equal(setupWantDaemon, s.runner.calls[2].stdin) + s.Equal(0, s.clock.sleeps) +} + +func (s *setupSuite) TestHostsEntryAddedOnlyWhenMissing() { + tests := []struct { + name string + hosts string + wantTee bool + }{ + {name: "present", hosts: "127.0.0.1 localhost host.docker.internal\n"}, + {name: "commented out", hosts: "# 127.0.0.1 host.docker.internal\n", wantTee: true}, + {name: "missing", hosts: "127.0.0.1 localhost\n", wantTee: true}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + s.runner.calls = nil + s.Require().NoError(os.WriteFile(s.hosts, []byte(tt.hosts), 0o600)) + + s.Require().NoError(s.h.addHostsEntry(context.Background())) + + s.Equal(tt.wantTee, slices.Contains(s.runner.commands(), "sudo tee -a "+s.hosts)) + }) + } +} + +func (s *setupSuite) TestWaitForDockerTimesOut() { + s.runner.infoFails = 1000 + + err := s.h.apply(context.Background(), []string{actionInsecureRegistry}) + + s.ErrorIs(err, errDockerNotReady) + s.Equal(31, s.runner.infoCalls) + s.Equal(30*time.Second, s.clock.t.Sub(s.start)) +} + +func (s *setupSuite) TestWaitForDockerSucceedsAfterRetries() { + s.runner.infoFails = 3 + + s.Require().NoError(s.h.apply(context.Background(), []string{actionInsecureRegistry})) + + s.Equal(4, s.runner.infoCalls) + s.Equal(3, s.clock.sleeps) + s.Equal(3*time.Second, s.clock.t.Sub(s.start)) +} + +func (s *setupSuite) TestUnknownActionRunsNothing() { + err := s.h.apply(context.Background(), []string{actionRemoveDocker, "reboot-the-world"}) + + s.ErrorIs(err, errUnknownAction) + s.Empty(s.runner.calls) +} + +func (s *setupSuite) TestNonLinuxHostIsRejected() { + s.h.goos = "darwin" + + s.ErrorIs(s.h.apply(context.Background(), []string{actionRemoveDocker}), errNotLinux) + s.Empty(s.runner.calls) + s.NoError(s.h.apply(context.Background(), nil)) +} + +func (s *setupSuite) TestPodmanInstallExtractsAndCopies() { + data := buildSetupArchive(s.T(), podmanEntries()) + s.writeArchive(data) + p := s.newPodman("", setupSHA(data), true) + + s.Require().NoError(p.install(context.Background())) + + src := filepath.Join(s.extract, podmanPackageDir) + body, err := os.ReadFile(filepath.Join(src, "usr", "bin", "podman")) + s.Require().NoError(err) + s.Equal("podman-binary", string(body)) + if runtime.GOOS != osWindows { + info, err := os.Stat(filepath.Join(src, "usr", "bin", "podman")) + s.Require().NoError(err) + s.Equal(os.FileMode(0o755), info.Mode().Perm()) + target, err := os.Readlink(filepath.Join(src, "usr", "bin", setupDockerDir)) + s.Require().NoError(err) + s.Equal("podman", target) + } + s.Equal([]string{ + "sudo cp -r " + filepath.Join(src, "usr") + " " + filepath.Join(src, "etc") + " /", + }, s.runner.commands()) +} + +func (s *setupSuite) TestPodmanChecksumMismatchIsReported() { + data := buildSetupArchive(s.T(), podmanEntries()) + s.writeArchive(data) + wrong := strings.Repeat("0", sha256.Size*2) + + p := s.newPodman("", wrong, true) + err := p.install(context.Background()) + + s.EqualError(err, fmt.Sprintf( + "SHA256 mismatch for podman-static archive! Expected: %s, Got: %s", wrong, setupSHA(data))) + s.Empty(s.runner.calls) + s.NoDirExists(s.extract) +} + +func (s *setupSuite) TestPodmanCLIReportsChecksumMismatchAsAnnotation() { + data := buildSetupArchive(s.T(), podmanEntries()) + s.writeArchive(data) + wrong := strings.Repeat("f", sha256.Size*2) + var stderr bytes.Buffer + + code := setupCLI([]string{ + setupPodmanCmd, + "--archive", s.archive, + "--url", "http://127.0.0.1:1/unused", + "--sha256", wrong, + "--extract-dir", s.extract, + }, io.Discard, &stderr) + + s.Equal(1, code) + s.Contains( + stderr.String(), + "::error::SHA256 mismatch for podman-static archive! Expected: "+wrong, + ) +} + +func (s *setupSuite) TestPodmanDownloadsWhenArchiveAbsent() { + data := buildSetupArchive(s.T(), podmanEntries()) + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _, _ = w.Write(data) + })) + defer server.Close() + p := s.newPodman(server.URL+"/podman-linux-amd64.tar.gz", setupSHA(data), true) + p.client = server.Client() + + s.Require().NoError(p.install(context.Background())) + + s.Equal(int32(1), hits.Load()) + s.FileExists(s.archive) + s.NoFileExists(s.archive + partialSuffix) + s.FileExists(filepath.Join(s.extract, podmanPackageDir, "usr", "bin", "podman")) +} + +func (s *setupSuite) TestPodmanDownloadFailsOnErrorStatus() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "gone", http.StatusNotFound) + })) + defer server.Close() + p := s.newPodman(server.URL+"/missing.tar.gz", strings.Repeat("0", sha256.Size*2), true) + p.client = server.Client() + + err := p.install(context.Background()) + + s.ErrorContains(err, "404") + s.NoFileExists(s.archive) + s.Empty(s.runner.calls) +} + +func (s *setupSuite) TestPodmanRejectsEscapingEntries() { + tests := []struct { + name string + entries []setupTarEntry + }{ + { + name: "parent traversal", + entries: []setupTarEntry{ + {name: "../escape.txt", typeflag: tar.TypeReg, mode: 0o644, body: "x"}, + }, + }, + { + name: "traversal below the package directory", + entries: []setupTarEntry{ + { + name: "podman-linux-amd64/../../escape.txt", + typeflag: tar.TypeReg, + mode: 0o644, + body: "x", + }, + }, + }, + { + name: "absolute name", + entries: []setupTarEntry{ + { + name: filepath.Join(string(filepath.Separator), "escape.txt"), + typeflag: tar.TypeReg, + mode: 0o644, + }, + }, + }, + { + name: "symlink pointing outside", + entries: []setupTarEntry{{ + name: "podman-linux-amd64/usr/bin/evil", + typeflag: tar.TypeSymlink, + mode: 0o777, + linkname: "../../../../escape.txt", + }}, + }, + } + for _, tt := range tests { + s.Run(tt.name, func() { + if runtime.GOOS == osWindows && strings.Contains(tt.name, "symlink") { + s.T().Skip("creating symlinks needs privileges on Windows") + } + s.runner.calls = nil + data := buildSetupArchive(s.T(), tt.entries) + s.writeArchive(data) + p := s.newPodman("", setupSHA(data), true) + + err := p.install(context.Background()) + + s.ErrorIs(err, errEscapesDest) + s.NoFileExists(filepath.Join(filepath.Dir(s.extract), "escape.txt")) + s.Empty(s.runner.calls) + }) + } +} + +func (s *setupSuite) TestUIDMapInstalledOnlyWhenMissing() { + data := buildSetupArchive(s.T(), podmanEntries()) + s.writeArchive(data) + aptInstall := "sudo apt-get install -y uidmap" + + s.Run("present", func() { + s.runner.calls = nil + s.Require().NoError(s.newPodman("", setupSHA(data), true).install(context.Background())) + s.NotContains(s.runner.commands(), aptInstall) + }) + s.Run("missing", func() { + s.runner.calls = nil + s.Require().NoError(s.newPodman("", setupSHA(data), false).install(context.Background())) + s.Equal([]string{ + "sudo cp -r " + filepath.Join(s.extract, podmanPackageDir, "usr") + " " + + filepath.Join(s.extract, podmanPackageDir, "etc") + " /", + aptInstall, + }, s.runner.commands()) + }) +} + +func (s *setupSuite) TestUsageErrorsExitTwo() { + tests := [][]string{ + nil, + {"unknown"}, + {setupHostCmd}, + {setupHostCmd, setupLabelFlag, "context", setupBogusFlag}, + {setupPodmanCmd}, + {setupPodmanCmd, "--archive", s.archive}, + } + for _, args := range tests { + s.Equal(2, setupCLI(args, io.Discard, io.Discard), "args %v", args) + } +} + +func (s *setupSuite) TestHostCLIRunsNothingForEmptyHostSetup() { + s.Equal(0, setupCLI([]string{setupHostCmd, setupLabelFlag, "context"}, io.Discard, io.Discard)) +} + +func (s *setupSuite) TestHostCLIFailsForUnknownLabel() { + s.Equal( + 1, + setupCLI([]string{setupHostCmd, setupLabelFlag, "no-such-suite"}, io.Discard, io.Discard), + ) +} + +func (s *setupSuite) newPodman(url, sum string, uidmapPresent bool) *podmanInstall { + return &podmanInstall{ + archive: s.archive, + url: url, + sha256Hex: sum, + extractDir: s.extract, + client: http.DefaultClient, + runner: s.runner, + lookPath: func(string) (string, error) { + if uidmapPresent { + return "/usr/bin/newuidmap", nil + } + return "", exec.ErrNotFound + }, + stdout: io.Discard, + stderr: io.Discard, + } +} + +func (s *setupSuite) writeArchive(data []byte) { + s.Require().NoError(os.WriteFile(s.archive, data, 0o600)) +} + +func TestCISetupSuite(t *testing.T) { + t.Parallel() + suite.Run(t, new(setupSuite)) +} diff --git a/hack/ci/success.go b/hack/ci/success.go new file mode 100644 index 000000000..074f35b35 --- /dev/null +++ b/hack/ci/success.go @@ -0,0 +1,179 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "maps" + "os" + "slices" + "strings" +) + +const ( + defaultNeedsEnv = "NEEDS_JSON" + changesJob = "changes" + goOutput = "go" + resultSuccess = "success" + resultSkipped = "skipped" +) + +var ( + errNeedsEmpty = errors.New("needs variable is empty") + errNeedsInvalid = errors.New("needs variable is not valid JSON") + errNeedsNoJobs = errors.New("needs variable has no jobs") + errBadGoOutput = errors.New("changes job go output is not true or false") + errUnexpectedArgs = errors.New("unexpected arguments") +) + +// neededJob is one entry of the toJSON(needs) object. +type neededJob struct { + Result string `json:"result"` + Outputs map[string]string `json:"outputs"` +} + +// successOptions holds the parsed flags of `ci success`. +type successOptions struct { + needsEnv string + alwaysRun map[string]bool +} + +// successCheck folds the results of the needed jobs into one required status. +type successCheck struct { + getenv func(string) string + stdout io.Writer + stderr io.Writer +} + +func successCLI(args []string, stdout, stderr io.Writer) int { + check := successCheck{getenv: os.Getenv, stdout: stdout, stderr: stderr} + return check.run(args) +} + +func (c successCheck) run(args []string) int { + opts, err := parseSuccessArgs(args, c.stderr) + if err != nil { + return 2 + } + needs, err := c.loadNeeds(opts.needsEnv) + if err != nil { + _, _ = fmt.Fprintf(c.stdout, "::error::%v\n", err) + return 1 + } + return c.report(needs, opts) +} + +func parseSuccessArgs(args []string, output io.Writer) (successOptions, error) { + fs := flag.NewFlagSet("success", flag.ContinueOnError) + fs.SetOutput(output) + needsEnv := fs.String( + "needs-env", + defaultNeedsEnv, + "environment variable holding toJSON(needs)", + ) + alwaysRun := fs.String("always-run", "", "comma-separated jobs that must never be skipped") + if err := fs.Parse(args); err != nil { + return successOptions{}, fmt.Errorf("parse flags: %w", err) + } + if fs.NArg() > 0 { + _, _ = fmt.Fprintf(output, "%v: %s\n", errUnexpectedArgs, strings.Join(fs.Args(), " ")) + return successOptions{}, errUnexpectedArgs + } + return successOptions{needsEnv: *needsEnv, alwaysRun: splitJobList(*alwaysRun)}, nil +} + +// splitJobList turns "a, b,,c" into the set {a, b, c}. +func splitJobList(list string) map[string]bool { + jobs := make(map[string]bool) + for job := range strings.SplitSeq(list, ",") { + if job = strings.TrimSpace(job); job != "" { + jobs[job] = true + } + } + return jobs +} + +// loadNeeds reads and decodes the toJSON(needs) object held in the named variable. +func (c successCheck) loadNeeds(name string) (map[string]neededJob, error) { + raw := c.getenv(name) + if raw == "" { + return nil, fmt.Errorf("%w: %s", errNeedsEmpty, name) + } + var needs map[string]neededJob + if err := json.Unmarshal([]byte(raw), &needs); err != nil { + return nil, fmt.Errorf("%w: %s: %w", errNeedsInvalid, name, err) + } + if len(needs) == 0 { + return nil, fmt.Errorf("%w: %s", errNeedsNoJobs, name) + } + return needs, nil +} + +// report prints one ::error:: line per bad job, in sorted job-id order, and returns the exit code. +func (c successCheck) report(needs map[string]neededJob, opts successOptions) int { + failures := 0 + // Without a changes job nothing proves a skip was legitimate, so skips are rejected. + goChanged := true + if changes, ok := needs[changesJob]; ok { + changed, err := goChangedFrom(changes) + if err != nil { + _, _ = fmt.Fprintf(c.stdout, "::error::%v\n", err) + failures++ + } else { + goChanged = changed + } + } else { + _, _ = fmt.Fprintf(c.stdout, "::error::%s job missing from %s\n", changesJob, opts.needsEnv) + failures++ + } + + jobs := slices.Sorted(maps.Keys(needs)) + for _, job := range jobs { + if problem := jobProblem(job, needs[job], opts.alwaysRun[job], goChanged); problem != "" { + _, _ = fmt.Fprintf(c.stdout, "::error::%s\n", problem) + failures++ + } + } + + if failures > 0 { + return 1 + } + _, _ = fmt.Fprintf( + c.stdout, + "all %d required jobs passed or were legitimately skipped\n", + len(jobs), + ) + return 0 +} + +// jobProblem returns the error message for a needed job, or "" when the job is acceptable. +func jobProblem(job string, res neededJob, alwaysRun, goChanged bool) string { + switch res.Result { + case resultSuccess: + return "" + case resultSkipped: + if alwaysRun || goChanged { + return job + " ended with result=skipped unexpectedly" + } + return "" + default: + return job + " ended with result=" + res.Result + } +} + +// goChangedFrom reads the go output of the changes job, trusted only after it succeeded. +func goChangedFrom(changes neededJob) (bool, error) { + if changes.Result != resultSuccess { + return false, nil + } + switch value := changes.Outputs[goOutput]; value { + case "true": + return true, nil + case "false": + return false, nil + default: + return false, fmt.Errorf("%w: %q", errBadGoOutput, value) + } +} diff --git a/hack/ci/success_test.go b/hack/ci/success_test.go new file mode 100644 index 000000000..8b1be4d9b --- /dev/null +++ b/hack/ci/success_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +const ( + goTrue = `{"result":"success","outputs":{"go":"true"}}` + goFalse = `{"result":"success","outputs":{"go":"false"}}` + okJob = `{"result":"success","outputs":{}}` + skippedJob = `{"result":"skipped","outputs":{}}` + failedJob = `{"result":"failure","outputs":{}}` + cancelledJob = `{"result":"cancelled","outputs":{}}` +) + +// successResult is the outcome of one `ci success` invocation. +type successResult struct { + code int + stdout string + stderr string +} + +type successSuite struct { + suite.Suite +} + +func TestSuccessSuite(t *testing.T) { + t.Parallel() + suite.Run(t, new(successSuite)) +} + +// needsObject builds a toJSON(needs) object from job id and raw JSON pairs, in the given order. +func needsObject(jobs ...string) string { + return "{" + strings.Join(jobs, ",") + "}" +} + +func entry(name, body string) string { + return `"` + name + `":` + body +} + +func (s *successSuite) TestAllSuccessPasses() { + needs := needsObject(entry("changes", goTrue), entry("lint", okJob), entry("unit-tests", okJob)) + + res := s.runSuccess(needs) + + s.Equal(0, res.code) + s.Equal("all 3 required jobs passed or were legitimately skipped\n", res.stdout) + s.Empty(res.stderr) +} + +func (s *successSuite) TestSkippedWithoutGoChangesIsAllowed() { + needs := needsObject( + entry("changes", goFalse), + entry("unit-tests", skippedJob), + entry("lint", okJob), + ) + + res := s.runSuccess(needs) + + s.Equal(0, res.code) + s.Equal("all 3 required jobs passed or were legitimately skipped\n", res.stdout) +} + +func (s *successSuite) TestSkippedWithGoChangesIsRejected() { + needs := needsObject(entry("changes", goTrue), entry("unit-tests", skippedJob)) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::unit-tests ended with result=skipped unexpectedly\n") + s.NotContains(res.stdout, "all ") +} + +func (s *successSuite) TestAlwaysRunSkippedIsRejectedWithoutGoChanges() { + needs := needsObject( + entry("changes", goFalse), + entry("lint", skippedJob), + entry("precommit", skippedJob), + ) + + res := s.runSuccess(needs, "--always-run", " lint, precommit ,") + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::lint ended with result=skipped unexpectedly\n") + s.Contains(res.stdout, "::error::precommit ended with result=skipped unexpectedly\n") +} + +func (s *successSuite) TestAlwaysRunListIsPerJob() { + needs := needsObject( + entry("changes", goFalse), + entry("lint", skippedJob), + entry("unit-tests", skippedJob), + ) + + res := s.runSuccess(needs, "--always-run", "lint") + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::lint ended with result=skipped unexpectedly\n") + s.NotContains(res.stdout, "unit-tests") +} + +func (s *successSuite) TestFailureAndCancelledAreRejected() { + needs := needsObject( + entry("changes", goTrue), + entry("lint", failedJob), + entry("unit-tests", cancelledJob), + entry("build-cli", `{"result":"","outputs":{}}`), + ) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::build-cli ended with result=\n") + s.Contains(res.stdout, "::error::lint ended with result=failure\n") + s.Contains(res.stdout, "::error::unit-tests ended with result=cancelled\n") + s.NotContains(res.stdout, "all ") +} + +func (s *successSuite) TestMissingChangesJobFails() { + needs := needsObject(entry("lint", okJob)) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::changes job missing from NEEDS_JSON\n") +} + +func (s *successSuite) TestMissingChangesRejectsSkippedJobs() { + needs := needsObject(entry("lint", okJob), entry("unit-tests", skippedJob)) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::changes job missing from NEEDS_JSON\n") + s.Contains(res.stdout, "::error::unit-tests ended with result=skipped unexpectedly\n") +} + +func (s *successSuite) TestChangesGoOutputMustBeTrueOrFalse() { + tests := []struct { + name string + changes string + }{ + {name: "missing output", changes: `{"result":"success","outputs":{}}`}, + {name: "empty output", changes: `{"result":"success","outputs":{"go":""}}`}, + {name: "yes", changes: `{"result":"success","outputs":{"go":"yes"}}`}, + {name: "uppercase true", changes: `{"result":"success","outputs":{"go":"TRUE"}}`}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + needs := needsObject(entry("changes", tt.changes), entry("unit-tests", skippedJob)) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::changes job go output is not true or false") + s.Contains(res.stdout, "::error::unit-tests ended with result=skipped unexpectedly\n") + }) + } +} + +func (s *successSuite) TestFailedChangesJobSkipsOutputCheck() { + needs := needsObject(entry("changes", failedJob), entry("unit-tests", skippedJob)) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Equal("::error::changes ended with result=failure\n", res.stdout) +} + +func (s *successSuite) TestEmptyOrInvalidNeedsFails() { + tests := []struct { + name string + needs string + message string + }{ + { + name: "empty variable", + needs: "", + message: "::error::needs variable is empty: NEEDS_JSON\n", + }, + { + name: "invalid JSON", + needs: "{not json", + message: "::error::needs variable is not valid JSON: NEEDS_JSON", + }, + { + name: "empty object", + needs: "{}", + message: "::error::needs variable has no jobs: NEEDS_JSON\n", + }, + {name: "null", needs: "null", message: "::error::needs variable has no jobs: NEEDS_JSON\n"}, + } + for _, tt := range tests { + s.Run(tt.name, func() { + res := s.runSuccess(tt.needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, tt.message) + s.NotContains(res.stdout, "all ") + }) + } +} + +func (s *successSuite) TestUnsetNeedsVariableNameFails() { + needs := needsObject(entry("changes", goTrue), entry("lint", okJob)) + + res := s.runSuccess(needs, "--needs-env", "MISSING_VAR") + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::needs variable is empty: MISSING_VAR\n") +} + +func (s *successSuite) TestNeedsEnvFlagSelectsVariable() { + var stdout, stderr bytes.Buffer + check := successCheck{ + getenv: func(name string) string { + if name == "CUSTOM_NEEDS" { + return needsObject(entry("changes", goTrue), entry("lint", okJob)) + } + return "" + }, + stdout: &stdout, + stderr: &stderr, + } + + code := check.run([]string{"--needs-env", "CUSTOM_NEEDS"}) + + s.Equal(0, code) + s.Equal("all 2 required jobs passed or were legitimately skipped\n", stdout.String()) +} + +func (s *successSuite) TestOutputIsSortedByJobID() { + needs := needsObject( + entry("zeta", failedJob), + entry("changes", goTrue), + entry("mid", cancelledJob), + entry("alpha", skippedJob), + ) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Equal( + "::error::alpha ended with result=skipped unexpectedly\n"+ + "::error::mid ended with result=cancelled\n"+ + "::error::zeta ended with result=failure\n", + res.stdout, + ) +} + +func (s *successSuite) TestFailureDoesNotHideOtherBadJobs() { + needs := needsObject( + entry("changes", goTrue), + entry("alpha", failedJob), + entry("beta", okJob), + entry("zeta", skippedJob), + ) + + res := s.runSuccess(needs) + + s.Equal(1, res.code) + s.Contains(res.stdout, "::error::alpha ended with result=failure\n") + s.Contains(res.stdout, "::error::zeta ended with result=skipped unexpectedly\n") + s.NotContains(res.stdout, "beta") +} + +func (s *successSuite) TestUnexpectedArgumentsIsUsageError() { + needs := needsObject(entry("changes", goTrue)) + + res := s.runSuccess(needs, "extra") + + s.Equal(2, res.code) + s.Contains(res.stderr, "unexpected arguments: extra") + s.Empty(res.stdout) +} + +func (s *successSuite) TestUnknownFlagIsUsageError() { + needs := needsObject(entry("changes", goTrue)) + + res := s.runSuccess(needs, "--bogus") + + s.Equal(2, res.code) + s.Contains(res.stderr, "flag provided but not defined") + s.Empty(res.stdout) +} + +// runSuccess runs `ci success` with args, serving needs from the NEEDS_JSON variable only. +func (s *successSuite) runSuccess(needs string, args ...string) successResult { + var stdout, stderr bytes.Buffer + check := successCheck{ + getenv: func(name string) string { + if name == defaultNeedsEnv { + return needs + } + return "" + }, + stdout: &stdout, + stderr: &stderr, + } + return successResult{code: check.run(args), stdout: stdout.String(), stderr: stderr.String()} +} diff --git a/hack/ci/suites.yaml b/hack/ci/suites.yaml new file mode 100644 index 000000000..337b874f9 --- /dev/null +++ b/hack/ci/suites.yaml @@ -0,0 +1,259 @@ +# Test tier manifest for the PR workflow (.github/workflows/pr-ci.yml). +# +# This file is the single place that defines what tests exist, where they run +# and how they run. The workflow contains no per-suite conditions; it asks +# `hack/ci matrix` for its job matrices and `hack/ci unit|e2e|setup` for the +# behaviour. hack/ci/manifest_test.go fails when a Ginkgo label or a Go test +# package is not assigned to a tier here. +# +# To add a test: +# - Go unit test in an ordinary package: nothing to do, it is discovered. +# - Go test in a package under test/ or e2e/ that must run as a unit test: +# add the package to a `unit.steps` entry. +# - Ginkgo spec: add its label to `e2e.suites` (or to `e2e.ungated` with a +# reason when another workflow runs it). + +unit: + # One job per runner; every runner executes the steps whose `os` matches. + runners: + - { runner: ubuntu-latest, timeout_minutes: 30 } + - { runner: macos-latest, timeout_minutes: 45 } + - { runner: windows-latest, timeout_minutes: 30 } + + # Steps run in order on each runner. `os` uses GOOS values; empty means all. + steps: + - name: packages + description: every package outside test/ and e2e/ directories, with the race detector + os: [linux, darwin] + discover: true + args: [-race, -short] + + - name: agent-bootstrap-busybox + description: agent bootstrap script under BusyBox sh + os: [linux] + host_shell: busybox + packages: [./pkg/inject] + run: '^(TestBootstrapSuite|TestInjectScript_RealShell.*)$' + args: [-race, -count=1, -timeout=90s] + + - name: e2e-runtime-helpers + description: E2E runtime helper packages + packages: [./e2e/framework/runenv/..., ./e2e/framework/dockerwsl, ./e2e/framework/ginkgoreport, ./e2e/cmd/devsy-ci] + args: [-race, -count=1, -timeout=5m] + + - name: e2e-framework + description: shared E2E framework helpers + os: [linux, darwin] + packages: [./e2e/framework] + args: [-race, -short, -count=1, -timeout=5m] + + - name: compose-uid-matrix + description: Compose UID expectation matrix + os: [linux, darwin] + packages: [./e2e/tests/up-docker-compose] + run: '^TestExpectedUIDMapping$' + require_tests: [TestExpectedUIDMapping] + args: [-race, -short, -count=1, -v] + + - name: workspace-fixture-helpers + description: workspace fixture helpers + os: [linux, darwin] + packages: [./e2e/tests/up] + args: [-race, -short, -count=1, -timeout=90s] + + - name: registry-fixture + description: Linux registry fixture + os: [linux] + packages: [./e2e/framework/registryfixture] + args: [-race, -count=1, -timeout=90s] + + - name: github-api-oracle + description: Linux GitHub API oracle fixture + os: [linux] + packages: [./e2e/framework/ghfixture] + args: [-race, -count=1, -timeout=90s] + + - name: windows-transport + description: native Windows Docker transport + os: [windows] + packages: [./pkg/docker] + run: '^(TestInvocationSuite|TestProviderDaemonOverrides.*|TestStdioTransportSuite|TestLaunch.*|TestEnvBlock.*)$' + args: [-race, -count=1, -v, -timeout=5m] + + - name: ci-tooling-windows + description: Go CI helpers that also run on Windows + os: [windows] + packages: [./hack/ci, ./hack/microsandbox, ./hack/podman_ci, ./hack/provider_update_ci] + args: [-race, -count=1, -timeout=10m] + + # Packages that contain tests but are deliberately not unit tests. + excluded: + - package: ./e2e + reason: Ginkgo E2E suite, compiled to e2e.test and run by the e2e tier + - package: ./e2e/tests/... + reason: Ginkgo specs run by the e2e tier; helper tests are listed in unit.steps + +e2e: + # Defaults per group. A suite inherits these and overrides what differs. + groups: + unprivileged: + description: runs as the runner user and needs no container runtime privileges + runner: ubuntu-latest + privileged: false + test_timeout: 1500s + job_timeout_minutes: 45 + flake_attempts: 1 + integration: + description: runs the e2e binary with sudo on Linux + runner: ubuntu-latest + privileged: true + test_timeout: 1500s + job_timeout_minutes: 45 + flake_attempts: 1 + microsandbox: + description: runs through the MicroSandbox CI helper, which owns the report checks + runner: ubuntu-latest + privileged: false + test_timeout: 1200s + job_timeout_minutes: 25 + flake_attempts: 1 + + # Labels executed by another workflow or intentionally not run in PR CI. + ungated: + - label: gh-forwarding + reason: sub-label that always accompanies the ssh suite label + - label: up-provider-podman-machine-bind + reason: darwin-only spec that needs a local Podman machine; run manually on macOS + - label: gpg + reason: sub-label that always accompanies a gated suite label + - label: microsandbox-builtin + reason: sub-label that always accompanies a microsandbox suite label + - label: microsandbox-external + reason: sub-label that always accompanies a microsandbox suite label + + - label: depends-on + reason: sub-label that always accompanies the up-features suite label + - label: legacy-id + reason: sub-label that always accompanies the up-features suite label + - label: lockfile + reason: sub-label that always accompanies the up-features suite label + - label: mixed + reason: sub-label that always accompanies the up-features suite label + - label: override + reason: sub-label that always accompanies the up-features suite label + - label: secret-option + reason: sub-label that always accompanies the up-features suite label + - label: version-aware + reason: sub-label that always accompanies the up-features suite label + - label: up-provider-kubernetes-dockerless-path + reason: sub-label that always accompanies the up-provider-kubernetes suite label + - label: up-provider-kubernetes-ignore + reason: sub-label that always accompanies the up-provider-kubernetes suite label + + suites: + # --- unprivileged --- + - { group: unprivileged, label: context } + - { group: unprivileged, label: extends } + - { group: unprivileged, label: outdated } + - { group: unprivileged, label: machine } + - { group: unprivileged, label: self-update } + - { group: unprivileged, label: upgrade } + + # --- integration: Linux --- + - { group: integration, label: docker-context, test_timeout: 300s, job_timeout_minutes: 10 } + - { group: integration, label: ide } + - { group: integration, label: machine-provider } + - { group: integration, label: integration } + - { group: integration, label: tunnel, flake_attempts: 2 } + - { group: integration, label: mcp } + - { group: integration, label: provider, free_disk_space: true, install_kind: true, flake_attempts: 2 } + - { group: integration, label: ssh, fixtures: [ghfixture-linux] } + - { group: integration, label: ssh-agent-forward } + - { group: integration, label: ssh-ports-attributes } + - { group: integration, label: ssh-tunnel-mode } + - { group: integration, label: task-cancel } + - { group: integration, label: ssh-credentials-server-race } + - { group: integration, label: build, install_kind: true, requires_secret: true } + - group: integration + label: docker-install + host_setup: [remove-docker] + post_checks: [docker-installed] + - { group: integration, label: exec } + - { group: integration, label: ci } + - { group: integration, label: feature } + - { group: integration, label: extends-up } + - { group: integration, label: down } + - { group: integration, label: rename } + - { group: integration, label: delivery } + - { group: integration, label: logs } + - { group: integration, label: templates } + - { group: integration, label: config-apply } + - { group: integration, label: run-user-commands } + - { group: integration, label: config-read } + - { group: integration, label: up-workspaces, free_disk_space: true, install_kind: true } + - { group: integration, label: up-behaviors } + - { group: integration, label: up-handle-errors, free_disk_space: true, install_kind: true } + - { group: integration, label: up-private-token, requires_secret: true } + - { group: integration, label: up-provider-kubernetes, free_disk_space: true, install_kind: true } + - { group: integration, label: up-provider-kubernetes-restricted-scc, free_disk_space: true, install_kind: true } + - { group: integration, label: up-provider-docker, install_kind: true } + - { group: integration, label: snapshot, host_setup: [insecure-registry] } + - group: integration + label: up-docker-compose + # Runs first, as the existing non-root runner user, before the sudo run. + focused_checks: + - name: compose-uid-nonroot + focus: 'implements updateRemoteUserUID with root container user$' + expect_leaf: implements updateRemoteUserUID with root container user + require_uid: 1001 + require_gid: 1001 + - { group: integration, label: up-features, install_kind: true } + + # --- integration: Podman on Linux --- + - { group: integration, label: up-provider-podman-rootless-basic, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootless-exec, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootless-lifecycle, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootless-config, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootless-features, runner: ubuntu-24.04, install_podman: rootless, privileged: false, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootful-basic, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootful-lifecycle, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootful-lifecycle-2, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootful-config, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + - { group: integration, label: up-provider-podman-rootful-features, runner: ubuntu-24.04, install_podman: rootful, test_timeout: 600s, job_timeout_minutes: 20 } + + # --- integration: Windows (Docker API served by Podman) --- + - { group: integration, label: provider, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: up-docker-wsl, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: up-features, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: ssh-proxy-command, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: ssh-tunnel-mode, runner: windows-latest, install_kind: true, test_timeout: 900s, job_timeout_minutes: 20 } + - { group: integration, label: task-cancel, runner: windows-latest, install_podman: rootful, test_timeout: 900s, job_timeout_minutes: 20 } + + # --- integration: Windows with a Docker Engine in WSL2, reached over SSH --- + # The workflow provisions the daemon with `devsy-ci docker-wsl setup`; the + # report must show exactly expected_spec_count specs passing first try. + - group: integration + label: docker-internal-buildkit + runner: windows-latest + runtime: docker-wsl + expected_spec_count: 6 + test_timeout: 900s + job_timeout_minutes: 25 + + # --- microsandbox --- + - { group: microsandbox, label: up-provider-microsandbox, expected_spec_count: 2, performance_capture: true } + - { group: microsandbox, label: up-provider-microsandbox-images, expected_spec_count: 6 } + - { group: microsandbox, label: up-provider-microsandbox-mounts, expected_spec_count: 10 } + - { group: microsandbox, label: up-provider-microsandbox-resources, expected_spec_count: 12 } + - { group: microsandbox, label: up-provider-microsandbox-storage, expected_spec_count: 6 } + - { group: microsandbox, label: up-provider-microsandbox-ephemeral, expected_spec_count: 6 } + - { group: microsandbox, label: up-provider-microsandbox-egress, expected_spec_count: 4 } + - { group: microsandbox, label: up-provider-microsandbox-prebuilds, expected_spec_count: 4 } + - { group: microsandbox, label: up-provider-microsandbox-dockerless, expected_spec_count: 8 } + - { group: microsandbox, label: up-provider-microsandbox-recreate, expected_spec_count: 4 } + - { group: microsandbox, label: up-provider-microsandbox-managed-recreate, expected_spec_count: 4 } + - { group: microsandbox, label: up-provider-microsandbox-logs-cancel, expected_spec_count: 6 } + - { group: microsandbox, label: up-provider-microsandbox-compatibility, expected_spec_count: 2 } + - { group: microsandbox, label: up-provider-microsandbox-registry-consumer, expected_spec_count: 3, provider_mode: external } + - { group: microsandbox, label: up-provider-microsandbox-remote-agent-binaries, expected_spec_count: 1, provider_mode: external } + - { group: microsandbox, label: up-provider-microsandbox-source-migration, expected_spec_count: 2, provider_mode: external } diff --git a/hack/ci/unit.go b/hack/ci/unit.go new file mode 100644 index 000000000..dc1e3435a --- /dev/null +++ b/hack/ci/unit.go @@ -0,0 +1,386 @@ +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +const ( + goListFormat = "{{.ImportPath}}|{{.Dir}}" + gitHubActionsEnv = "GITHUB_ACTIONS" + hostShellBusybox = "busybox" + shellPath = "/bin/sh" + busyboxPath = "/bin/busybox" + cmdGo = "go" + cmdSudo = "sudo" + cmdReadlink = "readlink" + cmdLn = "ln" + unitFlagSetName = "unit" + unitUsageExitCode = 2 + unitFailureExitCode = 1 + unitSuccessExitCode = 0 + unitTestPatternFormat = "^(%s)$" +) + +var ( + errNoUnitSteps = errors.New("no unit steps selected") + errNoUnitTestPackages = errors.New("no unit-test packages found") + errOutsideRepository = errors.New("package directory is outside the repository") + errMalformedGoList = errors.New("unexpected go list line") + errRequiredTestMissing = errors.New("required test not listed by go test -list") + errUnknownHostShell = errors.New("unknown host_shell") + errHostShellNeedsLinux = errors.New("host_shell needs Linux") + errShellNotSymlink = errors.New("shell is not a symlink") + errUnexpectedUnitArgs = errors.New("unexpected positional arguments") + excludedPackagePattern = regexp.MustCompile(`(^|/)(test|e2e)(/|$)`) +) + +// unitFlags holds the parsed flags of `ci unit`. +type unitFlags struct { + goos string + step string + coverprofile string + list bool +} + +// unitRunner runs the selected unit steps through an injected command runner. +type unitRunner struct { + runner commandRunner + goos string + root string + env func(string) string + coverprofile string + stdout io.Writer + stderr io.Writer +} + +func unitCLI(args []string, stdout, stderr io.Writer) int { + f, err := parseUnitFlags(args, stderr) + if errors.Is(err, flag.ErrHelp) { + return unitSuccessExitCode + } + if err != nil { + return unitUsageExitCode + } + m, err := LoadManifest() + if err != nil { + return reportUnitError(stderr, err) + } + steps, err := selectUnitSteps(m.Unit.Steps, f.goos, f.step) + if err != nil { + return reportUnitError(stderr, err) + } + root, err := os.Getwd() + if err != nil { + return reportUnitError(stderr, err) + } + u := unitRunner{ + runner: execRunner{}, + goos: f.goos, + root: root, + env: os.Getenv, + coverprofile: f.coverprofile, + stdout: stdout, + stderr: stderr, + } + return u.runSelected(steps, f.list) +} + +func reportUnitError(stderr io.Writer, err error) int { + _, _ = fmt.Fprintln(stderr, err) + return unitFailureExitCode +} + +func parseUnitFlags(args []string, stderr io.Writer) (unitFlags, error) { + var f unitFlags + fs := flag.NewFlagSet(unitFlagSetName, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.StringVar(&f.goos, "os", runtime.GOOS, "GOOS whose steps are selected") + fs.StringVar(&f.step, "step", "", "run only the step with this name") + fs.StringVar(&f.coverprofile, "coverprofile", "", "coverage profile for discovered packages") + fs.BoolVar(&f.list, "list", false, "print the selected step names and exit") + if err := fs.Parse(args); err != nil { + return unitFlags{}, err + } + if fs.NArg() != 0 { + _, _ = fmt.Fprintf(stderr, "unit: unexpected arguments: %s\n", strings.Join(fs.Args(), " ")) + return unitFlags{}, errUnexpectedUnitArgs + } + return f, nil +} + +// selectUnitSteps returns the steps that run on goos, narrowed to name when +// name is not empty. Manifest order is kept. +func selectUnitSteps(steps []UnitStep, goos, name string) ([]UnitStep, error) { + var selected []UnitStep + for _, step := range steps { + if step.RunsOn(goos) && (name == "" || step.Name == name) { + selected = append(selected, step) + } + } + if len(selected) == 0 { + return nil, fmt.Errorf("%w: os %s, step %q", errNoUnitSteps, goos, name) + } + return selected, nil +} + +func (u unitRunner) runSelected(steps []UnitStep, list bool) int { + if list { + for _, step := range steps { + _, _ = fmt.Fprintln(u.stdout, step.Name) + } + return unitSuccessExitCode + } + return u.runAll(steps) +} + +// runAll runs every step, even after a failure, and reports each failed step. +func (u unitRunner) runAll(steps []UnitStep) int { + var failures []error + for _, step := range steps { + if err := u.runStep(step); err != nil { + failures = append(failures, fmt.Errorf("%s: %w", step.Name, err)) + } + } + for _, err := range failures { + _, _ = fmt.Fprintf(u.stderr, "FAILED: %v\n", err) + } + if len(failures) > 0 { + return unitFailureExitCode + } + return unitSuccessExitCode +} + +func (u unitRunner) runStep(step UnitStep) error { + u.begin(step) + defer u.end() + if err := u.checkHostShell(step); err != nil { + return err + } + packages, err := u.packagesFor(step) + if err != nil { + return err + } + if err = u.checkRequiredTests(step, packages); err != nil { + return err + } + return u.withHostShell(step, func() error { return u.goTest(step, packages) }) +} + +func (u unitRunner) inActions() bool { + return u.env(gitHubActionsEnv) == "true" +} + +func (u unitRunner) begin(step UnitStep) { + if u.inActions() { + _, _ = fmt.Fprintf(u.stdout, "::group::%s\n", step.Name) + return + } + _, _ = fmt.Fprintf(u.stdout, "=== %s: %s\n", step.Name, step.Description) +} + +func (u unitRunner) end() { + if u.inActions() { + _, _ = fmt.Fprintln(u.stdout, "::endgroup::") + } +} + +// checkHostShell rejects host_shell values this runner cannot provide, before +// any command runs. +func (u unitRunner) checkHostShell(step UnitStep) error { + switch step.HostShell { + case "": + return nil + case hostShellBusybox: + if u.goos != osLinux { + return fmt.Errorf("%w: %s on %s", errHostShellNeedsLinux, hostShellBusybox, u.goos) + } + return nil + default: + return fmt.Errorf("%w: %q", errUnknownHostShell, step.HostShell) + } +} + +func (u unitRunner) withHostShell(step UnitStep, run func() error) error { + if step.HostShell != hostShellBusybox { + return run() + } + if !u.inActions() { + _, _ = fmt.Fprintf( + u.stdout, + "skipping %s: host_shell %s runs only in GitHub Actions\n", + step.Name, + hostShellBusybox, + ) + return nil + } + return u.withBusyboxShell(run) +} + +// withBusyboxShell points /bin/sh at BusyBox for the duration of run. The +// original target is restored even when run fails. +func (u unitRunner) withBusyboxShell(run func() error) (err error) { + if err = u.run(cmdSudo, "apt-get", "update", "-qq"); err != nil { + return err + } + if err = u.run(cmdSudo, "apt-get", "install", "-y", "busybox-static"); err != nil { + return err + } + original, err := u.shellTarget() + if err != nil { + return err + } + defer func() { + if restoreErr := u.run(cmdSudo, cmdLn, "-sf", original, shellPath); restoreErr != nil { + err = errors.Join(err, restoreErr) + } + }() + if err = u.run(cmdSudo, cmdLn, "-sf", busyboxPath, shellPath); err != nil { + return err + } + return run() +} + +func (u unitRunner) shellTarget() (string, error) { + out, err := u.capture(cmdReadlink, shellPath) + if err != nil { + return "", fmt.Errorf("readlink %s: %w", shellPath, err) + } + target := strings.TrimSpace(out) + if target == "" { + return "", fmt.Errorf("%w: %s", errShellNotSymlink, shellPath) + } + return target, nil +} + +func (u unitRunner) packagesFor(step UnitStep) ([]string, error) { + if !step.Discover { + return step.Packages, nil + } + goList, err := u.capture(cmdGo, "list", "-f", goListFormat, "./...") + if err != nil { + return nil, fmt.Errorf("go list: %w", err) + } + return filterUnitTestPackages(goList, u.root) +} + +// checkRequiredTests fails the step when a required test name is not an exact +// line of `go test -list`, so a renamed test cannot silently match nothing. +func (u unitRunner) checkRequiredTests(step UnitStep, packages []string) error { + if len(step.RequireTests) == 0 { + return nil + } + args := append([]string{"test"}, packages...) + args = append(args, "-list", unitTestPattern(step)) + listing, err := u.capture(cmdGo, args...) + if err != nil { + return fmt.Errorf("go test -list: %w", err) + } + if missing := missingTests(step.RequireTests, listing); len(missing) > 0 { + return fmt.Errorf("%w: %s", errRequiredTestMissing, strings.Join(missing, ", ")) + } + return nil +} + +func unitTestPattern(step UnitStep) string { + if step.Run != "" { + return step.Run + } + return fmt.Sprintf(unitTestPatternFormat, strings.Join(step.RequireTests, "|")) +} + +// missingTests returns the required names that are not an exact line of listing. +func missingTests(required []string, listing string) []string { + present := map[string]bool{} + for line := range strings.SplitSeq(listing, "\n") { + present[strings.TrimSuffix(line, "\r")] = true + } + var missing []string + for _, name := range required { + if !present[name] { + missing = append(missing, name) + } + } + return missing +} + +func (u unitRunner) goTest(step UnitStep, packages []string) error { + return u.run(cmdGo, u.goTestArgs(step, packages)...) +} + +func (u unitRunner) goTestArgs(step UnitStep, packages []string) []string { + args := append([]string{"test"}, step.Args...) + if step.Run != "" { + args = append(args, "-run", step.Run) + } + if u.coverprofile != "" && step.Discover { + args = append(args, "-coverprofile="+u.coverprofile, "-covermode=atomic") + } + return append(args, packages...) +} + +// filterUnitTestPackages takes `go list -f '{{.ImportPath}}|{{.Dir}}'` output +// and returns the import paths of packages under root, skipping any package +// whose directory has a test or e2e path segment below root. +func filterUnitTestPackages(goList, root string) ([]string, error) { + var packages []string + for line := range strings.SplitSeq(goList, "\n") { + if line == "" { + continue + } + importPath, dir, ok := strings.Cut(line, "|") + if !ok { + return nil, fmt.Errorf("%w: %q", errMalformedGoList, line) + } + if !isUnderRoot(dir, root) { + return nil, fmt.Errorf("%w: %s", errOutsideRepository, dir) + } + relative := filepath.ToSlash(strings.TrimPrefix(dir, root)) + if excludedPackagePattern.MatchString(relative) { + continue + } + packages = append(packages, importPath) + } + if len(packages) == 0 { + return nil, errNoUnitTestPackages + } + return packages, nil +} + +// isUnderRoot reports whether dir is root or a directory inside it. +// A sibling sharing root as a string prefix (root+"2") is not under root. +func isUnderRoot(dir, root string) bool { + return dir == root || strings.HasPrefix(dir, root+string(filepath.Separator)) +} + +func (u unitRunner) run(name string, args ...string) error { + return u.runner.Run(context.Background(), commandSpec{ + Name: name, + Args: args, + Dir: u.root, + Stdout: u.stdout, + Stderr: u.stderr, + }) +} + +// capture runs a command and returns its stdout. Stderr is passed through. +func (u unitRunner) capture(name string, args ...string) (string, error) { + var out bytes.Buffer + err := u.runner.Run(context.Background(), commandSpec{ + Name: name, + Args: args, + Dir: u.root, + Stdout: &out, + Stderr: u.stderr, + }) + return out.String(), err +} diff --git a/hack/ci/unit_test.go b/hack/ci/unit_test.go new file mode 100644 index 000000000..cd72f523e --- /dev/null +++ b/hack/ci/unit_test.go @@ -0,0 +1,587 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/suite" +) + +const unitTestManifest = ` +unit: + steps: + - name: packages + description: discovered packages + os: [linux, darwin] + discover: true + args: [-race, -short] + - name: linux-busybox + description: busybox shell + os: [linux] + host_shell: busybox + packages: [./pkg/inject] + run: '^TestBootstrap$' + args: [-race, -count=1] + - name: windows-transport + description: windows only + os: [windows] + packages: [./pkg/docker] + args: [-count=1] + - name: shared + description: every OS + packages: [./e2e/tests/up] + args: [-count=1] + - name: compose + description: required test + os: [linux, darwin] + packages: [./e2e/tests/up-docker-compose] + run: '^TestExpectedUIDMapping$' + require_tests: [TestExpectedUIDMapping] + args: [-race, -count=1, -v] +` + +const ( + unitSharedStep = "shared" + unitGoListCommand = "go list -f {{.ImportPath}}|{{.Dir}} ./..." + unitRequiredTest = "TestA" + unitPkgA = "./a" + unitProfile = "p.out" + unitListFlag = "--list" +) + +var errUnitFake = errors.New("fake failure") + +// unitFakeRunner records every command line and answers it through respond. +type unitFakeRunner struct { + calls []string + respond func(line string) (string, error) +} + +func (f *unitFakeRunner) Run(_ context.Context, spec commandSpec) error { + line := strings.Join(append([]string{spec.Name}, spec.Args...), " ") + f.calls = append(f.calls, line) + if f.respond == nil { + return nil + } + out, err := f.respond(line) + if spec.Stdout != nil { + _, _ = io.WriteString(spec.Stdout, out) + } + return err +} + +// unitHarness is a unitRunner wired to a fake runner and captured output. +type unitHarness struct { + fake *unitFakeRunner + stdout *bytes.Buffer + stderr *bytes.Buffer + runner unitRunner +} + +func newUnitHarness( + root, goos string, + env map[string]string, + respond func(string) (string, error), +) *unitHarness { + h := &unitHarness{ + fake: &unitFakeRunner{respond: respond}, + stdout: &bytes.Buffer{}, + stderr: &bytes.Buffer{}, + } + h.runner = unitRunner{ + runner: h.fake, + goos: goos, + root: root, + env: func(key string) string { return env[key] }, + stdout: h.stdout, + stderr: h.stderr, + } + return h +} + +// unitFailOn returns a responder that fails every command starting with prefix. +func unitFailOn(prefix string) func(string) (string, error) { + return func(line string) (string, error) { + if strings.HasPrefix(line, prefix) { + return "", errUnitFake + } + return "", nil + } +} + +// unitFilterFixture returns a repo root that does not exist on disk and a helper +// that joins slash-separated relative paths onto it. +func unitFilterFixture(root string) func(string) string { + return func(rel string) string { return filepath.Join(root, filepath.FromSlash(rel)) } +} + +func unitNames(steps []UnitStep) []string { + names := make([]string, 0, len(steps)) + for _, step := range steps { + names = append(names, step.Name) + } + return names +} + +type unitSuite struct { + suite.Suite + root string + steps []UnitStep +} + +func TestUnitSuite(t *testing.T) { + suite.Run(t, new(unitSuite)) +} + +func (s *unitSuite) SetupSuite() { + m, err := ParseManifest([]byte(unitTestManifest)) + s.Require().NoError(err) + s.steps = m.Unit.Steps +} + +func (s *unitSuite) SetupTest() { + s.root = filepath.Join(s.T().TempDir(), "devsy") +} + +// listing returns a go list responder answering with output and running every other command. +func listing(output string) func(string) (string, error) { + return func(line string) (string, error) { + if strings.HasPrefix(line, "go list") { + return output, nil + } + return "", nil + } +} + +func (s *unitSuite) TestSelectUnitStepsByOS() { + tests := []struct { + goos string + want []string + }{ + {goos: osLinux, want: []string{"packages", "linux-busybox", unitSharedStep, "compose"}}, + {goos: osDarwin, want: []string{"packages", unitSharedStep, "compose"}}, + {goos: osWindows, want: []string{"windows-transport", unitSharedStep}}, + } + for _, tc := range tests { + s.Run(tc.goos, func() { + got, err := selectUnitSteps(s.steps, tc.goos, "") + s.Require().NoError(err) + s.Equal(tc.want, unitNames(got)) + }) + } +} + +func (s *unitSuite) TestSelectUnitStepsByName() { + got, err := selectUnitSteps(s.steps, osWindows, unitSharedStep) + s.Require().NoError(err) + s.Equal([]string{unitSharedStep}, unitNames(got)) + + _, err = selectUnitSteps(s.steps, osDarwin, "linux-busybox") + s.ErrorIs(err, errNoUnitSteps) +} + +func (s *unitSuite) TestDiscoverRunsFilteredPackages() { + h := s.harness(osLinux, listing(strings.Join([]string{ + "example.com/devsy/pkg/foo|" + s.at("pkg/foo"), + "example.com/devsy/pkg/test|" + s.at("pkg/test"), + }, "\n")+"\n")) + + s.Equal(unitSuccessExitCode, h.runner.runAll([]UnitStep{s.step("packages")})) + s.Equal([]string{ + unitGoListCommand, + "go test -race -short example.com/devsy/pkg/foo", + }, h.fake.calls) +} + +func (s *unitSuite) TestCoverageOnlyForDiscoverSteps() { + h := s.harness(osLinux, listing("example.com/devsy/pkg/foo|"+s.at("pkg/foo")+"\n")) + h.runner.coverprofile = "dist/profile.out" + + s.Equal( + unitSuccessExitCode, + h.runner.runAll([]UnitStep{s.step("packages"), s.step(unitSharedStep)}), + ) + s.Equal([]string{ + unitGoListCommand, + "go test -race -short -coverprofile=dist/profile.out -covermode=atomic example.com/devsy/pkg/foo", + "go test -count=1 ./e2e/tests/up", + }, h.fake.calls) +} + +func (s *unitSuite) TestDiscoverFailureFailsStep() { + h := s.harness(osLinux, unitFailOn("go list")) + + err := h.runner.runStep(s.step("packages")) + s.ErrorIs(err, errUnitFake) + s.Equal([]string{unitGoListCommand}, h.fake.calls) +} + +func (s *unitSuite) TestRequireTestsRunsWhenListed() { + h := s.harness(osLinux, func(line string) (string, error) { + if strings.Contains(line, "-list") { + return "TestExpectedUIDMapping\nok \texample.com/x\t0.1s\n", nil + } + return "", nil + }) + + s.Equal(unitSuccessExitCode, h.runner.runAll([]UnitStep{s.step("compose")})) + s.Equal([]string{ + "go test ./e2e/tests/up-docker-compose -list ^TestExpectedUIDMapping$", + "go test -race -count=1 -v -run ^TestExpectedUIDMapping$ ./e2e/tests/up-docker-compose", + }, h.fake.calls) +} + +func (s *unitSuite) TestRequireTestsFailsWhenMissing() { + h := s.harness(osLinux, func(line string) (string, error) { + if strings.Contains(line, "-list") { + return "TestExpectedUIDMappingRenamed\n", nil + } + return "", nil + }) + + err := h.runner.runStep(s.step("compose")) + s.ErrorIs(err, errRequiredTestMissing) + s.Contains(err.Error(), "TestExpectedUIDMapping") + s.Equal([]string{ + "go test ./e2e/tests/up-docker-compose -list ^TestExpectedUIDMapping$", + }, h.fake.calls) +} + +func (s *unitSuite) TestRequireTestsPatternWithoutRun() { + step := UnitStep{RequireTests: []string{unitRequiredTest, "TestB"}} + s.Equal("^(TestA|TestB)$", unitTestPattern(step)) + s.Equal( + "^TestC$", + unitTestPattern(UnitStep{Run: "^TestC$", RequireTests: []string{unitRequiredTest}}), + ) +} + +func (s *unitSuite) TestMissingTests() { + tests := []struct { + name string + listing string + want []string + }{ + {name: "exact lines", listing: "TestA\nTestB\n", want: nil}, + {name: "crlf lines", listing: "TestA\r\nTestB\r\n", want: nil}, + {name: "prefix is not a match", listing: "TestAlpha\n", want: []string{unitRequiredTest}}, + {name: "empty listing", listing: "", want: []string{unitRequiredTest}}, + } + for _, tc := range tests { + s.Run(tc.name, func() { + got := missingTests([]string{unitRequiredTest}, tc.listing) + s.Equal(tc.want, got) + }) + } +} + +func (s *unitSuite) TestBusyboxRestoresShellWhenTestFails() { + h := s.actionsHarness(osLinux, func(line string) (string, error) { + if strings.HasPrefix(line, "readlink") { + return "dash\n", nil + } + if strings.HasPrefix(line, "go test") { + return "", errUnitFake + } + return "", nil + }) + + s.Equal(unitFailureExitCode, h.runner.runAll([]UnitStep{s.step("linux-busybox")})) + s.Equal([]string{ + "sudo apt-get update -qq", + "sudo apt-get install -y busybox-static", + "readlink /bin/sh", + "sudo ln -sf /bin/busybox /bin/sh", + "go test -race -count=1 -run ^TestBootstrap$ ./pkg/inject", + "sudo ln -sf dash /bin/sh", + }, h.fake.calls) + s.Contains(h.stderr.String(), "FAILED: linux-busybox: ") +} + +func (s *unitSuite) TestBusyboxRestoreFailureIsReported() { + h := s.actionsHarness(osLinux, func(line string) (string, error) { + if strings.HasPrefix(line, "readlink") { + return "dash\n", nil + } + if strings.HasPrefix(line, "sudo ln -sf dash") { + return "", errUnitFake + } + return "", nil + }) + + err := h.runner.runStep(s.step("linux-busybox")) + s.ErrorIs(err, errUnitFake) +} + +func (s *unitSuite) TestBusyboxFailsWhenShellIsNotSymlink() { + h := s.actionsHarness(osLinux, func(line string) (string, error) { + if strings.HasPrefix(line, "readlink") { + return "\n", nil + } + return "", nil + }) + + err := h.runner.runStep(s.step("linux-busybox")) + s.ErrorIs(err, errShellNotSymlink) + s.NotContains(h.fake.calls, "sudo ln -sf /bin/busybox /bin/sh") +} + +func (s *unitSuite) TestBusyboxSkippedOutsideActions() { + h := s.harness(osLinux, func(line string) (string, error) { + if strings.Contains(line, "-list") { + return "TestBootstrap\n", nil + } + return "", nil + }) + + s.NoError(h.runner.runStep(s.step("linux-busybox"))) + s.NotContains(h.fake.calls, "sudo apt-get update -qq") + s.NotContains(h.fake.calls, "sudo ln -sf /bin/busybox /bin/sh") + s.NotContains(h.fake.calls, "go test -race -count=1 -run ^TestBootstrap$ ./pkg/inject") + s.Contains( + h.stdout.String(), + "skipping linux-busybox: host_shell busybox runs only in GitHub Actions", + ) +} + +func (s *unitSuite) TestBusyboxRejectedOffLinux() { + h := s.harness(osDarwin, nil) + + err := h.runner.runStep(s.step("linux-busybox")) + s.ErrorIs(err, errHostShellNeedsLinux) + s.Empty(h.fake.calls) +} + +func (s *unitSuite) TestUnknownHostShellRejected() { + h := s.harness(osLinux, nil) + + err := h.runner.runStep(UnitStep{Name: "odd", Packages: []string{unitPkgA}, HostShell: "zsh"}) + s.ErrorIs(err, errUnknownHostShell) + s.Empty(h.fake.calls) +} + +func (s *unitSuite) TestRunAllContinuesAfterFailure() { + h := s.harness(osLinux, unitFailOn("go test ./a")) + steps := []UnitStep{ + {Name: "first", Packages: []string{unitPkgA}}, + {Name: "second", Packages: []string{"./b"}}, + } + + s.Equal(unitFailureExitCode, h.runner.runAll(steps)) + s.Equal([]string{"go test ./a", "go test ./b"}, h.fake.calls) + s.Contains(h.stderr.String(), "FAILED: first: ") + s.NotContains(h.stderr.String(), "second") +} + +func (s *unitSuite) TestRunAllSucceeds() { + h := s.harness(osLinux, nil) + + s.Equal(unitSuccessExitCode, h.runner.runAll([]UnitStep{s.step(unitSharedStep)})) + s.Empty(h.stderr.String()) +} + +func (s *unitSuite) TestPlainHeaderOutsideActions() { + h := newUnitHarness(s.root, osLinux, nil, nil) + + s.Equal(unitSuccessExitCode, h.runner.runAll([]UnitStep{s.step(unitSharedStep)})) + s.Equal("=== shared: every OS\n", h.stdout.String()) +} + +func (s *unitSuite) TestGitHubActionsGroups() { + h := newUnitHarness(s.root, osLinux, map[string]string{gitHubActionsEnv: "true"}, nil) + + s.Equal(unitSuccessExitCode, h.runner.runAll([]UnitStep{s.step(unitSharedStep)})) + s.Equal("::group::shared\n::endgroup::\n", h.stdout.String()) +} + +func (s *unitSuite) TestListPrintsWithoutRunning() { + h := s.harness(osLinux, nil) + steps, err := selectUnitSteps(s.steps, osLinux, "") + s.Require().NoError(err) + + s.Equal(unitSuccessExitCode, h.runner.runSelected(steps, true)) + s.Equal("packages\nlinux-busybox\nshared\ncompose\n", h.stdout.String()) + s.Empty(h.fake.calls) +} + +func (s *unitSuite) TestGoTestArgs() { + h := s.harness(osLinux, nil) + h.runner.coverprofile = unitProfile + + s.Equal([]string{"test", "-count=1", "-run", "^X$", unitPkgA}, + h.runner.goTestArgs(UnitStep{Args: []string{"-count=1"}, Run: "^X$"}, []string{unitPkgA})) + s.Equal([]string{"test", "-race", "-coverprofile=p.out", "-covermode=atomic", unitPkgA}, + h.runner.goTestArgs(UnitStep{Args: []string{"-race"}, Discover: true}, []string{unitPkgA})) +} + +func (s *unitSuite) TestParseUnitFlags() { + var stderr bytes.Buffer + f, err := parseUnitFlags([]string{ + "--os", osWindows, "--step", unitSharedStep, "--coverprofile", unitProfile, unitListFlag, + }, &stderr) + s.Require().NoError(err) + s.Equal( + unitFlags{goos: osWindows, step: unitSharedStep, coverprofile: unitProfile, list: true}, + f, + ) + + _, err = parseUnitFlags([]string{"extra"}, &stderr) + s.ErrorIs(err, errUnexpectedUnitArgs) + + _, err = parseUnitFlags([]string{"--bogus"}, &stderr) + s.Error(err) +} + +func (s *unitSuite) TestUnitCLIListsRealManifest() { + var stdout, stderr bytes.Buffer + + s.Equal( + unitSuccessExitCode, + unitCLI([]string{unitListFlag, "--os", osWindows}, &stdout, &stderr), + ) + s.Contains(stdout.String(), "windows-transport\n") + s.NotContains(stdout.String(), "linux-busybox") + + s.Equal( + unitFailureExitCode, + unitCLI([]string{unitListFlag, "--step", "no-such-step"}, &stdout, &stderr), + ) + s.Equal(unitUsageExitCode, unitCLI([]string{"--bogus"}, &stdout, &stderr)) +} + +func (s *unitSuite) TestFilterUnitTestPackagesKeepsPackages() { + root := s.root + at := unitFilterFixture(root) + + tests := []struct { + name string + goList string + want []string + }{ + { + name: "keeps package under root", + goList: "example.com/devsy/pkg/foo|" + at("pkg/foo") + "\n", + want: []string{"example.com/devsy/pkg/foo"}, + }, + { + name: "keeps root package", + goList: "example.com/devsy|" + root + "\n", + want: []string{"example.com/devsy"}, + }, + { + name: "excludes test and e2e directories", + goList: strings.Join([]string{ + "example.com/devsy/test/helpers|" + at("test/helpers"), + "example.com/devsy/pkg/test|" + at("pkg/test"), + "example.com/devsy/e2e|" + at("e2e"), + "example.com/devsy/e2e/framework|" + at("e2e/framework"), + "example.com/devsy/pkg/e2e/x|" + at("pkg/e2e/x"), + "example.com/devsy/pkg/foo|" + at("pkg/foo"), + }, "\n") + "\n", + want: []string{"example.com/devsy/pkg/foo"}, + }, + { + name: "keeps names that only contain test or e2e", + goList: strings.Join([]string{ + "example.com/devsy/pkg/testing|" + at("pkg/testing"), + "example.com/devsy/pkg/tests|" + at("pkg/tests"), + "example.com/devsy/pkg/e2equal|" + at("pkg/e2equal"), + }, "\n") + "\n", + want: []string{ + "example.com/devsy/pkg/testing", + "example.com/devsy/pkg/tests", + "example.com/devsy/pkg/e2equal", + }, + }, + } + + for _, tc := range tests { + s.Run(tc.name, func() { + got, err := filterUnitTestPackages(tc.goList, root) + s.Require().NoError(err) + s.Equal(tc.want, got) + }) + } +} + +func (s *unitSuite) TestFilterUnitTestPackagesRejects() { + root := s.root + at := unitFilterFixture(root) + sibling := root + "2" + + tests := []struct { + name string + goList string + wantErr error + }{ + { + name: "rejects sibling sharing root prefix", + goList: "example.com/x|" + filepath.Join(sibling, "x") + "\n", + wantErr: errOutsideRepository, + }, + { + name: "rejects directory outside root", + goList: "example.com/elsewhere|" + filepath.Join( + filepath.Dir(root), + "elsewhere", + ) + "\n", + wantErr: errOutsideRepository, + }, + { + name: "rejects listing where every package is excluded", + goList: "example.com/devsy/e2e|" + at("e2e") + "\n", + wantErr: errNoUnitTestPackages, + }, + { + name: "rejects empty listing", + goList: "", + wantErr: errNoUnitTestPackages, + }, + { + name: "rejects line without separator", + goList: "example.com/devsy/pkg/foo\n", + wantErr: errMalformedGoList, + }, + } + + for _, tc := range tests { + s.Run(tc.name, func() { + _, err := filterUnitTestPackages(tc.goList, root) + s.ErrorIs(err, tc.wantErr) + }) + } +} + +func (s *unitSuite) TestIsUnderRoot() { + root := s.T().TempDir() + s.True(isUnderRoot(root, root)) + s.True(isUnderRoot(filepath.Join(root, "pkg"), root)) + s.False(isUnderRoot(filepath.Join(root+"2", "pkg"), root)) +} + +func (s *unitSuite) at(rel string) string { + return unitFilterFixture(s.root)(rel) +} + +func (s *unitSuite) step(name string) UnitStep { + for _, step := range s.steps { + if step.Name == name { + return step + } + } + s.FailNow("no step named " + name) + return UnitStep{} +} + +func (s *unitSuite) harness(goos string, respond func(string) (string, error)) *unitHarness { + return newUnitHarness(s.root, goos, nil, respond) +} + +func (s *unitSuite) actionsHarness(goos string, respond func(string) (string, error)) *unitHarness { + return newUnitHarness(s.root, goos, map[string]string{gitHubActionsEnv: "true"}, respond) +} diff --git a/hack/unit_test_packages/main.go b/hack/unit_test_packages/main.go deleted file mode 100644 index fc9605161..000000000 --- a/hack/unit_test_packages/main.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "bytes" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" -) - -const goListFormat = "{{.ImportPath}}|{{.Dir}}" - -var ( - errNoUnitTestPackages = errors.New("no unit-test packages found") - errOutsideRepository = errors.New("package directory is outside the repository") - errMalformedGoList = errors.New("unexpected go list line") - excludedPackagePattern = regexp.MustCompile(`(^|/)(test|e2e)(/|$)`) -) - -func main() { - os.Exit(runCLI(os.Args[1:], os.Stdout, os.Stderr)) -} - -func runCLI(args []string, stdout, stderr io.Writer) int { - if len(args) != 0 { - _, _ = fmt.Fprintln(stderr, "usage: unit_test_packages (run from the repository root)") - return 2 - } - root, err := os.Getwd() - if err != nil { - _, _ = fmt.Fprintln(stderr, err) - return 1 - } - goList, err := runGoList(stderr) - if err != nil { - _, _ = fmt.Fprintln(stderr, err) - return 1 - } - packages, err := filterUnitTestPackages(goList, root) - if err != nil { - _, _ = fmt.Fprintln(stderr, err) - return 1 - } - for _, p := range packages { - _, _ = fmt.Fprintln(stdout, p) - } - return 0 -} - -func runGoList(stderr io.Writer) (string, error) { - var out bytes.Buffer - cmd := exec.Command("go", "list", "-f", goListFormat, "./...") - cmd.Stdout = &out - cmd.Stderr = stderr - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("go list: %w", err) - } - return out.String(), nil -} - -// filterUnitTestPackages takes `go list -f '{{.ImportPath}}|{{.Dir}}'` output -// and returns the import paths of packages under root, skipping any package -// whose directory has a test or e2e path segment below root. -func filterUnitTestPackages(goList, root string) ([]string, error) { - var packages []string - for line := range strings.SplitSeq(goList, "\n") { - if line == "" { - continue - } - importPath, dir, ok := strings.Cut(line, "|") - if !ok { - return nil, fmt.Errorf("%w: %q", errMalformedGoList, line) - } - if !isUnderRoot(dir, root) { - return nil, fmt.Errorf("%w: %s", errOutsideRepository, dir) - } - relative := filepath.ToSlash(strings.TrimPrefix(dir, root)) - if excludedPackagePattern.MatchString(relative) { - continue - } - packages = append(packages, importPath) - } - if len(packages) == 0 { - return nil, errNoUnitTestPackages - } - return packages, nil -} - -// isUnderRoot reports whether dir is root or a directory inside it. -// A sibling sharing root as a string prefix (root+"2") is not under root. -func isUnderRoot(dir, root string) bool { - return dir == root || strings.HasPrefix(dir, root+string(filepath.Separator)) -} diff --git a/hack/unit_test_packages/main_test.go b/hack/unit_test_packages/main_test.go deleted file mode 100644 index a1ed80a50..000000000 --- a/hack/unit_test_packages/main_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "errors" - "path/filepath" - "slices" - "strings" - "testing" -) - -// filterFixture returns a repo root that does not exist on disk and a helper -// that joins slash-separated relative paths onto it. -func filterFixture(t *testing.T) (root string, at func(string) string) { - t.Helper() - root = filepath.Join(t.TempDir(), "devsy") - at = func(rel string) string { return filepath.Join(root, filepath.FromSlash(rel)) } - return root, at -} - -func TestFilterUnitTestPackagesKeepsPackages(t *testing.T) { - t.Parallel() - - root, at := filterFixture(t) - - tests := []struct { - name string - goList string - want []string - }{ - { - name: "keeps package under root", - goList: "example.com/devsy/pkg/foo|" + at("pkg/foo") + "\n", - want: []string{"example.com/devsy/pkg/foo"}, - }, - { - name: "keeps root package", - goList: "example.com/devsy|" + root + "\n", - want: []string{"example.com/devsy"}, - }, - { - name: "excludes test and e2e directories", - goList: strings.Join([]string{ - "example.com/devsy/test/helpers|" + at("test/helpers"), - "example.com/devsy/pkg/test|" + at("pkg/test"), - "example.com/devsy/e2e|" + at("e2e"), - "example.com/devsy/e2e/framework|" + at("e2e/framework"), - "example.com/devsy/pkg/e2e/x|" + at("pkg/e2e/x"), - "example.com/devsy/pkg/foo|" + at("pkg/foo"), - }, "\n") + "\n", - want: []string{"example.com/devsy/pkg/foo"}, - }, - { - name: "keeps names that only contain test or e2e", - goList: strings.Join([]string{ - "example.com/devsy/pkg/testing|" + at("pkg/testing"), - "example.com/devsy/pkg/tests|" + at("pkg/tests"), - "example.com/devsy/pkg/e2equal|" + at("pkg/e2equal"), - }, "\n") + "\n", - want: []string{ - "example.com/devsy/pkg/testing", - "example.com/devsy/pkg/tests", - "example.com/devsy/pkg/e2equal", - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - got, err := filterUnitTestPackages(tc.goList, root) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !slices.Equal(got, tc.want) { - t.Fatalf("packages = %v, want %v", got, tc.want) - } - }) - } -} - -func TestFilterUnitTestPackagesRejects(t *testing.T) { - t.Parallel() - - root, at := filterFixture(t) - sibling := root + "2" - - tests := []struct { - name string - goList string - wantErr error - }{ - { - name: "rejects sibling sharing root prefix", - goList: "example.com/x|" + filepath.Join(sibling, "x") + "\n", - wantErr: errOutsideRepository, - }, - { - name: "rejects directory outside root", - goList: "example.com/elsewhere|" + filepath.Join( - filepath.Dir(root), - "elsewhere", - ) + "\n", - wantErr: errOutsideRepository, - }, - { - name: "rejects listing where every package is excluded", - goList: "example.com/devsy/e2e|" + at("e2e") + "\n", - wantErr: errNoUnitTestPackages, - }, - { - name: "rejects empty listing", - goList: "", - wantErr: errNoUnitTestPackages, - }, - { - name: "rejects line without separator", - goList: "example.com/devsy/pkg/foo\n", - wantErr: errMalformedGoList, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - _, err := filterUnitTestPackages(tc.goList, root) - if !errors.Is(err, tc.wantErr) { - t.Fatalf("error = %v, want %v", err, tc.wantErr) - } - }) - } -} diff --git a/mise.toml b/mise.toml index ddc8a952e..c33e6109b 100644 --- a/mise.toml +++ b/mise.toml @@ -112,12 +112,11 @@ protoc -I . tunnel.proto --go_out=. --go_opt=paths=source_relative --go-grpc_out ''' [tasks."cli:test"] -description = "run devsy unit tests" +description = "run the unit-test tier defined in hack/ci/suites.yaml" shell = "bash -c" run = ''' set -euo pipefail -packages="$(go run ./hack/unit_test_packages)" -go test -race -coverprofile=dist/profile.out -covermode=atomic $packages +go run ./hack/ci unit --coverprofile=dist/profile.out ''' [tasks."cli:test:e2e:build"]