From 0d84529add4e5e6b8cbf6cd7b11ede17fbe4c297 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 11:57:47 -0400 Subject: [PATCH 1/8] Implement immutable 31-payload candidate pipeline --- .github/workflows/candidate.yml | 307 +++++++++- .github/workflows/phase6-candidate-gate.yml | 150 +++++ .../workflows/phase6-live-verification.yml | 67 +++ catalog/buildsets/initial-warehouse-v1.json | 1 + docs/publishing.md | 41 +- schema/build-set-v1.schema.json | 73 ++- scripts/check_workflow_policy.py | 12 +- scripts/generate_phase6_buildset.py | 94 +++ scripts/github_phase6.py | 278 +++++++++ scripts/phase6_candidate.py | 541 ++++++++++++++++++ scripts/validate_catalog.py | 60 +- tests/test_phase6_candidate.py | 145 +++++ tests/test_policy.py | 84 ++- 13 files changed, 1781 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/phase6-candidate-gate.yml create mode 100644 .github/workflows/phase6-live-verification.yml create mode 100644 catalog/buildsets/initial-warehouse-v1.json create mode 100644 scripts/generate_phase6_buildset.py create mode 100644 scripts/github_phase6.py create mode 100644 scripts/phase6_candidate.py create mode 100644 tests/test_phase6_candidate.py diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index 5881829..e5b69a8 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -7,57 +7,232 @@ on: description: Committed catalog/buildsets ID (no path or URL) required: true type: string + build-set-sha256: + description: Exact SHA-256 of the committed build-set JSON + required: true + type: string permissions: contents: read + actions: read concurrency: - group: forge-candidate-${{ inputs.build-set-id }} + group: forge-candidate-publication cancel-in-progress: false jobs: - validate-committed-build-set: + validate-protected-source: runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 15 + permissions: + contents: read + actions: read + outputs: + build-set-path: ${{ steps.validate.outputs.build-set-path }} steps: - - name: Check out protected-main source + - name: Check out exact protected-main source uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false - - name: Validate inert build-set identifier and committed manifest + - name: Validate exact committed build set and source ancestry + id: validate env: BUILD_SET_ID: ${{ inputs.build-set-id }} + BUILD_SET_SHA256: ${{ inputs.build-set-sha256 }} run: | - case "$BUILD_SET_ID" in - ''|*[!a-z0-9._-]*) exit 2 ;; - esac - test "${GITHUB_REF}" = 'refs/heads/main' - test -f "catalog/buildsets/${BUILD_SET_ID}.json" + set -euo pipefail + case "$BUILD_SET_ID" in ''|*[!a-z0-9._-]*) exit 2 ;; esac + case "$BUILD_SET_SHA256" in *[!0-9a-f]*|'') exit 2 ;; esac + test "${#BUILD_SET_SHA256}" -eq 64 + test "$GITHUB_REF" = 'refs/heads/main' + test "$GITHUB_EVENT_NAME" = 'workflow_dispatch' + BUILD_SET_PATH="catalog/buildsets/${BUILD_SET_ID}.json" + test -f "$BUILD_SET_PATH" + test "$(sha256sum "$BUILD_SET_PATH" | cut -d' ' -f1)" = "$BUILD_SET_SHA256" python3 -m pip install --requirement requirements-ci.txt - python3 scripts/validate_catalog.py build-set "catalog/buildsets/${BUILD_SET_ID}.json" --root . + python3 scripts/phase6_candidate.py validate-buildset --build-set "$BUILD_SET_PATH" --require-git-ancestry + python3 scripts/github_phase6.py verify-source --expected-sha "$GITHUB_SHA" --output protected-source.json + printf 'build-set-path=%s\n' "$BUILD_SET_PATH" >> "$GITHUB_OUTPUT" + - name: Retain protected-source evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-protected-source-${{ github.sha }} + path: protected-source.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false + + assemble-verified-content: + needs: validate-protected-source + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + outputs: + staging-artifact-id: ${{ steps.stage.outputs.artifact-id }} + staging-name: ${{ steps.assemble.outputs.staging-name }} + content-list-digest: ${{ steps.assemble.outputs.content-list-digest }} + steps: + - name: Check out exact source without credentials + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Capture and verify live audited-input metadata + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/github_phase6.py capture-inputs --build-set "${{ needs.validate-protected-source.outputs.build-set-path }}" --output input-live-metadata.json + python3 scripts/phase6_candidate.py verify-live-inputs --build-set "${{ needs.validate-protected-source.outputs.build-set-path }}" --metadata input-live-metadata.json + - name: Download exact Phase-3p proof-data artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33170546601 + artifact-ids: 9685464135 + path: input-artifacts/phase3p-proof-data + - name: Download exact Phase-4 Celestia appd artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688244894 + path: input-artifacts/phase4-celestia-appd-linux-arm64 + - name: Download exact Phase-4 Celestia node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688243729 + path: input-artifacts/phase4-celestia-node-linux-arm64 + - name: Download exact Phase-4 Midnight node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688330126 + path: input-artifacts/phase4-node-linux-arm64 + - name: Download exact Phase-4 toolkit Linux amd64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688263793 + path: input-artifacts/phase4-toolkit-linux-amd64 + - name: Download exact Phase-4 toolkit Linux arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688255774 + path: input-artifacts/phase4-toolkit-linux-arm64 + - name: Download exact Phase-4 toolkit macOS arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9689647047 + path: input-artifacts/phase4-toolkit-macos-arm64 + - name: Download exact Phase-5 verified indexer aggregate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33176004154 + artifact-ids: 9690093579 + path: input-artifacts/phase5-indexer + - name: Assemble and non-executingly verify exact 31-payload content + id: assemble + env: + BUILD_SET_PATH: ${{ needs.validate-protected-source.outputs.build-set-path }} + run: | + set -euo pipefail + python3 scripts/phase6_candidate.py assemble --build-set "$BUILD_SET_PATH" --input-root input-artifacts --output verified-content | tail -n 1 > candidate-verification.json + python3 scripts/phase6_candidate.py verify --build-set "$BUILD_SET_PATH" --content verified-content + CONTENT_LIST_DIGEST="$(jq -r '.contentAssetListSha256' candidate-verification.json)" + test "$(printf '%s' "$CONTENT_LIST_DIGEST" | wc -c)" -eq 64 + STAGING_NAME="verified-content-${{ inputs.build-set-id }}-${CONTENT_LIST_DIGEST}" + printf 'content-list-digest=%s\n' "$CONTENT_LIST_DIGEST" >> "$GITHUB_OUTPUT" + printf 'staging-name=%s\n' "$STAGING_NAME" >> "$GITHUB_OUTPUT" + - name: Upload exact immutable staging content + id: stage + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ${{ steps.assemble.outputs.staging-name }} + path: verified-content/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false + - name: Capture exact staging artifact identity + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py capture-staging --artifact-id "${{ steps.stage.outputs.artifact-id }}" --expected-name "${{ steps.assemble.outputs.staging-name }}" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --output staging.json + - name: Retain verified input and staging metadata + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-assembly-evidence-${{ github.sha }} + path: | + candidate-verification.json + input-live-metadata.json + staging.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false - # Phase 1 freezes the privilege boundary and attestation inputs. Phase 6 activates these jobs only - # after exact build/mirror payloads and the two verifier handoffs exist. Keeping them false here - # prevents a manifest-only scaffold from allocating or publishing a release. create-protected-draft: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: validate-committed-build-set + needs: [validate-protected-source, assemble-verified-content] runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 15 environment: candidate-publish permissions: contents: write + actions: read steps: - - name: Allocate and API-read-back empty draft - run: echo 'Phase 6 supplies the hash-bound draft allocation implementation' && exit 2 + - name: Check out exact source without credentials + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Download protected-source evidence + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + name: phase6-protected-source-${{ github.sha }} + path: protected-source + - name: Allocate API-time unique empty draft and read it back + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py allocate-draft --expected-sha "$GITHUB_SHA" --source protected-source/protected-source.json --output draft.json + - name: Retain exact draft identity + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-draft-${{ github.sha }} + path: draft.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false fresh-final-claims-verifier: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: create-protected-draft + needs: [validate-protected-source, assemble-verified-content, create-protected-draft] runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 90 permissions: contents: read + actions: read outputs: claims-digest: ${{ steps.verify.outputs.claims-digest }} steps: @@ -65,20 +240,54 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false - - name: Re-verify inert content and final draft-bound claims + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Redownload exact staging by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + path: verified-content + - name: Download assembly, draft, and protected-source evidence + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + pattern: phase6-*-${{ github.sha }} + path: handoff + merge-multiple: true + - name: Reverify content and create canonical draft-bound claims id: verify + env: + BUILD_SET_PATH: ${{ needs.validate-protected-source.outputs.build-set-path }} run: | + set -euo pipefail + python3 scripts/phase6_candidate.py verify --build-set "$BUILD_SET_PATH" --content verified-content + WORKFLOW_SHA="$(jq -r '.workflowSha' handoff/protected-source.json)" + python3 scripts/phase6_candidate.py make-claims --build-set "$BUILD_SET_PATH" --content verified-content --draft handoff/draft.json --staging handoff/staging.json --commit-sha "$GITHUB_SHA" --workflow-sha "$WORKFLOW_SHA" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --output "promotion-claims-${{ inputs.build-set-id }}.json" python3 scripts/publisher_guard.py verify-claims-content --claims "promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context printf 'claims-digest=sha256:%s\n' "$(python3 scripts/canonical_json.py sha256 "promotion-claims-${{ inputs.build-set-id }}.json")" >> "$GITHUB_OUTPUT" + - name: Retain canonical claims + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-claims-${{ github.sha }} + path: promotion-claims-${{ inputs.build-set-id }}.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false protected-publisher: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: fresh-final-claims-verifier + needs: [validate-protected-source, assemble-verified-content, create-protected-draft, fresh-final-claims-verifier] runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 90 environment: candidate-publish permissions: contents: write + actions: read id-token: write attestations: write artifact-metadata: write @@ -87,8 +296,25 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false + - name: Redownload exact staging by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + path: verified-content + - name: Download exact claims and draft handoffs + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + pattern: phase6-*-${{ github.sha }} + path: handoff + merge-multiple: true - name: Recheck inert claims/content before attestation - run: python3 scripts/publisher_guard.py verify-claims-content --claims "promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context + run: python3 scripts/publisher_guard.py verify-claims-content --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context - name: Attest exact canonical claims predicate id: attest uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 @@ -96,7 +322,28 @@ jobs: subject-name: promotion-claims-${{ inputs.build-set-id }} subject-digest: ${{ needs.fresh-final-claims-verifier.outputs.claims-digest }} predicate-type: https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1 - predicate-path: promotion-claims-${{ inputs.build-set-id }}.json + predicate-path: handoff/promotion-claims-${{ inputs.build-set-id }}.json show-summary: false - - name: Bind bundle, recheck inert transport, upload/read-back, and publish - run: echo 'Phase 6 supplies the exact bundle-path handoff and release transaction' && exit 2 + - name: Cryptographically verify bundle and materialize frozen envelope + env: + GITHUB_TOKEN: ${{ github.token }} + ATTEST_BUNDLE_PATH: ${{ steps.attest.outputs.bundle-path }} + run: | + set -euo pipefail + cp -- "$ATTEST_BUNDLE_PATH" "attestation-${{ inputs.build-set-id }}.sigstore.json" + gh attestation verify "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --repo "$GITHUB_REPOSITORY" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + python3 scripts/materialize_envelope.py --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --output "promotion-envelope-${{ inputs.build-set-id }}.json" + python3 scripts/publisher_guard.py verify-transport --envelope "promotion-envelope-${{ inputs.build-set-id }}.json" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --content-dir verified-content --require-context + - name: Upload/read back, publish immutable, and prove mutation rejection + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py publish --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --content verified-content --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --envelope "promotion-envelope-${{ inputs.build-set-id }}.json" --draft handoff/draft.json --output published.json + - name: Retain post-publication handoff + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: published-candidate-${{ inputs.build-set-id }} + path: published.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false diff --git a/.github/workflows/phase6-candidate-gate.yml b/.github/workflows/phase6-candidate-gate.yml new file mode 100644 index 0000000..7c98801 --- /dev/null +++ b/.github/workflows/phase6-candidate-gate.yml @@ -0,0 +1,150 @@ +name: Phase 6 immutable candidate gate + +on: + pull_request: + paths: + - '.github/workflows/**' + - 'catalog/**' + - 'schema/**' + - 'scripts/**' + - 'tests/**' + - 'requirements-ci.txt' + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: phase6-candidate-gate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + assemble-and-verify: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + steps: + - name: Check out exact candidate source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Install verifier dependencies and validate exact build set + run: | + python3 -m pip install --requirement requirements-ci.txt + python3 scripts/generate_phase6_buildset.py --check + python3 scripts/phase6_candidate.py validate-buildset --build-set catalog/buildsets/initial-warehouse-v1.json --require-git-ancestry + - name: Capture and verify live audited-input metadata + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py capture-inputs --build-set catalog/buildsets/initial-warehouse-v1.json --output input-live-metadata.json + - name: Download exact Phase-3p proof-data artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33170546601 + artifact-ids: 9685464135 + path: input-artifacts/phase3p-proof-data + - name: Download exact Phase-4 Celestia appd artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688244894 + path: input-artifacts/phase4-celestia-appd-linux-arm64 + - name: Download exact Phase-4 Celestia node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688243729 + path: input-artifacts/phase4-celestia-node-linux-arm64 + - name: Download exact Phase-4 Midnight node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688330126 + path: input-artifacts/phase4-node-linux-arm64 + - name: Download exact Phase-4 toolkit Linux amd64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688263793 + path: input-artifacts/phase4-toolkit-linux-amd64 + - name: Download exact Phase-4 toolkit Linux arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688255774 + path: input-artifacts/phase4-toolkit-linux-arm64 + - name: Download exact Phase-4 toolkit macOS arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9689647047 + path: input-artifacts/phase4-toolkit-macos-arm64 + - name: Download exact Phase-5 verified indexer aggregate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33176004154 + artifact-ids: 9690093579 + path: input-artifacts/phase5-indexer + - name: Assemble and non-executingly verify exact candidate + run: | + set -euo pipefail + python3 scripts/phase6_candidate.py assemble --build-set catalog/buildsets/initial-warehouse-v1.json --input-root input-artifacts --output verified-content | tail -n 1 > candidate-verification.json + python3 scripts/phase6_candidate.py verify --build-set catalog/buildsets/initial-warehouse-v1.json --content verified-content + - name: Retain exact verified content for a fresh job + id: upload + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-pr-verified-content-${{ github.sha }} + path: verified-content/ + if-no-files-found: error + retention-days: 7 + compression-level: 0 + overwrite: false + + fresh-non-executing-verifier: + needs: assemble-and-verify + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + steps: + - name: Check out exact candidate source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Redownload exact staged content by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-and-verify.outputs.artifact-id }} + path: verified-content + - name: Stream and verify every payload/member without execution + run: | + python3 -m pip install --requirement requirements-ci.txt + python3 scripts/phase6_candidate.py verify --build-set catalog/buildsets/initial-warehouse-v1.json --content verified-content diff --git a/.github/workflows/phase6-live-verification.yml b/.github/workflows/phase6-live-verification.yml new file mode 100644 index 0000000..2534f37 --- /dev/null +++ b/.github/workflows/phase6-live-verification.yml @@ -0,0 +1,67 @@ +name: Phase 6 published-candidate live verification + +on: + workflow_run: + workflows: ['Immutable forge candidate'] + types: [completed] + +permissions: + contents: read + actions: read + attestations: read + +concurrency: + group: phase6-published-candidate-live-verification + cancel-in-progress: false + +jobs: + verify-published-candidate: + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'workflow_dispatch' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out exact candidate commit + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ github.event.workflow_run.head_sha }} + persist-credentials: false + - name: Download exact post-publication handoff + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.event.workflow_run.id }} + name: published-candidate-initial-warehouse-v1 + path: handoff + - name: Download released envelope and attestation bundle + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + TAG="$(jq -r '.tag' handoff/published.json)" + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --pattern 'promotion-envelope-initial-warehouse-v1.json' --pattern 'attestation-initial-warehouse-v1.sigstore.json' --dir transport + - name: Verify attestation, immutable release, and independent byte read-back + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 - <<'PY' + import sys + from pathlib import Path + sys.path.insert(0, 'scripts') + import canonical_json + envelope = canonical_json.load_json(Path('transport/promotion-envelope-initial-warehouse-v1.json')) + Path('promotion-claims-initial-warehouse-v1').write_bytes(canonical_json.canonical_bytes(envelope['claims'])) + PY + gh attestation verify promotion-claims-initial-warehouse-v1 --repo "$GITHUB_REPOSITORY" --bundle transport/attestation-initial-warehouse-v1.sigstore.json --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + python3 scripts/github_phase6.py capture-live --envelope transport/promotion-envelope-initial-warehouse-v1.json --bundle transport/attestation-initial-warehouse-v1.sigstore.json --run-id "${{ github.event.workflow_run.id }}" --output live-evidence.json + python3 scripts/canonical_json.py verify-live transport/promotion-envelope-initial-warehouse-v1.json transport/attestation-initial-warehouse-v1.sigstore.json live-evidence.json --require-staging-live + - name: Retain canonical live verification evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-published-live-evidence-${{ github.event.workflow_run.id }} + path: live-evidence.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false diff --git a/catalog/buildsets/initial-warehouse-v1.json b/catalog/buildsets/initial-warehouse-v1.json new file mode 100644 index 0000000..b65f2ee --- /dev/null +++ b/catalog/buildsets/initial-warehouse-v1.json @@ -0,0 +1 @@ +{"buildSetId":"initial-warehouse-v1","candidatePolicy":{"checksumsTemplate":"sha256sums-.txt","destinationCredentialAllowed":false,"immutableReleaseRequired":true,"inputArtifactPinningRequired":true,"protectedDefaultBranchRequired":true,"sourceManifestTemplate":"source-manifest-.json","typedAssetListRequired":true},"components":[{"componentId":"celestia-appd-6.4.10-linux-arm64","manifestPath":"catalog/components/celestia-appd-6.4.10-linux-arm64.json","manifestSha256":"59c65eabeae2179c273741c27a0af5ad5bb4c77e4310d7dc0cb6a26176b828be"},{"componentId":"celestia-node-0.28.4-linux-arm64","manifestPath":"catalog/components/celestia-node-0.28.4-linux-arm64.json","manifestSha256":"7bd9b7c734d2300a390b347f7e65f1e489deefdcdc1572c53bf5f660a5b4a8cf"},{"componentId":"indexer-standalone-linux-amd64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-linux-amd64-4.4.0-rc.3.json","manifestSha256":"5ec8fef85f9742ee9589ae48e17ca5e3e8f62a1d3b7c07ac3293eb752c0e5531"},{"componentId":"indexer-standalone-linux-arm64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-linux-arm64-4.4.0-rc.3.json","manifestSha256":"6b7cb632e29dc36e820da5d6ef54b01b51e18c6c82bc1a6d540f291956f5aaad"},{"componentId":"indexer-standalone-macos-amd64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-macos-amd64-4.4.0-rc.3.json","manifestSha256":"63457c7efd879b907d6abe3127f7c26b99740393984d74701c35409ef3c6ff60"},{"componentId":"indexer-standalone-macos-arm64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-macos-arm64-4.4.0-rc.3.json","manifestSha256":"26fdbecc07d2a2d3980c38901049f675bc388b565e633c5250aa162c8c30f53e"},{"componentId":"midnight-ledger-static-9.0.0","manifestPath":"catalog/components/midnight-ledger-static-9.0.0.json","manifestSha256":"3da06a2a85992212127070101c18b39f51620b8c8605aa7d869278b6f98716c1"},{"componentId":"midnight-node-2.0.0-rc.4-linux-arm64","manifestPath":"catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json","manifestSha256":"4a55943a745af8817a1bfeb71f657cbacb6473a8346c96d38a25af8ba05207c2"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-amd64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json","manifestSha256":"fde7a617ac19a3cd56f65b9462ea2cc481152b7b264211623eeb8eb7d7868cea"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-arm64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json","manifestSha256":"aa5d5d598d4c538e1e4cc855a2d7786c4835fccdff722599154d2c15a69fd558"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-macos-arm64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json","manifestSha256":"5eb01a7ca32a1860525b0abf0719d99abc85f55fd5eafa14181a3be0eba13e63"},{"componentId":"midnight-srs-k0","manifestPath":"catalog/components/midnight-srs-k0.json","manifestSha256":"96b93dc4670efd7765480986a3348428fd4e8ad24539ed84cf6624f35f9bffea"},{"componentId":"midnight-srs-k1","manifestPath":"catalog/components/midnight-srs-k1.json","manifestSha256":"b32784a7645f41b445afd05c9cd80dd5bb925e5c6d0ac78777043b02d4784765"},{"componentId":"midnight-srs-k10","manifestPath":"catalog/components/midnight-srs-k10.json","manifestSha256":"cb9fd03572750a000b2e293e19a229dc45bd66e9259b3449fadf3bf7680c00e9"},{"componentId":"midnight-srs-k11","manifestPath":"catalog/components/midnight-srs-k11.json","manifestSha256":"ad4b0c160ca6844aa561b8a1d223943ee491b6038558d320d5c8fd97592c0e4f"},{"componentId":"midnight-srs-k12","manifestPath":"catalog/components/midnight-srs-k12.json","manifestSha256":"571cec78e30abf1838e475a0226efda56ec1388f16b831658babfc6f7f9f167f"},{"componentId":"midnight-srs-k13","manifestPath":"catalog/components/midnight-srs-k13.json","manifestSha256":"0f3db81343cebc66f3282ed31f58e4f2c8314890ea9f3a0981a2940ce21f601f"},{"componentId":"midnight-srs-k14","manifestPath":"catalog/components/midnight-srs-k14.json","manifestSha256":"c959d6b179d7da155e9d54e47258106696c9c6a670dc4bbcbe7cf6ed9fb63720"},{"componentId":"midnight-srs-k15","manifestPath":"catalog/components/midnight-srs-k15.json","manifestSha256":"96e7fb7101fb3f0721c76be00a74d3eb905ef54a2641cbd1fac266af4d667fb7"},{"componentId":"midnight-srs-k16","manifestPath":"catalog/components/midnight-srs-k16.json","manifestSha256":"1f33014e3f1d2e7abaaa693dbfe2fe0094ea427201d07b44336c79d6b3c37de4"},{"componentId":"midnight-srs-k17","manifestPath":"catalog/components/midnight-srs-k17.json","manifestSha256":"fe59fb43db14585bcadb8eb4f3a13e9471df967a34bec2dbd40283814d9e454d"},{"componentId":"midnight-srs-k18","manifestPath":"catalog/components/midnight-srs-k18.json","manifestSha256":"d7b31231be36ce2905e9b478421b8967b7a4cbededb11413adc02386ad37ad07"},{"componentId":"midnight-srs-k19","manifestPath":"catalog/components/midnight-srs-k19.json","manifestSha256":"4ed42289e8e3b24b255dfc00ee868c2341b189979290a2adb2d89c81bbacf447"},{"componentId":"midnight-srs-k2","manifestPath":"catalog/components/midnight-srs-k2.json","manifestSha256":"ca323861f22c673af8ebca56a846eb20d9a1859859e029bca542240c56b63eb3"},{"componentId":"midnight-srs-k3","manifestPath":"catalog/components/midnight-srs-k3.json","manifestSha256":"50171b4c63116d4ce0ccdd830c85b069d51fd3c5655ed977de8c38be9ea5b110"},{"componentId":"midnight-srs-k4","manifestPath":"catalog/components/midnight-srs-k4.json","manifestSha256":"d37c242136b53b12a96617bb90e02ab9756cb6e9be7f0d5803694b0df94e6d5d"},{"componentId":"midnight-srs-k5","manifestPath":"catalog/components/midnight-srs-k5.json","manifestSha256":"108579a6fd1a411b0177215a051fc9c941632108b6d7483eb0c27e6cae5b9498"},{"componentId":"midnight-srs-k6","manifestPath":"catalog/components/midnight-srs-k6.json","manifestSha256":"89455050237cdc39ee6308836b1781422e1bcbe03651e89ccf4c9f8ec2e2cd17"},{"componentId":"midnight-srs-k7","manifestPath":"catalog/components/midnight-srs-k7.json","manifestSha256":"067c1e0c42f84c4ba86dc351c562a19727a9ec9f24f0dc3ed9c3ecd62c4fabc2"},{"componentId":"midnight-srs-k8","manifestPath":"catalog/components/midnight-srs-k8.json","manifestSha256":"4ea018017431291c216e2060dab31c8cd8acd3e76a56161afb48c42a9dc343e0"},{"componentId":"midnight-srs-k9","manifestPath":"catalog/components/midnight-srs-k9.json","manifestSha256":"4e4b149df4d8d18d601ccd88fcdb788cafd754ba49066e7d51517c46849536c9"}],"coveragePolicy":{"desired":["linux/arm64"],"optional":["macos/amd64"],"proofDataPlatform":"noarch","required":["linux/amd64","macos/arm64"]},"destination":{"distributionTier":"development-only","releaseMutability":"mutable-warehouse","repository":"effectstream/binaries","tag":"0.3.120"},"existingCoverage":[{"arch":"amd64","assetId":378640596,"assetNodeId":"RA_kwDOQpztJs4WkZjU","family":"celestia-appd","name":"celestia-appd-linux-amd64-v6.4.10.tar.gz","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"fa1826187a91514c6d506e22f1fcce25afbd551b98458caf899801b662a0c8da","size":198844329,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"6.4.10"},{"arch":"arm64","assetId":378640593,"assetNodeId":"RA_kwDOQpztJs4WkZjR","family":"celestia-appd","name":"celestia-appd-macos-arm64-v6.4.10.tar.gz","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"832f4b919d79768b960e55f6d778366d9afbb95df6a860e344dffbb92e85b5be","size":196072247,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"6.4.10"},{"arch":"amd64","assetId":378640594,"assetNodeId":"RA_kwDOQpztJs4WkZjS","family":"celestia-node","name":"celestia-node-linux-amd64-v0.28.4.tar.gz","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"bb8b9fd2ac859945fdf2f8c84ba038f75e01be54e56d1a946cbbd4b540f120d9","size":78842331,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"0.28.4"},{"arch":"arm64","assetId":378640595,"assetNodeId":"RA_kwDOQpztJs4WkZjT","family":"celestia-node","name":"celestia-node-macos-arm64-v0.28.4.tar.gz","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"d2d637282489ea1ffc37a5fded4997be7140a7e77a43f0ea61a68f0f73069821","size":76777711,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"0.28.4"},{"arch":"amd64","assetId":526514404,"assetNodeId":"RA_kwDOQpztJs4fYfjk","family":"midnight-node","name":"midnight-node-linux-amd64-2.0.0-rc.4.zip","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"8f53e9dfb2c70ec2fb98fd6958466ef107685774ca4d93660bc63e7686948879","size":84751613,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"2.0.0-rc.4"},{"arch":"arm64","assetId":526513732,"assetNodeId":"RA_kwDOQpztJs4fYfZE","family":"midnight-node","name":"midnight-node-macos-arm64-2.0.0-rc.4.zip","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"4ee77c1043dec716f7a1b133f0ebb8f23bbc3a704f348ae5708a6b58b330ed8c","size":78815030,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"2.0.0-rc.4"}],"inputArtifacts":[{"archiveSha256":"b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f","artifactId":9685464135,"artifactName":"proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1","artifactSize":222975770,"expiresAt":"2026-09-04T12:20:54Z","key":"phase3p-proof-data","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33170546601,"sourceHeadSha":"508aab47ab266344aedd6359fe971928bed309b5","sourceRef":"codex/00002-phase3p-proof-data","workflowPath":".github/workflows/proof-data-q8b.yml"},{"archiveSha256":"3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56","artifactId":9688244894,"artifactName":"phase4-celestia-appd-linux-arm64","artifactSize":181312477,"expiresAt":"2026-09-27T13:54:20Z","key":"phase4-celestia-appd-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba","artifactId":9688243729,"artifactName":"phase4-celestia-node-linux-arm64","artifactSize":72131099,"expiresAt":"2026-09-27T13:54:18Z","key":"phase4-celestia-node-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e","artifactId":9688330126,"artifactName":"phase4-node-linux-arm64","artifactSize":88276678,"expiresAt":"2026-09-27T13:56:54Z","key":"phase4-node-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b","artifactId":9688263793,"artifactName":"phase4-toolkit-linux-amd64","artifactSize":53080691,"expiresAt":"2026-09-27T13:54:55Z","key":"phase4-toolkit-linux-amd64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd","artifactId":9688255774,"artifactName":"phase4-toolkit-linux-arm64","artifactSize":51636493,"expiresAt":"2026-09-27T13:54:40Z","key":"phase4-toolkit-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e","artifactId":9689647047,"artifactName":"phase4-toolkit-macos-arm64","artifactSize":97365627,"expiresAt":"2026-09-27T14:35:47Z","key":"phase4-toolkit-macos-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9","artifactId":9690093579,"artifactName":"phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a","artifactSize":377650526,"expiresAt":"2026-09-27T14:49:01Z","key":"phase5-indexer","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33176004154,"sourceHeadSha":"e581add8952bae5ffeac39fb07e6b5c6f482862d","sourceRef":"codex/00002-phase5-indexer","workflowPath":".github/workflows/phase5-indexer.yml"}],"payloadCount":31,"payloads":[{"artifactKind":"proof-data","componentId":"midnight-srs-k0","container":"raw","installMode":"0644","k":0,"name":"bls_midnight_2p0","platform":"noarch","role":"payload","sha256":"59b30b3114a34ccbbfb599376e178fb8d9b3366cae2174c2f1da20e75847f823","size":580,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p0","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k1","container":"raw","installMode":"0644","k":1,"name":"bls_midnight_2p1","platform":"noarch","role":"payload","sha256":"bbe04fe3c70d0c138447cb086b4baddc30cb8bb2a004114bc02e6f739516280e","size":772,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p1","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k10","container":"raw","installMode":"0644","k":10,"name":"bls_midnight_2p10","platform":"noarch","role":"payload","sha256":"46b2290933cbed4c378889e4ba971f1a92888331ffb09466acd4ff61a1e2cb42","size":196996,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p10","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k11","container":"raw","installMode":"0644","k":11,"name":"bls_midnight_2p11","platform":"noarch","role":"payload","sha256":"9901589d7956ff58be0d85569b2f455b77b58c3758026ffb5bbe4807000b96d1","size":393604,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p11","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k12","container":"raw","installMode":"0644","k":12,"name":"bls_midnight_2p12","platform":"noarch","role":"payload","sha256":"ef08eb3fcf62df8f72c515cffa027e681808b530cb016eea104115545ef6d5c8","size":786820,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p12","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k13","container":"raw","installMode":"0644","k":13,"name":"bls_midnight_2p13","platform":"noarch","role":"payload","sha256":"d3324910969c4cc54143b8045b649e5c3a4bd5fb7b8f85fe1b770f640ce1c803","size":1573252,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p13","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k14","container":"raw","installMode":"0644","k":14,"name":"bls_midnight_2p14","platform":"noarch","role":"payload","sha256":"fc253016885ec830e97808c9ec920bb5cab5c21af590380a6cb5eb0538e2b244","size":3146116,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p14","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k15","container":"raw","installMode":"0644","k":15,"name":"bls_midnight_2p15","platform":"noarch","role":"payload","sha256":"724c7c3d779148bb113c7ee9c034b2f27db16e6bdf315fde90105a9bad00b1de","size":6291844,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p15","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k16","container":"raw","installMode":"0644","k":16,"name":"bls_midnight_2p16","platform":"noarch","role":"payload","sha256":"09c877216d6589b370263e18af40a030a901b41a7a7c37ef58c9901db41f05c6","size":12583300,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p16","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k17","container":"raw","installMode":"0644","k":17,"name":"bls_midnight_2p17","platform":"noarch","role":"payload","sha256":"4a9ef6c7c0619aab74eede44b13e753e3ba54508a02dd3b7106a949aabb73b74","size":25166212,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p17","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k18","container":"raw","installMode":"0644","k":18,"name":"bls_midnight_2p18","platform":"noarch","role":"payload","sha256":"e8436dc5d8b598f169c127c745135d889744007e6d384ff126df8d1332522f86","size":50332036,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p18","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k19","container":"raw","installMode":"0644","k":19,"name":"bls_midnight_2p19","platform":"noarch","role":"payload","sha256":"8e8dc15c4362f05c912f1e770559a3945db3e58a374def416ed5d3e65ad5b10e","size":100663684,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p19","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k2","container":"raw","installMode":"0644","k":2,"name":"bls_midnight_2p2","platform":"noarch","role":"payload","sha256":"80e15568fa1a0117db893239be7fa5e34a6bcc3a8c3bfa7709534b9cb88eb6c1","size":1156,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p2","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k3","container":"raw","installMode":"0644","k":3,"name":"bls_midnight_2p3","platform":"noarch","role":"payload","sha256":"4be827a6472193df80d8f08b4b25a85baef436fdd1965d89b6af89f4ec4e99e2","size":1924,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p3","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k4","container":"raw","installMode":"0644","k":4,"name":"bls_midnight_2p4","platform":"noarch","role":"payload","sha256":"232f401fad10c7ddf8828d2aa4c85c6506c5da09795998cecaeb9f75fc8f6ada","size":3460,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p4","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k5","container":"raw","installMode":"0644","k":5,"name":"bls_midnight_2p5","platform":"noarch","role":"payload","sha256":"0a1c9229f315fc1868ff25f668fb83aec4d09f4f23a706b5197c692c619d72c6","size":6532,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p5","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k6","container":"raw","installMode":"0644","k":6,"name":"bls_midnight_2p6","platform":"noarch","role":"payload","sha256":"cf2ad6be7d0fedf5bec2aaa35f6be4aca33053d74268fdf5aa54fcb2891ea6df","size":12676,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p6","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k7","container":"raw","installMode":"0644","k":7,"name":"bls_midnight_2p7","platform":"noarch","role":"payload","sha256":"e82ae890c080188355f37feaffe91372584cd810615082d9143d4dec0453fd9d","size":24964,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p7","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k8","container":"raw","installMode":"0644","k":8,"name":"bls_midnight_2p8","platform":"noarch","role":"payload","sha256":"909b707551eaaea79828e883cde6fc46ab15986c3b1d791bed462c9e2805c933","size":49540,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p8","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k9","container":"raw","installMode":"0644","k":9,"name":"bls_midnight_2p9","platform":"noarch","role":"payload","sha256":"b9009f1098bcefffec3c461ab3a5e3a17f7e5599f0f08c70fcdc55a89227bcbd","size":98692,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p9","tier":"noarch"},{"arch":"arm64","artifactKind":"software","componentId":"celestia-appd-6.4.10-linux-arm64","container":"tar.gz","installMode":"0755","name":"celestia-appd-linux-arm64-v6.4.10.tar.gz","os":"linux","role":"payload","sha256":"52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b","size":180685852,"sourceArtifactKey":"phase4-celestia-appd-linux-arm64","sourcePath":"payloads/celestia-appd-linux-arm64-v6.4.10.tar.gz","tier":"desired"},{"arch":"arm64","artifactKind":"software","componentId":"celestia-node-0.28.4-linux-arm64","container":"tar.gz","installMode":"0755","name":"celestia-node-linux-arm64-v0.28.4.tar.gz","os":"linux","role":"payload","sha256":"09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c","size":71184641,"sourceArtifactKey":"phase4-celestia-node-linux-arm64","sourcePath":"payloads/celestia-node-linux-arm64-v0.28.4.tar.gz","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"indexer-standalone-linux-amd64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-linux-amd64-v4.4.0-rc.3.zip","os":"linux","role":"payload","sha256":"4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434","size":31479027,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-linux-amd64-v4.4.0-rc.3.zip","tier":"required"},{"arch":"arm64","artifactKind":"software","componentId":"indexer-standalone-linux-arm64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-linux-arm64-v4.4.0-rc.3.zip","os":"linux","role":"payload","sha256":"eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0","size":29782570,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-linux-arm64-v4.4.0-rc.3.zip","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"indexer-standalone-macos-amd64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-macos-amd64-v4.4.0-rc.3.zip","os":"macos","role":"payload","sha256":"28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc","size":30713420,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-macos-amd64-v4.4.0-rc.3.zip","tier":"optional"},{"arch":"arm64","artifactKind":"software","componentId":"indexer-standalone-macos-arm64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-macos-arm64-v4.4.0-rc.3.zip","os":"macos","role":"payload","sha256":"b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2","size":29072181,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-macos-arm64-v4.4.0-rc.3.zip","tier":"required"},{"artifactKind":"proof-data","componentId":"midnight-ledger-static-9.0.0","container":"zip","installMode":"0644","ledgerStaticSemver":"9.0.0","name":"midnight-ledger-static-noarch-9.0.0.zip","platform":"noarch","role":"payload","sha256":"d7e8ccfdbc55a2b7139aadd4797d665f888a4502b63ebae24d23314eeee341b2","size":21601265,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/midnight-ledger-static-noarch-9.0.0.zip","tier":"noarch"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-2.0.0-rc.4-linux-arm64","container":"zip","installMode":"0755","name":"midnight-node-linux-arm64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c","size":82544614,"sourceArtifactKey":"phase4-node-linux-arm64","sourcePath":"payloads/midnight-node-linux-arm64-2.0.0-rc.4.zip","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-amd64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c","size":50017428,"sourceArtifactKey":"phase4-toolkit-linux-amd64","sourcePath":"payloads/midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip","tier":"required"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-arm64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528","size":48585850,"sourceArtifactKey":"phase4-toolkit-linux-arm64","sourcePath":"payloads/midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip","tier":"desired"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-macos-arm64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip","os":"macos","role":"payload","sha256":"8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c","size":45553847,"sourceArtifactKey":"phase4-toolkit-macos-arm64","sourcePath":"payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip","tier":"required"}],"schemaVersion":"build-set-v1","sourceFullSha":"19de8be5f434225dbf17126e86b6c3cc6aacc4fe"} \ No newline at end of file diff --git a/docs/publishing.md b/docs/publishing.md index 50d2f0f..e2373b2 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -5,23 +5,30 @@ writes. `effectstream/binaries@0.3.120` is updated manually outside Actions. ## Candidate lifecycle -Phase 1 installs and tests the boundary contract but deliberately cannot allocate or publish a -release: `PHASE6_CANDIDATE_ENABLED` is absent and all draft/final-claims/publisher jobs are guarded; -their transaction steps also fail closed. Phase 6 replaces those stubs with the exact hash-bound -implementation through a reviewed main-branch PR, runs the live gate, and only then enables the -repository variable. Setting the variable against the Phase-1 scaffold cannot publish a draft. +Phase 6 replaces the Phase-1 scaffold with an exact hash-bound implementation. Publication has no +repository-variable switch: the workflow exists only after reviewed protected-main integration, +accepts a build-set ID plus the exact committed build-set SHA-256, and rechecks protected main, +the workflow blob, immutable-release policy, PR-only/no-bypass rules, and the protected publisher +environment before allocating a draft. The workflow contains no destination credential reference. 1. Merge exact component/build-set manifests through protected main. Reject abbreviated/floating refs, Compact warehouse components, incomplete required platform coverage, and proof-data scope outside K0–K19 plus Ledger static 9. -2. Dispatch `.github/workflows/candidate.yml` with one committed build-set ID. The workflow resolves - the file from the triggering full SHA; free-form manifest JSON or URLs are not inputs. -3. Native build/mirror jobs run without secrets/write/OIDC. They emit payloads and content evidence - with exact roles. Software gets SBOM/provenance; proof data gets lineage/member evidence and no - fabricated signing/SBOM fields. -4. Upload content to short-lived staging. A fresh pre-draft verifier downloads it with no cache or - credentials, verifies every exact content byte, and emits only the content-list digest required - to allocate the draft. It does not emit final claims. +2. Dispatch `.github/workflows/candidate.yml` with build-set ID `initial-warehouse-v1` and the exact + SHA-256 of `catalog/buildsets/initial-warehouse-v1.json`. Free-form manifest JSON/URLs are not + inputs. Global `forge-candidate-publication` concurrency serializes every build-set/tag. +3. The no-write assembler downloads only the eight pinned, audited Actions artifacts by numeric + repository/run/artifact identity and verifies their API run event/conclusion/full source SHA, + name/size/wrapper digest and expiry. It emits exactly ten software payloads and Q8=B's 21 noarch + proof payloads (K0–K19 plus Ledger-static 9), never Compact or a platform/proof-server duplicate. + The retained Phase-3p artifact expires on 2026-09-04; if it expires before an accepted candidate, + rerun the exact audited Q8=B workflow from reviewed main, require identical 21 payload bytes and + a new independent audit, then update numeric pins through another PR. Never substitute by name. +4. The flat inert candidate also carries canonical source/checksum manifests, Apache license and + development notice, one SPDX SBOM per software payload, software archive-member policies, + proof lineage/cache/member evidence, signing evidence (including the actual unsigned/adhoc + macOS state), and provenance. Every evidence basename has one frozen envelope-v1 role. Upload + this content to short-lived staging with its complete typed-list digest in the artifact name. 5. The protected publisher creates an empty `forge-YYYY.MM.DD.N` draft, then reads back and freezes its repository/tag/numeric+node ID/target/URL identity. It passes only that inert identity to a second fresh no-write/no-OIDC verifier. @@ -35,9 +42,11 @@ repository variable. Setting the variable against the Phase-1 scaffold cannot pu 8. Upload content plus exactly the two predeclared transport files to the draft. Re-download every draft asset through the API, hash it, verify the exact complete name/count/byte set, then publish. API-read immutable state. Any upload/read-back/policy mismatch leaves a draft and fails. -9. Emit live evidence and verify workflow blob/run/artifact/protected-main/release identity plus every - independently downloaded byte, including the raw envelope and bundle. Prove a mutation attempt is - rejected by immutable release policy. +9. Prove a no-op release metadata mutation is rejected after immutable publication. The separate + read-only `phase6-live-verification.yml` workflow runs only after the candidate workflow reports + success; this sequencing lets it truthfully bind `status=completed, conclusion=success`. It + cryptographically verifies the released bundle and independently re-downloads/hashes every + released byte before emitting canonical live evidence. ## Workflow permissions diff --git a/schema/build-set-v1.schema.json b/schema/build-set-v1.schema.json index f211707..9b039c4 100644 --- a/schema/build-set-v1.schema.json +++ b/schema/build-set-v1.schema.json @@ -10,6 +10,7 @@ "sourceFullSha", "destination", "components", + "inputArtifacts", "existingCoverage", "payloads", "payloadCount", @@ -45,6 +46,12 @@ } } }, + "inputArtifacts": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { "$ref": "#/$defs/inputArtifact" } + }, "existingCoverage": { "type": "array", "items": { "$ref": "#/$defs/coverage" } @@ -75,6 +82,7 @@ "typedAssetListRequired", "sourceManifestTemplate", "checksumsTemplate", + "inputArtifactPinningRequired", "destinationCredentialAllowed" ], "properties": { @@ -83,11 +91,56 @@ "typedAssetListRequired": { "const": true }, "sourceManifestTemplate": { "const": "source-manifest-.json" }, "checksumsTemplate": { "const": "sha256sums-.txt" }, + "inputArtifactPinningRequired": { "const": true }, "destinationCredentialAllowed": { "const": false } } } }, "$defs": { + "inputArtifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "repository", + "repositoryId", + "workflowPath", + "runId", + "runAttempt", + "runEvent", + "runConclusion", + "sourceRef", + "sourceHeadSha", + "artifactId", + "artifactName", + "artifactSize", + "archiveSha256", + "expiresAt" + ], + "properties": { + "key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, + "repository": { "const": "acedward/midnight-binary-forge" }, + "repositoryId": { "const": 1349127482 }, + "workflowPath": { + "enum": [ + ".github/workflows/proof-data-q8b.yml", + ".github/workflows/phase4-payloads.yml", + ".github/workflows/phase5-indexer.yml" + ] + }, + "runId": { "type": "integer", "minimum": 1 }, + "runAttempt": { "const": 1 }, + "runEvent": { "enum": ["pull_request", "push", "workflow_dispatch"] }, + "runConclusion": { "const": "success" }, + "sourceRef": { "type": "string", "pattern": "^[A-Za-z0-9._/-]+$" }, + "sourceHeadSha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "artifactId": { "type": "integer", "minimum": 1 }, + "artifactName": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$" }, + "artifactSize": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "archiveSha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "expiresAt": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" } + } + }, "coverage": { "type": "object", "additionalProperties": false, @@ -137,13 +190,31 @@ "payload": { "type": "object", "additionalProperties": false, - "required": ["name", "role", "artifactKind", "componentId", "tier"], + "required": [ + "name", + "role", + "artifactKind", + "componentId", + "tier", + "container", + "size", + "sha256", + "sourceArtifactKey", + "sourcePath", + "installMode" + ], "properties": { "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, "role": { "const": "payload" }, "artifactKind": { "enum": ["software", "proof-data"] }, "componentId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, "tier": { "enum": ["required", "desired", "optional", "noarch"] }, + "container": { "enum": ["raw", "zip", "tar.gz"] }, + "size": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "sourceArtifactKey": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, + "sourcePath": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,511}$" }, + "installMode": { "enum": ["0644", "0755"] }, "os": { "enum": ["linux", "macos"] }, "arch": { "enum": ["amd64", "arm64"] }, "platform": { "const": "noarch" }, diff --git a/scripts/check_workflow_policy.py b/scripts/check_workflow_policy.py index e050a6e..8e326f1 100755 --- a/scripts/check_workflow_policy.py +++ b/scripts/check_workflow_policy.py @@ -42,12 +42,20 @@ def validate_workflow(path: Path) -> None: expect("id-token: write" not in text and "attestations: write" not in text and "contents: write" not in text, f"{path.name}: non-candidate workflow requests write/OIDC authority") expect("secrets:" not in text, f"{path.name}: non-candidate workflow declares secrets") else: - expect("environment: candidate-publish" in text, "candidate.yml: protected publisher environment missing") + expect(text.count("environment: candidate-publish") == 2, "candidate.yml: draft and publisher must use the protected environment") expect("id-token: write" in text and "attestations: write" in text and "contents: write" in text, "candidate.yml: forge publisher permission contract missing") - expect(text.count("if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }}") == 3, "candidate.yml: all Phase-1 privileged/claims jobs must remain fail-closed until explicit Phase 6 activation") + expect("PHASE6_CANDIDATE_ENABLED" not in text and "vars." not in text, "candidate.yml: publication cannot depend on an unpinned repository variable") + expect("group: forge-candidate-publication" in text and "cancel-in-progress: false" in text, "candidate.yml: global non-cancelling publication concurrency missing") + expect("actions: read" in text and "build-set-sha256" in text, "candidate.yml: exact build-set/actions read contract missing") + expect("verify-source --expected-sha \"$GITHUB_SHA\"" in text, "candidate.yml: live protected-main full-SHA gate missing") expect("actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6" in text, "candidate.yml: frozen attest action pin missing") expect("predicate-type: https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1" in text, "candidate.yml: frozen predicate type missing") expect("subject-name: promotion-claims-${{ inputs.build-set-id }}" in text, "candidate.yml: frozen subject name missing") + expect("gh attestation verify" in text and "scripts/materialize_envelope.py" in text, "candidate.yml: cryptographic bundle verification/envelope materialization missing") + expect("scripts/github_phase6.py publish" in text, "candidate.yml: create-only read-back publisher missing") + expect("--clobber" not in text and "delete release" not in text.casefold() and "-X DELETE" not in text, "candidate.yml: destructive release mutation token is forbidden") + for artifact_id in (9685464135, 9688244894, 9688243729, 9688330126, 9688263793, 9688255774, 9689647047, 9690093579): + expect(str(artifact_id) in text, f"candidate.yml: audited input artifact pin missing: {artifact_id}") def main() -> int: diff --git a/scripts/generate_phase6_buildset.py b/scripts/generate_phase6_buildset.py new file mode 100644 index 0000000..1f817f5 --- /dev/null +++ b/scripts/generate_phase6_buildset.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Generate the reviewed, exact Phase-6 initial warehouse build set.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from forge_io import canonical_bytes, create_file_atomic, load_json, sha256_file + + +ROOT = Path(__file__).resolve().parents[1] +BASE_SHA = "19de8be5f434225dbf17126e86b6c3cc6aacc4fe" +SNAPSHOT_PATH = "evidence/phase0/warehouse-release-0.3.120.json" +SNAPSHOT_SHA = "6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1" +INPUTS = [ + ("phase3p-proof-data", ".github/workflows/proof-data-q8b.yml", 33170546601, "pull_request", "codex/00002-phase3p-proof-data", "508aab47ab266344aedd6359fe971928bed309b5", 9685464135, "proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1", 222975770, "b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f", "2026-09-04T12:20:54Z"), + ("phase4-celestia-appd-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688244894, "phase4-celestia-appd-linux-arm64", 181312477, "3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56", "2026-09-27T13:54:20Z"), + ("phase4-celestia-node-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688243729, "phase4-celestia-node-linux-arm64", 72131099, "b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba", "2026-09-27T13:54:18Z"), + ("phase4-node-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688330126, "phase4-node-linux-arm64", 88276678, "ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e", "2026-09-27T13:56:54Z"), + ("phase4-toolkit-linux-amd64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688263793, "phase4-toolkit-linux-amd64", 53080691, "9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b", "2026-09-27T13:54:55Z"), + ("phase4-toolkit-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688255774, "phase4-toolkit-linux-arm64", 51636493, "e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd", "2026-09-27T13:54:40Z"), + ("phase4-toolkit-macos-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9689647047, "phase4-toolkit-macos-arm64", 97365627, "ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e", "2026-09-27T14:35:47Z"), + ("phase5-indexer", ".github/workflows/phase5-indexer.yml", 33176004154, "pull_request", "codex/00002-phase5-indexer", "e581add8952bae5ffeac39fb07e6b5c6f482862d", 9690093579, "phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a", 377650526, "eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9", "2026-09-27T14:49:01Z"), +] +SOFTWARE = [ + ("celestia-appd-linux-arm64-v6.4.10.tar.gz", "celestia-appd-6.4.10-linux-arm64", "desired", "linux", "arm64", 180685852, "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b", "phase4-celestia-appd-linux-arm64"), + ("celestia-node-linux-arm64-v0.28.4.tar.gz", "celestia-node-0.28.4-linux-arm64", "desired", "linux", "arm64", 71184641, "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c", "phase4-celestia-node-linux-arm64"), + ("indexer-standalone-linux-amd64-v4.4.0-rc.3.zip", "indexer-standalone-linux-amd64-4.4.0-rc.3", "required", "linux", "amd64", 31479027, "4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434", "phase5-indexer"), + ("indexer-standalone-linux-arm64-v4.4.0-rc.3.zip", "indexer-standalone-linux-arm64-4.4.0-rc.3", "desired", "linux", "arm64", 29782570, "eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0", "phase5-indexer"), + ("indexer-standalone-macos-amd64-v4.4.0-rc.3.zip", "indexer-standalone-macos-amd64-4.4.0-rc.3", "optional", "macos", "amd64", 30713420, "28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc", "phase5-indexer"), + ("indexer-standalone-macos-arm64-v4.4.0-rc.3.zip", "indexer-standalone-macos-arm64-4.4.0-rc.3", "required", "macos", "arm64", 29072181, "b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2", "phase5-indexer"), + ("midnight-node-linux-arm64-2.0.0-rc.4.zip", "midnight-node-2.0.0-rc.4-linux-arm64", "desired", "linux", "arm64", 82544614, "490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c", "phase4-node-linux-arm64"), + ("midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-linux-amd64", "required", "linux", "amd64", 50017428, "92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c", "phase4-toolkit-linux-amd64"), + ("midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-linux-arm64", "desired", "linux", "arm64", 48585850, "4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528", "phase4-toolkit-linux-arm64"), + ("midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-macos-arm64", "required", "macos", "arm64", 45553847, "8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c", "phase4-toolkit-macos-arm64"), +] +EXISTING_NAMES = { + "celestia-appd-linux-amd64-v6.4.10.tar.gz": ("celestia-appd", "6.4.10", "linux", "amd64", "required"), + "celestia-appd-macos-arm64-v6.4.10.tar.gz": ("celestia-appd", "6.4.10", "macos", "arm64", "required"), + "celestia-node-linux-amd64-v0.28.4.tar.gz": ("celestia-node", "0.28.4", "linux", "amd64", "required"), + "celestia-node-macos-arm64-v0.28.4.tar.gz": ("celestia-node", "0.28.4", "macos", "arm64", "required"), + "midnight-node-linux-amd64-2.0.0-rc.4.zip": ("midnight-node", "2.0.0-rc.4", "linux", "amd64", "required"), + "midnight-node-macos-arm64-2.0.0-rc.4.zip": ("midnight-node", "2.0.0-rc.4", "macos", "arm64", "required"), +} + + +def build(root: Path) -> dict: + component_paths = sorted((root / "catalog/components").glob("*.json")) + components = [] + component_by_id = {} + for path in component_paths: + component = load_json(path) + component_by_id[component["componentId"]] = component + components.append({"componentId": component["componentId"], "manifestPath": path.relative_to(root).as_posix(), "manifestSha256": sha256_file(path)[0]}) + inputs = [] + for key, workflow, run, event, ref, head, artifact, name, size, digest, expires in INPUTS: + inputs.append({"key": key, "repository": "acedward/midnight-binary-forge", "repositoryId": 1349127482, "workflowPath": workflow, "runId": run, "runAttempt": 1, "runEvent": event, "runConclusion": "success", "sourceRef": ref, "sourceHeadSha": head, "artifactId": artifact, "artifactName": name, "artifactSize": size, "archiveSha256": digest, "expiresAt": expires}) + payloads = [] + for name, component_id, tier, os_name, arch, size, digest, source_key in SOFTWARE: + component = component_by_id[component_id] + prefix = "payload" if source_key == "phase5-indexer" else "payloads" + payloads.append({"name": name, "role": "payload", "artifactKind": "software", "componentId": component_id, "tier": tier, "container": component["naming"]["container"], "size": size, "sha256": digest, "sourceArtifactKey": source_key, "sourcePath": f"{prefix}/{name}", "installMode": component["install"]["mode"], "os": os_name, "arch": arch}) + proof = load_json(root / "catalog/proof-data/q8b-v1.json") + for row in proof["srs"]: + payloads.append({"name": row["releaseName"], "role": "payload", "artifactKind": "proof-data", "componentId": row["componentId"], "tier": "noarch", "container": "raw", "size": row["size"], "sha256": row["sha256"], "sourceArtifactKey": "phase3p-proof-data", "sourcePath": f"payloads/{row['releaseName']}", "installMode": row["mode"], "platform": "noarch", "k": row["k"]}) + ledger = proof["ledgerStatic"] + payloads.append({"name": ledger["releaseName"], "role": "payload", "artifactKind": "proof-data", "componentId": ledger["componentId"], "tier": "noarch", "container": "zip", "size": ledger["archiveSize"], "sha256": ledger["archiveSha256"], "sourceArtifactKey": "phase3p-proof-data", "sourcePath": f"payloads/{ledger['releaseName']}", "installMode": "0644", "platform": "noarch", "ledgerStaticSemver": "9.0.0"}) + snapshot = load_json(root / SNAPSHOT_PATH) + assets = {row["name"]: row for row in snapshot["assets"]} + existing = [] + for name, (family, version, os_name, arch, tier) in sorted(EXISTING_NAMES.items()): + asset = assets[name] + existing.append({"family": family, "version": version, "os": os_name, "arch": arch, "tier": tier, "source": "warehouse-existing", "repository": "effectstream/binaries", "repositoryId": 1117580582, "repositoryNodeId": "R_kgDOQpztJg", "releaseTag": "0.3.120", "releaseId": 270761136, "releaseNodeId": "RE_kwDOQpztJs4QI3yw", "snapshotPath": SNAPSHOT_PATH, "snapshotSha256": SNAPSHOT_SHA, "assetId": asset["id"], "assetNodeId": asset["nodeId"], "name": name, "size": asset["size"], "sha256": asset["digest"].removeprefix("sha256:")}) + return {"schemaVersion": "build-set-v1", "buildSetId": "initial-warehouse-v1", "sourceFullSha": BASE_SHA, "destination": {"repository": "effectstream/binaries", "tag": "0.3.120", "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, "components": components, "inputArtifacts": inputs, "existingCoverage": existing, "payloads": sorted(payloads, key=lambda row: row["name"]), "payloadCount": len(payloads), "coveragePolicy": {"required": ["linux/amd64", "macos/arm64"], "desired": ["linux/arm64"], "optional": ["macos/amd64"], "proofDataPlatform": "noarch"}, "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "inputArtifactPinningRequired": True, "destinationCredentialAllowed": False}} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=ROOT / "catalog/buildsets/initial-warehouse-v1.json") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + rendered = canonical_bytes(build(ROOT)) + if args.check: + if args.output.read_bytes() != rendered: + raise SystemExit("generated Phase-6 build set differs from committed bytes") + else: + create_file_atomic(args.output, rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/github_phase6.py b/scripts/github_phase6.py new file mode 100644 index 0000000..cf58579 --- /dev/null +++ b/scripts/github_phase6.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Fail-closed GitHub API boundary for Phase-6 candidate publication.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email.utils +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +import canonical_json +import phase6_candidate +import publisher_guard +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, load_json, sha256_file + + +API = "https://api.github.com" +REPOSITORY = phase6_candidate.REPOSITORY +REPOSITORY_ID = phase6_candidate.REPOSITORY_ID +MAIN_REF = "refs/heads/main" +WORKFLOW_PATH = ".github/workflows/candidate.yml" + + +def token() -> str: + value = os.environ.get("GITHUB_TOKEN", "") + expect(bool(value), "GITHUB_TOKEN is required") + return value + + +def request(path: str, method: str = "GET", body: Any | None = None, accept: str = "application/vnd.github+json") -> tuple[Any, str]: + data = None if body is None else canonical_bytes(body) + req = urllib.request.Request( + API + path, + data=data, + method=method, + headers={"Accept": accept, "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": "2022-11-28", **({"Content-Type": "application/json"} if data is not None else {})}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read() + date = response.headers.get("Date", "") + except urllib.error.HTTPError as exc: + detail = exc.read(4096).decode("utf-8", "replace") + raise ForgeError(f"GitHub API {method} {path} failed with HTTP {exc.code}: {detail}") from exc + return (json.loads(raw) if raw else None), date + + +def api_time(header: str) -> str: + expect(bool(header), "GitHub API response omitted Date") + parsed = email.utils.parsedate_to_datetime(header) + expect(parsed is not None and parsed.tzinfo is not None, "GitHub API Date is invalid") + return parsed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def pagination(path: str) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for page in range(1, 12): + join = "&" if "?" in path else "?" + value, _ = request(f"{path}{join}per_page=100&page={page}") + expect(isinstance(value, list), f"GitHub API pagination response is not an array: {path}") + result.extend(value) + if len(value) < 100: + return result + raise ForgeError(f"GitHub API pagination ceiling exceeded: {path}") + + +def require_actions_context() -> None: + expect(os.environ.get("GITHUB_ACTIONS") == "true", "Phase-6 GitHub mutation is Actions-only") + expect(os.environ.get("GITHUB_REPOSITORY") == REPOSITORY, "Phase-6 Actions repository mismatch") + expect(os.environ.get("GITHUB_REF") == MAIN_REF, "Phase-6 mutation requires protected main") + expect(os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch", "Phase-6 mutation requires workflow_dispatch") + + +def capture_inputs(buildset_path: Path, output: Path) -> None: + buildset, _ = phase6_candidate.validate_buildset(buildset_path) + repository, date = request(f"/repos/{REPOSITORY}") + runs = [] + for run_id in sorted({row["runId"] for row in buildset["inputArtifacts"]}): + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + runs.append(run) + artifacts = [] + for artifact_id in sorted(row["artifactId"] for row in buildset["inputArtifacts"]): + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}") + artifacts.append(artifact) + value = {"schemaVersion": "phase6-input-live-metadata-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"]}, "runs": runs, "artifacts": artifacts} + phase6_candidate.verify_live_metadata(buildset, value) + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def verify_source(expected_sha: str, output: Path) -> None: + require_actions_context() + expect(os.environ.get("GITHUB_SHA") == expected_sha and re.fullmatch(r"[0-9a-f]{40}", expected_sha) is not None, "protected source full SHA mismatch") + repository, date = request(f"/repos/{REPOSITORY}") + expect(repository["id"] == REPOSITORY_ID and repository["full_name"] == REPOSITORY and repository["default_branch"] == "main", "live repository/default branch identity mismatch") + expect(repository.get("immutable_releases_enabled") is True, "forge immutable releases are not enabled") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + expect(branch.get("protected") is True and branch.get("commit", {}).get("sha") == expected_sha, "current source is not live protected main") + rulesets = pagination(f"/repos/{REPOSITORY}/rulesets") + active = [row for row in rulesets if row.get("enforcement") == "active"] + expect(active, "no active repository ruleset") + details = [request(f"/repos/{REPOSITORY}/rulesets/{row['id']}")[0] for row in active] + expect(any(any(rule.get("type") == "pull_request" for rule in row.get("rules", [])) and not row.get("bypass_actors") for row in details), "active PR-only/no-bypass ruleset absent") + environment, _ = request(f"/repos/{REPOSITORY}/environments/candidate-publish") + policy = environment.get("deployment_branch_policy") or {} + expect(policy.get("protected_branches") is True and policy.get("custom_branch_policies") is False, "candidate-publish is not protected-branch-only") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={expected_sha}") + expect(workflow.get("type") == "file" and re.fullmatch(r"[0-9a-f]{40}", workflow.get("sha", "")), "cannot bind candidate workflow blob") + value = {"schemaVersion": "phase6-protected-source-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"], "immutableReleasesEnabled": True}, "ref": MAIN_REF, "commitSha": expected_sha, "protected": True, "workflowPath": WORKFLOW_PATH, "workflowSha": workflow["sha"], "rulesetIds": sorted(row["id"] for row in details), "candidateEnvironment": "candidate-publish", "protectedBranchesOnly": True, "referencedRepositoryVariables": 0, "referencedDestinationSecrets": 0} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def capture_staging(artifact_id: int, expected_name: str, run_id: int, run_attempt: int, output: Path) -> None: + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}") + workflow = artifact.get("workflow_run", {}) + expect(artifact.get("id") == artifact_id and artifact.get("name") == expected_name, "staging artifact ID/name mismatch") + expect(artifact.get("expired") is False and workflow.get("id") == run_id and workflow.get("repository_id") == REPOSITORY_ID, "staging artifact is expired or belongs to another run/repository") + digest = artifact.get("digest", "") + expect(re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is not None, "staging artifact has no SHA-256 digest") + value = {"artifactId": artifact_id, "artifactName": expected_name, "archiveSha256": digest.removeprefix("sha256:"), "expiresAt": artifact["expires_at"], "runId": run_id, "runAttempt": run_attempt} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def allocate_draft(expected_sha: str, source_path: Path, output: Path) -> None: + require_actions_context() + source = load_json(source_path) + expect(source.get("commitSha") == expected_sha and source.get("repository", {}).get("immutableReleasesEnabled") is True, "protected-source evidence mismatch") + _, date_header = request("/rate_limit") + date = api_time(date_header)[:10].replace("-", ".") + releases = pagination(f"/repos/{REPOSITORY}/releases") + tags = pagination(f"/repos/{REPOSITORY}/tags") + occupied = {row.get("tag_name") for row in releases} | {row.get("name") for row in tags} + sequence = next(number for number in range(1, 10000) if f"forge-{date}.{number}" not in occupied) + tag = f"forge-{date}.{sequence}" + body = phase6_candidate.WARNING + "\n\nThis candidate is immutable supply-chain evidence. Only typed role=payload assets are eligible for the separately reviewed manual warehouse transaction." + created, _ = request(f"/repos/{REPOSITORY}/releases", "POST", {"tag_name": tag, "target_commitish": expected_sha, "name": tag, "body": body, "draft": True, "prerelease": False}) + reread, _ = request(f"/repos/{REPOSITORY}/releases/{created['id']}") + expect(reread["id"] == created["id"] and reread["tag_name"] == tag and reread["target_commitish"] == expected_sha and reread["draft"] is True and reread["prerelease"] is False and reread.get("assets") == [], "allocated draft read-back mismatch") + value = {key: reread[key] for key in ("id", "node_id", "html_url", "tag_name", "target_commitish")} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def download_asset(asset_id: int, output: Path) -> None: + req = urllib.request.Request(API + f"/repos/{REPOSITORY}/releases/assets/{asset_id}", headers={"Accept": "application/octet-stream", "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": "2022-11-28"}) + with urllib.request.urlopen(req, timeout=120) as response, output.open("xb") as stream: + while True: + block = response.read(1024 * 1024) + if not block: + break + stream.write(block) + + +def publish(claims_path: Path, content: Path, bundle: Path, envelope: Path, draft_path: Path, output: Path) -> None: + require_actions_context() + publisher_guard.verify_transport(envelope, bundle, content) + claims = load_json(claims_path) + draft = load_json(draft_path) + expect(claims == load_json(envelope)["claims"] and draft["id"] == claims["candidateDraft"]["releaseId"], "publisher draft/claims/envelope mismatch") + release, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}") + expect(release["draft"] is True and release["tag_name"] == draft["tag_name"] and release["target_commitish"] == claims["issuer"]["commitSha"], "publisher draft state mismatch") + expect(pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") == [], "publisher draft is not empty") + paths = {path.name: path for path in content.iterdir()} + paths[bundle.name] = bundle + paths[envelope.name] = envelope + expect(sorted(paths) == claims["completeAssetNames"], "publisher complete local asset-name set mismatch") + expected = {row["name"]: (row["size"], row["sha256"]) for row in claims["contentAssets"]} + expected[bundle.name] = (bundle.stat().st_size, sha256_file(bundle)[0]) + expected[envelope.name] = (envelope.stat().st_size, sha256_file(envelope)[0]) + env = {**os.environ, "GH_TOKEN": token()} + for name in sorted(paths): + subprocess.run(["gh", "release", "upload", draft["tag_name"], str(paths[name]), "--repo", REPOSITORY], check=True, env=env, timeout=600) + rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") + observed = next((row for row in rows if row.get("name") == name), None) + expect(observed is not None and observed.get("state") == "uploaded", f"uploaded asset read-back missing: {name}") + size, digest = expected[name] + expect(observed.get("size") == size and observed.get("digest") == f"sha256:{digest}", f"uploaded asset API identity mismatch: {name}") + temporary = Path(tempfile.gettempdir()) / f"phase6-readback-{os.getpid()}-{observed['id']}" + expect(not temporary.exists(), "read-back temporary path collision") + try: + download_asset(observed["id"], temporary) + actual_digest, actual_size = sha256_file(temporary, 2**31 - 1) + expect((actual_size, actual_digest) == (size, digest), f"uploaded asset independent download mismatch: {name}") + finally: + temporary.unlink(missing_ok=True) + rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") + expect(sorted(row["name"] for row in rows) == claims["completeAssetNames"], "complete uploaded asset set mismatch") + published, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"draft": False}) + expect(published["draft"] is False and published["prerelease"] is False and published.get("immutable") is True, "published release is not immutable") + mutation_rejected = False + try: + request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"name": published["name"]}) + except ForgeError: + mutation_rejected = True + expect(mutation_rejected, "immutable published release accepted a no-op metadata mutation probe") + value = {"schemaVersion": "phase6-published-candidate-v1", "releaseId": published["id"], "tag": published["tag_name"], "targetCommitish": published["target_commitish"], "immutable": True, "assetCount": len(rows), "completeAssetNameListSha256": claims["completeAssetNameListSha256"], "mutationRejected": True} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def capture_live(envelope_path: Path, bundle_path: Path, run_id: int, output: Path) -> None: + envelope = canonical_json.load_json(envelope_path) + canonical_json.verify_envelope(envelope) + claims = envelope["claims"] + expect(claims["staging"]["runId"] == run_id, "candidate run ID differs from signed staging run") + repository, date = request(f"/repos/{REPOSITORY}") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={claims['issuer']['commitSha']}") + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{claims['staging']['artifactId']}") + release, _ = request(f"/repos/{REPOSITORY}/releases/{claims['candidateDraft']['releaseId']}") + assets = pagination(f"/repos/{REPOSITORY}/releases/{release['id']}/assets") + asset_rows = [] + for asset in sorted(assets, key=lambda row: row["name"]): + temporary = Path(tempfile.gettempdir()) / f"phase6-live-{os.getpid()}-{asset['id']}" + expect(not temporary.exists(), "live-evidence temporary path collision") + try: + download_asset(asset["id"], temporary) + digest, size = sha256_file(temporary, 2**31 - 1) + expect(asset.get("state") == "uploaded" and asset.get("size") == size and asset.get("digest") == f"sha256:{digest}", f"live release asset API/download mismatch: {asset['name']}") + asset_rows.append({"name": asset["name"], "size": size, "sha256": digest}) + finally: + temporary.unlink(missing_ok=True) + value = { + "schemaVersion": "promotion-live-evidence-v1", + "capturedAt": api_time(date), + "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, + "protectedRef": {"ref": MAIN_REF, "commitSha": branch["commit"]["sha"], "protected": branch["protected"]}, + "workflowFile": {"path": WORKFLOW_PATH, "commitSha": claims["issuer"]["commitSha"], "blobSha": workflow["sha"]}, + "run": {"id": run["id"], "attempt": run["run_attempt"], "repository": run["repository"]["full_name"], "workflowPath": run["path"], "event": run["event"], "headSha": run["head_sha"], "headRef": run["head_branch"], "status": run["status"], "conclusion": run["conclusion"]}, + "stagingArtifact": {"id": artifact["id"], "runId": artifact["workflow_run"]["id"], "runAttempt": claims["staging"]["runAttempt"], "name": artifact["name"], "archiveSha256": artifact["digest"].removeprefix("sha256:"), "expired": artifact["expired"], "expiresAt": artifact["expires_at"]}, + "release": {"id": release["id"], "nodeId": release["node_id"], "repository": REPOSITORY, "tag": release["tag_name"], "targetCommitish": release["target_commitish"], "url": release["html_url"], "draft": release["draft"], "prerelease": release["prerelease"], "immutable": release.get("immutable")}, + "releaseAssets": asset_rows, + } + canonical_json.verify_live_evidence(envelope, value, envelope_path, bundle_path, allow_expired_staging=False) + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + capture = sub.add_parser("capture-inputs") + capture.add_argument("--build-set", type=Path, required=True); capture.add_argument("--output", type=Path, required=True) + source = sub.add_parser("verify-source") + source.add_argument("--expected-sha", required=True); source.add_argument("--output", type=Path, required=True) + staging = sub.add_parser("capture-staging") + staging.add_argument("--artifact-id", type=int, required=True); staging.add_argument("--expected-name", required=True); staging.add_argument("--run-id", type=int, required=True); staging.add_argument("--run-attempt", type=int, required=True); staging.add_argument("--output", type=Path, required=True) + draft = sub.add_parser("allocate-draft") + draft.add_argument("--expected-sha", required=True); draft.add_argument("--source", type=Path, required=True); draft.add_argument("--output", type=Path, required=True) + publish_parser = sub.add_parser("publish") + publish_parser.add_argument("--claims", type=Path, required=True); publish_parser.add_argument("--content", type=Path, required=True); publish_parser.add_argument("--bundle", type=Path, required=True); publish_parser.add_argument("--envelope", type=Path, required=True); publish_parser.add_argument("--draft", type=Path, required=True); publish_parser.add_argument("--output", type=Path, required=True) + live = sub.add_parser("capture-live") + live.add_argument("--envelope", type=Path, required=True); live.add_argument("--bundle", type=Path, required=True); live.add_argument("--run-id", type=int, required=True); live.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "capture-inputs": capture_inputs(args.build_set, args.output) + elif args.command == "verify-source": verify_source(args.expected_sha, args.output) + elif args.command == "capture-staging": capture_staging(args.artifact_id, args.expected_name, args.run_id, args.run_attempt, args.output) + elif args.command == "allocate-draft": allocate_draft(args.expected_sha, args.source, args.output) + elif args.command == "publish": publish(args.claims, args.content, args.bundle, args.envelope, args.draft, args.output) + else: capture_live(args.envelope, args.bundle, args.run_id, args.output) + print(f"OK Phase-6 GitHub boundary {args.command}") + return 0 + except (ForgeError, canonical_json.ProtocolError, OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py new file mode 100644 index 0000000..81baafb --- /dev/null +++ b/scripts/phase6_candidate.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""Assemble and non-executingly verify the exact initial 31-payload candidate.""" + +from __future__ import annotations + +import argparse +import copy +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +from pathlib import Path, PurePosixPath +from typing import Any + +import canonical_json +import compare_phase5_indexer_builds +import consolidate_phase4_macos +import proof_data_pipeline +import validate_archive +import validate_catalog +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, load_json, safe_basename, sha256_file, sha256_stream, validate_regular_file, validate_unique_names + + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = "acedward/midnight-binary-forge" +REPOSITORY_ID = 1349127482 +REVIEWED_BASE_SHA = "19de8be5f434225dbf17126e86b6c3cc6aacc4fe" +WARNING = "DEVELOPMENT ONLY — NOT FOR PRODUCTION USE. Release 0.3.120 is mutable; verify every downloaded SHA-256 against committed metadata before installation or execution." +EXPECTED_INPUTS = { + "phase3p-proof-data": (33170546601, 9685464135, "508aab47ab266344aedd6359fe971928bed309b5", "proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1", "b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f"), + "phase4-celestia-appd-linux-arm64": (33177534764, 9688244894, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-celestia-appd-linux-arm64", "3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56"), + "phase4-celestia-node-linux-arm64": (33177534764, 9688243729, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-celestia-node-linux-arm64", "b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba"), + "phase4-node-linux-arm64": (33177534764, 9688330126, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-node-linux-arm64", "ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e"), + "phase4-toolkit-linux-amd64": (33177534764, 9688263793, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-linux-amd64", "9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b"), + "phase4-toolkit-linux-arm64": (33177534764, 9688255774, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-linux-arm64", "e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd"), + "phase4-toolkit-macos-arm64": (33177534764, 9689647047, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-macos-arm64", "ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e"), + "phase5-indexer": (33176004154, 9690093579, "e581add8952bae5ffeac39fb07e6b5c6f482862d", "phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a", "eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9"), +} +EXPECTED_PAYLOAD_NAMES = { + *{f"bls_midnight_2p{k}" for k in range(20)}, + "celestia-appd-linux-arm64-v6.4.10.tar.gz", + "celestia-node-linux-arm64-v0.28.4.tar.gz", + "indexer-standalone-linux-amd64-v4.4.0-rc.3.zip", + "indexer-standalone-linux-arm64-v4.4.0-rc.3.zip", + "indexer-standalone-macos-amd64-v4.4.0-rc.3.zip", + "indexer-standalone-macos-arm64-v4.4.0-rc.3.zip", + "midnight-ledger-static-noarch-9.0.0.zip", + "midnight-node-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip", +} +EXPECTED_SOFTWARE_IDENTITIES = { + "celestia-appd-linux-arm64-v6.4.10.tar.gz": (180685852, "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b", "phase4-celestia-appd-linux-arm64", "payloads/celestia-appd-linux-arm64-v6.4.10.tar.gz"), + "celestia-node-linux-arm64-v0.28.4.tar.gz": (71184641, "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c", "phase4-celestia-node-linux-arm64", "payloads/celestia-node-linux-arm64-v0.28.4.tar.gz"), + "indexer-standalone-linux-amd64-v4.4.0-rc.3.zip": (31479027, "4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434", "phase5-indexer", "payload/indexer-standalone-linux-amd64-v4.4.0-rc.3.zip"), + "indexer-standalone-linux-arm64-v4.4.0-rc.3.zip": (29782570, "eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0", "phase5-indexer", "payload/indexer-standalone-linux-arm64-v4.4.0-rc.3.zip"), + "indexer-standalone-macos-amd64-v4.4.0-rc.3.zip": (30713420, "28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc", "phase5-indexer", "payload/indexer-standalone-macos-amd64-v4.4.0-rc.3.zip"), + "indexer-standalone-macos-arm64-v4.4.0-rc.3.zip": (29072181, "b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2", "phase5-indexer", "payload/indexer-standalone-macos-arm64-v4.4.0-rc.3.zip"), + "midnight-node-linux-arm64-2.0.0-rc.4.zip": (82544614, "490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c", "phase4-node-linux-arm64", "payloads/midnight-node-linux-arm64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip": (50017428, "92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c", "phase4-toolkit-linux-amd64", "payloads/midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip": (48585850, "4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528", "phase4-toolkit-linux-arm64", "payloads/midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip": (45553847, "8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c", "phase4-toolkit-macos-arm64", "payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip"), +} +EVIDENCE_ROLE_BY_PREFIX = { + "LICENSE-": "license", + "NOTICE-": "notice", + "license-evidence-": "license", + "provenance-": "provenance", + "signing-evidence-": "provenance", + "software-sbom-": "sbom-spdx", + "software-member-manifests-": "member-manifest", + "proof-data-lineage-": "lineage-manifest", + "proof-cache-content-manifest-": "lineage-manifest", + "ledger-static-member-manifest-": "member-manifest", + "source-manifest-": "source-manifest", + "sha256sums-": "checksums", +} + + +def identity(path: Path, ceiling: int = 2**31 - 1) -> dict[str, Any]: + validate_regular_file(path) + digest, size = sha256_file(path, ceiling) + return {"name": path.name, "size": size, "sha256": digest} + + +def strict_relative(value: str) -> PurePosixPath: + path = PurePosixPath(value) + expect(value == path.as_posix() and not path.is_absolute() and all(part not in {"", ".", ".."} for part in path.parts), f"unsafe relative path: {value!r}") + return path + + +def media_type(name: str) -> str: + if name.endswith(".spdx.json"): + return "application/spdx+json" + if name.endswith(".json"): + return "application/json" + if name.endswith(".zip"): + return "application/zip" + if name.endswith(".tar.gz"): + return "application/gzip" + if name.endswith(".txt"): + return "text/plain" + return "application/octet-stream" + + +def evidence_role(name: str) -> str: + matches = [role for prefix, role in EVIDENCE_ROLE_BY_PREFIX.items() if name.startswith(prefix)] + expect(len(matches) == 1, f"candidate evidence name has no unique typed role: {name}") + return matches[0] + + +def expected_evidence_names(build_id: str) -> set[str]: + return { + "LICENSE-Apache-2.0.txt", + "NOTICE-DEVELOPMENT-ONLY.txt", + f"license-evidence-{build_id}.json", + f"signing-evidence-{build_id}.json", + f"provenance-{build_id}.json", + f"software-member-manifests-{build_id}.json", + f"proof-data-lineage-{build_id}.json", + f"proof-cache-content-manifest-{build_id}.json", + f"ledger-static-member-manifest-{build_id}.json", + f"source-manifest-{build_id}.json", + f"sha256sums-{build_id}.txt", + *{f"software-sbom-{name}.spdx.json" for name in EXPECTED_SOFTWARE_IDENTITIES}, + } + + +def validate_buildset(buildset_path: Path, root: Path = ROOT, require_git: bool = False) -> tuple[dict[str, Any], dict[str, Any]]: + buildset = load_json(buildset_path) + report = validate_catalog.validate_build_set(buildset, root, require_source_head=require_git) + expect(buildset["buildSetId"] == "initial-warehouse-v1", "unexpected Phase-6 build-set ID") + expect(buildset["sourceFullSha"] == REVIEWED_BASE_SHA, "unexpected reviewed Phase-6 input baseline") + inputs = {row["key"]: row for row in buildset["inputArtifacts"]} + expect(set(inputs) == set(EXPECTED_INPUTS), "Phase-6 input artifact set differs from exact audited allowlist") + for key, expected in EXPECTED_INPUTS.items(): + row = inputs[key] + actual = (row["runId"], row["artifactId"], row["sourceHeadSha"], row["artifactName"], row["archiveSha256"]) + expect(actual == expected, f"Phase-6 input identity differs: {key}") + expect(row["repository"] == REPOSITORY and row["repositoryId"] == REPOSITORY_ID, f"Phase-6 input repository differs: {key}") + payloads = buildset["payloads"] + names = [row["name"] for row in payloads] + expect(len(payloads) == 31 and set(names) == EXPECTED_PAYLOAD_NAMES, "Phase-6 payload allowlist/count differs from exact 31") + expect(sum(row["artifactKind"] == "software" for row in payloads) == 10, "Phase-6 binary payload count must be ten") + expect(sum(row["artifactKind"] == "proof-data" for row in payloads) == 21, "Phase-6 proof-data payload count must be 21") + expect(not any("compact" in row["name"].casefold() or "compact" in row["componentId"].casefold() for row in payloads), "Compact compiler payload forbidden") + proof = [row for row in payloads if row["artifactKind"] == "proof-data"] + software = {row["name"]: row for row in payloads if row["artifactKind"] == "software"} + expect(set(software) == set(EXPECTED_SOFTWARE_IDENTITIES), "Phase-6 software payload allowlist differs") + for name, expected in EXPECTED_SOFTWARE_IDENTITIES.items(): + row = software[name] + expect((row["size"], row["sha256"], row["sourceArtifactKey"], row["sourcePath"]) == expected, f"Phase-6 software payload identity differs: {name}") + q8b = load_json(root / "catalog/proof-data/q8b-v1.json") + expected_proof = {row["releaseName"]: (row["size"], row["sha256"], row["componentId"], row["mode"]) for row in q8b["srs"]} + ledger = q8b["ledgerStatic"] + expected_proof[ledger["releaseName"]] = (ledger["archiveSize"], ledger["archiveSha256"], ledger["componentId"], "0644") + expect({row["name"] for row in proof} == set(expected_proof), "Phase-6 proof payload allowlist differs") + for row in proof: + expected = expected_proof[row["name"]] + expect((row["size"], row["sha256"], row["componentId"], row["installMode"]) == expected, f"Phase-6 proof payload identity differs: {row['name']}") + expect(row["sourceArtifactKey"] == "phase3p-proof-data" and row["sourcePath"] == f"payloads/{row['name']}", f"Phase-6 proof payload source differs: {row['name']}") + expect(not any("linux" in row["name"] or "macos" in row["name"] or "rc.5" in row["name"] for row in proof), "proof data cannot be duplicated by platform or proof-server release") + expect({row["sourceArtifactKey"] for row in payloads} == set(inputs), "every exact audited input must contribute an approved payload") + return buildset, report + + +def verify_live_metadata(buildset: dict[str, Any], metadata: dict[str, Any], require_live: bool = True) -> None: + expect(metadata.get("schemaVersion") == "phase6-input-live-metadata-v1", "wrong Phase-6 input metadata schema") + expect(metadata.get("repository") == {"fullName": REPOSITORY, "id": REPOSITORY_ID}, "input metadata repository mismatch") + runs = {row.get("id"): row for row in metadata.get("runs", []) if isinstance(row, dict)} + artifacts = {row.get("id"): row for row in metadata.get("artifacts", []) if isinstance(row, dict)} + expect(len(runs) == 3 and len(artifacts) == 8, "input live metadata count mismatch") + now = canonical_json.parse_time(metadata["capturedAt"], "input metadata capturedAt") + for expected in buildset["inputArtifacts"]: + run = runs.get(expected["runId"]) + artifact = artifacts.get(expected["artifactId"]) + expect(run is not None and artifact is not None, f"input live metadata missing: {expected['key']}") + expect(run.get("run_attempt") == expected["runAttempt"] and run.get("event") == expected["runEvent"], f"input run attempt/event mismatch: {expected['key']}") + expect(run.get("status") == "completed" and run.get("conclusion") == expected["runConclusion"], f"input run is not successful: {expected['key']}") + expect(run.get("head_sha") == expected["sourceHeadSha"] and run.get("head_branch") == expected["sourceRef"], f"input run source identity mismatch: {expected['key']}") + expect(run.get("path") == expected["workflowPath"], f"input run workflow mismatch: {expected['key']}") + repo = run.get("repository", {}) + expect(repo.get("full_name") == REPOSITORY and repo.get("id") == REPOSITORY_ID, f"input run repository mismatch: {expected['key']}") + expect(artifact.get("name") == expected["artifactName"] and artifact.get("size_in_bytes") == expected["artifactSize"], f"input artifact name/size mismatch: {expected['key']}") + expect(artifact.get("digest") == f"sha256:{expected['archiveSha256']}", f"input artifact archive digest mismatch: {expected['key']}") + expect(artifact.get("expired") is False and artifact.get("expires_at") == expected["expiresAt"], f"input artifact expiry/state mismatch: {expected['key']}") + workflow = artifact.get("workflow_run", {}) + expect(workflow.get("id") == expected["runId"] and workflow.get("head_sha") == expected["sourceHeadSha"] and workflow.get("repository_id") == REPOSITORY_ID, f"input artifact/run relation mismatch: {expected['key']}") + if require_live: + expect(now < canonical_json.parse_time(expected["expiresAt"], f"{expected['key']} expiresAt"), f"input artifact expired before assembly: {expected['key']}") + + +def git_ancestry(buildset: dict[str, Any], root: Path) -> None: + head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + for source in [buildset["sourceFullSha"], *[row["sourceHeadSha"] for row in buildset["inputArtifacts"]]]: + result = subprocess.run(["git", "-C", str(root), "merge-base", "--is-ancestor", source, head], check=False, capture_output=True) + expect(result.returncode == 0, f"reviewed input SHA is not reachable from candidate source HEAD: {source}") + + +def copy_inert(source: Path, destination: Path) -> dict[str, Any]: + validate_regular_file(source) + expect(not destination.exists() and destination.parent.is_dir(), f"unsafe/duplicate candidate destination: {destination}") + shutil.copyfile(source, destination) + os.chmod(destination, 0o644) + return identity(destination) + + +def phase4_record(root: Path, payload_name: str) -> tuple[dict[str, Any], dict[str, Any], Path]: + record = load_json(root / "evidence/payload-evidence.json") + expect(record.get("schemaVersion") == "phase4-payload-evidence-v1" and record.get("payload", {}).get("name") == payload_name, f"Phase-4 payload evidence mismatch: {payload_name}") + observed = identity(root / "payloads" / payload_name) + expect(record["payload"] == observed, f"Phase-4 payload bytes differ from evidence: {payload_name}") + member_manifest = load_json(root / "evidence/member-manifest.json") + sbom = next((path for path in (root / "sbom").iterdir() if path.name.endswith(".spdx.json")), None) + expect(sbom is not None, f"Phase-4 SPDX SBOM missing: {payload_name}") + return record, member_manifest, sbom + + +def checksum_manifest(root: Path) -> None: + compare_phase5_indexer_builds.validate_checksum_manifest(root) + + +def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = ROOT) -> dict[str, Any]: + buildset, coverage = validate_buildset(buildset_path, root) + expect(input_root.is_dir() and not input_root.is_symlink(), "input root is missing or unsafe") + expect(not output.exists() and output.parent.is_dir(), "candidate output must not already exist") + inputs = {row["key"]: input_root / row["key"] for row in buildset["inputArtifacts"]} + expect(set(path.name for path in input_root.iterdir()) == set(inputs), "downloaded input directory set differs from build set") + expect(all(path.is_dir() and not path.is_symlink() for path in inputs.values()), "downloaded input artifact root is unsafe") + proof_data_pipeline.verify_output(root / "catalog/proof-data/q8b-v1.json", inputs["phase3p-proof-data"]) + checksum_manifest(inputs["phase5-indexer"]) + consolidate_phase4_macos.verify(inputs["phase4-toolkit-macos-arm64"]) + + temporary = output.parent / f".{output.name}.tmp-{os.getpid()}" + expect(not temporary.exists(), "candidate temporary output collision") + temporary.mkdir(mode=0o700) + payload_rows: list[dict[str, Any]] = [] + archive_rows: list[dict[str, Any]] = [] + signing_rows: list[dict[str, Any]] = [] + sbom_sources: dict[str, Path] = {} + phase_records: dict[str, Any] = {} + try: + components = {row["componentId"]: load_json(root / row["manifestPath"]) for row in buildset["components"]} + for payload in buildset["payloads"]: + source = inputs[payload["sourceArtifactKey"]] / strict_relative(payload["sourcePath"]) + observed = identity(source) + expect(observed["name"] == payload["name"] and observed["size"] == payload["size"] and observed["sha256"] == payload["sha256"], f"payload input identity mismatch: {payload['name']}") + copied = copy_inert(source, temporary / payload["name"]) + row = copy.deepcopy(payload) + row.update({"size": copied["size"], "sha256": copied["sha256"]}) + payload_rows.append(row) + component = components[payload["componentId"]] + if payload["artifactKind"] == "software" and payload["sourceArtifactKey"].startswith("phase4-"): + record, members, sbom = phase4_record(inputs[payload["sourceArtifactKey"]], payload["name"]) + phase_records[payload["name"]] = record + archive_rows.append({"name": payload["name"], "container": payload["container"], "limits": component["naming"]["limits"], "members": members["members"]}) + sbom_sources[payload["name"]] = sbom + signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": record["signing"]}) + elif payload["artifactKind"] == "software": + os_name, arch = payload["os"], payload["arch"] + evidence = inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/build1/evidence" + reproduction = load_json(inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/reproducibility.json") + binary = reproduction["binary"] + archive_rows.append({"name": payload["name"], "container": "zip", "limits": component["naming"]["limits"], "members": [{"path": binary["name"], "type": "file", "mode": "0755", "size": binary["size"], "sha256": binary["sha256"]}]}) + sbom_sources[payload["name"]] = evidence / "sbom-indexer-standalone.spdx.json" + signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": load_json(evidence / "signing-evidence.json")}) + + build_id = buildset["buildSetId"] + sbom_evidence: list[dict[str, Any]] = [] + for payload_name, source in sorted(sbom_sources.items()): + evidence_name = f"software-sbom-{payload_name}.spdx.json" + record = copy_inert(source, temporary / evidence_name) + record["role"] = "sbom-spdx" + record["payloadName"] = payload_name + sbom_evidence.append(record) + expect(len(sbom_evidence) == 10, "candidate must retain exactly one SPDX SBOM per software payload") + + member_name = f"software-member-manifests-{build_id}.json" + create_file_atomic(temporary / member_name, canonical_bytes({"schemaVersion": "phase6-software-member-manifests-v1", "archives": sorted(archive_rows, key=lambda row: row["name"])}) ) + proof_lineage_name = f"proof-data-lineage-{build_id}.json" + copy_inert(inputs["phase3p-proof-data"] / "evidence/proof-data-lineage-v1.json", temporary / proof_lineage_name) + proof_content_name = f"proof-cache-content-manifest-{build_id}.json" + copy_inert(inputs["phase3p-proof-data"] / "evidence/proof-cache-content-manifest-v1.json", temporary / proof_content_name) + ledger_member_name = f"ledger-static-member-manifest-{build_id}.json" + source_ledger = root / "catalog/proof-data/ledger-static-9-member-manifest.json" + copy_inert(source_ledger, temporary / ledger_member_name) + + license_name = "LICENSE-Apache-2.0.txt" + copy_inert(root / "LICENSE", temporary / license_name) + license_evidence_name = f"license-evidence-{build_id}.json" + license_rows = [{"componentId": key, "license": components[key]["license"]} for key in sorted(components)] + create_file_atomic(temporary / license_evidence_name, canonical_bytes({"schemaVersion": "phase6-license-evidence-v1", "licenses": license_rows, "proofDataOwnerAcceptanceRequiredBeforeWarehouseUpload": True, "proofDataOwnerAcceptanceStatus": "pending"})) + signing_name = f"signing-evidence-{build_id}.json" + create_file_atomic(temporary / signing_name, canonical_bytes({"schemaVersion": "phase6-signing-evidence-v1", "payloads": sorted(signing_rows, key=lambda row: row["name"]), "macosPolicy": "UNSIGNED_DEVELOPMENT_ONLY; actual codeSignatureKind is recorded per payload; later Developer ID bytes require a distinct name/version"})) + notice_name = "NOTICE-DEVELOPMENT-ONLY.txt" + create_file_atomic(temporary / notice_name, (WARNING + "\nCompact 0.34 is consumed directly from official LFDT-Minokawa assets and is not a candidate payload.\n").encode("utf-8")) + provenance_name = f"provenance-{build_id}.json" + provenance = { + "_type": "https://in-toto.io/Statement/v1", + "predicateType": "https://github.com/acedward/midnight-binary-forge/predicates/phase6-candidate/v1", + "subject": [{"name": row["name"], "digest": {"sha256": row["sha256"]}} for row in payload_rows], + "predicate": {"buildSetId": build_id, "reviewedInputBaselineSha": buildset["sourceFullSha"], "inputArtifacts": buildset["inputArtifacts"], "phase4PayloadEvidence": phase_records, "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, + } + create_file_atomic(temporary / provenance_name, canonical_bytes(provenance)) + + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + evidence_names = sorted([path.name for path in temporary.iterdir() if path.name not in EXPECTED_PAYLOAD_NAMES] + [source_name, checksums_name]) + source_manifest = { + "schemaVersion": "phase6-source-manifest-v1", + "buildSetId": build_id, + "buildSet": {"path": buildset_path.relative_to(root).as_posix(), "size": buildset_path.stat().st_size, "sha256": sha256_file(buildset_path)[0]}, + "reviewedInputBaselineSha": buildset["sourceFullSha"], + "inputArtifacts": buildset["inputArtifacts"], + "destination": buildset["destination"], + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "warning": WARNING, + "payloadCount": 31, + "binaryPayloadCount": 10, + "proofDataPayloadCount": 21, + "payloadNameListSha256": hashlib.sha256(canonical_bytes(sorted(EXPECTED_PAYLOAD_NAMES))).hexdigest(), + "payloads": payload_rows, + "coverage": coverage, + "evidenceAssetNames": evidence_names, + "evidenceCount": len(evidence_names), + "compactCompilerPayloadCount": 0, + "proofDataScope": {"platform": "noarch", "k": list(range(20)), "ledgerStaticSemver": "9.0.0", "customProvingKeys": False}, + } + create_file_atomic(temporary / source_name, canonical_bytes(source_manifest)) + rows = [] + for path in sorted(temporary.iterdir(), key=lambda item: item.name): + if path.name != checksums_name: + digest, _ = sha256_file(path, 2**31 - 1) + rows.append(f"{digest} {path.name}\n") + create_file_atomic(temporary / checksums_name, "".join(rows).encode("utf-8")) + os.chmod(temporary, 0o755) + os.replace(temporary, output) + result = verify_candidate(buildset_path, output, root) + return result + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def verify_checksums(content: Path, checksums_name: str) -> None: + rows: dict[str, str] = {} + previous = "" + for line in (content / checksums_name).read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._-]{0,255})", line) + expect(match is not None, f"malformed candidate checksum row: {line!r}") + digest, name = match.groups() + expect(name > previous and name not in rows and name != checksums_name, "candidate checksum names must be unique and sorted") + previous = name + rows[name] = digest + files = {path.name: path for path in content.iterdir() if path.is_file() and not path.is_symlink() and path.name != checksums_name} + expect(set(rows) == set(files), "candidate checksum closure differs from exact content") + for name, path in files.items(): + expect(sha256_file(path, 2**31 - 1)[0] == rows[name], f"candidate checksum mismatch: {name}") + + +def verify_one_archive(path: Path, policy: dict[str, Any]) -> None: + compressed = path.stat().st_size + limits = policy["limits"] + expect(compressed <= limits["maxCompressedBytes"], f"archive exceeds compressed bound: {path.name}") + iterator = validate_archive.zip_members if policy["container"] == "zip" else validate_archive.tar_members + members = list(iterator(path)) + expect(len(members) <= limits["maxMembers"], f"archive exceeds member bound: {path.name}") + validate_unique_names(member.path for member in members) + expected = {row["path"]: row for row in policy["members"]} + expect(len(expected) == len(policy["members"]) and set(expected) == {member.path for member in members}, f"archive member set mismatch: {path.name}") + expanded = sum(member.size for member in members) + expect(expanded <= limits["maxExpandedBytes"] and expanded / max(compressed, 1) <= limits["maxExpansionRatio"], f"archive expansion bound exceeded: {path.name}") + for member in members: + row = expected[member.path] + expect(member.type == row["type"] and member.mode == row["mode"], f"archive member type/mode mismatch: {path.name}:{member.path}") + if member.type == "file": + expect(member.size == row["size"], f"archive member size mismatch: {path.name}:{member.path}") + context = member.opener() + expect(context is not None, f"archive member cannot be streamed: {path.name}:{member.path}") + with context as stream: + digest, size = sha256_stream(stream, row["size"]) + expect(size == row["size"] and digest == row["sha256"], f"archive member digest mismatch: {path.name}:{member.path}") + + +def content_assets(buildset: dict[str, Any], content: Path) -> list[dict[str, Any]]: + payload_by_name = {row["name"]: row for row in buildset["payloads"]} + rows = [] + for path in sorted(content.iterdir(), key=lambda item: item.name): + safe_basename(path.name, "candidate asset name") + observed = identity(path) + row: dict[str, Any] = {**observed, "mediaType": media_type(path.name)} + if path.name in payload_by_name: + payload = payload_by_name[path.name] + row.update({"role": "payload", "artifactKind": payload["artifactKind"], "componentId": payload["componentId"]}) + else: + row["role"] = evidence_role(path.name) + rows.append(row) + return rows + + +def verify_candidate(buildset_path: Path, content: Path, root: Path = ROOT) -> dict[str, Any]: + buildset, _ = validate_buildset(buildset_path, root) + expect(content.is_dir() and not content.is_symlink(), "candidate content root is unsafe") + for path in content.iterdir(): + validate_regular_file(path, "0644") + safe_basename(path.name, "candidate content name") + build_id = buildset["buildSetId"] + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + verify_checksums(content, checksums_name) + source = load_json(content / source_name) + expect(source.get("schemaVersion") == "phase6-source-manifest-v1" and source.get("payloadCount") == 31 and source.get("binaryPayloadCount") == 10 and source.get("proofDataPayloadCount") == 21, "candidate source-manifest counts differ") + expect(source.get("distributionTier") == "development-only" and source.get("releaseMutability") == "mutable-warehouse" and source.get("warning") == WARNING, "candidate distribution warning/policy differs") + expected_evidence = expected_evidence_names(build_id) + expect(len(expected_evidence) == 21 and source.get("evidenceCount") == 21 and source.get("evidenceAssetNames") == sorted(expected_evidence), "candidate exact evidence allowlist/count differs") + payloads = source.get("payloads") + expect(isinstance(payloads, list) and {row.get("name") for row in payloads if isinstance(row, dict)} == EXPECTED_PAYLOAD_NAMES, "candidate source-manifest payload allowlist differs") + for row in payloads: + observed = identity(content / row["name"]) + expect(observed["size"] == row["size"] and observed["sha256"] == row["sha256"], f"candidate payload identity mismatch: {row['name']}") + expect(source.get("compactCompilerPayloadCount") == 0 and not any("compact" in name.casefold() for name in EXPECTED_PAYLOAD_NAMES), "Compact compiler leaked into candidate") + raw_names = {f"bls_midnight_2p{k}" for k in range(20)} + q8b = load_json(root / "catalog/proof-data/q8b-v1.json") + srs = {row["releaseName"]: row for row in q8b["srs"]} + for name in raw_names: + observed = identity(content / name) + expect(observed["size"] == srs[name]["size"] and observed["sha256"] == srs[name]["sha256"], f"raw proof payload differs: {name}") + policies = load_json(content / f"software-member-manifests-{build_id}.json") + expect(policies.get("schemaVersion") == "phase6-software-member-manifests-v1" and len(policies.get("archives", [])) == 10, "software archive policy set differs") + for policy in policies["archives"]: + verify_one_archive(content / policy["name"], policy) + ledger = content / "midnight-ledger-static-noarch-9.0.0.zip" + ledger_policy = {"name": ledger.name, "container": "zip", "limits": load_json(root / "catalog/components/midnight-ledger-static-9.0.0.json")["naming"]["limits"], "members": load_json(root / "catalog/proof-data/ledger-static-9-zip-layout-manifest.json")["members"]} + verify_one_archive(ledger, ledger_policy) + lineage = load_json(content / f"proof-data-lineage-{build_id}.json") + expect(lineage.get("payloadCount") == 21 and len(lineage.get("payloads", [])) == 21 and lineage.get("softwareSbom") == "not-applicable", "proof-data lineage differs") + signing = load_json(content / f"signing-evidence-{build_id}.json") + expect(len(signing.get("payloads", [])) == 10, "candidate signing evidence count differs") + for row in signing["payloads"]: + state = row["signing"].get("distributionSigningState") + expect(state in {"NOT_APPLICABLE", "UNSIGNED_DEVELOPMENT_ONLY"}, f"candidate signing state forbidden: {row['name']}") + if row["name"].startswith("indexer-standalone-macos-") or row["name"].startswith("midnight-node-toolkit-macos-"): + expect(state == "UNSIGNED_DEVELOPMENT_ONLY" and row["signing"].get("codeSignatureKind") in {"none", "linker-adhoc"}, f"macOS signature metadata incomplete: {row['name']}") + assets = content_assets(buildset, content) + payload_count = sum(row["role"] == "payload" for row in assets) + evidence_count = len(assets) - payload_count + expect(payload_count == 31 and evidence_count == 21 and set(row["name"] for row in assets if row["role"] != "payload") == expected_evidence and len(assets) == 52, "candidate typed asset allowlist/count differs") + result = {"schemaVersion": "phase6-candidate-verification-v1", "payloadCount": payload_count, "evidenceCount": evidence_count, "contentAssetCount": len(assets), "contentAssetListSha256": canonical_json.digest(assets), "payloadNameListSha256": canonical_json.digest(sorted(EXPECTED_PAYLOAD_NAMES))} + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return result + + +def make_claims(buildset_path: Path, content: Path, draft: dict[str, Any], staging: dict[str, Any], commit_sha: str, workflow_sha: str, run_id: int, run_attempt: int, output: Path, root: Path = ROOT) -> None: + buildset, _ = validate_buildset(buildset_path, root) + verification = verify_candidate(buildset_path, content, root) + assets = content_assets(buildset, content) + build_id = buildset["buildSetId"] + by_name = {row["name"]: row for row in assets} + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + complete_names = sorted([row["name"] for row in assets] + [f"promotion-envelope-{build_id}.json", f"attestation-{build_id}.sigstore.json"]) + claims = { + "issuer": {"repository": REPOSITORY, "repositoryId": REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "workflowPath": canonical_json.WORKFLOW_PATH, "workflowSha": workflow_sha, "ref": canonical_json.MAIN_REF, "commitSha": commit_sha}, + "staging": {"provider": "github-actions-artifact", "runId": run_id, "runAttempt": run_attempt, "artifactId": staging["artifactId"], "artifactName": staging["artifactName"], "archiveSha256": staging["archiveSha256"], "expiresAt": staging["expiresAt"]}, + "candidateDraft": {"repository": REPOSITORY, "repositoryId": REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "tag": draft["tag_name"], "targetCommitish": commit_sha, "releaseId": draft["id"], "releaseNodeId": draft["node_id"], "releaseUrl": draft["html_url"], "liveImmutableVerificationRequired": True}, + "buildSet": {"id": build_id, "manifestName": source_name, "manifestSha256": by_name[source_name]["sha256"], "checksumsName": checksums_name, "checksumsSha256": by_name[checksums_name]["sha256"]}, + "transport": {"envelopeName": f"promotion-envelope-{build_id}.json", "attestationBundleName": f"attestation-{build_id}.sigstore.json"}, + "contentAssets": assets, + "contentAssetListSha256": verification["contentAssetListSha256"], + "completeAssetNames": complete_names, + "completeAssetNameListSha256": canonical_json.digest(complete_names), + "payloadCount": 31, + "contentEvidenceCount": verification["evidenceCount"], + "transportAssetCount": 2, + "totalAssetCount": len(complete_names), + } + expect(len(complete_names) == 54 and verification["evidenceCount"] == 21, "Phase-6 complete candidate must contain 31 payload, 21 evidence and two transport assets") + canonical_json.verify_claims(claims) + create_file_atomic(output, canonical_json.canonical_bytes(claims)) + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + validate = sub.add_parser("validate-buildset") + validate.add_argument("--build-set", type=Path, required=True) + validate.add_argument("--root", type=Path, default=ROOT) + validate.add_argument("--require-git-ancestry", action="store_true") + metadata = sub.add_parser("verify-live-inputs") + metadata.add_argument("--build-set", type=Path, required=True) + metadata.add_argument("--metadata", type=Path, required=True) + metadata.add_argument("--allow-expired", action="store_true") + assemble_parser = sub.add_parser("assemble") + assemble_parser.add_argument("--build-set", type=Path, required=True) + assemble_parser.add_argument("--input-root", type=Path, required=True) + assemble_parser.add_argument("--output", type=Path, required=True) + verify_parser = sub.add_parser("verify") + verify_parser.add_argument("--build-set", type=Path, required=True) + verify_parser.add_argument("--content", type=Path, required=True) + claims = sub.add_parser("make-claims") + claims.add_argument("--build-set", type=Path, required=True) + claims.add_argument("--content", type=Path, required=True) + claims.add_argument("--draft", type=Path, required=True) + claims.add_argument("--staging", type=Path, required=True) + claims.add_argument("--commit-sha", required=True) + claims.add_argument("--workflow-sha", required=True) + claims.add_argument("--run-id", type=int, required=True) + claims.add_argument("--run-attempt", type=int, required=True) + claims.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "validate-buildset": + buildset, report = validate_buildset(args.build_set, args.root, args.require_git_ancestry) + if args.require_git_ancestry: + git_ancestry(buildset, args.root) + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + elif args.command == "verify-live-inputs": + buildset, _ = validate_buildset(args.build_set) + verify_live_metadata(buildset, load_json(args.metadata), not args.allow_expired) + print("OK exact audited Phase-6 input metadata") + elif args.command == "assemble": + assemble(args.build_set, args.input_root, args.output) + elif args.command == "verify": + verify_candidate(args.build_set, args.content) + else: + make_claims(args.build_set, args.content, load_json(args.draft), load_json(args.staging), args.commit_sha, args.workflow_sha, args.run_id, args.run_attempt, args.output) + return 0 + except (ForgeError, canonical_json.ProtocolError, OSError, KeyError, TypeError, ValueError, json.JSONDecodeError, subprocess.SubprocessError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_catalog.py b/scripts/validate_catalog.py index 7dc4fa2..ecef0ce 100755 --- a/scripts/validate_catalog.py +++ b/scripts/validate_catalog.py @@ -8,7 +8,7 @@ import json import re import sys -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from forge_io import ForgeError, expect, load_json, safe_basename, sha256_file @@ -265,12 +265,39 @@ def validate_build_set(build_set: dict[str, Any], root: Path, require_source_hea import subprocess result = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10) - expect(result.returncode == 0 and result.stdout.strip() == build_set["sourceFullSha"], "build set is not bound to current full source HEAD") + expect(result.returncode == 0 and len(result.stdout.strip()) == 40, "cannot resolve current full source HEAD") + ancestor = subprocess.run( + ["git", "-C", str(root), "merge-base", "--is-ancestor", build_set["sourceFullSha"], result.stdout.strip()], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + expect(ancestor.returncode == 0, "build-set reviewed input baseline is not reachable from current full source HEAD") components = load_components(root, build_set) + input_artifacts = build_set["inputArtifacts"] + input_keys = [row["key"] for row in input_artifacts] + expect(input_keys == sorted(input_keys), "input artifact keys must be lexically sorted") + expect(len(input_keys) == len(set(input_keys)), "duplicate input artifact key") + expect(len({row["artifactId"] for row in input_artifacts}) == len(input_artifacts), "duplicate input artifact ID") + for row in input_artifacts: + expect(row["workflowPath"] in { + ".github/workflows/proof-data-q8b.yml", + ".github/workflows/phase4-payloads.yml", + ".github/workflows/phase5-indexer.yml", + }, "unapproved input workflow") + expect(row["sourceHeadSha"] != "0" * 40, "input source head cannot be a sentinel SHA") + input_key_set = set(input_keys) declared_semantics: dict[tuple[str, str, str, str], str] = {} + family_templates: dict[tuple[str, str], str] = {} for component in components.values(): if component["artifactKind"] != "software": continue + family_key = (component["family"], component["version"]) + template = component["naming"]["outerTemplate"] + prior_template = family_templates.get(family_key) + expect(prior_template is None or prior_template == template, f"software family/version has conflicting public name templates: {family_key}") + family_templates[family_key] = template for target in component["targets"]: key = (component["family"], component["version"], target["os"], target["arch"]) prior = declared_semantics.get(key) @@ -284,9 +311,19 @@ def validate_build_set(build_set: dict[str, Any], root: Path, require_source_hea candidate_coverage: dict[tuple[str, str, str, str], tuple[str, str]] = {} for payload in payloads: safe_basename(payload["name"], "payload name") + expect(payload["sourceArtifactKey"] in input_key_set, "payload references unknown input artifact") + source_path = PurePosixPath(payload["sourcePath"]) + expect( + payload["sourcePath"] == source_path.as_posix() + and not source_path.is_absolute() + and all(part not in {"", ".", ".."} for part in source_path.parts), + "unsafe payload source path", + ) expect(payload["componentId"] in components, f"payload references unknown component: {payload['componentId']}") component = components[payload["componentId"]] expect(payload["artifactKind"] == component["artifactKind"], "payload/component artifactKind mismatch") + expect(payload["container"] == component["naming"]["container"], "payload/component archive container mismatch") + expect(payload["installMode"] == component["install"]["mode"], "payload/component install mode mismatch") if payload["artifactKind"] == "software": pair = (payload["os"], payload["arch"]) target = next((row for row in component["targets"] if (row["os"], row["arch"]) == pair), None) @@ -328,19 +365,12 @@ def validate_build_set(build_set: dict[str, Any], root: Path, require_source_hea expect(asset.get("digest") == f"sha256:{row['sha256']}", "existing coverage asset digest mismatch") key = (row["family"], row["version"], row["os"], row["arch"]) expect(key not in all_coverage, f"duplicate software semantic tuple across candidate/existing coverage: {key}") - component = next( - ( - value - for value in components.values() - if value["artifactKind"] == "software" - and value["family"] == row["family"] - and value["version"] == row["version"] - and any((target["os"], target["arch"]) == (row["os"], row["arch"]) for target in value["targets"]) - ), - None, - ) - expect(component is not None, "existing coverage tuple is not declared by a pinned component") - expected_name = render_name(component["naming"]["outerTemplate"], version=row["version"], os_name=row["os"], arch=row["arch"]) + template = family_templates.get((row["family"], row["version"])) + expect(template is not None, "existing coverage family/version is not declared by a pinned component") + # A legacy exact warehouse asset may prove coverage even when no new-build component + # truthfully claims that old target. The reviewed family/version naming contract, + # pinned snapshot identity and exact asset digest remain mandatory. + expected_name = render_name(template, version=row["version"], os_name=row["os"], arch=row["arch"]) expect(row["name"] == expected_name, "existing coverage asset name is not the component's canonical rendered name") all_coverage.add(key) diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py new file mode 100644 index 0000000..1ec8d66 --- /dev/null +++ b/tests/test_phase6_candidate.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import copy +import hashlib +import json +import stat +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import phase6_candidate # noqa: E402 +import github_phase6 # noqa: E402 +from forge_io import ForgeError, load_json # noqa: E402 + + +BUILD_SET = ROOT / "catalog/buildsets/initial-warehouse-v1.json" + + +class Phase6BuildSetTest(unittest.TestCase): + def test_exact_31_payload_build_set_and_generator_are_closed(self) -> None: + buildset, report = phase6_candidate.validate_buildset(BUILD_SET) + self.assertEqual(buildset["payloadCount"], 31) + self.assertEqual(len(buildset["inputArtifacts"]), 8) + self.assertEqual(len(buildset["existingCoverage"]), 6) + self.assertEqual(len(report["families"]), 5) + self.assertEqual(len(phase6_candidate.expected_evidence_names(buildset["buildSetId"])), 21) + self.assertFalse(any("compact" in row["name"].casefold() for row in buildset["payloads"])) + + def test_input_payload_and_compact_substitutions_fail(self) -> None: + base = load_json(BUILD_SET) + mutations = [] + wrong_input = copy.deepcopy(base) + wrong_input["inputArtifacts"][0]["artifactId"] += 1 + mutations.append(wrong_input) + wrong_payload = copy.deepcopy(base) + wrong_payload["payloads"][0]["sha256"] = "0" * 64 + mutations.append(wrong_payload) + extra_compact = copy.deepcopy(base) + compact = copy.deepcopy(extra_compact["payloads"][0]) + compact["name"] = "compactc-linux-amd64-v0.34.0.zip" + extra_compact["payloads"].append(compact) + extra_compact["payloads"].sort(key=lambda row: row["name"]) + extra_compact["payloadCount"] += 1 + mutations.append(extra_compact) + platform_proof = copy.deepcopy(base) + proof = next(row for row in platform_proof["payloads"] if row["artifactKind"] == "proof-data") + proof["platform"] = "linux-amd64" + mutations.append(platform_proof) + for index, value in enumerate(mutations): + with self.subTest(index=index), tempfile.TemporaryDirectory() as text: + path = Path(text) / "buildset.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaises(ForgeError): + phase6_candidate.validate_buildset(path) + + def test_live_metadata_exactness_and_mutations(self) -> None: + buildset = load_json(BUILD_SET) + run_by_id = {} + artifacts = [] + for row in buildset["inputArtifacts"]: + run_by_id[row["runId"]] = { + "id": row["runId"], "run_attempt": row["runAttempt"], "event": row["runEvent"], + "status": "completed", "conclusion": row["runConclusion"], "head_sha": row["sourceHeadSha"], + "head_branch": row["sourceRef"], "path": row["workflowPath"], + "repository": {"full_name": row["repository"], "id": row["repositoryId"]}, + } + artifacts.append({ + "id": row["artifactId"], "name": row["artifactName"], "size_in_bytes": row["artifactSize"], + "digest": f"sha256:{row['archiveSha256']}", "expired": False, "expires_at": row["expiresAt"], + "workflow_run": {"id": row["runId"], "head_sha": row["sourceHeadSha"], "repository_id": row["repositoryId"]}, + }) + live = {"schemaVersion": "phase6-input-live-metadata-v1", "capturedAt": "2026-08-28T00:00:00Z", "repository": {"fullName": phase6_candidate.REPOSITORY, "id": phase6_candidate.REPOSITORY_ID}, "runs": sorted(run_by_id.values(), key=lambda row: row["id"]), "artifacts": sorted(artifacts, key=lambda row: row["id"])} + phase6_candidate.verify_live_metadata(buildset, live) + mutations = [ + ("runs", 0, "event", "push"), + ("runs", 0, "path", ".github/workflows/candidate.yml"), + ("runs", 0, "head_sha", "0" * 40), + ("artifacts", 0, "digest", "sha256:" + "0" * 64), + ("artifacts", 0, "expired", True), + ] + for section, index, field, value in mutations: + adversarial = copy.deepcopy(live) + adversarial[section][index][field] = value + with self.subTest(field=field), self.assertRaises(ForgeError): + phase6_candidate.verify_live_metadata(buildset, adversarial) + + def test_unexpected_actions_repository_branch_and_event_fail(self) -> None: + valid = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": phase6_candidate.REPOSITORY, "GITHUB_REF": "refs/heads/main", "GITHUB_EVENT_NAME": "workflow_dispatch"} + with mock.patch.dict("os.environ", valid, clear=True): + github_phase6.require_actions_context() + for field, value in (("GITHUB_REPOSITORY", "attacker/fork"), ("GITHUB_REF", "refs/heads/topic"), ("GITHUB_EVENT_NAME", "pull_request")): + adversarial = {**valid, field: value} + with self.subTest(field=field), mock.patch.dict("os.environ", adversarial, clear=True), self.assertRaises(ForgeError): + github_phase6.require_actions_context() + + def test_staging_artifact_identity_is_api_bound(self) -> None: + artifact = {"id": 7, "name": "verified-content-fixture", "expired": False, "digest": "sha256:" + "a" * 64, "expires_at": "2026-09-01T00:00:00Z", "workflow_run": {"id": 11, "repository_id": phase6_candidate.REPOSITORY_ID}} + with tempfile.TemporaryDirectory() as text: + root = Path(text) + with mock.patch("github_phase6.request", return_value=(artifact, "")): + github_phase6.capture_staging(7, artifact["name"], 11, 1, root / "staging.json") + for field, value in (("name", "substituted"), ("digest", "sha256:malformed"), ("expired", True)): + adversarial = copy.deepcopy(artifact) + adversarial[field] = value + with self.subTest(field=field), mock.patch("github_phase6.request", return_value=(adversarial, "")), self.assertRaises(ForgeError): + github_phase6.capture_staging(7, artifact["name"], 11, 1, root / f"failed-{field}.json") + + +class Phase6StreamingVerifierTest(unittest.TestCase): + def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, dict]: + path = root / "fixture.zip" + info = zipfile.ZipInfo("fixture") + info.create_system = 3 + info.external_attr = (stat.S_IFREG | mode) << 16 + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(info, value) + policy = {"container": "zip", "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 2, "maxExpansionRatio": 100}, "members": [{"path": "fixture", "type": "file", "mode": f"{mode:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()}]} + return path, policy + + def test_streamed_member_identity_and_substitution(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + archive, policy = self._archive(root, b"reviewed bytes") + phase6_candidate.verify_one_archive(archive, policy) + policy["members"][0]["sha256"] = "0" * 64 + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(archive, policy) + + def test_all_evidence_names_are_typed(self) -> None: + self.assertEqual(phase6_candidate.evidence_role("provenance-initial-warehouse-v1.json"), "provenance") + with self.assertRaises(ForgeError): + phase6_candidate.evidence_role("untyped-evidence.json") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_policy.py b/tests/test_policy.py index 346b03c..64555f2 100644 --- a/tests/test_policy.py +++ b/tests/test_policy.py @@ -191,12 +191,13 @@ def _make_build_root(self, directory: Path) -> tuple[Path, dict]: target.write_bytes((POLICY_FIXTURES / name).read_bytes()) components[value["componentId"]] = value references.append({"componentId": value["componentId"], "manifestPath": f"catalog/components/{name}", "manifestSha256": sha256(target)}) + common = {"size": 1, "sha256": "b" * 64, "sourceArtifactKey": "fixture-input", "installMode": "0755"} payloads = [ - {"name": "bls_midnight_2p0", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-srs-k0", "tier": "noarch", "platform": "noarch", "k": 0}, - {"name": "fixture-tool-linux-amd64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "linux", "arch": "amd64"}, - {"name": "fixture-tool-linux-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "desired", "os": "linux", "arch": "arm64"}, - {"name": "fixture-tool-macos-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "macos", "arch": "arm64"}, - {"name": "midnight-ledger-static-noarch-9.0.0.zip", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-ledger-static-9.0.0", "tier": "noarch", "platform": "noarch", "ledgerStaticSemver": "9.0.0"}, + {"name": "bls_midnight_2p0", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-srs-k0", "tier": "noarch", "platform": "noarch", "k": 0, **common, "container": "raw", "sourcePath": "payloads/bls_midnight_2p0", "installMode": "0644"}, + {"name": "fixture-tool-linux-amd64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "linux", "arch": "amd64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-linux-amd64-v1.0.0.zip"}, + {"name": "fixture-tool-linux-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "desired", "os": "linux", "arch": "arm64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-linux-arm64-v1.0.0.zip"}, + {"name": "fixture-tool-macos-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "macos", "arch": "arm64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-macos-arm64-v1.0.0.zip"}, + {"name": "midnight-ledger-static-noarch-9.0.0.zip", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-ledger-static-9.0.0", "tier": "noarch", "platform": "noarch", "ledgerStaticSemver": "9.0.0", **common, "container": "zip", "sourcePath": "payloads/midnight-ledger-static-noarch-9.0.0.zip", "installMode": "0644"}, ] build_set = { "schemaVersion": "build-set-v1", @@ -204,11 +205,18 @@ def _make_build_root(self, directory: Path) -> tuple[Path, dict]: "sourceFullSha": "a" * 40, "destination": {"repository": "effectstream/binaries", "tag": "0.3.120", "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, "components": references, + "inputArtifacts": [{ + "key": "fixture-input", "repository": "acedward/midnight-binary-forge", "repositoryId": 1349127482, + "workflowPath": ".github/workflows/phase4-payloads.yml", "runId": 1, "runAttempt": 1, + "runEvent": "pull_request", "runConclusion": "success", "sourceRef": "fixture", + "sourceHeadSha": "c" * 40, "artifactId": 1, "artifactName": "fixture-input", + "artifactSize": 1, "archiveSha256": "d" * 64, "expiresAt": "2026-09-27T00:00:00Z", + }], "existingCoverage": [], "payloads": sorted(payloads, key=lambda row: row["name"]), "payloadCount": len(payloads), "coveragePolicy": {"required": ["linux/amd64", "macos/arm64"], "desired": ["linux/arm64"], "optional": ["macos/amd64"], "proofDataPlatform": "noarch"}, - "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "destinationCredentialAllowed": False}, + "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "inputArtifactPinningRequired": True, "destinationCredentialAllowed": False}, } return build_dir, build_set @@ -249,6 +257,12 @@ def test_existing_coverage_is_bound_to_pinned_warehouse_asset(self) -> None: "tier": "required", "os": "linux", "arch": "amd64", + "container": "tar.gz", + "size": 1, + "sha256": "e" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": "payloads/celestia-appd-linux-amd64-v6.4.10.tar.gz", + "installMode": "0755", } build_set["payloads"].append(payload) build_set["payloads"].sort(key=lambda row: row["name"]) @@ -270,6 +284,51 @@ def test_existing_coverage_is_bound_to_pinned_warehouse_asset(self) -> None: with self.subTest(case=mutation["name"]), self.assertRaises(ForgeError): validate_catalog.validate_build_set(adversarial, root) + def test_existing_coverage_can_complete_a_reviewed_family_without_fabricated_target_component(self) -> None: + """Legacy exact bytes prove coverage; they are not retroactively declared as new native builds.""" + coverage_fixture = fixture("adversarial-existing-coverage.json") + with tempfile.TemporaryDirectory() as text: + root = Path(text) + _, build_set = self._make_build_root(root) + component = fixture("valid-existing-coverage-software.json") + component["targets"] = [{"os": "linux", "arch": "amd64", "tier": "required", "runner": "ubuntu-24.04", "native": True}] + component["signing"] = { + "applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE", + "codeSignatureKind": "none", "cdHash": None, "authorities": [], + "teamId": None, "hardenedRuntime": None, "strictVerification": False, + } + component_path = root / "catalog/components/valid-existing-coverage-software.json" + write_json(component_path, component) + build_set["components"].append({ + "componentId": component["componentId"], + "manifestPath": "catalog/components/valid-existing-coverage-software.json", + "manifestSha256": sha256(component_path), + }) + candidate_name = "celestia-appd-linux-amd64-v6.4.10.tar.gz" + build_set["payloads"].append({ + "name": candidate_name, + "role": "payload", + "artifactKind": "software", + "componentId": component["componentId"], + "tier": "required", + "os": "linux", + "arch": "amd64", + "container": "tar.gz", + "size": 1, + "sha256": "e" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": f"payloads/{candidate_name}", + "installMode": "0755", + }) + build_set["payloads"].sort(key=lambda row: row["name"]) + build_set["payloadCount"] += 1 + coverage = copy.deepcopy(coverage_fixture["positive"]) + build_set["existingCoverage"] = [coverage] + report = validate_catalog.validate_build_set(build_set, root) + celestia = next(row for row in report["families"] if row["family"] == "celestia-appd") + self.assertEqual(celestia["required"]["present"], ["linux/amd64", "macos/arm64"]) + self.assertEqual(celestia["required"]["missing"], []) + def test_duplicate_software_semantic_tuple_across_names_and_components(self) -> None: adversarial = fixture("adversarial-duplicate-software-tuple.json") with tempfile.TemporaryDirectory() as text: @@ -285,10 +344,19 @@ def test_duplicate_software_semantic_tuple_across_names_and_components(self) -> "manifestPath": "catalog/components/duplicate-software-semantic-tuple.json", "manifestSha256": sha256(path), }) - build_set["payloads"].append(adversarial["duplicatePayload"]) + duplicate_payload = copy.deepcopy(adversarial["duplicatePayload"]) + duplicate_payload.update({ + "container": "zip", + "size": 1, + "sha256": "f" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": f"payloads/{duplicate_payload['name']}", + "installMode": "0755", + }) + build_set["payloads"].append(duplicate_payload) build_set["payloads"].sort(key=lambda row: row["name"]) build_set["payloadCount"] += 1 - with self.assertRaisesRegex(ForgeError, "duplicate software semantic tuple"): + with self.assertRaisesRegex(ForgeError, "duplicate software semantic tuple|conflicting public name"): validate_catalog.validate_build_set(build_set, root) From 4103bb1bb38dbde4402ff2d72c7f053f2db3d18d Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 11:59:13 -0400 Subject: [PATCH 2/8] Fetch reviewed ancestry in candidate gates --- .github/workflows/candidate.yml | 1 + .github/workflows/phase6-candidate-gate.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index e5b69a8..114624b 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -33,6 +33,7 @@ jobs: - name: Check out exact protected-main source uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: + fetch-depth: 0 persist-credentials: false - name: Validate exact committed build set and source ancestry id: validate diff --git a/.github/workflows/phase6-candidate-gate.yml b/.github/workflows/phase6-candidate-gate.yml index 7c98801..762c0f1 100644 --- a/.github/workflows/phase6-candidate-gate.yml +++ b/.github/workflows/phase6-candidate-gate.yml @@ -32,6 +32,7 @@ jobs: - name: Check out exact candidate source uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: + fetch-depth: 0 persist-credentials: false - name: Install verifier dependencies and validate exact build set run: | From 2ffc5ded991099de24dfdcb539eb00488a174db0 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 12:02:48 -0400 Subject: [PATCH 3/8] Flatten and validate exact input artifact layouts --- .github/workflows/candidate.yml | 10 +++++- .github/workflows/phase6-candidate-gate.yml | 10 +++++- scripts/phase6_candidate.py | 37 ++++++++++++++++++--- tests/test_phase6_candidate.py | 28 ++++++++++++++++ 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index 114624b..4e7d2f0 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -95,6 +95,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33170546601 artifact-ids: 9685464135 + merge-multiple: true path: input-artifacts/phase3p-proof-data - name: Download exact Phase-4 Celestia appd artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -103,6 +104,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688244894 + merge-multiple: true path: input-artifacts/phase4-celestia-appd-linux-arm64 - name: Download exact Phase-4 Celestia node artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -111,6 +113,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688243729 + merge-multiple: true path: input-artifacts/phase4-celestia-node-linux-arm64 - name: Download exact Phase-4 Midnight node artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -119,6 +122,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688330126 + merge-multiple: true path: input-artifacts/phase4-node-linux-arm64 - name: Download exact Phase-4 toolkit Linux amd64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -127,6 +131,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688263793 + merge-multiple: true path: input-artifacts/phase4-toolkit-linux-amd64 - name: Download exact Phase-4 toolkit Linux arm64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -135,6 +140,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688255774 + merge-multiple: true path: input-artifacts/phase4-toolkit-linux-arm64 - name: Download exact Phase-4 toolkit macOS arm64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -143,6 +149,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9689647047 + merge-multiple: true path: input-artifacts/phase4-toolkit-macos-arm64 - name: Download exact Phase-5 verified indexer aggregate uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -151,6 +158,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33176004154 artifact-ids: 9690093579 + merge-multiple: true path: input-artifacts/phase5-indexer - name: Assemble and non-executingly verify exact 31-payload content id: assemble @@ -158,7 +166,7 @@ jobs: BUILD_SET_PATH: ${{ needs.validate-protected-source.outputs.build-set-path }} run: | set -euo pipefail - python3 scripts/phase6_candidate.py assemble --build-set "$BUILD_SET_PATH" --input-root input-artifacts --output verified-content | tail -n 1 > candidate-verification.json + python3 scripts/phase6_candidate.py assemble --build-set "$BUILD_SET_PATH" --input-root input-artifacts --output verified-content --result-output candidate-verification.json python3 scripts/phase6_candidate.py verify --build-set "$BUILD_SET_PATH" --content verified-content CONTENT_LIST_DIGEST="$(jq -r '.contentAssetListSha256' candidate-verification.json)" test "$(printf '%s' "$CONTENT_LIST_DIGEST" | wc -c)" -eq 64 diff --git a/.github/workflows/phase6-candidate-gate.yml b/.github/workflows/phase6-candidate-gate.yml index 762c0f1..1a0ebe5 100644 --- a/.github/workflows/phase6-candidate-gate.yml +++ b/.github/workflows/phase6-candidate-gate.yml @@ -50,6 +50,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33170546601 artifact-ids: 9685464135 + merge-multiple: true path: input-artifacts/phase3p-proof-data - name: Download exact Phase-4 Celestia appd artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -58,6 +59,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688244894 + merge-multiple: true path: input-artifacts/phase4-celestia-appd-linux-arm64 - name: Download exact Phase-4 Celestia node artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -66,6 +68,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688243729 + merge-multiple: true path: input-artifacts/phase4-celestia-node-linux-arm64 - name: Download exact Phase-4 Midnight node artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -74,6 +77,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688330126 + merge-multiple: true path: input-artifacts/phase4-node-linux-arm64 - name: Download exact Phase-4 toolkit Linux amd64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -82,6 +86,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688263793 + merge-multiple: true path: input-artifacts/phase4-toolkit-linux-amd64 - name: Download exact Phase-4 toolkit Linux arm64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -90,6 +95,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9688255774 + merge-multiple: true path: input-artifacts/phase4-toolkit-linux-arm64 - name: Download exact Phase-4 toolkit macOS arm64 artifact uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -98,6 +104,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33177534764 artifact-ids: 9689647047 + merge-multiple: true path: input-artifacts/phase4-toolkit-macos-arm64 - name: Download exact Phase-5 verified indexer aggregate uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -106,11 +113,12 @@ jobs: repository: acedward/midnight-binary-forge run-id: 33176004154 artifact-ids: 9690093579 + merge-multiple: true path: input-artifacts/phase5-indexer - name: Assemble and non-executingly verify exact candidate run: | set -euo pipefail - python3 scripts/phase6_candidate.py assemble --build-set catalog/buildsets/initial-warehouse-v1.json --input-root input-artifacts --output verified-content | tail -n 1 > candidate-verification.json + python3 scripts/phase6_candidate.py assemble --build-set catalog/buildsets/initial-warehouse-v1.json --input-root input-artifacts --output verified-content --result-output candidate-verification.json python3 scripts/phase6_candidate.py verify --build-set catalog/buildsets/initial-warehouse-v1.json --content verified-content - name: Retain exact verified content for a fresh job id: upload diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py index 81baafb..a1cd24b 100644 --- a/scripts/phase6_candidate.py +++ b/scripts/phase6_candidate.py @@ -226,13 +226,37 @@ def checksum_manifest(root: Path) -> None: compare_phase5_indexer_builds.validate_checksum_manifest(root) +def validate_input_layout(buildset: dict[str, Any], input_root: Path) -> dict[str, Path]: + expect(input_root.is_dir() and not input_root.is_symlink(), "input root is missing or unsafe") + expected_keys = {row["key"] for row in buildset["inputArtifacts"]} + expect({path.name for path in input_root.iterdir()} == expected_keys, "downloaded input directory set differs from build set") + inputs = {key: input_root / key for key in expected_keys} + expected_top = { + "phase3p-proof-data": {"payloads", "evidence"}, + "phase4-celestia-appd-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-celestia-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-amd64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-macos-arm64": {"SHA256SUMS", "payloads", "evidence", "sbom", "independent-builds"}, + "phase5-indexer": {"SHA256SUMS", "payload", "evidence"}, + } + expect(set(expected_top) == expected_keys, "Phase-6 input-layout policy differs from pinned inputs") + for key, root in inputs.items(): + expect(root.is_dir() and not root.is_symlink(), f"downloaded input artifact root is unsafe: {key}") + children = {path.name: path for path in root.iterdir()} + expect(set(children) == expected_top[key], f"downloaded input top-level layout differs: {key}") + for name, path in children.items(): + expect(not path.is_symlink(), f"downloaded input top-level symlink forbidden: {key}/{name}") + expected_file = name == "SHA256SUMS" + expect(path.is_file() if expected_file else path.is_dir(), f"downloaded input top-level type differs: {key}/{name}") + return inputs + + def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = ROOT) -> dict[str, Any]: buildset, coverage = validate_buildset(buildset_path, root) - expect(input_root.is_dir() and not input_root.is_symlink(), "input root is missing or unsafe") expect(not output.exists() and output.parent.is_dir(), "candidate output must not already exist") - inputs = {row["key"]: input_root / row["key"] for row in buildset["inputArtifacts"]} - expect(set(path.name for path in input_root.iterdir()) == set(inputs), "downloaded input directory set differs from build set") - expect(all(path.is_dir() and not path.is_symlink() for path in inputs.values()), "downloaded input artifact root is unsafe") + inputs = validate_input_layout(buildset, input_root) proof_data_pipeline.verify_output(root / "catalog/proof-data/q8b-v1.json", inputs["phase3p-proof-data"]) checksum_manifest(inputs["phase5-indexer"]) consolidate_phase4_macos.verify(inputs["phase4-toolkit-macos-arm64"]) @@ -501,6 +525,7 @@ def main() -> int: assemble_parser.add_argument("--build-set", type=Path, required=True) assemble_parser.add_argument("--input-root", type=Path, required=True) assemble_parser.add_argument("--output", type=Path, required=True) + assemble_parser.add_argument("--result-output", type=Path) verify_parser = sub.add_parser("verify") verify_parser.add_argument("--build-set", type=Path, required=True) verify_parser.add_argument("--content", type=Path, required=True) @@ -526,7 +551,9 @@ def main() -> int: verify_live_metadata(buildset, load_json(args.metadata), not args.allow_expired) print("OK exact audited Phase-6 input metadata") elif args.command == "assemble": - assemble(args.build_set, args.input_root, args.output) + result = assemble(args.build_set, args.input_root, args.output) + if args.result_output is not None: + create_file_atomic(args.result_output, canonical_bytes(result), 0o644) elif args.command == "verify": verify_candidate(args.build_set, args.content) else: diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py index 1ec8d66..e0d2e54 100644 --- a/tests/test_phase6_candidate.py +++ b/tests/test_phase6_candidate.py @@ -114,6 +114,34 @@ def test_staging_artifact_identity_is_api_bound(self) -> None: with self.subTest(field=field), mock.patch("github_phase6.request", return_value=(adversarial, "")), self.assertRaises(ForgeError): github_phase6.capture_staging(7, artifact["name"], 11, 1, root / f"failed-{field}.json") + def test_downloaded_input_layout_is_exact_and_rejects_artifact_name_nesting(self) -> None: + buildset = load_json(BUILD_SET) + top = { + "phase3p-proof-data": {"payloads", "evidence"}, + "phase4-celestia-appd-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-celestia-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-amd64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-macos-arm64": {"SHA256SUMS", "payloads", "evidence", "sbom", "independent-builds"}, + "phase5-indexer": {"SHA256SUMS", "payload", "evidence"}, + } + with tempfile.TemporaryDirectory() as text: + root = Path(text) + for key, children in top.items(): + directory = root / key + directory.mkdir() + for name in children: + if name == "SHA256SUMS": + (directory / name).write_text("fixture\n", encoding="utf-8") + else: + (directory / name).mkdir() + phase6_candidate.validate_input_layout(buildset, root) + proof = root / "phase3p-proof-data" + (proof / "payloads").rename(proof / "proof-data-q8b-wrapper") + with self.assertRaisesRegex(ForgeError, "top-level layout"): + phase6_candidate.validate_input_layout(buildset, root) + class Phase6StreamingVerifierTest(unittest.TestCase): def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, dict]: From ed4c428f117928ab4fa4ffe82cc73e27cb49a521 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 12:06:13 -0400 Subject: [PATCH 4/8] fix: normalize phase6 repository paths --- scripts/phase6_candidate.py | 25 ++++++++++++++++++++++++- tests/test_phase6_candidate.py | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py index a1cd24b..f4cf9ee 100644 --- a/scripts/phase6_candidate.py +++ b/scripts/phase6_candidate.py @@ -95,6 +95,28 @@ def strict_relative(value: str) -> PurePosixPath: return path +def repository_file(path: Path, root: Path = ROOT, label: str = "repository file") -> tuple[Path, PurePosixPath]: + """Resolve a CLI path from the caller's CWD without permitting escape or symlinks.""" + repository = root.resolve(strict=True) + expect(repository.is_dir() and not root.is_symlink(), "repository root is missing or unsafe") + lexical = Path(os.path.abspath(os.fspath(path))) + try: + lexical_relative = lexical.relative_to(repository) + except ValueError as exc: + raise ForgeError(f"{label} is outside the repository root: {path}") from exc + cursor = repository + for part in lexical_relative.parts: + cursor = cursor / part + expect(not cursor.is_symlink(), f"{label} traverses a symlink: {path}") + resolved = lexical.resolve(strict=True) + try: + relative = resolved.relative_to(repository) + except ValueError as exc: + raise ForgeError(f"{label} resolves outside the repository root: {path}") from exc + validate_regular_file(resolved) + return resolved, PurePosixPath(relative.as_posix()) + + def media_type(name: str) -> str: if name.endswith(".spdx.json"): return "application/spdx+json" @@ -254,6 +276,7 @@ def validate_input_layout(buildset: dict[str, Any], input_root: Path) -> dict[st def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = ROOT) -> dict[str, Any]: + buildset_path, buildset_relative = repository_file(buildset_path, root, "build-set path") buildset, coverage = validate_buildset(buildset_path, root) expect(not output.exists() and output.parent.is_dir(), "candidate output must not already exist") inputs = validate_input_layout(buildset, input_root) @@ -339,7 +362,7 @@ def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = R source_manifest = { "schemaVersion": "phase6-source-manifest-v1", "buildSetId": build_id, - "buildSet": {"path": buildset_path.relative_to(root).as_posix(), "size": buildset_path.stat().st_size, "sha256": sha256_file(buildset_path)[0]}, + "buildSet": {"path": buildset_relative.as_posix(), "size": buildset_path.stat().st_size, "sha256": sha256_file(buildset_path)[0]}, "reviewedInputBaselineSha": buildset["sourceFullSha"], "inputArtifacts": buildset["inputArtifacts"], "destination": buildset["destination"], diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py index e0d2e54..37f1df8 100644 --- a/tests/test_phase6_candidate.py +++ b/tests/test_phase6_candidate.py @@ -5,6 +5,7 @@ import copy import hashlib import json +import os import stat import sys import tempfile @@ -142,6 +143,30 @@ def test_downloaded_input_layout_is_exact_and_rejects_artifact_name_nesting(self with self.assertRaisesRegex(ForgeError, "top-level layout"): phase6_candidate.validate_input_layout(buildset, root) + def test_relative_cwd_buildset_path_resolves_inside_repository(self) -> None: + previous = Path.cwd() + try: + os.chdir(ROOT) + resolved, relative = phase6_candidate.repository_file(Path("catalog/buildsets/initial-warehouse-v1.json")) + finally: + os.chdir(previous) + self.assertEqual(resolved, BUILD_SET) + self.assertEqual(relative.as_posix(), "catalog/buildsets/initial-warehouse-v1.json") + + def test_out_of_root_and_symlink_buildset_paths_fail(self) -> None: + with tempfile.TemporaryDirectory() as repository_text, tempfile.TemporaryDirectory() as outside_text: + repository = Path(repository_text) + inside = repository / "buildset.json" + inside.write_text("{}\n", encoding="utf-8") + outside = Path(outside_text) / "buildset.json" + outside.write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(ForgeError, "outside the repository root"): + phase6_candidate.repository_file(outside, repository) + link = repository / "buildset-link.json" + link.symlink_to(inside) + with self.assertRaisesRegex(ForgeError, "traverses a symlink"): + phase6_candidate.repository_file(link, repository) + class Phase6StreamingVerifierTest(unittest.TestCase): def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, dict]: From 586c778113aaba13680eb2d94771f25fdeba4216 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 12:16:18 -0400 Subject: [PATCH 5/8] fix: bind identity mirror tar headers --- scripts/phase6_candidate.py | 68 ++++++++++++++++++++++++++++++---- scripts/validate_archive.py | 14 +++++-- tests/test_phase6_candidate.py | 48 +++++++++++++++++++++++- 3 files changed, 117 insertions(+), 13 deletions(-) diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py index f4cf9ee..fa637a4 100644 --- a/scripts/phase6_candidate.py +++ b/scripts/phase6_candidate.py @@ -67,6 +67,24 @@ "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip": (48585850, "4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528", "phase4-toolkit-linux-arm64", "payloads/midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip"), "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip": (45553847, "8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c", "phase4-toolkit-macos-arm64", "payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip"), } +EXPECTED_IDENTITY_MIRROR_TAR_HEADERS = { + "celestia-appd-linux-arm64-v6.4.10.tar.gz": { + "componentId": "celestia-appd-6.4.10-linux-arm64", + "headers": [ + {"path": "LICENSE", "uid": 1001, "gid": 1001, "uname": "", "gname": "", "mtime": 1770110835}, + {"path": "README.md", "uid": 1001, "gid": 1001, "uname": "", "gname": "", "mtime": 1770110835}, + {"path": "celestia-appd", "uid": 0, "gid": 0, "uname": "root", "gname": "root", "mtime": 1770111631}, + ], + }, + "celestia-node-linux-arm64-v0.28.4.tar.gz": { + "componentId": "celestia-node-0.28.4-linux-arm64", + "headers": [ + {"path": "LICENSE", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256280}, + {"path": "README.md", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256280}, + {"path": "celestia", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256973}, + ], + }, +} EVIDENCE_ROLE_BY_PREFIX = { "LICENSE-": "license", "NOTICE-": "notice", @@ -306,7 +324,8 @@ def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = R if payload["artifactKind"] == "software" and payload["sourceArtifactKey"].startswith("phase4-"): record, members, sbom = phase4_record(inputs[payload["sourceArtifactKey"]], payload["name"]) phase_records[payload["name"]] = record - archive_rows.append({"name": payload["name"], "container": payload["container"], "limits": component["naming"]["limits"], "members": members["members"]}) + mirror = EXPECTED_IDENTITY_MIRROR_TAR_HEADERS.get(payload["name"]) + archive_rows.append({"name": payload["name"], "componentId": payload["componentId"], "operation": component["operation"], "container": payload["container"], "wholeArchive": {"size": copied["size"], "sha256": copied["sha256"]}, "limits": component["naming"]["limits"], "members": members["members"], "tarHeaders": mirror["headers"] if mirror is not None else []}) sbom_sources[payload["name"]] = sbom signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": record["signing"]}) elif payload["artifactKind"] == "software": @@ -314,7 +333,7 @@ def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = R evidence = inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/build1/evidence" reproduction = load_json(inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/reproducibility.json") binary = reproduction["binary"] - archive_rows.append({"name": payload["name"], "container": "zip", "limits": component["naming"]["limits"], "members": [{"path": binary["name"], "type": "file", "mode": "0755", "size": binary["size"], "sha256": binary["sha256"]}]}) + archive_rows.append({"name": payload["name"], "componentId": payload["componentId"], "operation": component["operation"], "container": "zip", "wholeArchive": {"size": copied["size"], "sha256": copied["sha256"]}, "limits": component["naming"]["limits"], "members": [{"path": binary["name"], "type": "file", "mode": "0755", "size": binary["size"], "sha256": binary["sha256"]}], "tarHeaders": []}) sbom_sources[payload["name"]] = evidence / "sbom-indexer-standalone.spdx.json" signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": load_json(evidence / "signing-evidence.json")}) @@ -412,12 +431,40 @@ def verify_checksums(content: Path, checksums_name: str) -> None: expect(sha256_file(path, 2**31 - 1)[0] == rows[name], f"candidate checksum mismatch: {name}") -def verify_one_archive(path: Path, policy: dict[str, Any]) -> None: +def verify_one_archive(path: Path, policy: dict[str, Any], expected_payload: dict[str, Any] | None = None, component: dict[str, Any] | None = None) -> None: + required = {"name", "componentId", "operation", "container", "wholeArchive", "limits", "members", "tarHeaders"} + expect(set(policy) == required and policy["name"] == path.name, f"archive evidence fields/name differ: {path.name}") + whole = identity(path) + expect(policy["wholeArchive"] == {"size": whole["size"], "sha256": whole["sha256"]}, f"whole-archive identity differs: {path.name}") + if expected_payload is not None: + expect(policy["componentId"] == expected_payload["componentId"], f"archive component differs: {path.name}") + expect(policy["container"] == expected_payload["container"], f"archive container differs: {path.name}") + expect(policy["wholeArchive"] == {"size": expected_payload["size"], "sha256": expected_payload["sha256"]}, f"archive differs from exact build-set payload: {path.name}") + if component is not None: + expect(policy["componentId"] == component["componentId"] and policy["operation"] == component["operation"], f"archive operation/component differs from manifest: {path.name}") + expect(policy["container"] == component["naming"]["container"] and policy["limits"] == component["naming"]["limits"], f"archive container/limits differ from component: {path.name}") + member_contract = [{key: row[key] for key in ("path", "type", "mode")} for row in policy["members"]] + component_contract = [{key: row[key] for key in ("path", "type", "mode")} for row in component["naming"]["members"]] + component_by_path = {row["path"]: row for row in component_contract} + member_by_path = {row["path"]: row for row in member_contract} + expect(all(member_by_path.get(name) == row for name, row in component_by_path.items()), f"archive misses/differs from component member contract: {path.name}") + for row in member_contract: + if row["path"] not in component_by_path: + expect(any(parent["type"] == "directory" and row["path"].startswith(parent["path"] + "/") for parent in component_contract), f"archive member is outside component contract: {path.name}:{row['path']}") compressed = path.stat().st_size limits = policy["limits"] expect(compressed <= limits["maxCompressedBytes"], f"archive exceeds compressed bound: {path.name}") - iterator = validate_archive.zip_members if policy["container"] == "zip" else validate_archive.tar_members - members = list(iterator(path)) + mirror = EXPECTED_IDENTITY_MIRROR_TAR_HEADERS.get(path.name) + if mirror is None: + expect(policy["tarHeaders"] == [], f"tar-header exception forbidden for build/repackage archive: {path.name}") + iterator = validate_archive.zip_members if policy["container"] == "zip" else validate_archive.tar_members + members = list(iterator(path)) + else: + expect(policy["container"] == "tar.gz" and policy["operation"] in {"identity-mirror", "rename-only"}, f"upstream tar-header policy requires identity-mirror/rename-only: {path.name}") + expect(policy["componentId"] == mirror["componentId"] and policy["tarHeaders"] == mirror["headers"], f"upstream tar-header allowlist differs: {path.name}") + members = list(validate_archive.tar_members(path, require_canonical_owner=False)) + headers = [{"path": member.path, "uid": member.uid, "gid": member.gid, "uname": member.uname, "gname": member.gname, "mtime": member.mtime} for member in members] + expect(headers == mirror["headers"], f"pinned upstream tar headers differ: {path.name}") expect(len(members) <= limits["maxMembers"], f"archive exceeds member bound: {path.name}") validate_unique_names(member.path for member in members) expected = {row["path"]: row for row in policy["members"]} @@ -481,11 +528,16 @@ def verify_candidate(buildset_path: Path, content: Path, root: Path = ROOT) -> d expect(observed["size"] == srs[name]["size"] and observed["sha256"] == srs[name]["sha256"], f"raw proof payload differs: {name}") policies = load_json(content / f"software-member-manifests-{build_id}.json") expect(policies.get("schemaVersion") == "phase6-software-member-manifests-v1" and len(policies.get("archives", [])) == 10, "software archive policy set differs") + component_by_id = {row["componentId"]: load_json(root / row["manifestPath"]) for row in buildset["components"]} for policy in policies["archives"]: - verify_one_archive(content / policy["name"], policy) + expected = next((row for row in buildset["payloads"] if row["name"] == policy["name"]), None) + expect(expected is not None, f"archive evidence names an unapproved payload: {policy['name']}") + verify_one_archive(content / policy["name"], policy, expected, component_by_id[expected["componentId"]]) ledger = content / "midnight-ledger-static-noarch-9.0.0.zip" - ledger_policy = {"name": ledger.name, "container": "zip", "limits": load_json(root / "catalog/components/midnight-ledger-static-9.0.0.json")["naming"]["limits"], "members": load_json(root / "catalog/proof-data/ledger-static-9-zip-layout-manifest.json")["members"]} - verify_one_archive(ledger, ledger_policy) + ledger_component = load_json(root / "catalog/components/midnight-ledger-static-9.0.0.json") + ledger_payload = next(row for row in buildset["payloads"] if row["name"] == ledger.name) + ledger_policy = {"name": ledger.name, "componentId": ledger_payload["componentId"], "operation": ledger_component["operation"], "container": "zip", "wholeArchive": {"size": ledger_payload["size"], "sha256": ledger_payload["sha256"]}, "limits": ledger_component["naming"]["limits"], "members": load_json(root / "catalog/proof-data/ledger-static-9-zip-layout-manifest.json")["members"], "tarHeaders": []} + verify_one_archive(ledger, ledger_policy, ledger_payload, ledger_component) lineage = load_json(content / f"proof-data-lineage-{build_id}.json") expect(lineage.get("payloadCount") == 21 and len(lineage.get("payloads", [])) == 21 and lineage.get("softwareSbom") == "not-applicable", "proof-data lineage differs") signing = load_json(content / f"signing-evidence-{build_id}.json") diff --git a/scripts/validate_archive.py b/scripts/validate_archive.py index e6018e0..a474803 100644 --- a/scripts/validate_archive.py +++ b/scripts/validate_archive.py @@ -36,6 +36,11 @@ class Member: size: int compressed_size: int opener: callable + uid: int | None = None + gid: int | None = None + uname: str | None = None + gname: str | None = None + mtime: int | float | None = None @contextlib.contextmanager @@ -79,7 +84,7 @@ def zip_members(path: Path) -> Iterator[Member]: archive.close() -def tar_members(path: Path) -> Iterator[Member]: +def tar_members(path: Path, require_canonical_owner: bool = True) -> Iterator[Member]: archive = tarfile.open(path, "r:gz") try: expect(not archive.pax_headers, "global PAX metadata forbidden") @@ -88,12 +93,13 @@ def tar_members(path: Path) -> Iterator[Member]: safe_member_name(name) expect(not info.pax_headers, f"PAX metadata forbidden: {name}") expect(not info.issym() and not info.islnk(), f"archive links forbidden: {name}") - expect(info.uid in {0} and info.gid in {0}, f"non-canonical tar owner forbidden: {name}") + if require_canonical_owner: + expect(info.uid in {0} and info.gid in {0}, f"non-canonical tar owner forbidden: {name}") if info.isdir(): - yield Member(name, "directory", f"{info.mode:04o}", 0, 0, lambda: None) + yield Member(name, "directory", f"{info.mode:04o}", 0, 0, lambda: None, info.uid, info.gid, info.uname, info.gname, info.mtime) else: expect(info.isfile(), f"unsafe tar member type: {name}") - yield Member(name, "file", f"{info.mode:04o}", info.size, info.size, lambda member_name=info.name: open_tar_member(path, member_name)) + yield Member(name, "file", f"{info.mode:04o}", info.size, info.size, lambda member_name=info.name: open_tar_member(path, member_name), info.uid, info.gid, info.uname, info.gname, info.mtime) finally: archive.close() diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py index 37f1df8..f378770 100644 --- a/tests/test_phase6_candidate.py +++ b/tests/test_phase6_candidate.py @@ -4,11 +4,13 @@ import copy import hashlib +import io import json import os import stat import sys import tempfile +import tarfile import unittest import zipfile from pathlib import Path @@ -176,7 +178,7 @@ def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, d info.external_attr = (stat.S_IFREG | mode) << 16 with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: archive.writestr(info, value) - policy = {"container": "zip", "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 2, "maxExpansionRatio": 100}, "members": [{"path": "fixture", "type": "file", "mode": f"{mode:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()}]} + policy = {"name": path.name, "componentId": "fixture-component", "operation": "build", "container": "zip", "wholeArchive": {"size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}, "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 2, "maxExpansionRatio": 100}, "members": [{"path": "fixture", "type": "file", "mode": f"{mode:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()}], "tarHeaders": []} return path, policy def test_streamed_member_identity_and_substitution(self) -> None: @@ -188,6 +190,50 @@ def test_streamed_member_identity_and_substitution(self) -> None: with self.assertRaises(ForgeError): phase6_candidate.verify_one_archive(archive, policy) + def test_pinned_upstream_tar_headers_and_owner_header_digest_mutations(self) -> None: + name = "celestia-appd-linux-arm64-v6.4.10.tar.gz" + expected = phase6_candidate.EXPECTED_IDENTITY_MIRROR_TAR_HEADERS[name] + values = {"LICENSE": b"license", "README.md": b"readme", "celestia-appd": b"binary"} + modes = {"LICENSE": 0o644, "README.md": 0o644, "celestia-appd": 0o755} + with tempfile.TemporaryDirectory() as text: + path = Path(text) / name + with tarfile.open(path, "w:gz", format=tarfile.GNU_FORMAT) as archive: + for header in expected["headers"]: + value = values[header["path"]] + info = tarfile.TarInfo(header["path"]) + info.size = len(value) + info.mode = modes[header["path"]] + info.uid, info.gid = header["uid"], header["gid"] + info.uname, info.gname = header["uname"], header["gname"] + info.mtime = header["mtime"] + archive.addfile(info, io.BytesIO(value)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + members = [{"path": item, "type": "file", "mode": f"{modes[item]:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()} for item, value in values.items()] + members.sort(key=lambda row: row["path"]) + policy = {"name": name, "componentId": expected["componentId"], "operation": "rename-only", "container": "tar.gz", "wholeArchive": {"size": path.stat().st_size, "sha256": digest}, "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 3, "maxExpansionRatio": 100}, "members": members, "tarHeaders": expected["headers"]} + payload = {"componentId": expected["componentId"], "container": "tar.gz", "size": path.stat().st_size, "sha256": digest} + component = {"componentId": expected["componentId"], "operation": "rename-only", "naming": {"container": "tar.gz", "limits": policy["limits"], "members": [{key: row[key] for key in ("path", "type", "mode")} for row in members]}} + phase6_candidate.verify_one_archive(path, policy, payload, component) + for field, value in (("uid", 0), ("mtime", 0)): + mutation = copy.deepcopy(policy) + mutation["tarHeaders"][0][field] = value + with self.subTest(field=field), self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, mutation, payload, component) + mutation = copy.deepcopy(policy) + mutation["wholeArchive"]["sha256"] = "0" * 64 + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, mutation, payload, component) + mutation = copy.deepcopy(component) + mutation["operation"] = "build" + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, policy, payload, mutation) + canonical_only = copy.deepcopy(policy) + canonical_only.update({"name": "built.tar.gz", "componentId": "built-component", "operation": "build", "tarHeaders": []}) + built = path.with_name("built.tar.gz") + built.write_bytes(path.read_bytes()) + with self.assertRaisesRegex(ForgeError, "non-canonical tar owner"): + phase6_candidate.verify_one_archive(built, canonical_only) + def test_all_evidence_names_are_typed(self) -> None: self.assertEqual(phase6_candidate.evidence_role("provenance-initial-warehouse-v1.json"), "provenance") with self.assertRaises(ForgeError): From d8db4327958bc057db685714d116579a00947317 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 12:21:48 -0400 Subject: [PATCH 6/8] fix: flatten staged candidate redownloads --- .github/workflows/candidate.yml | 2 ++ .github/workflows/phase6-candidate-gate.yml | 1 + scripts/phase6_candidate.py | 15 +++++++++++---- tests/test_phase6_candidate.py | 17 +++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index 4e7d2f0..cf6770f 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -258,6 +258,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: ${{ github.run_id }} artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + merge-multiple: true path: verified-content - name: Download assembly, draft, and protected-source evidence uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 @@ -312,6 +313,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: ${{ github.run_id }} artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + merge-multiple: true path: verified-content - name: Download exact claims and draft handoffs uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 diff --git a/.github/workflows/phase6-candidate-gate.yml b/.github/workflows/phase6-candidate-gate.yml index 1a0ebe5..230131b 100644 --- a/.github/workflows/phase6-candidate-gate.yml +++ b/.github/workflows/phase6-candidate-gate.yml @@ -152,6 +152,7 @@ jobs: repository: acedward/midnight-binary-forge run-id: ${{ github.run_id }} artifact-ids: ${{ needs.assemble-and-verify.outputs.artifact-id }} + merge-multiple: true path: verified-content - name: Stream and verify every payload/member without execution run: | diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py index fa637a4..3229e0a 100644 --- a/scripts/phase6_candidate.py +++ b/scripts/phase6_candidate.py @@ -499,13 +499,20 @@ def content_assets(buildset: dict[str, Any], content: Path) -> list[dict[str, An return rows -def verify_candidate(buildset_path: Path, content: Path, root: Path = ROOT) -> dict[str, Any]: - buildset, _ = validate_buildset(buildset_path, root) +def validate_staged_content_layout(content: Path, build_id: str) -> None: expect(content.is_dir() and not content.is_symlink(), "candidate content root is unsafe") - for path in content.iterdir(): + expected = EXPECTED_PAYLOAD_NAMES | expected_evidence_names(build_id) + children = {path.name: path for path in content.iterdir()} + expect(len(expected) == 52 and set(children) == expected, "staged content must be the exact flat 31-payload plus 21-evidence closure") + for name, path in children.items(): validate_regular_file(path, "0644") - safe_basename(path.name, "candidate content name") + safe_basename(name, "candidate content name") + + +def verify_candidate(buildset_path: Path, content: Path, root: Path = ROOT) -> dict[str, Any]: + buildset, _ = validate_buildset(buildset_path, root) build_id = buildset["buildSetId"] + validate_staged_content_layout(content, build_id) source_name = f"source-manifest-{build_id}.json" checksums_name = f"sha256sums-{build_id}.txt" verify_checksums(content, checksums_name) diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py index f378770..fdcdc56 100644 --- a/tests/test_phase6_candidate.py +++ b/tests/test_phase6_candidate.py @@ -145,6 +145,23 @@ def test_downloaded_input_layout_is_exact_and_rejects_artifact_name_nesting(self with self.assertRaisesRegex(ForgeError, "top-level layout"): phase6_candidate.validate_input_layout(buildset, root) + def test_staged_content_layout_is_exact_and_rejects_artifact_name_nesting(self) -> None: + build_id = "initial-warehouse-v1" + names = phase6_candidate.EXPECTED_PAYLOAD_NAMES | phase6_candidate.expected_evidence_names(build_id) + self.assertEqual(len(names), 52) + with tempfile.TemporaryDirectory() as text: + root = Path(text) + for name in names: + (root / name).write_bytes(b"") + (root / name).chmod(0o644) + phase6_candidate.validate_staged_content_layout(root, build_id) + wrapper = root / "phase6-pr-verified-content-wrapper" + wrapper.mkdir() + for name in names: + (root / name).rename(wrapper / name) + with self.assertRaisesRegex(ForgeError, "exact flat"): + phase6_candidate.validate_staged_content_layout(root, build_id) + def test_relative_cwd_buildset_path_resolves_inside_repository(self) -> None: previous = Path.cwd() try: From 030bbc624ef4079c09b7ab793d47d4abae0359a2 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 13:09:56 -0400 Subject: [PATCH 7/8] fix: reject shallow ancestry verification --- scripts/phase6_candidate.py | 7 +++- scripts/validate_catalog.py | 25 ++++++++++-- tests/test_phase6_candidate.py | 72 ++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py index 3229e0a..900887a 100644 --- a/scripts/phase6_candidate.py +++ b/scripts/phase6_candidate.py @@ -237,9 +237,12 @@ def verify_live_metadata(buildset: dict[str, Any], metadata: dict[str, Any], req def git_ancestry(buildset: dict[str, Any], root: Path) -> None: - head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + validate_catalog.require_complete_git_history(root) + resolved_head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10) + expect(resolved_head.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}\n", resolved_head.stdout) is not None and resolved_head.stderr == "", "cannot resolve candidate full source HEAD") + head = resolved_head.stdout.rstrip("\n") for source in [buildset["sourceFullSha"], *[row["sourceHeadSha"] for row in buildset["inputArtifacts"]]]: - result = subprocess.run(["git", "-C", str(root), "merge-base", "--is-ancestor", source, head], check=False, capture_output=True) + result = subprocess.run(["git", "-C", str(root), "merge-base", "--is-ancestor", source, head], check=False, capture_output=True, timeout=10) expect(result.returncode == 0, f"reviewed input SHA is not reachable from candidate source HEAD: {source}") diff --git a/scripts/validate_catalog.py b/scripts/validate_catalog.py index ecef0ce..3ec13d8 100755 --- a/scripts/validate_catalog.py +++ b/scripts/validate_catalog.py @@ -7,6 +7,7 @@ import hashlib import json import re +import subprocess import sys from pathlib import Path, PurePosixPath from typing import Any @@ -257,13 +258,29 @@ def load_components(root: Path, build_set: dict[str, Any]) -> dict[str, dict[str return components +def require_complete_git_history(root: Path) -> None: + """Reject ancestry claims unless Git proves this exact checkout is non-shallow.""" + expect(root.is_dir(), "source-head verification requires a repository directory") + git_metadata = root / ".git" + expect(git_metadata.exists() and not git_metadata.is_symlink(), "source-head verification requires an exact Git checkout") + try: + shallow = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-shallow-repository"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ForgeError("cannot prove complete Git history") from exc + expect(shallow.returncode == 0, "cannot prove complete Git history") + expect(shallow.stdout == "false\n" and shallow.stderr == "", "source-head verification requires exact non-shallow Git history") + + def validate_build_set(build_set: dict[str, Any], root: Path, require_source_head: bool = False) -> dict[str, Any]: schema_validate(build_set, "build-set-v1.schema.json") if require_source_head: - head_file = root / ".git" - expect(head_file.exists(), "source-head verification requires a Git checkout") - import subprocess - + require_complete_git_history(root) result = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10) expect(result.returncode == 0 and len(result.stdout.strip()) == 40, "cannot resolve current full source HEAD") ancestor = subprocess.run( diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py index fdcdc56..57db01a 100644 --- a/tests/test_phase6_candidate.py +++ b/tests/test_phase6_candidate.py @@ -8,6 +8,7 @@ import json import os import stat +import subprocess import sys import tempfile import tarfile @@ -22,12 +23,23 @@ import phase6_candidate # noqa: E402 import github_phase6 # noqa: E402 +import validate_catalog # noqa: E402 from forge_io import ForgeError, load_json # noqa: E402 BUILD_SET = ROOT / "catalog/buildsets/initial-warehouse-v1.json" +def run_git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + class Phase6BuildSetTest(unittest.TestCase): def test_exact_31_payload_build_set_and_generator_are_closed(self) -> None: buildset, report = phase6_candidate.validate_buildset(BUILD_SET) @@ -187,6 +199,66 @@ def test_out_of_root_and_symlink_buildset_paths_fail(self) -> None: phase6_candidate.repository_file(link, repository) +class FullHistoryAncestryTest(unittest.TestCase): + def _repository_with_three_commits(self, root: Path) -> tuple[Path, list[str]]: + repository = root / "origin" + repository.mkdir() + run_git(repository, "init", "--initial-branch=main") + run_git(repository, "config", "user.email", "phase6-test@example.invalid") + run_git(repository, "config", "user.name", "Phase 6 test") + commits = [] + for index in range(3): + (repository / "value.txt").write_text(f"{index}\n", encoding="utf-8") + run_git(repository, "add", "value.txt") + run_git(repository, "commit", "-m", f"fixture {index}") + commits.append(run_git(repository, "rev-parse", "HEAD")) + return repository, commits + + def test_full_history_positive_and_real_shallow_clone_negative(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + complete, commits = self._repository_with_three_commits(root) + buildset = {"sourceFullSha": commits[0], "inputArtifacts": [{"sourceHeadSha": commits[1]}]} + validate_catalog.require_complete_git_history(complete) + phase6_candidate.git_ancestry(buildset, complete) + + shallow = root / "shallow" + subprocess.run( + ["git", "clone", "--depth=2", complete.as_uri(), str(shallow)], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(run_git(shallow, "rev-parse", "--is-shallow-repository"), "true") + self.assertEqual(run_git(shallow, "merge-base", "--is-ancestor", commits[1], commits[2]), "") + self.assertTrue((shallow / ".git/shallow").is_file()) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(shallow) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + phase6_candidate.git_ancestry(buildset, shallow) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.validate_build_set(load_json(BUILD_SET), shallow, require_source_head=True) + + def test_malformed_failed_and_missing_git_context_are_rejected(self) -> None: + completed = subprocess.CompletedProcess([], 0, stdout="false\n", stderr="") + with mock.patch("validate_catalog.subprocess.run", return_value=completed): + validate_catalog.require_complete_git_history(ROOT) + for output in ("", "true\n", "false", "false \n", "false\nextra\n"): + malformed = subprocess.CompletedProcess([], 0, stdout=output, stderr="") + with self.subTest(output=repr(output)), mock.patch("validate_catalog.subprocess.run", return_value=malformed), self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(ROOT) + noisy = subprocess.CompletedProcess([], 0, stdout="false\n", stderr="warning\n") + with mock.patch("validate_catalog.subprocess.run", return_value=noisy), self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(ROOT) + failed = subprocess.CompletedProcess([], 128, stdout="", stderr="fatal\n") + with mock.patch("validate_catalog.subprocess.run", return_value=failed), self.assertRaisesRegex(ForgeError, "cannot prove"): + validate_catalog.require_complete_git_history(ROOT) + with mock.patch("validate_catalog.subprocess.run", side_effect=subprocess.TimeoutExpired(["git"], 10)), self.assertRaisesRegex(ForgeError, "cannot prove"): + validate_catalog.require_complete_git_history(ROOT) + with tempfile.TemporaryDirectory() as text, self.assertRaisesRegex(ForgeError, "exact Git checkout"): + validate_catalog.require_complete_git_history(Path(text)) + + class Phase6StreamingVerifierTest(unittest.TestCase): def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, dict]: path = root / "fixture.zip" From 0170c0c4dbad8707e3b34e4daf029d3377d47077 Mon Sep 17 00:00:00 2001 From: "Edward A." Date: Fri, 28 Aug 2026 14:10:16 -0400 Subject: [PATCH 8/8] Close Phase 6 release boundary recovery gaps --- .../workflows/phase6-live-verification.yml | 54 +++ docs/publishing.md | 38 +- docs/rollback.md | 22 +- scripts/check_workflow_policy.py | 6 + scripts/github_phase6.py | 269 ++++++++++++- tests/test_phase6_release_boundaries.py | 379 ++++++++++++++++++ 6 files changed, 748 insertions(+), 20 deletions(-) create mode 100644 tests/test_phase6_release_boundaries.py diff --git a/.github/workflows/phase6-live-verification.yml b/.github/workflows/phase6-live-verification.yml index 2534f37..29e6803 100644 --- a/.github/workflows/phase6-live-verification.yml +++ b/.github/workflows/phase6-live-verification.yml @@ -65,3 +65,57 @@ jobs: retention-days: 30 compression-level: 0 overwrite: false + + recover-post-publication-handoff-loss: + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'workflow_dispatch' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + attestations: read + steps: + - name: Check out exact failed candidate commit + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Classify and reconstruct only an existing exact immutable publication + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 scripts/github_phase6.py recover-publication \ + --run-id "${{ github.event.workflow_run.id }}" \ + --expected-head "${{ github.event.workflow_run.head_sha }}" \ + --build-set-id initial-warehouse-v1 \ + --output-dir recovered + - name: Cryptographically verify recovered original-run claims attestation + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + cp -- recovered/promotion-claims-initial-warehouse-v1.json recovered/promotion-claims-initial-warehouse-v1 + gh attestation verify recovered/promotion-claims-initial-warehouse-v1 \ + --repo "$GITHUB_REPOSITORY" \ + --bundle recovered/attestation-initial-warehouse-v1.sigstore.json \ + --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + rm -- recovered/promotion-claims-initial-warehouse-v1 + python3 scripts/github_phase6.py verify-recovery \ + --claims recovered/promotion-claims-initial-warehouse-v1.json \ + --draft recovered/draft.json \ + --envelope recovered/promotion-envelope-initial-warehouse-v1.json \ + --bundle recovered/attestation-initial-warehouse-v1.sigstore.json \ + --evidence recovered/recovered-publication.json + - name: Retain canonical read-only recovered-publication evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-recovered-publication-${{ github.event.workflow_run.id }} + path: recovered/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false diff --git a/docs/publishing.md b/docs/publishing.md index e2373b2..a739932 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -8,8 +8,11 @@ writes. `effectstream/binaries@0.3.120` is updated manually outside Actions. Phase 6 replaces the Phase-1 scaffold with an exact hash-bound implementation. Publication has no repository-variable switch: the workflow exists only after reviewed protected-main integration, accepts a build-set ID plus the exact committed build-set SHA-256, and rechecks protected main, -the workflow blob, immutable-release policy, PR-only/no-bypass rules, and the protected publisher +the workflow blob, the dedicated immutable-release setting, PR-only/no-bypass rules, and the protected publisher environment before allocating a draft. The workflow contains no destination credential reference. +Every authoritative setting check calls `GET /repos/acedward/midnight-binary-forge/immutable-releases` +with GitHub API version `2026-03-10` and requires the typed response `enabled: true`; the general +repository response is not an immutable-release authority. 1. Merge exact component/build-set manifests through protected main. Reject abbreviated/floating refs, Compact warehouse components, incomplete required platform coverage, and proof-data scope @@ -29,9 +32,10 @@ environment before allocating a draft. The workflow contains no destination cred proof lineage/cache/member evidence, signing evidence (including the actual unsigned/adhoc macOS state), and provenance. Every evidence basename has one frozen envelope-v1 role. Upload this content to short-lived staging with its complete typed-list digest in the artifact name. -5. The protected publisher creates an empty `forge-YYYY.MM.DD.N` draft, then reads back and freezes - its repository/tag/numeric+node ID/target/URL identity. It passes only that inert identity to a - second fresh no-write/no-OIDC verifier. +5. Immediately before the draft POST, the protected publisher rechecks the dedicated immutable- + release setting. It creates an empty `forge-YYYY.MM.DD.N` draft, then reads back and freezes its + repository/tag/numeric+node ID/target/URL identity. It passes only that inert identity to a second + fresh no-write/no-OIDC verifier. 6. The final-claims verifier independently downloads staging again, requires exact content, checks liveness against authenticated GitHub API server time, and emits canonical draft-bound claims and predicate. It cannot mutate the draft or attest, and no later step may change those claims. @@ -39,14 +43,25 @@ environment before allocating a draft. The workflow contains no destination cred subject name and custom predicate contract. Download the detached bundle, bind its digest in the canonical envelope, download staging again, and require exact inert name/count/size/digest equality. Never extract or execute candidate files. -8. Upload content plus exactly the two predeclared transport files to the draft. Re-download every - draft asset through the API, hash it, verify the exact complete name/count/byte set, then publish. - API-read immutable state. Any upload/read-back/policy mismatch leaves a draft and fails. +8. Recheck the dedicated setting immediately before the first asset upload. Upload content plus + exactly the two predeclared transport files to the draft. Re-download every draft asset through + the API, hash it, and verify the exact complete name/count/byte set. Recheck the setting again + immediately before the public transition, publish, recheck the setting once more, and read back + `draft=false`, `prerelease=false`, `immutable=true` before proceeding. 9. Prove a no-op release metadata mutation is rejected after immutable publication. The separate - read-only `phase6-live-verification.yml` workflow runs only after the candidate workflow reports - success; this sequencing lets it truthfully bind `status=completed, conclusion=success`. It - cryptographically verifies the released bundle and independently re-downloads/hashes every + read-only `phase6-live-verification.yml` workflow handles both terminal outcomes. On source-workflow + success it binds `status=completed, conclusion=success`, rechecks the dedicated setting, + cryptographically verifies the released bundle, and independently re-downloads/hashes every released byte before emitting canonical live evidence. +10. If the immutable publication succeeded but the final `published.json` Actions-artifact upload + failed, the source workflow is truthfully failed. The failure lane may recover evidence only when + the exact original run, retained claims/draft/staging artifact IDs and digests, signed release + ID/node/tag/target/URL, missing normal handoff, current immutable setting, and all 54 existing + release assets agree. It performs GET/download operations only, cryptographically verifies the + original claims bundle, and retains canonical `phase6-recovered-publication-v1` evidence. It never + creates a second release, resumes an upload, patches, republishes, reuploads, deletes, or changes + the already immutable release. A still-draft/mutable release is a prepublication failure and is + abandoned without recovery mutation. ## Workflow permissions @@ -68,3 +83,6 @@ read-back mismatch, or mutable candidate state. Failed drafts are never treated as candidates. Do not reuse their tag or claim immutability. Record the failure, fix manifests/tooling through a new PR, and allocate a new monotonically increasing tag. +After a public transition, do not assume a failed workflow means the release remained a draft. First +classify it through the exact read-only recovery lane; any missing, ambiguous, expired, substituted, +or inconsistent retained artifact/release byte is a hard stop for operator review. diff --git a/docs/rollback.md b/docs/rollback.md index 304391f..b588650 100644 --- a/docs/rollback.md +++ b/docs/rollback.md @@ -5,11 +5,31 @@ an existing asset. ## Before forge publication -- A build/verifier/publisher failure leaves only ephemeral staging or a failed draft. +- A build/verifier failure before the public transition leaves only ephemeral staging or a failed + draft. A generic publisher/workflow failure does not prove that state: it may instead be a + post-publication Actions-artifact handoff failure after the release became immutable. - Delete/expire staging under its normal retention policy only after retaining non-secret logs and manifest/digest evidence. A failed draft is not promoted and its tag is not reused. - Correct source/build/packaging metadata on a new reviewed commit and create a new candidate tag. +## Immutable publication succeeded but final handoff failed + +- The read-only workflow-run recovery lane is permitted only for a completed failed candidate run + whose normal `published-candidate-` handoff is absent. It must bind the exact original + run/head/workflow, retained canonical claims and draft artifacts, staging artifact ID/digest, and + the signed release ID/node/tag/target/URL. +- It rechecks the dedicated immutable-release setting, requires the existing release to be + non-draft, non-prerelease, and immutable, then downloads and hashes the exact signed 54-asset set. + It cryptographically verifies the original attestation bundle and retains canonical recovered + evidence for live verification/audit. +- Recovery is strictly observational. Never create a second release, resume or re-upload assets, + patch or republish the release, delete a tag/release/asset, or replace the failed handoff. Wrong, + missing, ambiguous, expired, or substituted run/artifact/claims/draft/release/asset evidence is a + hard stop for human review. +- If the bound release remains a draft or mutable, classify it as a before-publication failure and + abandon the recovery path without mutation. Correct the cause on a new reviewed commit and use a + new monotonically increasing candidate tag. + ## After immutable forge publication, before warehouse append - Stop destination work. Open a reviewed advisory/manifest PR explaining the affected candidate and diff --git a/scripts/check_workflow_policy.py b/scripts/check_workflow_policy.py index 8e326f1..eecb4cc 100755 --- a/scripts/check_workflow_policy.py +++ b/scripts/check_workflow_policy.py @@ -56,6 +56,12 @@ def validate_workflow(path: Path) -> None: expect("--clobber" not in text and "delete release" not in text.casefold() and "-X DELETE" not in text, "candidate.yml: destructive release mutation token is forbidden") for artifact_id in (9685464135, 9688244894, 9688243729, 9688330126, 9688263793, 9688255774, 9689647047, 9690093579): expect(str(artifact_id) in text, f"candidate.yml: audited input artifact pin missing: {artifact_id}") + if path.name == "phase6-live-verification.yml": + expect("github.event.workflow_run.conclusion == 'failure'" in text, "phase6-live-verification.yml: failed-source recovery trigger missing") + expect("scripts/github_phase6.py recover-publication" in text and "scripts/github_phase6.py verify-recovery" in text, "phase6-live-verification.yml: read-only recovery/verifier commands missing") + expect("phase6-recovered-publication-${{ github.event.workflow_run.id }}" in text, "phase6-live-verification.yml: recovered evidence retention missing") + expect("cp -- recovered/promotion-claims-initial-warehouse-v1.json recovered/promotion-claims-initial-warehouse-v1" in text, "phase6-live-verification.yml: exact attestation subject materialization missing") + expect("gh attestation verify recovered/promotion-claims-initial-warehouse-v1" in text and "rm -- recovered/promotion-claims-initial-warehouse-v1" in text, "phase6-live-verification.yml: recovered claims cryptographic verification/cleanup missing") def main() -> int: diff --git a/scripts/github_phase6.py b/scripts/github_phase6.py index cf58579..2088547 100644 --- a/scripts/github_phase6.py +++ b/scripts/github_phase6.py @@ -16,6 +16,7 @@ import urllib.error import urllib.parse import urllib.request +import zipfile from pathlib import Path from typing import Any @@ -30,6 +31,9 @@ REPOSITORY_ID = phase6_candidate.REPOSITORY_ID MAIN_REF = "refs/heads/main" WORKFLOW_PATH = ".github/workflows/candidate.yml" +DEFAULT_API_VERSION = "2022-11-28" +IMMUTABLE_RELEASES_API_VERSION = "2026-03-10" +IMMUTABLE_RELEASES_PATH = f"/repos/{REPOSITORY}/immutable-releases" def token() -> str: @@ -38,13 +42,13 @@ def token() -> str: return value -def request(path: str, method: str = "GET", body: Any | None = None, accept: str = "application/vnd.github+json") -> tuple[Any, str]: +def request(path: str, method: str = "GET", body: Any | None = None, accept: str = "application/vnd.github+json", api_version: str = DEFAULT_API_VERSION) -> tuple[Any, str]: data = None if body is None else canonical_bytes(body) req = urllib.request.Request( API + path, data=data, method=method, - headers={"Accept": accept, "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": "2022-11-28", **({"Content-Type": "application/json"} if data is not None else {})}, + headers={"Accept": accept, "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": api_version, **({"Content-Type": "application/json"} if data is not None else {})}, ) try: with urllib.request.urlopen(req, timeout=60) as response: @@ -63,6 +67,31 @@ def api_time(header: str) -> str: return parsed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def verify_immutable_policy_evidence(value: dict[str, Any], boundary: str) -> None: + expect(isinstance(value, dict) and set(value) == {"boundary", "endpoint", "apiVersion", "capturedAt", "enabled", "enforcedByOwner"}, f"immutable-release policy evidence is malformed at {boundary}") + expect(value["boundary"] == boundary and value["endpoint"] == IMMUTABLE_RELEASES_PATH and value["apiVersion"] == IMMUTABLE_RELEASES_API_VERSION, f"immutable-release policy authority differs at {boundary}") + canonical_json.parse_time(value["capturedAt"], f"immutable-release policy capturedAt at {boundary}") + expect(value["enabled"] is True, f"immutable releases are not enabled at {boundary}") + expect(type(value["enforcedByOwner"]) is bool, f"immutable-release owner-enforcement state has wrong type at {boundary}") + + +def immutable_release_policy(boundary: str) -> dict[str, Any]: + value, date = request(IMMUTABLE_RELEASES_PATH, api_version=IMMUTABLE_RELEASES_API_VERSION) + expect(isinstance(value, dict) and set(value) == {"enabled", "enforced_by_owner"}, f"immutable-release policy response is malformed at {boundary}") + expect(value["enabled"] is True, f"immutable releases are not enabled at {boundary}") + expect(type(value["enforced_by_owner"]) is bool, f"immutable-release owner-enforcement state has wrong type at {boundary}") + evidence = { + "boundary": boundary, + "endpoint": IMMUTABLE_RELEASES_PATH, + "apiVersion": IMMUTABLE_RELEASES_API_VERSION, + "capturedAt": api_time(date), + "enabled": True, + "enforcedByOwner": value["enforced_by_owner"], + } + verify_immutable_policy_evidence(evidence, boundary) + return evidence + + def pagination(path: str) -> list[dict[str, Any]]: result: list[dict[str, Any]] = [] for page in range(1, 12): @@ -103,7 +132,7 @@ def verify_source(expected_sha: str, output: Path) -> None: expect(os.environ.get("GITHUB_SHA") == expected_sha and re.fullmatch(r"[0-9a-f]{40}", expected_sha) is not None, "protected source full SHA mismatch") repository, date = request(f"/repos/{REPOSITORY}") expect(repository["id"] == REPOSITORY_ID and repository["full_name"] == REPOSITORY and repository["default_branch"] == "main", "live repository/default branch identity mismatch") - expect(repository.get("immutable_releases_enabled") is True, "forge immutable releases are not enabled") + immutable_policy = immutable_release_policy("protected-source-admission") branch, _ = request(f"/repos/{REPOSITORY}/branches/main") expect(branch.get("protected") is True and branch.get("commit", {}).get("sha") == expected_sha, "current source is not live protected main") rulesets = pagination(f"/repos/{REPOSITORY}/rulesets") @@ -116,7 +145,7 @@ def verify_source(expected_sha: str, output: Path) -> None: expect(policy.get("protected_branches") is True and policy.get("custom_branch_policies") is False, "candidate-publish is not protected-branch-only") workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={expected_sha}") expect(workflow.get("type") == "file" and re.fullmatch(r"[0-9a-f]{40}", workflow.get("sha", "")), "cannot bind candidate workflow blob") - value = {"schemaVersion": "phase6-protected-source-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"], "immutableReleasesEnabled": True}, "ref": MAIN_REF, "commitSha": expected_sha, "protected": True, "workflowPath": WORKFLOW_PATH, "workflowSha": workflow["sha"], "rulesetIds": sorted(row["id"] for row in details), "candidateEnvironment": "candidate-publish", "protectedBranchesOnly": True, "referencedRepositoryVariables": 0, "referencedDestinationSecrets": 0} + value = {"schemaVersion": "phase6-protected-source-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, "immutableReleasePolicy": immutable_policy, "ref": MAIN_REF, "commitSha": expected_sha, "protected": True, "workflowPath": WORKFLOW_PATH, "workflowSha": workflow["sha"], "rulesetIds": sorted(row["id"] for row in details), "candidateEnvironment": "candidate-publish", "protectedBranchesOnly": True, "referencedRepositoryVariables": 0, "referencedDestinationSecrets": 0} create_file_atomic(output, canonical_bytes(value), 0o600) @@ -134,7 +163,9 @@ def capture_staging(artifact_id: int, expected_name: str, run_id: int, run_attem def allocate_draft(expected_sha: str, source_path: Path, output: Path) -> None: require_actions_context() source = load_json(source_path) - expect(source.get("commitSha") == expected_sha and source.get("repository", {}).get("immutableReleasesEnabled") is True, "protected-source evidence mismatch") + policy = source.get("immutableReleasePolicy", {}) + expect(source.get("commitSha") == expected_sha, "protected-source evidence mismatch") + verify_immutable_policy_evidence(policy, "protected-source-admission") _, date_header = request("/rate_limit") date = api_time(date_header)[:10].replace("-", ".") releases = pagination(f"/repos/{REPOSITORY}/releases") @@ -143,10 +174,12 @@ def allocate_draft(expected_sha: str, source_path: Path, output: Path) -> None: sequence = next(number for number in range(1, 10000) if f"forge-{date}.{number}" not in occupied) tag = f"forge-{date}.{sequence}" body = phase6_candidate.WARNING + "\n\nThis candidate is immutable supply-chain evidence. Only typed role=payload assets are eligible for the separately reviewed manual warehouse transaction." + before_create = immutable_release_policy("immediately-before-draft-create") created, _ = request(f"/repos/{REPOSITORY}/releases", "POST", {"tag_name": tag, "target_commitish": expected_sha, "name": tag, "body": body, "draft": True, "prerelease": False}) reread, _ = request(f"/repos/{REPOSITORY}/releases/{created['id']}") expect(reread["id"] == created["id"] and reread["tag_name"] == tag and reread["target_commitish"] == expected_sha and reread["draft"] is True and reread["prerelease"] is False and reread.get("assets") == [], "allocated draft read-back mismatch") value = {key: reread[key] for key in ("id", "node_id", "html_url", "tag_name", "target_commitish")} + value["immutableReleasePolicyBeforeCreate"] = before_create create_file_atomic(output, canonical_bytes(value), 0o600) @@ -160,11 +193,217 @@ def download_asset(asset_id: int, output: Path) -> None: stream.write(block) +def run_artifacts(run_id: int) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + total_count: int | None = None + for page in range(1, 12): + value, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}/artifacts?per_page=100&page={page}") + expect(isinstance(value, dict) and set(value) >= {"total_count", "artifacts"}, "run-artifact response is malformed") + expect(type(value["total_count"]) is int and value["total_count"] >= 0 and isinstance(value["artifacts"], list), "run-artifact pagination fields are malformed") + if total_count is None: + total_count = value["total_count"] + expect(value["total_count"] == total_count, "run-artifact total changed during pagination") + rows.extend(value["artifacts"]) + if len(value["artifacts"]) < 100: + expect(len(rows) == total_count, "run-artifact pagination is incomplete") + return rows + raise ForgeError("run-artifact pagination ceiling exceeded") + + +def download_action_artifact(artifact_id: int, output: Path, max_bytes: int = 8 * 1024 * 1024) -> None: + req = urllib.request.Request( + API + f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}/zip", + headers={"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": DEFAULT_API_VERSION}, + ) + total = 0 + with urllib.request.urlopen(req, timeout=120) as response, output.open("xb") as stream: + while True: + block = response.read(1024 * 1024) + if not block: + break + total += len(block) + expect(total <= max_bytes, "retained JSON artifact exceeds recovery byte ceiling") + stream.write(block) + + +def retained_json_artifact(artifact: dict[str, Any], expected_inner_name: str, output: Path) -> tuple[Any, dict[str, Any]]: + expect(type(artifact.get("id")) is int and artifact["id"] > 0, "retained artifact ID is invalid") + expect(artifact.get("expired") is False, "retained recovery artifact is expired") + wrapper_digest = artifact.get("digest", "") + expect(isinstance(wrapper_digest, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", wrapper_digest) is not None, "retained artifact wrapper digest is missing") + with tempfile.TemporaryDirectory(prefix="phase6-recovery-artifact-") as temporary_text: + archive_path = Path(temporary_text) / "artifact.zip" + download_action_artifact(artifact["id"], archive_path) + actual_wrapper_digest, actual_wrapper_size = sha256_file(archive_path, 8 * 1024 * 1024) + expect(actual_wrapper_digest == wrapper_digest.removeprefix("sha256:"), "retained artifact wrapper digest mismatch") + expect(artifact.get("size_in_bytes") == actual_wrapper_size, "retained artifact wrapper size mismatch") + with zipfile.ZipFile(archive_path) as archive: + infos = archive.infolist() + expect(len(infos) == 1 and infos[0].filename == expected_inner_name and not infos[0].is_dir(), "retained JSON artifact layout mismatch") + info = infos[0] + expect(info.flag_bits & 1 == 0 and 0 < info.file_size <= 4 * 1024 * 1024, "retained JSON artifact member is unsafe") + expect(info.compress_size <= 4 * 1024 * 1024 and (info.compress_size > 0 or info.file_size == 0), "retained JSON artifact compression bounds invalid") + raw = archive.read(info) + expect(len(raw) == info.file_size, "retained JSON artifact member size mismatch") + create_file_atomic(output, raw, 0o600) + value = canonical_json.load_json(output) + expect(raw == canonical_json.canonical_bytes(value), "retained JSON artifact is not canonical") + evidence = { + "id": artifact["id"], + "name": artifact["name"], + "archiveSize": actual_wrapper_size, + "archiveSha256": actual_wrapper_digest, + "expiresAt": artifact["expires_at"], + "innerName": expected_inner_name, + "innerSize": len(raw), + "innerSha256": hashlib.sha256(raw).hexdigest(), + } + return value, evidence + + +def verify_recovery_evidence(value: dict[str, Any], claims: dict[str, Any], draft: dict[str, Any], envelope: dict[str, Any], bundle_path: Path) -> None: + expect(set(value) == {"schemaVersion", "recoveryMode", "capturedAt", "repository", "protectedRef", "workflowFile", "originalRun", "retainedArtifacts", "missingPublishedHandoff", "immutableReleasePolicy", "release", "releaseAssets", "claimsSha256", "draftSha256", "envelopeSha256", "bundleSha256"}, "recovery evidence fields differ") + expect(value["schemaVersion"] == "phase6-recovered-publication-v1" and value["recoveryMode"] == "read-only-post-publication-handoff-loss", "wrong recovery evidence mode") + canonical_json.parse_time(value["capturedAt"], "recovery capturedAt") + canonical_json.verify_envelope(envelope) + expect(envelope["claims"] == claims and canonical_json.verify_claims(claims) == value["claimsSha256"], "recovered claims/envelope binding mismatch") + expect(value["draftSha256"] == canonical_json.digest(draft) and value["envelopeSha256"] == canonical_json.digest(envelope), "recovered canonical JSON digest mismatch") + bundle_digest, bundle_size = sha256_file(bundle_path, 2**31 - 1) + expect(bundle_size > 0 and bundle_digest == value["bundleSha256"] == envelope["attestation"]["bundleSha256"], "recovered bundle binding mismatch") + issuer = claims["issuer"] + staging = claims["staging"] + candidate = claims["candidateDraft"] + run = value["originalRun"] + expect(run == {"id": staging["runId"], "attempt": staging["runAttempt"], "repository": REPOSITORY, "workflowPath": WORKFLOW_PATH, "event": "workflow_dispatch", "headSha": issuer["commitSha"], "headRef": "main", "status": "completed", "conclusion": "failure"}, "original failed run identity differs from signed claims") + expect(value["repository"] == {"fullName": REPOSITORY, "id": REPOSITORY_ID, "nodeId": canonical_json.REPOSITORY_NODE_ID}, "recovery repository identity mismatch") + expect(value["protectedRef"] == {"ref": MAIN_REF, "commitSha": issuer["commitSha"], "protected": True}, "recovery protected source mismatch") + expect(value["workflowFile"] == {"path": WORKFLOW_PATH, "commitSha": issuer["commitSha"], "blobSha": issuer["workflowSha"]}, "recovery workflow binding mismatch") + expect(value["missingPublishedHandoff"] == {"name": f"published-candidate-{claims['buildSet']['id']}", "confirmedAbsent": True}, "recovery handoff-loss classification mismatch") + policy = value["immutableReleasePolicy"] + verify_immutable_policy_evidence(policy, "recovery-live-readback") + verify_immutable_policy_evidence(draft.get("immutableReleasePolicyBeforeCreate", {}), "immediately-before-draft-create") + expect(draft["id"] == candidate["releaseId"] and draft["node_id"] == candidate["releaseNodeId"] and draft["tag_name"] == candidate["tag"] and draft["target_commitish"] == issuer["commitSha"] and draft["html_url"] == candidate["releaseUrl"], "retained draft differs from signed candidate") + retained = value["retainedArtifacts"] + expect(set(retained) == {"claims", "draft", "staging"}, "recovery retained-artifact set differs") + for label, expected_artifact_name, expected_inner_name, expected_inner_digest in ( + ("claims", f"phase6-claims-{issuer['commitSha']}", f"promotion-claims-{claims['buildSet']['id']}.json", canonical_json.digest(claims)), + ("draft", f"phase6-draft-{issuer['commitSha']}", "draft.json", canonical_json.digest(draft)), + ): + artifact_record = retained[label] + expect(set(artifact_record) == {"id", "name", "archiveSize", "archiveSha256", "expiresAt", "innerName", "innerSize", "innerSha256"}, f"recovery retained {label} artifact fields differ") + expect(type(artifact_record["id"]) is int and artifact_record["id"] > 0 and type(artifact_record["archiveSize"]) is int and artifact_record["archiveSize"] > 0 and type(artifact_record["innerSize"]) is int and artifact_record["innerSize"] > 0, f"recovery retained {label} artifact sizes/ID invalid") + expect(artifact_record["name"] == expected_artifact_name and artifact_record["innerName"] == expected_inner_name and artifact_record["innerSha256"] == expected_inner_digest, f"recovery retained {label} artifact binding mismatch") + expect(re.fullmatch(r"[0-9a-f]{64}", artifact_record["archiveSha256"]) is not None, f"recovery retained {label} wrapper digest invalid") + canonical_json.parse_time(artifact_record["expiresAt"], f"recovery retained {label} expiresAt") + expect(artifact_record["innerSize"] == len(canonical_json.canonical_bytes(claims if label == "claims" else draft)), f"recovery retained {label} member size mismatch") + expect(retained["claims"]["id"] != retained["draft"]["id"], "recovery retained artifact IDs collide") + expect(set(retained["staging"]) == {"id", "name", "archiveSha256", "expiresAt", "runId"}, "recovery staging artifact fields differ") + expect(retained["staging"]["id"] == staging["artifactId"] and retained["staging"]["name"] == staging["artifactName"] and retained["staging"]["archiveSha256"] == staging["archiveSha256"] and retained["staging"]["runId"] == staging["runId"], "recovery staging artifact differs from claims") + canonical_json.parse_time(retained["staging"]["expiresAt"], "recovery staging expiresAt") + release = value["release"] + expect(release == {"id": candidate["releaseId"], "nodeId": candidate["releaseNodeId"], "repository": REPOSITORY, "tag": candidate["tag"], "targetCommitish": issuer["commitSha"], "url": candidate["releaseUrl"], "draft": False, "prerelease": False, "immutable": True}, "recovered release is not the exact signed immutable publication") + asset_rows = value["releaseAssets"] + expect(isinstance(asset_rows, list) and len(asset_rows) == claims["totalAssetCount"] == 54, "recovered release asset count mismatch") + expect([row["name"] for row in asset_rows] == claims["completeAssetNames"], "recovered release asset names differ") + by_name = {row["name"]: row for row in asset_rows} + expect(len(by_name) == len(asset_rows) and all(set(row) == {"id", "name", "size", "sha256"} and type(row["id"]) is int and row["id"] > 0 for row in asset_rows), "recovered release asset rows are malformed") + for content in claims["contentAssets"]: + expect((by_name[content["name"]]["size"], by_name[content["name"]]["sha256"]) == (content["size"], content["sha256"]), f"recovered content asset differs: {content['name']}") + expect((by_name[claims["transport"]["attestationBundleName"]]["size"], by_name[claims["transport"]["attestationBundleName"]]["sha256"]) == (bundle_size, bundle_digest), "recovered bundle asset differs") + envelope_size = len(canonical_json.canonical_bytes(envelope)) + expect((by_name[claims["transport"]["envelopeName"]]["size"], by_name[claims["transport"]["envelopeName"]]["sha256"]) == (envelope_size, canonical_json.digest(envelope)), "recovered envelope asset differs") + + +def recover_publication(run_id: int, expected_head: str, build_set_id: str, output_dir: Path) -> None: + expect(os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("GITHUB_REPOSITORY") == REPOSITORY and os.environ.get("GITHUB_EVENT_NAME") == "workflow_run", "Phase-6 recovery requires the exact read-only workflow_run context") + expect(re.fullmatch(r"[0-9a-f]{40}", expected_head) is not None and re.fullmatch(r"[a-z0-9][a-z0-9._-]{2,127}", build_set_id) is not None, "recovery input identity is malformed") + expect(not output_dir.exists(), "recovery output path already exists") + output_dir.mkdir(mode=0o700) + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + expect(run.get("id") == run_id and run.get("run_attempt") == 1 and run.get("event") == "workflow_dispatch" and run.get("status") == "completed" and run.get("conclusion") == "failure", "recovery is only for an exact completed failed candidate run") + expect(run.get("path") == WORKFLOW_PATH and run.get("head_sha") == expected_head and run.get("head_branch") == "main" and run.get("repository", {}).get("full_name") == REPOSITORY and run.get("repository", {}).get("id") == REPOSITORY_ID, "recovery run repository/workflow/head identity mismatch") + artifacts = run_artifacts(run_id) + for row in artifacts: + workflow = row.get("workflow_run", {}) + expect(workflow.get("id") == run_id and workflow.get("repository_id") == REPOSITORY_ID, "recovery artifact belongs to another run/repository") + by_name: dict[str, list[dict[str, Any]]] = {} + for row in artifacts: + by_name.setdefault(row.get("name", ""), []).append(row) + draft_name = f"phase6-draft-{expected_head}" + claims_name = f"phase6-claims-{expected_head}" + published_name = f"published-candidate-{build_set_id}" + expect(by_name.get(published_name, []) == [], "published handoff already exists; recovery is neither required nor allowed") + expect(len(by_name.get(draft_name, [])) == 1 and len(by_name.get(claims_name, [])) == 1, "exact retained draft/claims artifacts are missing or ambiguous") + draft, draft_artifact = retained_json_artifact(by_name[draft_name][0], "draft.json", output_dir / "draft.json") + claims_inner_name = f"promotion-claims-{build_set_id}.json" + claims, claims_artifact = retained_json_artifact(by_name[claims_name][0], claims_inner_name, output_dir / claims_inner_name) + canonical_json.verify_claims(claims) + expect(claims["issuer"]["commitSha"] == expected_head and claims["staging"]["runId"] == run_id and claims["staging"]["runAttempt"] == run["run_attempt"] and claims["buildSet"]["id"] == build_set_id, "retained claims differ from recovery run/input") + candidate = claims["candidateDraft"] + expect(draft.get("id") == candidate["releaseId"] and draft.get("tag_name") == candidate["tag"] and draft.get("target_commitish") == expected_head, "retained draft differs from signed claims") + staging_rows = by_name.get(claims["staging"]["artifactName"], []) + expect(len(staging_rows) == 1 and staging_rows[0].get("id") == claims["staging"]["artifactId"], "exact staging artifact is missing or substituted") + staging_row = staging_rows[0] + expect(staging_row.get("digest") == f"sha256:{claims['staging']['archiveSha256']}", "staging artifact wrapper digest differs from claims") + release, _ = request(f"/repos/{REPOSITORY}/releases/{candidate['releaseId']}") + expect(release.get("id") == candidate["releaseId"] and release.get("node_id") == candidate["releaseNodeId"] and release.get("tag_name") == candidate["tag"] and release.get("target_commitish") == expected_head and release.get("html_url") == candidate["releaseUrl"], "live release identity differs from retained claims/draft") + expect(release.get("draft") is False and release.get("prerelease") is False and release.get("immutable") is True, "candidate run failed before an immutable publication; abandon recovery without mutation") + policy = immutable_release_policy("recovery-live-readback") + release_assets = pagination(f"/repos/{REPOSITORY}/releases/{candidate['releaseId']}/assets") + expect(len(release_assets) == claims["totalAssetCount"] == 54 and sorted(row.get("name") for row in release_assets) == claims["completeAssetNames"], "live immutable release does not have the exact signed 54-asset set") + asset_evidence = [] + with tempfile.TemporaryDirectory(prefix="phase6-recovery-release-") as temporary_text: + temporary_root = Path(temporary_text) + for asset in sorted(release_assets, key=lambda row: row["name"]): + expect(type(asset.get("id")) is int and asset["id"] > 0 and asset.get("state") == "uploaded", "recovery release asset API identity is malformed") + path = temporary_root / str(asset["id"]) + download_asset(asset["id"], path) + digest, size = sha256_file(path, 2**31 - 1) + expect(asset.get("size") == size and asset.get("digest") == f"sha256:{digest}", f"recovery release asset API/download mismatch: {asset['name']}") + asset_evidence.append({"id": asset["id"], "name": asset["name"], "size": size, "sha256": digest}) + if asset["name"] in {claims["transport"]["envelopeName"], claims["transport"]["attestationBundleName"]}: + create_file_atomic(output_dir / asset["name"], path.read_bytes(), 0o600) + envelope_path = output_dir / claims["transport"]["envelopeName"] + bundle_path = output_dir / claims["transport"]["attestationBundleName"] + envelope = canonical_json.load_json(envelope_path) + canonical_json.verify_envelope(envelope) + expect(envelope["claims"] == claims, "released envelope claims differ from retained original-run claims") + repository, date = request(f"/repos/{REPOSITORY}") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={expected_head}") + retained = { + "claims": claims_artifact, + "draft": draft_artifact, + "staging": {"id": staging_row["id"], "name": staging_row["name"], "archiveSha256": staging_row["digest"].removeprefix("sha256:"), "expiresAt": staging_row["expires_at"], "runId": run_id}, + } + value = { + "schemaVersion": "phase6-recovered-publication-v1", + "recoveryMode": "read-only-post-publication-handoff-loss", + "capturedAt": api_time(date), + "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, + "protectedRef": {"ref": MAIN_REF, "commitSha": branch["commit"]["sha"], "protected": branch["protected"]}, + "workflowFile": {"path": WORKFLOW_PATH, "commitSha": expected_head, "blobSha": workflow["sha"]}, + "originalRun": {"id": run["id"], "attempt": run["run_attempt"], "repository": run["repository"]["full_name"], "workflowPath": run["path"], "event": run["event"], "headSha": run["head_sha"], "headRef": run["head_branch"], "status": run["status"], "conclusion": run["conclusion"]}, + "retainedArtifacts": retained, + "missingPublishedHandoff": {"name": published_name, "confirmedAbsent": True}, + "immutableReleasePolicy": policy, + "release": {"id": release["id"], "nodeId": release["node_id"], "repository": REPOSITORY, "tag": release["tag_name"], "targetCommitish": release["target_commitish"], "url": release["html_url"], "draft": release["draft"], "prerelease": release["prerelease"], "immutable": release["immutable"]}, + "releaseAssets": asset_evidence, + "claimsSha256": canonical_json.digest(claims), + "draftSha256": canonical_json.digest(draft), + "envelopeSha256": canonical_json.digest(envelope), + "bundleSha256": sha256_file(bundle_path, 2**31 - 1)[0], + } + verify_recovery_evidence(value, claims, draft, envelope, bundle_path) + create_file_atomic(output_dir / "recovered-publication.json", canonical_bytes(value), 0o600) + + def publish(claims_path: Path, content: Path, bundle: Path, envelope: Path, draft_path: Path, output: Path) -> None: require_actions_context() publisher_guard.verify_transport(envelope, bundle, content) claims = load_json(claims_path) draft = load_json(draft_path) + verify_immutable_policy_evidence(draft.get("immutableReleasePolicyBeforeCreate", {}), "immediately-before-draft-create") expect(claims == load_json(envelope)["claims"] and draft["id"] == claims["candidateDraft"]["releaseId"], "publisher draft/claims/envelope mismatch") release, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}") expect(release["draft"] is True and release["tag_name"] == draft["tag_name"] and release["target_commitish"] == claims["issuer"]["commitSha"], "publisher draft state mismatch") @@ -177,6 +416,7 @@ def publish(claims_path: Path, content: Path, bundle: Path, envelope: Path, draf expected[bundle.name] = (bundle.stat().st_size, sha256_file(bundle)[0]) expected[envelope.name] = (envelope.stat().st_size, sha256_file(envelope)[0]) env = {**os.environ, "GH_TOKEN": token()} + policy_checks = [immutable_release_policy("immediately-before-draft-asset-upload")] for name in sorted(paths): subprocess.run(["gh", "release", "upload", draft["tag_name"], str(paths[name]), "--repo", REPOSITORY], check=True, env=env, timeout=600) rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") @@ -194,15 +434,18 @@ def publish(claims_path: Path, content: Path, bundle: Path, envelope: Path, draf temporary.unlink(missing_ok=True) rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") expect(sorted(row["name"] for row in rows) == claims["completeAssetNames"], "complete uploaded asset set mismatch") + policy_checks.append(immutable_release_policy("immediately-before-public-transition")) published, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"draft": False}) - expect(published["draft"] is False and published["prerelease"] is False and published.get("immutable") is True, "published release is not immutable") + policy_checks.append(immutable_release_policy("immediately-after-publication")) + reread, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}") + expect(published["id"] == reread["id"] and reread["draft"] is False and reread["prerelease"] is False and reread.get("immutable") is True, "published release is not immutable") mutation_rejected = False try: - request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"name": published["name"]}) + request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"name": reread["name"]}) except ForgeError: mutation_rejected = True expect(mutation_rejected, "immutable published release accepted a no-op metadata mutation probe") - value = {"schemaVersion": "phase6-published-candidate-v1", "releaseId": published["id"], "tag": published["tag_name"], "targetCommitish": published["target_commitish"], "immutable": True, "assetCount": len(rows), "completeAssetNameListSha256": claims["completeAssetNameListSha256"], "mutationRejected": True} + value = {"schemaVersion": "phase6-published-candidate-v1", "releaseId": reread["id"], "tag": reread["tag_name"], "targetCommitish": reread["target_commitish"], "immutable": True, "assetCount": len(rows), "completeAssetNameListSha256": claims["completeAssetNameListSha256"], "immutableReleasePolicyChecks": policy_checks, "mutationRejected": True} create_file_atomic(output, canonical_bytes(value), 0o600) @@ -217,6 +460,7 @@ def capture_live(envelope_path: Path, bundle_path: Path, run_id: int, output: Pa run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{claims['staging']['artifactId']}") release, _ = request(f"/repos/{REPOSITORY}/releases/{claims['candidateDraft']['releaseId']}") + immutable_release_policy("live-verifier-readback") assets = pagination(f"/repos/{REPOSITORY}/releases/{release['id']}/assets") asset_rows = [] for asset in sorted(assets, key=lambda row: row["name"]): @@ -259,6 +503,10 @@ def main() -> int: publish_parser.add_argument("--claims", type=Path, required=True); publish_parser.add_argument("--content", type=Path, required=True); publish_parser.add_argument("--bundle", type=Path, required=True); publish_parser.add_argument("--envelope", type=Path, required=True); publish_parser.add_argument("--draft", type=Path, required=True); publish_parser.add_argument("--output", type=Path, required=True) live = sub.add_parser("capture-live") live.add_argument("--envelope", type=Path, required=True); live.add_argument("--bundle", type=Path, required=True); live.add_argument("--run-id", type=int, required=True); live.add_argument("--output", type=Path, required=True) + recover = sub.add_parser("recover-publication") + recover.add_argument("--run-id", type=int, required=True); recover.add_argument("--expected-head", required=True); recover.add_argument("--build-set-id", required=True); recover.add_argument("--output-dir", type=Path, required=True) + verify_recovery = sub.add_parser("verify-recovery") + verify_recovery.add_argument("--claims", type=Path, required=True); verify_recovery.add_argument("--draft", type=Path, required=True); verify_recovery.add_argument("--envelope", type=Path, required=True); verify_recovery.add_argument("--bundle", type=Path, required=True); verify_recovery.add_argument("--evidence", type=Path, required=True) args = parser.parse_args() try: if args.command == "capture-inputs": capture_inputs(args.build_set, args.output) @@ -266,7 +514,10 @@ def main() -> int: elif args.command == "capture-staging": capture_staging(args.artifact_id, args.expected_name, args.run_id, args.run_attempt, args.output) elif args.command == "allocate-draft": allocate_draft(args.expected_sha, args.source, args.output) elif args.command == "publish": publish(args.claims, args.content, args.bundle, args.envelope, args.draft, args.output) - else: capture_live(args.envelope, args.bundle, args.run_id, args.output) + elif args.command == "capture-live": capture_live(args.envelope, args.bundle, args.run_id, args.output) + elif args.command == "recover-publication": recover_publication(args.run_id, args.expected_head, args.build_set_id, args.output_dir) + else: + verify_recovery_evidence(load_json(args.evidence), load_json(args.claims), load_json(args.draft), canonical_json.load_json(args.envelope), args.bundle) print(f"OK Phase-6 GitHub boundary {args.command}") return 0 except (ForgeError, canonical_json.ProtocolError, OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as exc: diff --git a/tests/test_phase6_release_boundaries.py b/tests/test_phase6_release_boundaries.py new file mode 100644 index 0000000..4adb035 --- /dev/null +++ b/tests/test_phase6_release_boundaries.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import canonical_json # noqa: E402 +import github_phase6 # noqa: E402 +from forge_io import ForgeError, canonical_bytes # noqa: E402 + + +DATE = "Fri, 28 Aug 2026 17:00:00 GMT" +HEAD = "2" * 40 +WORKFLOW_SHA = "1" * 40 + + +def policy(boundary: str) -> dict: + return { + "boundary": boundary, + "endpoint": github_phase6.IMMUTABLE_RELEASES_PATH, + "apiVersion": github_phase6.IMMUTABLE_RELEASES_API_VERSION, + "capturedAt": "2026-08-28T17:00:00Z", + "enabled": True, + "enforcedByOwner": False, + } + + +def fixture_transport() -> tuple[dict, dict, bytes, dict[str, bytes]]: + content: dict[str, bytes] = {} + rows = [] + for index in range(31): + name = f"fixture-payload-{index:02d}.bin" + value = f"payload-{index}\n".encode() + content[name] = value + rows.append({"name": name, "role": "payload", "size": len(value), "sha256": hashlib.sha256(value).hexdigest(), "mediaType": "application/octet-stream", "artifactKind": "software", "componentId": f"fixture-{index:02d}"}) + evidence = [ + ("source-manifest-fixture-1.json", "source-manifest"), + ("sha256sums-fixture-1.txt", "checksums"), + *[(f"provenance-fixture-{index:02d}.json", "provenance") for index in range(19)], + ] + for name, role in evidence: + value = f"{role}:{name}\n".encode() + content[name] = value + rows.append({"name": name, "role": role, "size": len(value), "sha256": hashlib.sha256(value).hexdigest(), "mediaType": "text/plain" if role == "checksums" else "application/json"}) + rows.sort(key=lambda row: row["name"]) + content_list_digest = canonical_json.digest(rows) + bundle_name = "attestation-fixture-1.sigstore.json" + envelope_name = "promotion-envelope-fixture-1.json" + claims = { + "issuer": {"repository": github_phase6.REPOSITORY, "repositoryId": github_phase6.REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "workflowPath": github_phase6.WORKFLOW_PATH, "workflowSha": WORKFLOW_SHA, "ref": github_phase6.MAIN_REF, "commitSha": HEAD}, + "staging": {"provider": "github-actions-artifact", "runId": 100, "runAttempt": 1, "artifactId": 200, "artifactName": f"verified-content-fixture-1-{content_list_digest}", "archiveSha256": "3" * 64, "expiresAt": "2026-09-03T00:00:00Z"}, + "candidateDraft": {"repository": github_phase6.REPOSITORY, "repositoryId": github_phase6.REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "tag": "forge-2026.08.28.1", "targetCommitish": HEAD, "releaseId": 300, "releaseNodeId": "RE_fixture_node", "releaseUrl": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "liveImmutableVerificationRequired": True}, + "buildSet": {"id": "fixture-1", "manifestName": "source-manifest-fixture-1.json", "manifestSha256": next(row["sha256"] for row in rows if row["name"] == "source-manifest-fixture-1.json"), "checksumsName": "sha256sums-fixture-1.txt", "checksumsSha256": next(row["sha256"] for row in rows if row["name"] == "sha256sums-fixture-1.txt")}, + "transport": {"envelopeName": envelope_name, "attestationBundleName": bundle_name}, + "contentAssets": rows, + "contentAssetListSha256": content_list_digest, + "completeAssetNames": sorted([*content, envelope_name, bundle_name]), + "completeAssetNameListSha256": "", + "payloadCount": 31, + "contentEvidenceCount": 21, + "transportAssetCount": 2, + "totalAssetCount": 54, + } + claims["completeAssetNameListSha256"] = canonical_json.digest(claims["completeAssetNames"]) + claims_digest = canonical_json.verify_claims(claims) + bundle = b"fixture attestation bundle\n" + envelope = { + "schemaVersion": "promotion-envelope-v1", + "canonicalization": "forge-canonical-json-v1", + "claims": claims, + "claimsDigest": f"sha256:{claims_digest}", + "attestation": {"kind": "github-artifact-attestation", "predicateType": canonical_json.PREDICATE_TYPE, "predicateCanonicalization": "forge-canonical-json-v1", "predicateSha256": claims_digest, "subjectName": "promotion-claims-fixture-1", "bundleName": bundle_name, "bundleSha256": hashlib.sha256(bundle).hexdigest(), "subjectDigest": f"sha256:{claims_digest}", "issuer": "https://token.actions.githubusercontent.com", "identity": canonical_json.ATTESTATION_IDENTITY}, + } + canonical_json.verify_envelope(envelope) + released = {**content, envelope_name: canonical_json.canonical_bytes(envelope), bundle_name: bundle} + draft = {"id": 300, "node_id": "RE_fixture_node", "html_url": claims["candidateDraft"]["releaseUrl"], "tag_name": claims["candidateDraft"]["tag"], "target_commitish": HEAD, "immutableReleasePolicyBeforeCreate": policy("immediately-before-draft-create")} + return claims, envelope, bundle, released | {"__draft__": canonical_json.canonical_bytes(draft)} + + +class ImmutableReleaseSettingTest(unittest.TestCase): + def test_dedicated_exact_api_and_typed_response(self) -> None: + with mock.patch("github_phase6.request", return_value=({"enabled": True, "enforced_by_owner": False}, DATE)) as request: + observed = github_phase6.immutable_release_policy("fixture") + request.assert_called_once_with(github_phase6.IMMUTABLE_RELEASES_PATH, api_version="2026-03-10") + self.assertEqual(observed, policy("fixture")) + invalid = [ + {}, {"enabled": False, "enforced_by_owner": False}, {"enabled": None, "enforced_by_owner": False}, + {"enabled": 1, "enforced_by_owner": False}, {"enabled": True}, + {"enabled": True, "enforced_by_owner": None}, {"enabled": True, "enforced_by_owner": False, "extra": False}, [], None, + ] + for value in invalid: + with self.subTest(value=value), mock.patch("github_phase6.request", return_value=(value, DATE)), self.assertRaises(ForgeError): + github_phase6.immutable_release_policy("fixture") + with mock.patch("github_phase6.request", side_effect=ForgeError("HTTP/API-version failure")), self.assertRaises(ForgeError): + github_phase6.immutable_release_policy("fixture") + + def test_protected_source_ignores_absent_general_field_and_uses_dedicated_endpoint(self) -> None: + def api(path: str, method: str = "GET", body=None, accept="application/vnd.github+json", api_version=github_phase6.DEFAULT_API_VERSION): + if path == f"/repos/{github_phase6.REPOSITORY}": + return ({"id": github_phase6.REPOSITORY_ID, "full_name": github_phase6.REPOSITORY, "node_id": canonical_json.REPOSITORY_NODE_ID, "default_branch": "main"}, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + self.assertEqual(api_version, "2026-03-10") + return ({"enabled": True, "enforced_by_owner": False}, DATE) + if path.endswith("/branches/main"): + return ({"protected": True, "commit": {"sha": HEAD}}, DATE) + if "/rulesets/7" in path: + return ({"id": 7, "rules": [{"type": "pull_request"}], "bypass_actors": []}, DATE) + if path.endswith("/environments/candidate-publish"): + return ({"deployment_branch_policy": {"protected_branches": True, "custom_branch_policies": False}}, DATE) + if "/contents/" in path: + return ({"type": "file", "sha": WORKFLOW_SHA}, DATE) + raise AssertionError((path, method, body, accept, api_version)) + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_REF": github_phase6.MAIN_REF, "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_SHA": HEAD} + with tempfile.TemporaryDirectory() as text, mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", return_value=[{"id": 7, "enforcement": "active"}]): + output = Path(text) / "source.json" + github_phase6.verify_source(HEAD, output) + value = json.loads(output.read_text()) + self.assertEqual(value["immutableReleasePolicy"], policy("protected-source-admission")) + self.assertNotIn("immutable_releases_enabled", value["repository"]) + + +class WriteBoundaryStateFlipTest(unittest.TestCase): + def _source(self, root: Path) -> Path: + path = root / "source.json" + path.write_bytes(canonical_bytes({"commitSha": HEAD, "immutableReleasePolicy": policy("protected-source-admission")})) + return path + + def _publisher_fixture(self, root: Path) -> tuple[Path, Path, Path, Path, Path]: + content = root / "content" + content.mkdir() + bundle = root / "attestation-fixture.sigstore.json" + envelope = root / "promotion-envelope-fixture.json" + bundle.write_bytes(b"bundle") + claims = {"issuer": {"commitSha": HEAD}, "candidateDraft": {"releaseId": 7}, "contentAssets": [], "completeAssetNames": sorted([bundle.name, envelope.name]), "completeAssetNameListSha256": "4" * 64} + claims_path = root / "claims.json" + claims_path.write_bytes(canonical_bytes(claims)) + envelope.write_bytes(canonical_bytes({"claims": claims})) + draft = root / "draft.json" + draft.write_bytes(canonical_bytes({"id": 7, "tag_name": "forge-2026.08.28.1", "immutableReleasePolicyBeforeCreate": policy("immediately-before-draft-create")})) + return content, bundle, envelope, claims_path, draft + + @mock.patch("github_phase6.require_actions_context") + def test_setting_flip_before_draft_post_prevents_create(self, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + source = self._source(root) + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method)) + if path == "/rate_limit": + return ({}, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + return ({"enabled": False, "enforced_by_owner": False}, DATE) + raise AssertionError((path, method)) + + with mock.patch("github_phase6.pagination", return_value=[]), mock.patch("github_phase6.request", side_effect=api), self.assertRaisesRegex(ForgeError, "not enabled"): + github_phase6.allocate_draft(HEAD, source, root / "draft.json") + self.assertFalse(any(method == "POST" for _, method in calls)) + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_flip_before_upload_prevents_first_write(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + with mock.patch("github_phase6.request", return_value=({"draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE)), mock.patch("github_phase6.pagination", return_value=[]), mock.patch("github_phase6.immutable_release_policy", side_effect=ForgeError("disabled before upload")), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run") as upload, self.assertRaisesRegex(ForgeError, "disabled"): + github_phase6.publish(claims, content, bundle, envelope, draft, root / "published.json") + upload.assert_not_called() + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_flip_after_upload_prevents_public_patch(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + values = {bundle.name: bundle.read_bytes(), envelope.name: envelope.read_bytes()} + rows = [{"id": index + 1, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(values.items()))] + pages = [[], rows[:1], rows, rows] + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method)) + return ({"draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + + def download(asset_id: int, output: Path): + output.write_bytes(values[rows[asset_id - 1]["name"]]) + + with mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", side_effect=pages), mock.patch("github_phase6.immutable_release_policy", side_effect=[policy("immediately-before-draft-asset-upload"), ForgeError("disabled before public transition")]), mock.patch("github_phase6.download_asset", side_effect=download), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run"), self.assertRaisesRegex(ForgeError, "disabled"): + github_phase6.publish(claims, content, bundle, envelope, draft, root / "published.json") + self.assertFalse(any(method == "PATCH" for _, method in calls)) + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_rechecked_after_publication_and_immutable_readback(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + values = {bundle.name: bundle.read_bytes(), envelope.name: envelope.read_bytes()} + rows = [{"id": index + 1, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(values.items()))] + pages = [[], rows[:1], rows, rows] + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method, body)) + if method == "GET": + if len([call for call in calls if call[1] == "PATCH"]) == 0: + return ({"id": 7, "draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + return ({"id": 7, "name": "forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + if body == {"draft": False}: + return ({"id": 7}, DATE) + raise ForgeError("immutable mutation rejected") + + def download(asset_id: int, output: Path): + output.write_bytes(values[rows[asset_id - 1]["name"]]) + + checks = [policy("immediately-before-draft-asset-upload"), policy("immediately-before-public-transition"), policy("immediately-after-publication")] + output = root / "published.json" + with mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", side_effect=pages), mock.patch("github_phase6.immutable_release_policy", side_effect=checks) as setting, mock.patch("github_phase6.download_asset", side_effect=download), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run"): + github_phase6.publish(claims, content, bundle, envelope, draft, output) + self.assertEqual(setting.call_count, 3) + self.assertEqual(json.loads(output.read_text())["immutableReleasePolicyChecks"], checks) + self.assertEqual(sum(method == "PATCH" and body == {"draft": False} for _, method, body in calls), 1) + self.assertEqual(sum(method == "GET" for _, method, _ in calls), 2) + + +class ReadOnlyRecoveryTest(unittest.TestCase): + def _fixture(self, root: Path): + claims, envelope, bundle, released = fixture_transport() + draft = canonical_json.load_json(self._write(root / "draft-source.json", released.pop("__draft__"))) + claims_raw = canonical_json.canonical_bytes(claims) + draft_raw = canonical_json.canonical_bytes(draft) + retained = { + "claims": {"id": 501, "name": f"phase6-claims-{HEAD}", "archiveSize": 100, "archiveSha256": "a" * 64, "expiresAt": "2026-09-03T00:00:00Z", "innerName": "promotion-claims-fixture-1.json", "innerSize": len(claims_raw), "innerSha256": hashlib.sha256(claims_raw).hexdigest()}, + "draft": {"id": 502, "name": f"phase6-draft-{HEAD}", "archiveSize": 100, "archiveSha256": "b" * 64, "expiresAt": "2026-09-03T00:00:00Z", "innerName": "draft.json", "innerSize": len(draft_raw), "innerSha256": hashlib.sha256(draft_raw).hexdigest()}, + "staging": {"id": 200, "name": claims["staging"]["artifactName"], "archiveSha256": claims["staging"]["archiveSha256"], "expiresAt": claims["staging"]["expiresAt"], "runId": 100}, + } + release_assets = [{"id": index + 1000, "name": name, "size": len(value), "sha256": hashlib.sha256(value).hexdigest()} for index, (name, value) in enumerate(sorted(released.items()))] + value = { + "schemaVersion": "phase6-recovered-publication-v1", "recoveryMode": "read-only-post-publication-handoff-loss", "capturedAt": "2026-08-28T17:00:00Z", + "repository": {"fullName": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID, "nodeId": canonical_json.REPOSITORY_NODE_ID}, + "protectedRef": {"ref": github_phase6.MAIN_REF, "commitSha": HEAD, "protected": True}, + "workflowFile": {"path": github_phase6.WORKFLOW_PATH, "commitSha": HEAD, "blobSha": WORKFLOW_SHA}, + "originalRun": {"id": 100, "attempt": 1, "repository": github_phase6.REPOSITORY, "workflowPath": github_phase6.WORKFLOW_PATH, "event": "workflow_dispatch", "headSha": HEAD, "headRef": "main", "status": "completed", "conclusion": "failure"}, + "retainedArtifacts": retained, "missingPublishedHandoff": {"name": "published-candidate-fixture-1", "confirmedAbsent": True}, "immutableReleasePolicy": policy("recovery-live-readback"), + "release": {"id": 300, "nodeId": "RE_fixture_node", "repository": github_phase6.REPOSITORY, "tag": "forge-2026.08.28.1", "targetCommitish": HEAD, "url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True}, + "releaseAssets": release_assets, "claimsSha256": canonical_json.digest(claims), "draftSha256": canonical_json.digest(draft), "envelopeSha256": canonical_json.digest(envelope), "bundleSha256": hashlib.sha256(bundle).hexdigest(), + } + bundle_path = self._write(root / claims["transport"]["attestationBundleName"], bundle) + return claims, draft, envelope, bundle_path, value, released + + @staticmethod + def _write(path: Path, value: bytes) -> Path: + path.write_bytes(value) + return path + + def test_exact_recovery_evidence_and_identity_mutations(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, envelope, bundle, value, _ = self._fixture(root) + github_phase6.verify_recovery_evidence(value, claims, draft, envelope, bundle) + mutations = [] + for section, field, replacement in ( + ("originalRun", "id", 101), ("release", "id", 301), ("release", "tag", "forge-2026.08.28.2"), + ("protectedRef", "commitSha", "f" * 40), ("missingPublishedHandoff", "confirmedAbsent", False), + ): + changed = copy.deepcopy(value) + changed[section][field] = replacement + mutations.append(changed) + changed = copy.deepcopy(value) + changed["releaseAssets"][0]["sha256"] = "f" * 64 + mutations.append(changed) + changed_claims = copy.deepcopy(claims) + changed_claims["candidateDraft"]["releaseId"] = 301 + mutations.append((value, changed_claims)) + for index, mutation in enumerate(mutations): + candidate_value, candidate_claims = mutation if isinstance(mutation, tuple) else (mutation, claims) + with self.subTest(index=index), self.assertRaises((ForgeError, canonical_json.ProtocolError)): + github_phase6.verify_recovery_evidence(candidate_value, candidate_claims, draft, envelope, bundle) + changed_draft = copy.deepcopy(draft) + changed_draft["id"] = 301 + with self.assertRaises(ForgeError): + github_phase6.verify_recovery_evidence(value, claims, changed_draft, envelope, bundle) + + def test_post_publication_handoff_loss_recovers_with_gets_only(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, envelope, _, _, released = self._fixture(root) + workflow_relation = {"id": 100, "repository_id": github_phase6.REPOSITORY_ID} + artifacts = [ + {"id": 501, "name": f"phase6-claims-{HEAD}", "expired": False, "digest": "sha256:" + "a" * 64, "size_in_bytes": 100, "expires_at": "2026-09-03T00:00:00Z", "workflow_run": workflow_relation}, + {"id": 502, "name": f"phase6-draft-{HEAD}", "expired": False, "digest": "sha256:" + "b" * 64, "size_in_bytes": 100, "expires_at": "2026-09-03T00:00:00Z", "workflow_run": workflow_relation}, + {"id": 200, "name": claims["staging"]["artifactName"], "expired": False, "digest": "sha256:" + claims["staging"]["archiveSha256"], "size_in_bytes": 100, "expires_at": claims["staging"]["expiresAt"], "workflow_run": workflow_relation}, + ] + release_rows = [{"id": index + 1000, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(released.items()))] + by_id = {row["id"]: released[row["name"]] for row in release_rows} + run = {"id": 100, "run_attempt": 1, "event": "workflow_dispatch", "status": "completed", "conclusion": "failure", "path": github_phase6.WORKFLOW_PATH, "head_sha": HEAD, "head_branch": "main", "repository": {"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID}} + release = {"id": 300, "node_id": "RE_fixture_node", "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD, "html_url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True} + calls = [] + + def api(path: str, method: str = "GET", body=None, api_version=github_phase6.DEFAULT_API_VERSION, **kwargs): + calls.append((path, method, api_version)) + if path.endswith("/actions/runs/100"): + return (run, DATE) + if path.endswith("/releases/300"): + return (release, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + return ({"enabled": True, "enforced_by_owner": False}, DATE) + if path == f"/repos/{github_phase6.REPOSITORY}": + return ({"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID, "node_id": canonical_json.REPOSITORY_NODE_ID}, DATE) + if path.endswith("/branches/main"): + return ({"protected": True, "commit": {"sha": HEAD}}, DATE) + if "/contents/" in path: + return ({"sha": WORKFLOW_SHA}, DATE) + raise AssertionError(path) + + def retained(row, inner, output): + value = claims if inner.startswith("promotion-claims") else draft + output.write_bytes(canonical_json.canonical_bytes(value)) + raw = output.read_bytes() + return value, {"id": row["id"], "name": row["name"], "archiveSize": 100, "archiveSha256": row["digest"].removeprefix("sha256:"), "expiresAt": row["expires_at"], "innerName": inner, "innerSize": len(raw), "innerSha256": hashlib.sha256(raw).hexdigest()} + + def release_download(asset_id: int, output: Path): + output.write_bytes(by_id[asset_id]) + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_EVENT_NAME": "workflow_run"} + output = root / "recovered" + with mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.run_artifacts", return_value=artifacts), mock.patch("github_phase6.retained_json_artifact", side_effect=retained), mock.patch("github_phase6.pagination", return_value=release_rows), mock.patch("github_phase6.download_asset", side_effect=release_download): + github_phase6.recover_publication(100, HEAD, "fixture-1", output) + self.assertTrue((output / "recovered-publication.json").is_file()) + self.assertTrue(all(method == "GET" for _, method, _ in calls)) + recovered = json.loads((output / "recovered-publication.json").read_text()) + self.assertEqual(recovered["release"]["id"], 300) + self.assertEqual(len(recovered["releaseAssets"]), 54) + + def test_before_publication_failure_is_abandoned_without_release_write(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, _, _, _, _ = self._fixture(root) + relation = {"id": 100, "repository_id": github_phase6.REPOSITORY_ID} + artifacts = [ + {"id": 501, "name": f"phase6-claims-{HEAD}", "workflow_run": relation}, + {"id": 502, "name": f"phase6-draft-{HEAD}", "workflow_run": relation}, + {"id": 200, "name": claims["staging"]["artifactName"], "digest": "sha256:" + claims["staging"]["archiveSha256"], "expires_at": claims["staging"]["expiresAt"], "workflow_run": relation}, + ] + run = {"id": 100, "run_attempt": 1, "event": "workflow_dispatch", "status": "completed", "conclusion": "failure", "path": github_phase6.WORKFLOW_PATH, "head_sha": HEAD, "head_branch": "main", "repository": {"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID}} + release = {"id": 300, "node_id": "RE_fixture_node", "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD, "html_url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": True, "prerelease": False, "immutable": False} + + def api(path: str, method: str = "GET", body=None, **kwargs): + self.assertEqual(method, "GET") + return (run if path.endswith("/actions/runs/100") else release, DATE) + + def retained(row, inner, output): + value = claims if inner.startswith("promotion-claims") else draft + output.write_bytes(canonical_json.canonical_bytes(value)) + return value, {} + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_EVENT_NAME": "workflow_run"} + with mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.run_artifacts", return_value=artifacts), mock.patch("github_phase6.retained_json_artifact", side_effect=retained), mock.patch("github_phase6.immutable_release_policy") as setting, mock.patch("github_phase6.download_asset") as download, self.assertRaisesRegex(ForgeError, "failed before"): + github_phase6.recover_publication(100, HEAD, "fixture-1", root / "abandoned") + setting.assert_not_called() + download.assert_not_called() + + +if __name__ == "__main__": + unittest.main()