diff --git a/.github/workflows/candidate.yml b/.github/workflows/candidate.yml index 5881829..cf6770f 100644 --- a/.github/workflows/candidate.yml +++ b/.github/workflows/candidate.yml @@ -7,57 +7,241 @@ on: description: Committed catalog/buildsets ID (no path or URL) required: true type: string + build-set-sha256: + description: Exact SHA-256 of the committed build-set JSON + required: true + type: string permissions: contents: read + actions: read concurrency: - group: forge-candidate-${{ inputs.build-set-id }} + group: forge-candidate-publication cancel-in-progress: false jobs: - validate-committed-build-set: + validate-protected-source: runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 15 + permissions: + contents: read + actions: read + outputs: + build-set-path: ${{ steps.validate.outputs.build-set-path }} steps: - - name: Check out protected-main source + - name: Check out exact protected-main source uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: + fetch-depth: 0 persist-credentials: false - - name: Validate inert build-set identifier and committed manifest + - name: Validate exact committed build set and source ancestry + id: validate env: BUILD_SET_ID: ${{ inputs.build-set-id }} + BUILD_SET_SHA256: ${{ inputs.build-set-sha256 }} run: | - case "$BUILD_SET_ID" in - ''|*[!a-z0-9._-]*) exit 2 ;; - esac - test "${GITHUB_REF}" = 'refs/heads/main' - test -f "catalog/buildsets/${BUILD_SET_ID}.json" + set -euo pipefail + case "$BUILD_SET_ID" in ''|*[!a-z0-9._-]*) exit 2 ;; esac + case "$BUILD_SET_SHA256" in *[!0-9a-f]*|'') exit 2 ;; esac + test "${#BUILD_SET_SHA256}" -eq 64 + test "$GITHUB_REF" = 'refs/heads/main' + test "$GITHUB_EVENT_NAME" = 'workflow_dispatch' + BUILD_SET_PATH="catalog/buildsets/${BUILD_SET_ID}.json" + test -f "$BUILD_SET_PATH" + test "$(sha256sum "$BUILD_SET_PATH" | cut -d' ' -f1)" = "$BUILD_SET_SHA256" python3 -m pip install --requirement requirements-ci.txt - python3 scripts/validate_catalog.py build-set "catalog/buildsets/${BUILD_SET_ID}.json" --root . + python3 scripts/phase6_candidate.py validate-buildset --build-set "$BUILD_SET_PATH" --require-git-ancestry + python3 scripts/github_phase6.py verify-source --expected-sha "$GITHUB_SHA" --output protected-source.json + printf 'build-set-path=%s\n' "$BUILD_SET_PATH" >> "$GITHUB_OUTPUT" + - name: Retain protected-source evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-protected-source-${{ github.sha }} + path: protected-source.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false + + assemble-verified-content: + needs: validate-protected-source + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + outputs: + staging-artifact-id: ${{ steps.stage.outputs.artifact-id }} + staging-name: ${{ steps.assemble.outputs.staging-name }} + content-list-digest: ${{ steps.assemble.outputs.content-list-digest }} + steps: + - name: Check out exact source without credentials + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Capture and verify live audited-input metadata + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/github_phase6.py capture-inputs --build-set "${{ needs.validate-protected-source.outputs.build-set-path }}" --output input-live-metadata.json + python3 scripts/phase6_candidate.py verify-live-inputs --build-set "${{ needs.validate-protected-source.outputs.build-set-path }}" --metadata input-live-metadata.json + - name: Download exact Phase-3p proof-data artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33170546601 + artifact-ids: 9685464135 + merge-multiple: true + path: input-artifacts/phase3p-proof-data + - name: Download exact Phase-4 Celestia appd artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688244894 + merge-multiple: true + path: input-artifacts/phase4-celestia-appd-linux-arm64 + - name: Download exact Phase-4 Celestia node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688243729 + merge-multiple: true + path: input-artifacts/phase4-celestia-node-linux-arm64 + - name: Download exact Phase-4 Midnight node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688330126 + merge-multiple: true + path: input-artifacts/phase4-node-linux-arm64 + - name: Download exact Phase-4 toolkit Linux amd64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688263793 + merge-multiple: true + path: input-artifacts/phase4-toolkit-linux-amd64 + - name: Download exact Phase-4 toolkit Linux arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688255774 + merge-multiple: true + path: input-artifacts/phase4-toolkit-linux-arm64 + - name: Download exact Phase-4 toolkit macOS arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9689647047 + merge-multiple: true + path: input-artifacts/phase4-toolkit-macos-arm64 + - name: Download exact Phase-5 verified indexer aggregate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33176004154 + artifact-ids: 9690093579 + merge-multiple: true + path: input-artifacts/phase5-indexer + - name: Assemble and non-executingly verify exact 31-payload content + id: assemble + env: + BUILD_SET_PATH: ${{ needs.validate-protected-source.outputs.build-set-path }} + run: | + set -euo pipefail + python3 scripts/phase6_candidate.py assemble --build-set "$BUILD_SET_PATH" --input-root input-artifacts --output verified-content --result-output candidate-verification.json + python3 scripts/phase6_candidate.py verify --build-set "$BUILD_SET_PATH" --content verified-content + CONTENT_LIST_DIGEST="$(jq -r '.contentAssetListSha256' candidate-verification.json)" + test "$(printf '%s' "$CONTENT_LIST_DIGEST" | wc -c)" -eq 64 + STAGING_NAME="verified-content-${{ inputs.build-set-id }}-${CONTENT_LIST_DIGEST}" + printf 'content-list-digest=%s\n' "$CONTENT_LIST_DIGEST" >> "$GITHUB_OUTPUT" + printf 'staging-name=%s\n' "$STAGING_NAME" >> "$GITHUB_OUTPUT" + - name: Upload exact immutable staging content + id: stage + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ${{ steps.assemble.outputs.staging-name }} + path: verified-content/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false + - name: Capture exact staging artifact identity + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py capture-staging --artifact-id "${{ steps.stage.outputs.artifact-id }}" --expected-name "${{ steps.assemble.outputs.staging-name }}" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --output staging.json + - name: Retain verified input and staging metadata + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-assembly-evidence-${{ github.sha }} + path: | + candidate-verification.json + input-live-metadata.json + staging.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false - # Phase 1 freezes the privilege boundary and attestation inputs. Phase 6 activates these jobs only - # after exact build/mirror payloads and the two verifier handoffs exist. Keeping them false here - # prevents a manifest-only scaffold from allocating or publishing a release. create-protected-draft: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: validate-committed-build-set + needs: [validate-protected-source, assemble-verified-content] runs-on: ubuntu-24.04 - timeout-minutes: 10 + timeout-minutes: 15 environment: candidate-publish permissions: contents: write + actions: read steps: - - name: Allocate and API-read-back empty draft - run: echo 'Phase 6 supplies the hash-bound draft allocation implementation' && exit 2 + - name: Check out exact source without credentials + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Download protected-source evidence + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + name: phase6-protected-source-${{ github.sha }} + path: protected-source + - name: Allocate API-time unique empty draft and read it back + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py allocate-draft --expected-sha "$GITHUB_SHA" --source protected-source/protected-source.json --output draft.json + - name: Retain exact draft identity + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-draft-${{ github.sha }} + path: draft.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false fresh-final-claims-verifier: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: create-protected-draft + needs: [validate-protected-source, assemble-verified-content, create-protected-draft] runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 90 permissions: contents: read + actions: read outputs: claims-digest: ${{ steps.verify.outputs.claims-digest }} steps: @@ -65,20 +249,55 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false - - name: Re-verify inert content and final draft-bound claims + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Redownload exact staging by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + merge-multiple: true + path: verified-content + - name: Download assembly, draft, and protected-source evidence + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + pattern: phase6-*-${{ github.sha }} + path: handoff + merge-multiple: true + - name: Reverify content and create canonical draft-bound claims id: verify + env: + BUILD_SET_PATH: ${{ needs.validate-protected-source.outputs.build-set-path }} run: | + set -euo pipefail + python3 scripts/phase6_candidate.py verify --build-set "$BUILD_SET_PATH" --content verified-content + WORKFLOW_SHA="$(jq -r '.workflowSha' handoff/protected-source.json)" + python3 scripts/phase6_candidate.py make-claims --build-set "$BUILD_SET_PATH" --content verified-content --draft handoff/draft.json --staging handoff/staging.json --commit-sha "$GITHUB_SHA" --workflow-sha "$WORKFLOW_SHA" --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" --output "promotion-claims-${{ inputs.build-set-id }}.json" python3 scripts/publisher_guard.py verify-claims-content --claims "promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context printf 'claims-digest=sha256:%s\n' "$(python3 scripts/canonical_json.py sha256 "promotion-claims-${{ inputs.build-set-id }}.json")" >> "$GITHUB_OUTPUT" + - name: Retain canonical claims + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-claims-${{ github.sha }} + path: promotion-claims-${{ inputs.build-set-id }}.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false protected-publisher: - if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }} - needs: fresh-final-claims-verifier + needs: [validate-protected-source, assemble-verified-content, create-protected-draft, fresh-final-claims-verifier] runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 90 environment: candidate-publish permissions: contents: write + actions: read id-token: write attestations: write artifact-metadata: write @@ -87,8 +306,26 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false + - name: Redownload exact staging by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-verified-content.outputs.staging-artifact-id }} + merge-multiple: true + path: verified-content + - name: Download exact claims and draft handoffs + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + pattern: phase6-*-${{ github.sha }} + path: handoff + merge-multiple: true - name: Recheck inert claims/content before attestation - run: python3 scripts/publisher_guard.py verify-claims-content --claims "promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context + run: python3 scripts/publisher_guard.py verify-claims-content --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --content-dir verified-content --require-context - name: Attest exact canonical claims predicate id: attest uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 @@ -96,7 +333,28 @@ jobs: subject-name: promotion-claims-${{ inputs.build-set-id }} subject-digest: ${{ needs.fresh-final-claims-verifier.outputs.claims-digest }} predicate-type: https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1 - predicate-path: promotion-claims-${{ inputs.build-set-id }}.json + predicate-path: handoff/promotion-claims-${{ inputs.build-set-id }}.json show-summary: false - - name: Bind bundle, recheck inert transport, upload/read-back, and publish - run: echo 'Phase 6 supplies the exact bundle-path handoff and release transaction' && exit 2 + - name: Cryptographically verify bundle and materialize frozen envelope + env: + GITHUB_TOKEN: ${{ github.token }} + ATTEST_BUNDLE_PATH: ${{ steps.attest.outputs.bundle-path }} + run: | + set -euo pipefail + cp -- "$ATTEST_BUNDLE_PATH" "attestation-${{ inputs.build-set-id }}.sigstore.json" + gh attestation verify "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --repo "$GITHUB_REPOSITORY" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + python3 scripts/materialize_envelope.py --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --output "promotion-envelope-${{ inputs.build-set-id }}.json" + python3 scripts/publisher_guard.py verify-transport --envelope "promotion-envelope-${{ inputs.build-set-id }}.json" --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --content-dir verified-content --require-context + - name: Upload/read back, publish immutable, and prove mutation rejection + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py publish --claims "handoff/promotion-claims-${{ inputs.build-set-id }}.json" --content verified-content --bundle "attestation-${{ inputs.build-set-id }}.sigstore.json" --envelope "promotion-envelope-${{ inputs.build-set-id }}.json" --draft handoff/draft.json --output published.json + - name: Retain post-publication handoff + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: published-candidate-${{ inputs.build-set-id }} + path: published.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false diff --git a/.github/workflows/phase6-candidate-gate.yml b/.github/workflows/phase6-candidate-gate.yml new file mode 100644 index 0000000..230131b --- /dev/null +++ b/.github/workflows/phase6-candidate-gate.yml @@ -0,0 +1,160 @@ +name: Phase 6 immutable candidate gate + +on: + pull_request: + paths: + - '.github/workflows/**' + - 'catalog/**' + - 'schema/**' + - 'scripts/**' + - 'tests/**' + - 'requirements-ci.txt' + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: phase6-candidate-gate-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + assemble-and-verify: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + steps: + - name: Check out exact candidate source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + fetch-depth: 0 + persist-credentials: false + - name: Install verifier dependencies and validate exact build set + run: | + python3 -m pip install --requirement requirements-ci.txt + python3 scripts/generate_phase6_buildset.py --check + python3 scripts/phase6_candidate.py validate-buildset --build-set catalog/buildsets/initial-warehouse-v1.json --require-git-ancestry + - name: Capture and verify live audited-input metadata + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/github_phase6.py capture-inputs --build-set catalog/buildsets/initial-warehouse-v1.json --output input-live-metadata.json + - name: Download exact Phase-3p proof-data artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33170546601 + artifact-ids: 9685464135 + merge-multiple: true + path: input-artifacts/phase3p-proof-data + - name: Download exact Phase-4 Celestia appd artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688244894 + merge-multiple: true + path: input-artifacts/phase4-celestia-appd-linux-arm64 + - name: Download exact Phase-4 Celestia node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688243729 + merge-multiple: true + path: input-artifacts/phase4-celestia-node-linux-arm64 + - name: Download exact Phase-4 Midnight node artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688330126 + merge-multiple: true + path: input-artifacts/phase4-node-linux-arm64 + - name: Download exact Phase-4 toolkit Linux amd64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688263793 + merge-multiple: true + path: input-artifacts/phase4-toolkit-linux-amd64 + - name: Download exact Phase-4 toolkit Linux arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9688255774 + merge-multiple: true + path: input-artifacts/phase4-toolkit-linux-arm64 + - name: Download exact Phase-4 toolkit macOS arm64 artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33177534764 + artifact-ids: 9689647047 + merge-multiple: true + path: input-artifacts/phase4-toolkit-macos-arm64 + - name: Download exact Phase-5 verified indexer aggregate + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: 33176004154 + artifact-ids: 9690093579 + merge-multiple: true + path: input-artifacts/phase5-indexer + - name: Assemble and non-executingly verify exact candidate + run: | + set -euo pipefail + python3 scripts/phase6_candidate.py assemble --build-set catalog/buildsets/initial-warehouse-v1.json --input-root input-artifacts --output verified-content --result-output candidate-verification.json + python3 scripts/phase6_candidate.py verify --build-set catalog/buildsets/initial-warehouse-v1.json --content verified-content + - name: Retain exact verified content for a fresh job + id: upload + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-pr-verified-content-${{ github.sha }} + path: verified-content/ + if-no-files-found: error + retention-days: 7 + compression-level: 0 + overwrite: false + + fresh-non-executing-verifier: + needs: assemble-and-verify + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + steps: + - name: Check out exact candidate source + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Redownload exact staged content by artifact ID + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.run_id }} + artifact-ids: ${{ needs.assemble-and-verify.outputs.artifact-id }} + merge-multiple: true + path: verified-content + - name: Stream and verify every payload/member without execution + run: | + python3 -m pip install --requirement requirements-ci.txt + python3 scripts/phase6_candidate.py verify --build-set catalog/buildsets/initial-warehouse-v1.json --content verified-content diff --git a/.github/workflows/phase6-live-verification.yml b/.github/workflows/phase6-live-verification.yml new file mode 100644 index 0000000..29e6803 --- /dev/null +++ b/.github/workflows/phase6-live-verification.yml @@ -0,0 +1,121 @@ +name: Phase 6 published-candidate live verification + +on: + workflow_run: + workflows: ['Immutable forge candidate'] + types: [completed] + +permissions: + contents: read + actions: read + attestations: read + +concurrency: + group: phase6-published-candidate-live-verification + cancel-in-progress: false + +jobs: + verify-published-candidate: + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'workflow_dispatch' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out exact candidate commit + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ github.event.workflow_run.head_sha }} + persist-credentials: false + - name: Download exact post-publication handoff + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + github-token: ${{ github.token }} + repository: acedward/midnight-binary-forge + run-id: ${{ github.event.workflow_run.id }} + name: published-candidate-initial-warehouse-v1 + path: handoff + - name: Download released envelope and attestation bundle + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + TAG="$(jq -r '.tag' handoff/published.json)" + gh release download "$TAG" --repo "$GITHUB_REPOSITORY" --pattern 'promotion-envelope-initial-warehouse-v1.json' --pattern 'attestation-initial-warehouse-v1.sigstore.json' --dir transport + - name: Verify attestation, immutable release, and independent byte read-back + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 - <<'PY' + import sys + from pathlib import Path + sys.path.insert(0, 'scripts') + import canonical_json + envelope = canonical_json.load_json(Path('transport/promotion-envelope-initial-warehouse-v1.json')) + Path('promotion-claims-initial-warehouse-v1').write_bytes(canonical_json.canonical_bytes(envelope['claims'])) + PY + gh attestation verify promotion-claims-initial-warehouse-v1 --repo "$GITHUB_REPOSITORY" --bundle transport/attestation-initial-warehouse-v1.sigstore.json --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + python3 scripts/github_phase6.py capture-live --envelope transport/promotion-envelope-initial-warehouse-v1.json --bundle transport/attestation-initial-warehouse-v1.sigstore.json --run-id "${{ github.event.workflow_run.id }}" --output live-evidence.json + python3 scripts/canonical_json.py verify-live transport/promotion-envelope-initial-warehouse-v1.json transport/attestation-initial-warehouse-v1.sigstore.json live-evidence.json --require-staging-live + - name: Retain canonical live verification evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-published-live-evidence-${{ github.event.workflow_run.id }} + path: live-evidence.json + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false + + recover-post-publication-handoff-loss: + if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'workflow_dispatch' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + permissions: + contents: read + actions: read + attestations: read + steps: + - name: Check out exact failed candidate commit + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ github.event.workflow_run.head_sha }} + fetch-depth: 0 + persist-credentials: false + - name: Install verifier dependencies + run: python3 -m pip install --requirement requirements-ci.txt + - name: Classify and reconstruct only an existing exact immutable publication + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 scripts/github_phase6.py recover-publication \ + --run-id "${{ github.event.workflow_run.id }}" \ + --expected-head "${{ github.event.workflow_run.head_sha }}" \ + --build-set-id initial-warehouse-v1 \ + --output-dir recovered + - name: Cryptographically verify recovered original-run claims attestation + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + cp -- recovered/promotion-claims-initial-warehouse-v1.json recovered/promotion-claims-initial-warehouse-v1 + gh attestation verify recovered/promotion-claims-initial-warehouse-v1 \ + --repo "$GITHUB_REPOSITORY" \ + --bundle recovered/attestation-initial-warehouse-v1.sigstore.json \ + --predicate-type 'https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1' + rm -- recovered/promotion-claims-initial-warehouse-v1 + python3 scripts/github_phase6.py verify-recovery \ + --claims recovered/promotion-claims-initial-warehouse-v1.json \ + --draft recovered/draft.json \ + --envelope recovered/promotion-envelope-initial-warehouse-v1.json \ + --bundle recovered/attestation-initial-warehouse-v1.sigstore.json \ + --evidence recovered/recovered-publication.json + - name: Retain canonical read-only recovered-publication evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: phase6-recovered-publication-${{ github.event.workflow_run.id }} + path: recovered/ + if-no-files-found: error + retention-days: 30 + compression-level: 0 + overwrite: false diff --git a/catalog/buildsets/initial-warehouse-v1.json b/catalog/buildsets/initial-warehouse-v1.json new file mode 100644 index 0000000..b65f2ee --- /dev/null +++ b/catalog/buildsets/initial-warehouse-v1.json @@ -0,0 +1 @@ +{"buildSetId":"initial-warehouse-v1","candidatePolicy":{"checksumsTemplate":"sha256sums-.txt","destinationCredentialAllowed":false,"immutableReleaseRequired":true,"inputArtifactPinningRequired":true,"protectedDefaultBranchRequired":true,"sourceManifestTemplate":"source-manifest-.json","typedAssetListRequired":true},"components":[{"componentId":"celestia-appd-6.4.10-linux-arm64","manifestPath":"catalog/components/celestia-appd-6.4.10-linux-arm64.json","manifestSha256":"59c65eabeae2179c273741c27a0af5ad5bb4c77e4310d7dc0cb6a26176b828be"},{"componentId":"celestia-node-0.28.4-linux-arm64","manifestPath":"catalog/components/celestia-node-0.28.4-linux-arm64.json","manifestSha256":"7bd9b7c734d2300a390b347f7e65f1e489deefdcdc1572c53bf5f660a5b4a8cf"},{"componentId":"indexer-standalone-linux-amd64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-linux-amd64-4.4.0-rc.3.json","manifestSha256":"5ec8fef85f9742ee9589ae48e17ca5e3e8f62a1d3b7c07ac3293eb752c0e5531"},{"componentId":"indexer-standalone-linux-arm64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-linux-arm64-4.4.0-rc.3.json","manifestSha256":"6b7cb632e29dc36e820da5d6ef54b01b51e18c6c82bc1a6d540f291956f5aaad"},{"componentId":"indexer-standalone-macos-amd64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-macos-amd64-4.4.0-rc.3.json","manifestSha256":"63457c7efd879b907d6abe3127f7c26b99740393984d74701c35409ef3c6ff60"},{"componentId":"indexer-standalone-macos-arm64-4.4.0-rc.3","manifestPath":"catalog/components/indexer-standalone-macos-arm64-4.4.0-rc.3.json","manifestSha256":"26fdbecc07d2a2d3980c38901049f675bc388b565e633c5250aa162c8c30f53e"},{"componentId":"midnight-ledger-static-9.0.0","manifestPath":"catalog/components/midnight-ledger-static-9.0.0.json","manifestSha256":"3da06a2a85992212127070101c18b39f51620b8c8605aa7d869278b6f98716c1"},{"componentId":"midnight-node-2.0.0-rc.4-linux-arm64","manifestPath":"catalog/components/midnight-node-2.0.0-rc.4-linux-arm64.json","manifestSha256":"4a55943a745af8817a1bfeb71f657cbacb6473a8346c96d38a25af8ba05207c2"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-amd64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-amd64.json","manifestSha256":"fde7a617ac19a3cd56f65b9462ea2cc481152b7b264211623eeb8eb7d7868cea"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-arm64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-linux-arm64.json","manifestSha256":"aa5d5d598d4c538e1e4cc855a2d7786c4835fccdff722599154d2c15a69fd558"},{"componentId":"midnight-node-toolkit-2.0.0-rc.4-macos-arm64","manifestPath":"catalog/components/midnight-node-toolkit-2.0.0-rc.4-macos-arm64.json","manifestSha256":"5eb01a7ca32a1860525b0abf0719d99abc85f55fd5eafa14181a3be0eba13e63"},{"componentId":"midnight-srs-k0","manifestPath":"catalog/components/midnight-srs-k0.json","manifestSha256":"96b93dc4670efd7765480986a3348428fd4e8ad24539ed84cf6624f35f9bffea"},{"componentId":"midnight-srs-k1","manifestPath":"catalog/components/midnight-srs-k1.json","manifestSha256":"b32784a7645f41b445afd05c9cd80dd5bb925e5c6d0ac78777043b02d4784765"},{"componentId":"midnight-srs-k10","manifestPath":"catalog/components/midnight-srs-k10.json","manifestSha256":"cb9fd03572750a000b2e293e19a229dc45bd66e9259b3449fadf3bf7680c00e9"},{"componentId":"midnight-srs-k11","manifestPath":"catalog/components/midnight-srs-k11.json","manifestSha256":"ad4b0c160ca6844aa561b8a1d223943ee491b6038558d320d5c8fd97592c0e4f"},{"componentId":"midnight-srs-k12","manifestPath":"catalog/components/midnight-srs-k12.json","manifestSha256":"571cec78e30abf1838e475a0226efda56ec1388f16b831658babfc6f7f9f167f"},{"componentId":"midnight-srs-k13","manifestPath":"catalog/components/midnight-srs-k13.json","manifestSha256":"0f3db81343cebc66f3282ed31f58e4f2c8314890ea9f3a0981a2940ce21f601f"},{"componentId":"midnight-srs-k14","manifestPath":"catalog/components/midnight-srs-k14.json","manifestSha256":"c959d6b179d7da155e9d54e47258106696c9c6a670dc4bbcbe7cf6ed9fb63720"},{"componentId":"midnight-srs-k15","manifestPath":"catalog/components/midnight-srs-k15.json","manifestSha256":"96e7fb7101fb3f0721c76be00a74d3eb905ef54a2641cbd1fac266af4d667fb7"},{"componentId":"midnight-srs-k16","manifestPath":"catalog/components/midnight-srs-k16.json","manifestSha256":"1f33014e3f1d2e7abaaa693dbfe2fe0094ea427201d07b44336c79d6b3c37de4"},{"componentId":"midnight-srs-k17","manifestPath":"catalog/components/midnight-srs-k17.json","manifestSha256":"fe59fb43db14585bcadb8eb4f3a13e9471df967a34bec2dbd40283814d9e454d"},{"componentId":"midnight-srs-k18","manifestPath":"catalog/components/midnight-srs-k18.json","manifestSha256":"d7b31231be36ce2905e9b478421b8967b7a4cbededb11413adc02386ad37ad07"},{"componentId":"midnight-srs-k19","manifestPath":"catalog/components/midnight-srs-k19.json","manifestSha256":"4ed42289e8e3b24b255dfc00ee868c2341b189979290a2adb2d89c81bbacf447"},{"componentId":"midnight-srs-k2","manifestPath":"catalog/components/midnight-srs-k2.json","manifestSha256":"ca323861f22c673af8ebca56a846eb20d9a1859859e029bca542240c56b63eb3"},{"componentId":"midnight-srs-k3","manifestPath":"catalog/components/midnight-srs-k3.json","manifestSha256":"50171b4c63116d4ce0ccdd830c85b069d51fd3c5655ed977de8c38be9ea5b110"},{"componentId":"midnight-srs-k4","manifestPath":"catalog/components/midnight-srs-k4.json","manifestSha256":"d37c242136b53b12a96617bb90e02ab9756cb6e9be7f0d5803694b0df94e6d5d"},{"componentId":"midnight-srs-k5","manifestPath":"catalog/components/midnight-srs-k5.json","manifestSha256":"108579a6fd1a411b0177215a051fc9c941632108b6d7483eb0c27e6cae5b9498"},{"componentId":"midnight-srs-k6","manifestPath":"catalog/components/midnight-srs-k6.json","manifestSha256":"89455050237cdc39ee6308836b1781422e1bcbe03651e89ccf4c9f8ec2e2cd17"},{"componentId":"midnight-srs-k7","manifestPath":"catalog/components/midnight-srs-k7.json","manifestSha256":"067c1e0c42f84c4ba86dc351c562a19727a9ec9f24f0dc3ed9c3ecd62c4fabc2"},{"componentId":"midnight-srs-k8","manifestPath":"catalog/components/midnight-srs-k8.json","manifestSha256":"4ea018017431291c216e2060dab31c8cd8acd3e76a56161afb48c42a9dc343e0"},{"componentId":"midnight-srs-k9","manifestPath":"catalog/components/midnight-srs-k9.json","manifestSha256":"4e4b149df4d8d18d601ccd88fcdb788cafd754ba49066e7d51517c46849536c9"}],"coveragePolicy":{"desired":["linux/arm64"],"optional":["macos/amd64"],"proofDataPlatform":"noarch","required":["linux/amd64","macos/arm64"]},"destination":{"distributionTier":"development-only","releaseMutability":"mutable-warehouse","repository":"effectstream/binaries","tag":"0.3.120"},"existingCoverage":[{"arch":"amd64","assetId":378640596,"assetNodeId":"RA_kwDOQpztJs4WkZjU","family":"celestia-appd","name":"celestia-appd-linux-amd64-v6.4.10.tar.gz","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"fa1826187a91514c6d506e22f1fcce25afbd551b98458caf899801b662a0c8da","size":198844329,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"6.4.10"},{"arch":"arm64","assetId":378640593,"assetNodeId":"RA_kwDOQpztJs4WkZjR","family":"celestia-appd","name":"celestia-appd-macos-arm64-v6.4.10.tar.gz","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"832f4b919d79768b960e55f6d778366d9afbb95df6a860e344dffbb92e85b5be","size":196072247,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"6.4.10"},{"arch":"amd64","assetId":378640594,"assetNodeId":"RA_kwDOQpztJs4WkZjS","family":"celestia-node","name":"celestia-node-linux-amd64-v0.28.4.tar.gz","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"bb8b9fd2ac859945fdf2f8c84ba038f75e01be54e56d1a946cbbd4b540f120d9","size":78842331,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"0.28.4"},{"arch":"arm64","assetId":378640595,"assetNodeId":"RA_kwDOQpztJs4WkZjT","family":"celestia-node","name":"celestia-node-macos-arm64-v0.28.4.tar.gz","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"d2d637282489ea1ffc37a5fded4997be7140a7e77a43f0ea61a68f0f73069821","size":76777711,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"0.28.4"},{"arch":"amd64","assetId":526514404,"assetNodeId":"RA_kwDOQpztJs4fYfjk","family":"midnight-node","name":"midnight-node-linux-amd64-2.0.0-rc.4.zip","os":"linux","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"8f53e9dfb2c70ec2fb98fd6958466ef107685774ca4d93660bc63e7686948879","size":84751613,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"2.0.0-rc.4"},{"arch":"arm64","assetId":526513732,"assetNodeId":"RA_kwDOQpztJs4fYfZE","family":"midnight-node","name":"midnight-node-macos-arm64-2.0.0-rc.4.zip","os":"macos","releaseId":270761136,"releaseNodeId":"RE_kwDOQpztJs4QI3yw","releaseTag":"0.3.120","repository":"effectstream/binaries","repositoryId":1117580582,"repositoryNodeId":"R_kgDOQpztJg","sha256":"4ee77c1043dec716f7a1b133f0ebb8f23bbc3a704f348ae5708a6b58b330ed8c","size":78815030,"snapshotPath":"evidence/phase0/warehouse-release-0.3.120.json","snapshotSha256":"6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1","source":"warehouse-existing","tier":"required","version":"2.0.0-rc.4"}],"inputArtifacts":[{"archiveSha256":"b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f","artifactId":9685464135,"artifactName":"proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1","artifactSize":222975770,"expiresAt":"2026-09-04T12:20:54Z","key":"phase3p-proof-data","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33170546601,"sourceHeadSha":"508aab47ab266344aedd6359fe971928bed309b5","sourceRef":"codex/00002-phase3p-proof-data","workflowPath":".github/workflows/proof-data-q8b.yml"},{"archiveSha256":"3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56","artifactId":9688244894,"artifactName":"phase4-celestia-appd-linux-arm64","artifactSize":181312477,"expiresAt":"2026-09-27T13:54:20Z","key":"phase4-celestia-appd-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba","artifactId":9688243729,"artifactName":"phase4-celestia-node-linux-arm64","artifactSize":72131099,"expiresAt":"2026-09-27T13:54:18Z","key":"phase4-celestia-node-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e","artifactId":9688330126,"artifactName":"phase4-node-linux-arm64","artifactSize":88276678,"expiresAt":"2026-09-27T13:56:54Z","key":"phase4-node-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b","artifactId":9688263793,"artifactName":"phase4-toolkit-linux-amd64","artifactSize":53080691,"expiresAt":"2026-09-27T13:54:55Z","key":"phase4-toolkit-linux-amd64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd","artifactId":9688255774,"artifactName":"phase4-toolkit-linux-arm64","artifactSize":51636493,"expiresAt":"2026-09-27T13:54:40Z","key":"phase4-toolkit-linux-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e","artifactId":9689647047,"artifactName":"phase4-toolkit-macos-arm64","artifactSize":97365627,"expiresAt":"2026-09-27T14:35:47Z","key":"phase4-toolkit-macos-arm64","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33177534764,"sourceHeadSha":"656cd9664d23bda4ef0578d62c9e27392bff063e","sourceRef":"codex/00002-phase4-node-toolkit-celestia","workflowPath":".github/workflows/phase4-payloads.yml"},{"archiveSha256":"eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9","artifactId":9690093579,"artifactName":"phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a","artifactSize":377650526,"expiresAt":"2026-09-27T14:49:01Z","key":"phase5-indexer","repository":"acedward/midnight-binary-forge","repositoryId":1349127482,"runAttempt":1,"runConclusion":"success","runEvent":"pull_request","runId":33176004154,"sourceHeadSha":"e581add8952bae5ffeac39fb07e6b5c6f482862d","sourceRef":"codex/00002-phase5-indexer","workflowPath":".github/workflows/phase5-indexer.yml"}],"payloadCount":31,"payloads":[{"artifactKind":"proof-data","componentId":"midnight-srs-k0","container":"raw","installMode":"0644","k":0,"name":"bls_midnight_2p0","platform":"noarch","role":"payload","sha256":"59b30b3114a34ccbbfb599376e178fb8d9b3366cae2174c2f1da20e75847f823","size":580,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p0","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k1","container":"raw","installMode":"0644","k":1,"name":"bls_midnight_2p1","platform":"noarch","role":"payload","sha256":"bbe04fe3c70d0c138447cb086b4baddc30cb8bb2a004114bc02e6f739516280e","size":772,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p1","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k10","container":"raw","installMode":"0644","k":10,"name":"bls_midnight_2p10","platform":"noarch","role":"payload","sha256":"46b2290933cbed4c378889e4ba971f1a92888331ffb09466acd4ff61a1e2cb42","size":196996,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p10","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k11","container":"raw","installMode":"0644","k":11,"name":"bls_midnight_2p11","platform":"noarch","role":"payload","sha256":"9901589d7956ff58be0d85569b2f455b77b58c3758026ffb5bbe4807000b96d1","size":393604,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p11","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k12","container":"raw","installMode":"0644","k":12,"name":"bls_midnight_2p12","platform":"noarch","role":"payload","sha256":"ef08eb3fcf62df8f72c515cffa027e681808b530cb016eea104115545ef6d5c8","size":786820,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p12","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k13","container":"raw","installMode":"0644","k":13,"name":"bls_midnight_2p13","platform":"noarch","role":"payload","sha256":"d3324910969c4cc54143b8045b649e5c3a4bd5fb7b8f85fe1b770f640ce1c803","size":1573252,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p13","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k14","container":"raw","installMode":"0644","k":14,"name":"bls_midnight_2p14","platform":"noarch","role":"payload","sha256":"fc253016885ec830e97808c9ec920bb5cab5c21af590380a6cb5eb0538e2b244","size":3146116,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p14","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k15","container":"raw","installMode":"0644","k":15,"name":"bls_midnight_2p15","platform":"noarch","role":"payload","sha256":"724c7c3d779148bb113c7ee9c034b2f27db16e6bdf315fde90105a9bad00b1de","size":6291844,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p15","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k16","container":"raw","installMode":"0644","k":16,"name":"bls_midnight_2p16","platform":"noarch","role":"payload","sha256":"09c877216d6589b370263e18af40a030a901b41a7a7c37ef58c9901db41f05c6","size":12583300,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p16","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k17","container":"raw","installMode":"0644","k":17,"name":"bls_midnight_2p17","platform":"noarch","role":"payload","sha256":"4a9ef6c7c0619aab74eede44b13e753e3ba54508a02dd3b7106a949aabb73b74","size":25166212,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p17","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k18","container":"raw","installMode":"0644","k":18,"name":"bls_midnight_2p18","platform":"noarch","role":"payload","sha256":"e8436dc5d8b598f169c127c745135d889744007e6d384ff126df8d1332522f86","size":50332036,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p18","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k19","container":"raw","installMode":"0644","k":19,"name":"bls_midnight_2p19","platform":"noarch","role":"payload","sha256":"8e8dc15c4362f05c912f1e770559a3945db3e58a374def416ed5d3e65ad5b10e","size":100663684,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p19","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k2","container":"raw","installMode":"0644","k":2,"name":"bls_midnight_2p2","platform":"noarch","role":"payload","sha256":"80e15568fa1a0117db893239be7fa5e34a6bcc3a8c3bfa7709534b9cb88eb6c1","size":1156,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p2","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k3","container":"raw","installMode":"0644","k":3,"name":"bls_midnight_2p3","platform":"noarch","role":"payload","sha256":"4be827a6472193df80d8f08b4b25a85baef436fdd1965d89b6af89f4ec4e99e2","size":1924,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p3","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k4","container":"raw","installMode":"0644","k":4,"name":"bls_midnight_2p4","platform":"noarch","role":"payload","sha256":"232f401fad10c7ddf8828d2aa4c85c6506c5da09795998cecaeb9f75fc8f6ada","size":3460,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p4","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k5","container":"raw","installMode":"0644","k":5,"name":"bls_midnight_2p5","platform":"noarch","role":"payload","sha256":"0a1c9229f315fc1868ff25f668fb83aec4d09f4f23a706b5197c692c619d72c6","size":6532,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p5","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k6","container":"raw","installMode":"0644","k":6,"name":"bls_midnight_2p6","platform":"noarch","role":"payload","sha256":"cf2ad6be7d0fedf5bec2aaa35f6be4aca33053d74268fdf5aa54fcb2891ea6df","size":12676,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p6","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k7","container":"raw","installMode":"0644","k":7,"name":"bls_midnight_2p7","platform":"noarch","role":"payload","sha256":"e82ae890c080188355f37feaffe91372584cd810615082d9143d4dec0453fd9d","size":24964,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p7","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k8","container":"raw","installMode":"0644","k":8,"name":"bls_midnight_2p8","platform":"noarch","role":"payload","sha256":"909b707551eaaea79828e883cde6fc46ab15986c3b1d791bed462c9e2805c933","size":49540,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p8","tier":"noarch"},{"artifactKind":"proof-data","componentId":"midnight-srs-k9","container":"raw","installMode":"0644","k":9,"name":"bls_midnight_2p9","platform":"noarch","role":"payload","sha256":"b9009f1098bcefffec3c461ab3a5e3a17f7e5599f0f08c70fcdc55a89227bcbd","size":98692,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/bls_midnight_2p9","tier":"noarch"},{"arch":"arm64","artifactKind":"software","componentId":"celestia-appd-6.4.10-linux-arm64","container":"tar.gz","installMode":"0755","name":"celestia-appd-linux-arm64-v6.4.10.tar.gz","os":"linux","role":"payload","sha256":"52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b","size":180685852,"sourceArtifactKey":"phase4-celestia-appd-linux-arm64","sourcePath":"payloads/celestia-appd-linux-arm64-v6.4.10.tar.gz","tier":"desired"},{"arch":"arm64","artifactKind":"software","componentId":"celestia-node-0.28.4-linux-arm64","container":"tar.gz","installMode":"0755","name":"celestia-node-linux-arm64-v0.28.4.tar.gz","os":"linux","role":"payload","sha256":"09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c","size":71184641,"sourceArtifactKey":"phase4-celestia-node-linux-arm64","sourcePath":"payloads/celestia-node-linux-arm64-v0.28.4.tar.gz","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"indexer-standalone-linux-amd64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-linux-amd64-v4.4.0-rc.3.zip","os":"linux","role":"payload","sha256":"4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434","size":31479027,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-linux-amd64-v4.4.0-rc.3.zip","tier":"required"},{"arch":"arm64","artifactKind":"software","componentId":"indexer-standalone-linux-arm64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-linux-arm64-v4.4.0-rc.3.zip","os":"linux","role":"payload","sha256":"eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0","size":29782570,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-linux-arm64-v4.4.0-rc.3.zip","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"indexer-standalone-macos-amd64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-macos-amd64-v4.4.0-rc.3.zip","os":"macos","role":"payload","sha256":"28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc","size":30713420,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-macos-amd64-v4.4.0-rc.3.zip","tier":"optional"},{"arch":"arm64","artifactKind":"software","componentId":"indexer-standalone-macos-arm64-4.4.0-rc.3","container":"zip","installMode":"0755","name":"indexer-standalone-macos-arm64-v4.4.0-rc.3.zip","os":"macos","role":"payload","sha256":"b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2","size":29072181,"sourceArtifactKey":"phase5-indexer","sourcePath":"payload/indexer-standalone-macos-arm64-v4.4.0-rc.3.zip","tier":"required"},{"artifactKind":"proof-data","componentId":"midnight-ledger-static-9.0.0","container":"zip","installMode":"0644","ledgerStaticSemver":"9.0.0","name":"midnight-ledger-static-noarch-9.0.0.zip","platform":"noarch","role":"payload","sha256":"d7e8ccfdbc55a2b7139aadd4797d665f888a4502b63ebae24d23314eeee341b2","size":21601265,"sourceArtifactKey":"phase3p-proof-data","sourcePath":"payloads/midnight-ledger-static-noarch-9.0.0.zip","tier":"noarch"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-2.0.0-rc.4-linux-arm64","container":"zip","installMode":"0755","name":"midnight-node-linux-arm64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c","size":82544614,"sourceArtifactKey":"phase4-node-linux-arm64","sourcePath":"payloads/midnight-node-linux-arm64-2.0.0-rc.4.zip","tier":"desired"},{"arch":"amd64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-amd64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c","size":50017428,"sourceArtifactKey":"phase4-toolkit-linux-amd64","sourcePath":"payloads/midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip","tier":"required"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-linux-arm64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip","os":"linux","role":"payload","sha256":"4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528","size":48585850,"sourceArtifactKey":"phase4-toolkit-linux-arm64","sourcePath":"payloads/midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip","tier":"desired"},{"arch":"arm64","artifactKind":"software","componentId":"midnight-node-toolkit-2.0.0-rc.4-macos-arm64","container":"zip","installMode":"0755","name":"midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip","os":"macos","role":"payload","sha256":"8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c","size":45553847,"sourceArtifactKey":"phase4-toolkit-macos-arm64","sourcePath":"payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip","tier":"required"}],"schemaVersion":"build-set-v1","sourceFullSha":"19de8be5f434225dbf17126e86b6c3cc6aacc4fe"} \ No newline at end of file diff --git a/docs/publishing.md b/docs/publishing.md index 50d2f0f..a739932 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -5,26 +5,37 @@ writes. `effectstream/binaries@0.3.120` is updated manually outside Actions. ## Candidate lifecycle -Phase 1 installs and tests the boundary contract but deliberately cannot allocate or publish a -release: `PHASE6_CANDIDATE_ENABLED` is absent and all draft/final-claims/publisher jobs are guarded; -their transaction steps also fail closed. Phase 6 replaces those stubs with the exact hash-bound -implementation through a reviewed main-branch PR, runs the live gate, and only then enables the -repository variable. Setting the variable against the Phase-1 scaffold cannot publish a draft. +Phase 6 replaces the Phase-1 scaffold with an exact hash-bound implementation. Publication has no +repository-variable switch: the workflow exists only after reviewed protected-main integration, +accepts a build-set ID plus the exact committed build-set SHA-256, and rechecks protected main, +the workflow blob, the dedicated immutable-release setting, PR-only/no-bypass rules, and the protected publisher +environment before allocating a draft. The workflow contains no destination credential reference. +Every authoritative setting check calls `GET /repos/acedward/midnight-binary-forge/immutable-releases` +with GitHub API version `2026-03-10` and requires the typed response `enabled: true`; the general +repository response is not an immutable-release authority. 1. Merge exact component/build-set manifests through protected main. Reject abbreviated/floating refs, Compact warehouse components, incomplete required platform coverage, and proof-data scope outside K0–K19 plus Ledger static 9. -2. Dispatch `.github/workflows/candidate.yml` with one committed build-set ID. The workflow resolves - the file from the triggering full SHA; free-form manifest JSON or URLs are not inputs. -3. Native build/mirror jobs run without secrets/write/OIDC. They emit payloads and content evidence - with exact roles. Software gets SBOM/provenance; proof data gets lineage/member evidence and no - fabricated signing/SBOM fields. -4. Upload content to short-lived staging. A fresh pre-draft verifier downloads it with no cache or - credentials, verifies every exact content byte, and emits only the content-list digest required - to allocate the draft. It does not emit final claims. -5. The protected publisher creates an empty `forge-YYYY.MM.DD.N` draft, then reads back and freezes - its repository/tag/numeric+node ID/target/URL identity. It passes only that inert identity to a - second fresh no-write/no-OIDC verifier. +2. Dispatch `.github/workflows/candidate.yml` with build-set ID `initial-warehouse-v1` and the exact + SHA-256 of `catalog/buildsets/initial-warehouse-v1.json`. Free-form manifest JSON/URLs are not + inputs. Global `forge-candidate-publication` concurrency serializes every build-set/tag. +3. The no-write assembler downloads only the eight pinned, audited Actions artifacts by numeric + repository/run/artifact identity and verifies their API run event/conclusion/full source SHA, + name/size/wrapper digest and expiry. It emits exactly ten software payloads and Q8=B's 21 noarch + proof payloads (K0–K19 plus Ledger-static 9), never Compact or a platform/proof-server duplicate. + The retained Phase-3p artifact expires on 2026-09-04; if it expires before an accepted candidate, + rerun the exact audited Q8=B workflow from reviewed main, require identical 21 payload bytes and + a new independent audit, then update numeric pins through another PR. Never substitute by name. +4. The flat inert candidate also carries canonical source/checksum manifests, Apache license and + development notice, one SPDX SBOM per software payload, software archive-member policies, + proof lineage/cache/member evidence, signing evidence (including the actual unsigned/adhoc + macOS state), and provenance. Every evidence basename has one frozen envelope-v1 role. Upload + this content to short-lived staging with its complete typed-list digest in the artifact name. +5. Immediately before the draft POST, the protected publisher rechecks the dedicated immutable- + release setting. It creates an empty `forge-YYYY.MM.DD.N` draft, then reads back and freezes its + repository/tag/numeric+node ID/target/URL identity. It passes only that inert identity to a second + fresh no-write/no-OIDC verifier. 6. The final-claims verifier independently downloads staging again, requires exact content, checks liveness against authenticated GitHub API server time, and emits canonical draft-bound claims and predicate. It cannot mutate the draft or attest, and no later step may change those claims. @@ -32,12 +43,25 @@ repository variable. Setting the variable against the Phase-1 scaffold cannot pu subject name and custom predicate contract. Download the detached bundle, bind its digest in the canonical envelope, download staging again, and require exact inert name/count/size/digest equality. Never extract or execute candidate files. -8. Upload content plus exactly the two predeclared transport files to the draft. Re-download every - draft asset through the API, hash it, verify the exact complete name/count/byte set, then publish. - API-read immutable state. Any upload/read-back/policy mismatch leaves a draft and fails. -9. Emit live evidence and verify workflow blob/run/artifact/protected-main/release identity plus every - independently downloaded byte, including the raw envelope and bundle. Prove a mutation attempt is - rejected by immutable release policy. +8. Recheck the dedicated setting immediately before the first asset upload. Upload content plus + exactly the two predeclared transport files to the draft. Re-download every draft asset through + the API, hash it, and verify the exact complete name/count/byte set. Recheck the setting again + immediately before the public transition, publish, recheck the setting once more, and read back + `draft=false`, `prerelease=false`, `immutable=true` before proceeding. +9. Prove a no-op release metadata mutation is rejected after immutable publication. The separate + read-only `phase6-live-verification.yml` workflow handles both terminal outcomes. On source-workflow + success it binds `status=completed, conclusion=success`, rechecks the dedicated setting, + cryptographically verifies the released bundle, and independently re-downloads/hashes every + released byte before emitting canonical live evidence. +10. If the immutable publication succeeded but the final `published.json` Actions-artifact upload + failed, the source workflow is truthfully failed. The failure lane may recover evidence only when + the exact original run, retained claims/draft/staging artifact IDs and digests, signed release + ID/node/tag/target/URL, missing normal handoff, current immutable setting, and all 54 existing + release assets agree. It performs GET/download operations only, cryptographically verifies the + original claims bundle, and retains canonical `phase6-recovered-publication-v1` evidence. It never + creates a second release, resumes an upload, patches, republishes, reuploads, deletes, or changes + the already immutable release. A still-draft/mutable release is a prepublication failure and is + abandoned without recovery mutation. ## Workflow permissions @@ -59,3 +83,6 @@ read-back mismatch, or mutable candidate state. Failed drafts are never treated as candidates. Do not reuse their tag or claim immutability. Record the failure, fix manifests/tooling through a new PR, and allocate a new monotonically increasing tag. +After a public transition, do not assume a failed workflow means the release remained a draft. First +classify it through the exact read-only recovery lane; any missing, ambiguous, expired, substituted, +or inconsistent retained artifact/release byte is a hard stop for operator review. diff --git a/docs/rollback.md b/docs/rollback.md index 304391f..b588650 100644 --- a/docs/rollback.md +++ b/docs/rollback.md @@ -5,11 +5,31 @@ an existing asset. ## Before forge publication -- A build/verifier/publisher failure leaves only ephemeral staging or a failed draft. +- A build/verifier failure before the public transition leaves only ephemeral staging or a failed + draft. A generic publisher/workflow failure does not prove that state: it may instead be a + post-publication Actions-artifact handoff failure after the release became immutable. - Delete/expire staging under its normal retention policy only after retaining non-secret logs and manifest/digest evidence. A failed draft is not promoted and its tag is not reused. - Correct source/build/packaging metadata on a new reviewed commit and create a new candidate tag. +## Immutable publication succeeded but final handoff failed + +- The read-only workflow-run recovery lane is permitted only for a completed failed candidate run + whose normal `published-candidate-` handoff is absent. It must bind the exact original + run/head/workflow, retained canonical claims and draft artifacts, staging artifact ID/digest, and + the signed release ID/node/tag/target/URL. +- It rechecks the dedicated immutable-release setting, requires the existing release to be + non-draft, non-prerelease, and immutable, then downloads and hashes the exact signed 54-asset set. + It cryptographically verifies the original attestation bundle and retains canonical recovered + evidence for live verification/audit. +- Recovery is strictly observational. Never create a second release, resume or re-upload assets, + patch or republish the release, delete a tag/release/asset, or replace the failed handoff. Wrong, + missing, ambiguous, expired, or substituted run/artifact/claims/draft/release/asset evidence is a + hard stop for human review. +- If the bound release remains a draft or mutable, classify it as a before-publication failure and + abandon the recovery path without mutation. Correct the cause on a new reviewed commit and use a + new monotonically increasing candidate tag. + ## After immutable forge publication, before warehouse append - Stop destination work. Open a reviewed advisory/manifest PR explaining the affected candidate and diff --git a/schema/build-set-v1.schema.json b/schema/build-set-v1.schema.json index f211707..9b039c4 100644 --- a/schema/build-set-v1.schema.json +++ b/schema/build-set-v1.schema.json @@ -10,6 +10,7 @@ "sourceFullSha", "destination", "components", + "inputArtifacts", "existingCoverage", "payloads", "payloadCount", @@ -45,6 +46,12 @@ } } }, + "inputArtifacts": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { "$ref": "#/$defs/inputArtifact" } + }, "existingCoverage": { "type": "array", "items": { "$ref": "#/$defs/coverage" } @@ -75,6 +82,7 @@ "typedAssetListRequired", "sourceManifestTemplate", "checksumsTemplate", + "inputArtifactPinningRequired", "destinationCredentialAllowed" ], "properties": { @@ -83,11 +91,56 @@ "typedAssetListRequired": { "const": true }, "sourceManifestTemplate": { "const": "source-manifest-.json" }, "checksumsTemplate": { "const": "sha256sums-.txt" }, + "inputArtifactPinningRequired": { "const": true }, "destinationCredentialAllowed": { "const": false } } } }, "$defs": { + "inputArtifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "key", + "repository", + "repositoryId", + "workflowPath", + "runId", + "runAttempt", + "runEvent", + "runConclusion", + "sourceRef", + "sourceHeadSha", + "artifactId", + "artifactName", + "artifactSize", + "archiveSha256", + "expiresAt" + ], + "properties": { + "key": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, + "repository": { "const": "acedward/midnight-binary-forge" }, + "repositoryId": { "const": 1349127482 }, + "workflowPath": { + "enum": [ + ".github/workflows/proof-data-q8b.yml", + ".github/workflows/phase4-payloads.yml", + ".github/workflows/phase5-indexer.yml" + ] + }, + "runId": { "type": "integer", "minimum": 1 }, + "runAttempt": { "const": 1 }, + "runEvent": { "enum": ["pull_request", "push", "workflow_dispatch"] }, + "runConclusion": { "const": "success" }, + "sourceRef": { "type": "string", "pattern": "^[A-Za-z0-9._/-]+$" }, + "sourceHeadSha": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "artifactId": { "type": "integer", "minimum": 1 }, + "artifactName": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$" }, + "artifactSize": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "archiveSha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "expiresAt": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" } + } + }, "coverage": { "type": "object", "additionalProperties": false, @@ -137,13 +190,31 @@ "payload": { "type": "object", "additionalProperties": false, - "required": ["name", "role", "artifactKind", "componentId", "tier"], + "required": [ + "name", + "role", + "artifactKind", + "componentId", + "tier", + "container", + "size", + "sha256", + "sourceArtifactKey", + "sourcePath", + "installMode" + ], "properties": { "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, "role": { "const": "payload" }, "artifactKind": { "enum": ["software", "proof-data"] }, "componentId": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, "tier": { "enum": ["required", "desired", "optional", "noarch"] }, + "container": { "enum": ["raw", "zip", "tar.gz"] }, + "size": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "sourceArtifactKey": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{2,127}$" }, + "sourcePath": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]{0,511}$" }, + "installMode": { "enum": ["0644", "0755"] }, "os": { "enum": ["linux", "macos"] }, "arch": { "enum": ["amd64", "arm64"] }, "platform": { "const": "noarch" }, diff --git a/scripts/check_workflow_policy.py b/scripts/check_workflow_policy.py index e050a6e..eecb4cc 100755 --- a/scripts/check_workflow_policy.py +++ b/scripts/check_workflow_policy.py @@ -42,12 +42,26 @@ def validate_workflow(path: Path) -> None: expect("id-token: write" not in text and "attestations: write" not in text and "contents: write" not in text, f"{path.name}: non-candidate workflow requests write/OIDC authority") expect("secrets:" not in text, f"{path.name}: non-candidate workflow declares secrets") else: - expect("environment: candidate-publish" in text, "candidate.yml: protected publisher environment missing") + expect(text.count("environment: candidate-publish") == 2, "candidate.yml: draft and publisher must use the protected environment") expect("id-token: write" in text and "attestations: write" in text and "contents: write" in text, "candidate.yml: forge publisher permission contract missing") - expect(text.count("if: ${{ vars.PHASE6_CANDIDATE_ENABLED == 'true' }}") == 3, "candidate.yml: all Phase-1 privileged/claims jobs must remain fail-closed until explicit Phase 6 activation") + expect("PHASE6_CANDIDATE_ENABLED" not in text and "vars." not in text, "candidate.yml: publication cannot depend on an unpinned repository variable") + expect("group: forge-candidate-publication" in text and "cancel-in-progress: false" in text, "candidate.yml: global non-cancelling publication concurrency missing") + expect("actions: read" in text and "build-set-sha256" in text, "candidate.yml: exact build-set/actions read contract missing") + expect("verify-source --expected-sha \"$GITHUB_SHA\"" in text, "candidate.yml: live protected-main full-SHA gate missing") expect("actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6" in text, "candidate.yml: frozen attest action pin missing") expect("predicate-type: https://github.com/acedward/midnight-binary-forge/predicates/promotion-envelope/v1" in text, "candidate.yml: frozen predicate type missing") expect("subject-name: promotion-claims-${{ inputs.build-set-id }}" in text, "candidate.yml: frozen subject name missing") + expect("gh attestation verify" in text and "scripts/materialize_envelope.py" in text, "candidate.yml: cryptographic bundle verification/envelope materialization missing") + expect("scripts/github_phase6.py publish" in text, "candidate.yml: create-only read-back publisher missing") + expect("--clobber" not in text and "delete release" not in text.casefold() and "-X DELETE" not in text, "candidate.yml: destructive release mutation token is forbidden") + for artifact_id in (9685464135, 9688244894, 9688243729, 9688330126, 9688263793, 9688255774, 9689647047, 9690093579): + expect(str(artifact_id) in text, f"candidate.yml: audited input artifact pin missing: {artifact_id}") + if path.name == "phase6-live-verification.yml": + expect("github.event.workflow_run.conclusion == 'failure'" in text, "phase6-live-verification.yml: failed-source recovery trigger missing") + expect("scripts/github_phase6.py recover-publication" in text and "scripts/github_phase6.py verify-recovery" in text, "phase6-live-verification.yml: read-only recovery/verifier commands missing") + expect("phase6-recovered-publication-${{ github.event.workflow_run.id }}" in text, "phase6-live-verification.yml: recovered evidence retention missing") + expect("cp -- recovered/promotion-claims-initial-warehouse-v1.json recovered/promotion-claims-initial-warehouse-v1" in text, "phase6-live-verification.yml: exact attestation subject materialization missing") + expect("gh attestation verify recovered/promotion-claims-initial-warehouse-v1" in text and "rm -- recovered/promotion-claims-initial-warehouse-v1" in text, "phase6-live-verification.yml: recovered claims cryptographic verification/cleanup missing") def main() -> int: diff --git a/scripts/generate_phase6_buildset.py b/scripts/generate_phase6_buildset.py new file mode 100644 index 0000000..1f817f5 --- /dev/null +++ b/scripts/generate_phase6_buildset.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Generate the reviewed, exact Phase-6 initial warehouse build set.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from forge_io import canonical_bytes, create_file_atomic, load_json, sha256_file + + +ROOT = Path(__file__).resolve().parents[1] +BASE_SHA = "19de8be5f434225dbf17126e86b6c3cc6aacc4fe" +SNAPSHOT_PATH = "evidence/phase0/warehouse-release-0.3.120.json" +SNAPSHOT_SHA = "6cb1abbbcf3e693e85b1d6806569caec956d5627b4aeb18ba413b519216124e1" +INPUTS = [ + ("phase3p-proof-data", ".github/workflows/proof-data-q8b.yml", 33170546601, "pull_request", "codex/00002-phase3p-proof-data", "508aab47ab266344aedd6359fe971928bed309b5", 9685464135, "proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1", 222975770, "b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f", "2026-09-04T12:20:54Z"), + ("phase4-celestia-appd-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688244894, "phase4-celestia-appd-linux-arm64", 181312477, "3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56", "2026-09-27T13:54:20Z"), + ("phase4-celestia-node-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688243729, "phase4-celestia-node-linux-arm64", 72131099, "b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba", "2026-09-27T13:54:18Z"), + ("phase4-node-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688330126, "phase4-node-linux-arm64", 88276678, "ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e", "2026-09-27T13:56:54Z"), + ("phase4-toolkit-linux-amd64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688263793, "phase4-toolkit-linux-amd64", 53080691, "9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b", "2026-09-27T13:54:55Z"), + ("phase4-toolkit-linux-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9688255774, "phase4-toolkit-linux-arm64", 51636493, "e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd", "2026-09-27T13:54:40Z"), + ("phase4-toolkit-macos-arm64", ".github/workflows/phase4-payloads.yml", 33177534764, "pull_request", "codex/00002-phase4-node-toolkit-celestia", "656cd9664d23bda4ef0578d62c9e27392bff063e", 9689647047, "phase4-toolkit-macos-arm64", 97365627, "ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e", "2026-09-27T14:35:47Z"), + ("phase5-indexer", ".github/workflows/phase5-indexer.yml", 33176004154, "pull_request", "codex/00002-phase5-indexer", "e581add8952bae5ffeac39fb07e6b5c6f482862d", 9690093579, "phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a", 377650526, "eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9", "2026-09-27T14:49:01Z"), +] +SOFTWARE = [ + ("celestia-appd-linux-arm64-v6.4.10.tar.gz", "celestia-appd-6.4.10-linux-arm64", "desired", "linux", "arm64", 180685852, "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b", "phase4-celestia-appd-linux-arm64"), + ("celestia-node-linux-arm64-v0.28.4.tar.gz", "celestia-node-0.28.4-linux-arm64", "desired", "linux", "arm64", 71184641, "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c", "phase4-celestia-node-linux-arm64"), + ("indexer-standalone-linux-amd64-v4.4.0-rc.3.zip", "indexer-standalone-linux-amd64-4.4.0-rc.3", "required", "linux", "amd64", 31479027, "4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434", "phase5-indexer"), + ("indexer-standalone-linux-arm64-v4.4.0-rc.3.zip", "indexer-standalone-linux-arm64-4.4.0-rc.3", "desired", "linux", "arm64", 29782570, "eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0", "phase5-indexer"), + ("indexer-standalone-macos-amd64-v4.4.0-rc.3.zip", "indexer-standalone-macos-amd64-4.4.0-rc.3", "optional", "macos", "amd64", 30713420, "28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc", "phase5-indexer"), + ("indexer-standalone-macos-arm64-v4.4.0-rc.3.zip", "indexer-standalone-macos-arm64-4.4.0-rc.3", "required", "macos", "arm64", 29072181, "b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2", "phase5-indexer"), + ("midnight-node-linux-arm64-2.0.0-rc.4.zip", "midnight-node-2.0.0-rc.4-linux-arm64", "desired", "linux", "arm64", 82544614, "490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c", "phase4-node-linux-arm64"), + ("midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-linux-amd64", "required", "linux", "amd64", 50017428, "92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c", "phase4-toolkit-linux-amd64"), + ("midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-linux-arm64", "desired", "linux", "arm64", 48585850, "4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528", "phase4-toolkit-linux-arm64"), + ("midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip", "midnight-node-toolkit-2.0.0-rc.4-macos-arm64", "required", "macos", "arm64", 45553847, "8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c", "phase4-toolkit-macos-arm64"), +] +EXISTING_NAMES = { + "celestia-appd-linux-amd64-v6.4.10.tar.gz": ("celestia-appd", "6.4.10", "linux", "amd64", "required"), + "celestia-appd-macos-arm64-v6.4.10.tar.gz": ("celestia-appd", "6.4.10", "macos", "arm64", "required"), + "celestia-node-linux-amd64-v0.28.4.tar.gz": ("celestia-node", "0.28.4", "linux", "amd64", "required"), + "celestia-node-macos-arm64-v0.28.4.tar.gz": ("celestia-node", "0.28.4", "macos", "arm64", "required"), + "midnight-node-linux-amd64-2.0.0-rc.4.zip": ("midnight-node", "2.0.0-rc.4", "linux", "amd64", "required"), + "midnight-node-macos-arm64-2.0.0-rc.4.zip": ("midnight-node", "2.0.0-rc.4", "macos", "arm64", "required"), +} + + +def build(root: Path) -> dict: + component_paths = sorted((root / "catalog/components").glob("*.json")) + components = [] + component_by_id = {} + for path in component_paths: + component = load_json(path) + component_by_id[component["componentId"]] = component + components.append({"componentId": component["componentId"], "manifestPath": path.relative_to(root).as_posix(), "manifestSha256": sha256_file(path)[0]}) + inputs = [] + for key, workflow, run, event, ref, head, artifact, name, size, digest, expires in INPUTS: + inputs.append({"key": key, "repository": "acedward/midnight-binary-forge", "repositoryId": 1349127482, "workflowPath": workflow, "runId": run, "runAttempt": 1, "runEvent": event, "runConclusion": "success", "sourceRef": ref, "sourceHeadSha": head, "artifactId": artifact, "artifactName": name, "artifactSize": size, "archiveSha256": digest, "expiresAt": expires}) + payloads = [] + for name, component_id, tier, os_name, arch, size, digest, source_key in SOFTWARE: + component = component_by_id[component_id] + prefix = "payload" if source_key == "phase5-indexer" else "payloads" + payloads.append({"name": name, "role": "payload", "artifactKind": "software", "componentId": component_id, "tier": tier, "container": component["naming"]["container"], "size": size, "sha256": digest, "sourceArtifactKey": source_key, "sourcePath": f"{prefix}/{name}", "installMode": component["install"]["mode"], "os": os_name, "arch": arch}) + proof = load_json(root / "catalog/proof-data/q8b-v1.json") + for row in proof["srs"]: + payloads.append({"name": row["releaseName"], "role": "payload", "artifactKind": "proof-data", "componentId": row["componentId"], "tier": "noarch", "container": "raw", "size": row["size"], "sha256": row["sha256"], "sourceArtifactKey": "phase3p-proof-data", "sourcePath": f"payloads/{row['releaseName']}", "installMode": row["mode"], "platform": "noarch", "k": row["k"]}) + ledger = proof["ledgerStatic"] + payloads.append({"name": ledger["releaseName"], "role": "payload", "artifactKind": "proof-data", "componentId": ledger["componentId"], "tier": "noarch", "container": "zip", "size": ledger["archiveSize"], "sha256": ledger["archiveSha256"], "sourceArtifactKey": "phase3p-proof-data", "sourcePath": f"payloads/{ledger['releaseName']}", "installMode": "0644", "platform": "noarch", "ledgerStaticSemver": "9.0.0"}) + snapshot = load_json(root / SNAPSHOT_PATH) + assets = {row["name"]: row for row in snapshot["assets"]} + existing = [] + for name, (family, version, os_name, arch, tier) in sorted(EXISTING_NAMES.items()): + asset = assets[name] + existing.append({"family": family, "version": version, "os": os_name, "arch": arch, "tier": tier, "source": "warehouse-existing", "repository": "effectstream/binaries", "repositoryId": 1117580582, "repositoryNodeId": "R_kgDOQpztJg", "releaseTag": "0.3.120", "releaseId": 270761136, "releaseNodeId": "RE_kwDOQpztJs4QI3yw", "snapshotPath": SNAPSHOT_PATH, "snapshotSha256": SNAPSHOT_SHA, "assetId": asset["id"], "assetNodeId": asset["nodeId"], "name": name, "size": asset["size"], "sha256": asset["digest"].removeprefix("sha256:")}) + return {"schemaVersion": "build-set-v1", "buildSetId": "initial-warehouse-v1", "sourceFullSha": BASE_SHA, "destination": {"repository": "effectstream/binaries", "tag": "0.3.120", "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, "components": components, "inputArtifacts": inputs, "existingCoverage": existing, "payloads": sorted(payloads, key=lambda row: row["name"]), "payloadCount": len(payloads), "coveragePolicy": {"required": ["linux/amd64", "macos/arm64"], "desired": ["linux/arm64"], "optional": ["macos/amd64"], "proofDataPlatform": "noarch"}, "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "inputArtifactPinningRequired": True, "destinationCredentialAllowed": False}} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=ROOT / "catalog/buildsets/initial-warehouse-v1.json") + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + rendered = canonical_bytes(build(ROOT)) + if args.check: + if args.output.read_bytes() != rendered: + raise SystemExit("generated Phase-6 build set differs from committed bytes") + else: + create_file_atomic(args.output, rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/github_phase6.py b/scripts/github_phase6.py new file mode 100644 index 0000000..2088547 --- /dev/null +++ b/scripts/github_phase6.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 +"""Fail-closed GitHub API boundary for Phase-6 candidate publication.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email.utils +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from pathlib import Path +from typing import Any + +import canonical_json +import phase6_candidate +import publisher_guard +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, load_json, sha256_file + + +API = "https://api.github.com" +REPOSITORY = phase6_candidate.REPOSITORY +REPOSITORY_ID = phase6_candidate.REPOSITORY_ID +MAIN_REF = "refs/heads/main" +WORKFLOW_PATH = ".github/workflows/candidate.yml" +DEFAULT_API_VERSION = "2022-11-28" +IMMUTABLE_RELEASES_API_VERSION = "2026-03-10" +IMMUTABLE_RELEASES_PATH = f"/repos/{REPOSITORY}/immutable-releases" + + +def token() -> str: + value = os.environ.get("GITHUB_TOKEN", "") + expect(bool(value), "GITHUB_TOKEN is required") + return value + + +def request(path: str, method: str = "GET", body: Any | None = None, accept: str = "application/vnd.github+json", api_version: str = DEFAULT_API_VERSION) -> tuple[Any, str]: + data = None if body is None else canonical_bytes(body) + req = urllib.request.Request( + API + path, + data=data, + method=method, + headers={"Accept": accept, "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": api_version, **({"Content-Type": "application/json"} if data is not None else {})}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read() + date = response.headers.get("Date", "") + except urllib.error.HTTPError as exc: + detail = exc.read(4096).decode("utf-8", "replace") + raise ForgeError(f"GitHub API {method} {path} failed with HTTP {exc.code}: {detail}") from exc + return (json.loads(raw) if raw else None), date + + +def api_time(header: str) -> str: + expect(bool(header), "GitHub API response omitted Date") + parsed = email.utils.parsedate_to_datetime(header) + expect(parsed is not None and parsed.tzinfo is not None, "GitHub API Date is invalid") + return parsed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def verify_immutable_policy_evidence(value: dict[str, Any], boundary: str) -> None: + expect(isinstance(value, dict) and set(value) == {"boundary", "endpoint", "apiVersion", "capturedAt", "enabled", "enforcedByOwner"}, f"immutable-release policy evidence is malformed at {boundary}") + expect(value["boundary"] == boundary and value["endpoint"] == IMMUTABLE_RELEASES_PATH and value["apiVersion"] == IMMUTABLE_RELEASES_API_VERSION, f"immutable-release policy authority differs at {boundary}") + canonical_json.parse_time(value["capturedAt"], f"immutable-release policy capturedAt at {boundary}") + expect(value["enabled"] is True, f"immutable releases are not enabled at {boundary}") + expect(type(value["enforcedByOwner"]) is bool, f"immutable-release owner-enforcement state has wrong type at {boundary}") + + +def immutable_release_policy(boundary: str) -> dict[str, Any]: + value, date = request(IMMUTABLE_RELEASES_PATH, api_version=IMMUTABLE_RELEASES_API_VERSION) + expect(isinstance(value, dict) and set(value) == {"enabled", "enforced_by_owner"}, f"immutable-release policy response is malformed at {boundary}") + expect(value["enabled"] is True, f"immutable releases are not enabled at {boundary}") + expect(type(value["enforced_by_owner"]) is bool, f"immutable-release owner-enforcement state has wrong type at {boundary}") + evidence = { + "boundary": boundary, + "endpoint": IMMUTABLE_RELEASES_PATH, + "apiVersion": IMMUTABLE_RELEASES_API_VERSION, + "capturedAt": api_time(date), + "enabled": True, + "enforcedByOwner": value["enforced_by_owner"], + } + verify_immutable_policy_evidence(evidence, boundary) + return evidence + + +def pagination(path: str) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for page in range(1, 12): + join = "&" if "?" in path else "?" + value, _ = request(f"{path}{join}per_page=100&page={page}") + expect(isinstance(value, list), f"GitHub API pagination response is not an array: {path}") + result.extend(value) + if len(value) < 100: + return result + raise ForgeError(f"GitHub API pagination ceiling exceeded: {path}") + + +def require_actions_context() -> None: + expect(os.environ.get("GITHUB_ACTIONS") == "true", "Phase-6 GitHub mutation is Actions-only") + expect(os.environ.get("GITHUB_REPOSITORY") == REPOSITORY, "Phase-6 Actions repository mismatch") + expect(os.environ.get("GITHUB_REF") == MAIN_REF, "Phase-6 mutation requires protected main") + expect(os.environ.get("GITHUB_EVENT_NAME") == "workflow_dispatch", "Phase-6 mutation requires workflow_dispatch") + + +def capture_inputs(buildset_path: Path, output: Path) -> None: + buildset, _ = phase6_candidate.validate_buildset(buildset_path) + repository, date = request(f"/repos/{REPOSITORY}") + runs = [] + for run_id in sorted({row["runId"] for row in buildset["inputArtifacts"]}): + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + runs.append(run) + artifacts = [] + for artifact_id in sorted(row["artifactId"] for row in buildset["inputArtifacts"]): + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}") + artifacts.append(artifact) + value = {"schemaVersion": "phase6-input-live-metadata-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"]}, "runs": runs, "artifacts": artifacts} + phase6_candidate.verify_live_metadata(buildset, value) + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def verify_source(expected_sha: str, output: Path) -> None: + require_actions_context() + expect(os.environ.get("GITHUB_SHA") == expected_sha and re.fullmatch(r"[0-9a-f]{40}", expected_sha) is not None, "protected source full SHA mismatch") + repository, date = request(f"/repos/{REPOSITORY}") + expect(repository["id"] == REPOSITORY_ID and repository["full_name"] == REPOSITORY and repository["default_branch"] == "main", "live repository/default branch identity mismatch") + immutable_policy = immutable_release_policy("protected-source-admission") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + expect(branch.get("protected") is True and branch.get("commit", {}).get("sha") == expected_sha, "current source is not live protected main") + rulesets = pagination(f"/repos/{REPOSITORY}/rulesets") + active = [row for row in rulesets if row.get("enforcement") == "active"] + expect(active, "no active repository ruleset") + details = [request(f"/repos/{REPOSITORY}/rulesets/{row['id']}")[0] for row in active] + expect(any(any(rule.get("type") == "pull_request" for rule in row.get("rules", [])) and not row.get("bypass_actors") for row in details), "active PR-only/no-bypass ruleset absent") + environment, _ = request(f"/repos/{REPOSITORY}/environments/candidate-publish") + policy = environment.get("deployment_branch_policy") or {} + expect(policy.get("protected_branches") is True and policy.get("custom_branch_policies") is False, "candidate-publish is not protected-branch-only") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={expected_sha}") + expect(workflow.get("type") == "file" and re.fullmatch(r"[0-9a-f]{40}", workflow.get("sha", "")), "cannot bind candidate workflow blob") + value = {"schemaVersion": "phase6-protected-source-v1", "capturedAt": api_time(date), "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, "immutableReleasePolicy": immutable_policy, "ref": MAIN_REF, "commitSha": expected_sha, "protected": True, "workflowPath": WORKFLOW_PATH, "workflowSha": workflow["sha"], "rulesetIds": sorted(row["id"] for row in details), "candidateEnvironment": "candidate-publish", "protectedBranchesOnly": True, "referencedRepositoryVariables": 0, "referencedDestinationSecrets": 0} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def capture_staging(artifact_id: int, expected_name: str, run_id: int, run_attempt: int, output: Path) -> None: + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}") + workflow = artifact.get("workflow_run", {}) + expect(artifact.get("id") == artifact_id and artifact.get("name") == expected_name, "staging artifact ID/name mismatch") + expect(artifact.get("expired") is False and workflow.get("id") == run_id and workflow.get("repository_id") == REPOSITORY_ID, "staging artifact is expired or belongs to another run/repository") + digest = artifact.get("digest", "") + expect(re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is not None, "staging artifact has no SHA-256 digest") + value = {"artifactId": artifact_id, "artifactName": expected_name, "archiveSha256": digest.removeprefix("sha256:"), "expiresAt": artifact["expires_at"], "runId": run_id, "runAttempt": run_attempt} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def allocate_draft(expected_sha: str, source_path: Path, output: Path) -> None: + require_actions_context() + source = load_json(source_path) + policy = source.get("immutableReleasePolicy", {}) + expect(source.get("commitSha") == expected_sha, "protected-source evidence mismatch") + verify_immutable_policy_evidence(policy, "protected-source-admission") + _, date_header = request("/rate_limit") + date = api_time(date_header)[:10].replace("-", ".") + releases = pagination(f"/repos/{REPOSITORY}/releases") + tags = pagination(f"/repos/{REPOSITORY}/tags") + occupied = {row.get("tag_name") for row in releases} | {row.get("name") for row in tags} + sequence = next(number for number in range(1, 10000) if f"forge-{date}.{number}" not in occupied) + tag = f"forge-{date}.{sequence}" + body = phase6_candidate.WARNING + "\n\nThis candidate is immutable supply-chain evidence. Only typed role=payload assets are eligible for the separately reviewed manual warehouse transaction." + before_create = immutable_release_policy("immediately-before-draft-create") + created, _ = request(f"/repos/{REPOSITORY}/releases", "POST", {"tag_name": tag, "target_commitish": expected_sha, "name": tag, "body": body, "draft": True, "prerelease": False}) + reread, _ = request(f"/repos/{REPOSITORY}/releases/{created['id']}") + expect(reread["id"] == created["id"] and reread["tag_name"] == tag and reread["target_commitish"] == expected_sha and reread["draft"] is True and reread["prerelease"] is False and reread.get("assets") == [], "allocated draft read-back mismatch") + value = {key: reread[key] for key in ("id", "node_id", "html_url", "tag_name", "target_commitish")} + value["immutableReleasePolicyBeforeCreate"] = before_create + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def download_asset(asset_id: int, output: Path) -> None: + req = urllib.request.Request(API + f"/repos/{REPOSITORY}/releases/assets/{asset_id}", headers={"Accept": "application/octet-stream", "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": "2022-11-28"}) + with urllib.request.urlopen(req, timeout=120) as response, output.open("xb") as stream: + while True: + block = response.read(1024 * 1024) + if not block: + break + stream.write(block) + + +def run_artifacts(run_id: int) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + total_count: int | None = None + for page in range(1, 12): + value, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}/artifacts?per_page=100&page={page}") + expect(isinstance(value, dict) and set(value) >= {"total_count", "artifacts"}, "run-artifact response is malformed") + expect(type(value["total_count"]) is int and value["total_count"] >= 0 and isinstance(value["artifacts"], list), "run-artifact pagination fields are malformed") + if total_count is None: + total_count = value["total_count"] + expect(value["total_count"] == total_count, "run-artifact total changed during pagination") + rows.extend(value["artifacts"]) + if len(value["artifacts"]) < 100: + expect(len(rows) == total_count, "run-artifact pagination is incomplete") + return rows + raise ForgeError("run-artifact pagination ceiling exceeded") + + +def download_action_artifact(artifact_id: int, output: Path, max_bytes: int = 8 * 1024 * 1024) -> None: + req = urllib.request.Request( + API + f"/repos/{REPOSITORY}/actions/artifacts/{artifact_id}/zip", + headers={"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token()}", "User-Agent": "midnight-binary-forge/phase6", "X-GitHub-Api-Version": DEFAULT_API_VERSION}, + ) + total = 0 + with urllib.request.urlopen(req, timeout=120) as response, output.open("xb") as stream: + while True: + block = response.read(1024 * 1024) + if not block: + break + total += len(block) + expect(total <= max_bytes, "retained JSON artifact exceeds recovery byte ceiling") + stream.write(block) + + +def retained_json_artifact(artifact: dict[str, Any], expected_inner_name: str, output: Path) -> tuple[Any, dict[str, Any]]: + expect(type(artifact.get("id")) is int and artifact["id"] > 0, "retained artifact ID is invalid") + expect(artifact.get("expired") is False, "retained recovery artifact is expired") + wrapper_digest = artifact.get("digest", "") + expect(isinstance(wrapper_digest, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", wrapper_digest) is not None, "retained artifact wrapper digest is missing") + with tempfile.TemporaryDirectory(prefix="phase6-recovery-artifact-") as temporary_text: + archive_path = Path(temporary_text) / "artifact.zip" + download_action_artifact(artifact["id"], archive_path) + actual_wrapper_digest, actual_wrapper_size = sha256_file(archive_path, 8 * 1024 * 1024) + expect(actual_wrapper_digest == wrapper_digest.removeprefix("sha256:"), "retained artifact wrapper digest mismatch") + expect(artifact.get("size_in_bytes") == actual_wrapper_size, "retained artifact wrapper size mismatch") + with zipfile.ZipFile(archive_path) as archive: + infos = archive.infolist() + expect(len(infos) == 1 and infos[0].filename == expected_inner_name and not infos[0].is_dir(), "retained JSON artifact layout mismatch") + info = infos[0] + expect(info.flag_bits & 1 == 0 and 0 < info.file_size <= 4 * 1024 * 1024, "retained JSON artifact member is unsafe") + expect(info.compress_size <= 4 * 1024 * 1024 and (info.compress_size > 0 or info.file_size == 0), "retained JSON artifact compression bounds invalid") + raw = archive.read(info) + expect(len(raw) == info.file_size, "retained JSON artifact member size mismatch") + create_file_atomic(output, raw, 0o600) + value = canonical_json.load_json(output) + expect(raw == canonical_json.canonical_bytes(value), "retained JSON artifact is not canonical") + evidence = { + "id": artifact["id"], + "name": artifact["name"], + "archiveSize": actual_wrapper_size, + "archiveSha256": actual_wrapper_digest, + "expiresAt": artifact["expires_at"], + "innerName": expected_inner_name, + "innerSize": len(raw), + "innerSha256": hashlib.sha256(raw).hexdigest(), + } + return value, evidence + + +def verify_recovery_evidence(value: dict[str, Any], claims: dict[str, Any], draft: dict[str, Any], envelope: dict[str, Any], bundle_path: Path) -> None: + expect(set(value) == {"schemaVersion", "recoveryMode", "capturedAt", "repository", "protectedRef", "workflowFile", "originalRun", "retainedArtifacts", "missingPublishedHandoff", "immutableReleasePolicy", "release", "releaseAssets", "claimsSha256", "draftSha256", "envelopeSha256", "bundleSha256"}, "recovery evidence fields differ") + expect(value["schemaVersion"] == "phase6-recovered-publication-v1" and value["recoveryMode"] == "read-only-post-publication-handoff-loss", "wrong recovery evidence mode") + canonical_json.parse_time(value["capturedAt"], "recovery capturedAt") + canonical_json.verify_envelope(envelope) + expect(envelope["claims"] == claims and canonical_json.verify_claims(claims) == value["claimsSha256"], "recovered claims/envelope binding mismatch") + expect(value["draftSha256"] == canonical_json.digest(draft) and value["envelopeSha256"] == canonical_json.digest(envelope), "recovered canonical JSON digest mismatch") + bundle_digest, bundle_size = sha256_file(bundle_path, 2**31 - 1) + expect(bundle_size > 0 and bundle_digest == value["bundleSha256"] == envelope["attestation"]["bundleSha256"], "recovered bundle binding mismatch") + issuer = claims["issuer"] + staging = claims["staging"] + candidate = claims["candidateDraft"] + run = value["originalRun"] + expect(run == {"id": staging["runId"], "attempt": staging["runAttempt"], "repository": REPOSITORY, "workflowPath": WORKFLOW_PATH, "event": "workflow_dispatch", "headSha": issuer["commitSha"], "headRef": "main", "status": "completed", "conclusion": "failure"}, "original failed run identity differs from signed claims") + expect(value["repository"] == {"fullName": REPOSITORY, "id": REPOSITORY_ID, "nodeId": canonical_json.REPOSITORY_NODE_ID}, "recovery repository identity mismatch") + expect(value["protectedRef"] == {"ref": MAIN_REF, "commitSha": issuer["commitSha"], "protected": True}, "recovery protected source mismatch") + expect(value["workflowFile"] == {"path": WORKFLOW_PATH, "commitSha": issuer["commitSha"], "blobSha": issuer["workflowSha"]}, "recovery workflow binding mismatch") + expect(value["missingPublishedHandoff"] == {"name": f"published-candidate-{claims['buildSet']['id']}", "confirmedAbsent": True}, "recovery handoff-loss classification mismatch") + policy = value["immutableReleasePolicy"] + verify_immutable_policy_evidence(policy, "recovery-live-readback") + verify_immutable_policy_evidence(draft.get("immutableReleasePolicyBeforeCreate", {}), "immediately-before-draft-create") + expect(draft["id"] == candidate["releaseId"] and draft["node_id"] == candidate["releaseNodeId"] and draft["tag_name"] == candidate["tag"] and draft["target_commitish"] == issuer["commitSha"] and draft["html_url"] == candidate["releaseUrl"], "retained draft differs from signed candidate") + retained = value["retainedArtifacts"] + expect(set(retained) == {"claims", "draft", "staging"}, "recovery retained-artifact set differs") + for label, expected_artifact_name, expected_inner_name, expected_inner_digest in ( + ("claims", f"phase6-claims-{issuer['commitSha']}", f"promotion-claims-{claims['buildSet']['id']}.json", canonical_json.digest(claims)), + ("draft", f"phase6-draft-{issuer['commitSha']}", "draft.json", canonical_json.digest(draft)), + ): + artifact_record = retained[label] + expect(set(artifact_record) == {"id", "name", "archiveSize", "archiveSha256", "expiresAt", "innerName", "innerSize", "innerSha256"}, f"recovery retained {label} artifact fields differ") + expect(type(artifact_record["id"]) is int and artifact_record["id"] > 0 and type(artifact_record["archiveSize"]) is int and artifact_record["archiveSize"] > 0 and type(artifact_record["innerSize"]) is int and artifact_record["innerSize"] > 0, f"recovery retained {label} artifact sizes/ID invalid") + expect(artifact_record["name"] == expected_artifact_name and artifact_record["innerName"] == expected_inner_name and artifact_record["innerSha256"] == expected_inner_digest, f"recovery retained {label} artifact binding mismatch") + expect(re.fullmatch(r"[0-9a-f]{64}", artifact_record["archiveSha256"]) is not None, f"recovery retained {label} wrapper digest invalid") + canonical_json.parse_time(artifact_record["expiresAt"], f"recovery retained {label} expiresAt") + expect(artifact_record["innerSize"] == len(canonical_json.canonical_bytes(claims if label == "claims" else draft)), f"recovery retained {label} member size mismatch") + expect(retained["claims"]["id"] != retained["draft"]["id"], "recovery retained artifact IDs collide") + expect(set(retained["staging"]) == {"id", "name", "archiveSha256", "expiresAt", "runId"}, "recovery staging artifact fields differ") + expect(retained["staging"]["id"] == staging["artifactId"] and retained["staging"]["name"] == staging["artifactName"] and retained["staging"]["archiveSha256"] == staging["archiveSha256"] and retained["staging"]["runId"] == staging["runId"], "recovery staging artifact differs from claims") + canonical_json.parse_time(retained["staging"]["expiresAt"], "recovery staging expiresAt") + release = value["release"] + expect(release == {"id": candidate["releaseId"], "nodeId": candidate["releaseNodeId"], "repository": REPOSITORY, "tag": candidate["tag"], "targetCommitish": issuer["commitSha"], "url": candidate["releaseUrl"], "draft": False, "prerelease": False, "immutable": True}, "recovered release is not the exact signed immutable publication") + asset_rows = value["releaseAssets"] + expect(isinstance(asset_rows, list) and len(asset_rows) == claims["totalAssetCount"] == 54, "recovered release asset count mismatch") + expect([row["name"] for row in asset_rows] == claims["completeAssetNames"], "recovered release asset names differ") + by_name = {row["name"]: row for row in asset_rows} + expect(len(by_name) == len(asset_rows) and all(set(row) == {"id", "name", "size", "sha256"} and type(row["id"]) is int and row["id"] > 0 for row in asset_rows), "recovered release asset rows are malformed") + for content in claims["contentAssets"]: + expect((by_name[content["name"]]["size"], by_name[content["name"]]["sha256"]) == (content["size"], content["sha256"]), f"recovered content asset differs: {content['name']}") + expect((by_name[claims["transport"]["attestationBundleName"]]["size"], by_name[claims["transport"]["attestationBundleName"]]["sha256"]) == (bundle_size, bundle_digest), "recovered bundle asset differs") + envelope_size = len(canonical_json.canonical_bytes(envelope)) + expect((by_name[claims["transport"]["envelopeName"]]["size"], by_name[claims["transport"]["envelopeName"]]["sha256"]) == (envelope_size, canonical_json.digest(envelope)), "recovered envelope asset differs") + + +def recover_publication(run_id: int, expected_head: str, build_set_id: str, output_dir: Path) -> None: + expect(os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("GITHUB_REPOSITORY") == REPOSITORY and os.environ.get("GITHUB_EVENT_NAME") == "workflow_run", "Phase-6 recovery requires the exact read-only workflow_run context") + expect(re.fullmatch(r"[0-9a-f]{40}", expected_head) is not None and re.fullmatch(r"[a-z0-9][a-z0-9._-]{2,127}", build_set_id) is not None, "recovery input identity is malformed") + expect(not output_dir.exists(), "recovery output path already exists") + output_dir.mkdir(mode=0o700) + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + expect(run.get("id") == run_id and run.get("run_attempt") == 1 and run.get("event") == "workflow_dispatch" and run.get("status") == "completed" and run.get("conclusion") == "failure", "recovery is only for an exact completed failed candidate run") + expect(run.get("path") == WORKFLOW_PATH and run.get("head_sha") == expected_head and run.get("head_branch") == "main" and run.get("repository", {}).get("full_name") == REPOSITORY and run.get("repository", {}).get("id") == REPOSITORY_ID, "recovery run repository/workflow/head identity mismatch") + artifacts = run_artifacts(run_id) + for row in artifacts: + workflow = row.get("workflow_run", {}) + expect(workflow.get("id") == run_id and workflow.get("repository_id") == REPOSITORY_ID, "recovery artifact belongs to another run/repository") + by_name: dict[str, list[dict[str, Any]]] = {} + for row in artifacts: + by_name.setdefault(row.get("name", ""), []).append(row) + draft_name = f"phase6-draft-{expected_head}" + claims_name = f"phase6-claims-{expected_head}" + published_name = f"published-candidate-{build_set_id}" + expect(by_name.get(published_name, []) == [], "published handoff already exists; recovery is neither required nor allowed") + expect(len(by_name.get(draft_name, [])) == 1 and len(by_name.get(claims_name, [])) == 1, "exact retained draft/claims artifacts are missing or ambiguous") + draft, draft_artifact = retained_json_artifact(by_name[draft_name][0], "draft.json", output_dir / "draft.json") + claims_inner_name = f"promotion-claims-{build_set_id}.json" + claims, claims_artifact = retained_json_artifact(by_name[claims_name][0], claims_inner_name, output_dir / claims_inner_name) + canonical_json.verify_claims(claims) + expect(claims["issuer"]["commitSha"] == expected_head and claims["staging"]["runId"] == run_id and claims["staging"]["runAttempt"] == run["run_attempt"] and claims["buildSet"]["id"] == build_set_id, "retained claims differ from recovery run/input") + candidate = claims["candidateDraft"] + expect(draft.get("id") == candidate["releaseId"] and draft.get("tag_name") == candidate["tag"] and draft.get("target_commitish") == expected_head, "retained draft differs from signed claims") + staging_rows = by_name.get(claims["staging"]["artifactName"], []) + expect(len(staging_rows) == 1 and staging_rows[0].get("id") == claims["staging"]["artifactId"], "exact staging artifact is missing or substituted") + staging_row = staging_rows[0] + expect(staging_row.get("digest") == f"sha256:{claims['staging']['archiveSha256']}", "staging artifact wrapper digest differs from claims") + release, _ = request(f"/repos/{REPOSITORY}/releases/{candidate['releaseId']}") + expect(release.get("id") == candidate["releaseId"] and release.get("node_id") == candidate["releaseNodeId"] and release.get("tag_name") == candidate["tag"] and release.get("target_commitish") == expected_head and release.get("html_url") == candidate["releaseUrl"], "live release identity differs from retained claims/draft") + expect(release.get("draft") is False and release.get("prerelease") is False and release.get("immutable") is True, "candidate run failed before an immutable publication; abandon recovery without mutation") + policy = immutable_release_policy("recovery-live-readback") + release_assets = pagination(f"/repos/{REPOSITORY}/releases/{candidate['releaseId']}/assets") + expect(len(release_assets) == claims["totalAssetCount"] == 54 and sorted(row.get("name") for row in release_assets) == claims["completeAssetNames"], "live immutable release does not have the exact signed 54-asset set") + asset_evidence = [] + with tempfile.TemporaryDirectory(prefix="phase6-recovery-release-") as temporary_text: + temporary_root = Path(temporary_text) + for asset in sorted(release_assets, key=lambda row: row["name"]): + expect(type(asset.get("id")) is int and asset["id"] > 0 and asset.get("state") == "uploaded", "recovery release asset API identity is malformed") + path = temporary_root / str(asset["id"]) + download_asset(asset["id"], path) + digest, size = sha256_file(path, 2**31 - 1) + expect(asset.get("size") == size and asset.get("digest") == f"sha256:{digest}", f"recovery release asset API/download mismatch: {asset['name']}") + asset_evidence.append({"id": asset["id"], "name": asset["name"], "size": size, "sha256": digest}) + if asset["name"] in {claims["transport"]["envelopeName"], claims["transport"]["attestationBundleName"]}: + create_file_atomic(output_dir / asset["name"], path.read_bytes(), 0o600) + envelope_path = output_dir / claims["transport"]["envelopeName"] + bundle_path = output_dir / claims["transport"]["attestationBundleName"] + envelope = canonical_json.load_json(envelope_path) + canonical_json.verify_envelope(envelope) + expect(envelope["claims"] == claims, "released envelope claims differ from retained original-run claims") + repository, date = request(f"/repos/{REPOSITORY}") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={expected_head}") + retained = { + "claims": claims_artifact, + "draft": draft_artifact, + "staging": {"id": staging_row["id"], "name": staging_row["name"], "archiveSha256": staging_row["digest"].removeprefix("sha256:"), "expiresAt": staging_row["expires_at"], "runId": run_id}, + } + value = { + "schemaVersion": "phase6-recovered-publication-v1", + "recoveryMode": "read-only-post-publication-handoff-loss", + "capturedAt": api_time(date), + "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, + "protectedRef": {"ref": MAIN_REF, "commitSha": branch["commit"]["sha"], "protected": branch["protected"]}, + "workflowFile": {"path": WORKFLOW_PATH, "commitSha": expected_head, "blobSha": workflow["sha"]}, + "originalRun": {"id": run["id"], "attempt": run["run_attempt"], "repository": run["repository"]["full_name"], "workflowPath": run["path"], "event": run["event"], "headSha": run["head_sha"], "headRef": run["head_branch"], "status": run["status"], "conclusion": run["conclusion"]}, + "retainedArtifacts": retained, + "missingPublishedHandoff": {"name": published_name, "confirmedAbsent": True}, + "immutableReleasePolicy": policy, + "release": {"id": release["id"], "nodeId": release["node_id"], "repository": REPOSITORY, "tag": release["tag_name"], "targetCommitish": release["target_commitish"], "url": release["html_url"], "draft": release["draft"], "prerelease": release["prerelease"], "immutable": release["immutable"]}, + "releaseAssets": asset_evidence, + "claimsSha256": canonical_json.digest(claims), + "draftSha256": canonical_json.digest(draft), + "envelopeSha256": canonical_json.digest(envelope), + "bundleSha256": sha256_file(bundle_path, 2**31 - 1)[0], + } + verify_recovery_evidence(value, claims, draft, envelope, bundle_path) + create_file_atomic(output_dir / "recovered-publication.json", canonical_bytes(value), 0o600) + + +def publish(claims_path: Path, content: Path, bundle: Path, envelope: Path, draft_path: Path, output: Path) -> None: + require_actions_context() + publisher_guard.verify_transport(envelope, bundle, content) + claims = load_json(claims_path) + draft = load_json(draft_path) + verify_immutable_policy_evidence(draft.get("immutableReleasePolicyBeforeCreate", {}), "immediately-before-draft-create") + expect(claims == load_json(envelope)["claims"] and draft["id"] == claims["candidateDraft"]["releaseId"], "publisher draft/claims/envelope mismatch") + release, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}") + expect(release["draft"] is True and release["tag_name"] == draft["tag_name"] and release["target_commitish"] == claims["issuer"]["commitSha"], "publisher draft state mismatch") + expect(pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") == [], "publisher draft is not empty") + paths = {path.name: path for path in content.iterdir()} + paths[bundle.name] = bundle + paths[envelope.name] = envelope + expect(sorted(paths) == claims["completeAssetNames"], "publisher complete local asset-name set mismatch") + expected = {row["name"]: (row["size"], row["sha256"]) for row in claims["contentAssets"]} + expected[bundle.name] = (bundle.stat().st_size, sha256_file(bundle)[0]) + expected[envelope.name] = (envelope.stat().st_size, sha256_file(envelope)[0]) + env = {**os.environ, "GH_TOKEN": token()} + policy_checks = [immutable_release_policy("immediately-before-draft-asset-upload")] + for name in sorted(paths): + subprocess.run(["gh", "release", "upload", draft["tag_name"], str(paths[name]), "--repo", REPOSITORY], check=True, env=env, timeout=600) + rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") + observed = next((row for row in rows if row.get("name") == name), None) + expect(observed is not None and observed.get("state") == "uploaded", f"uploaded asset read-back missing: {name}") + size, digest = expected[name] + expect(observed.get("size") == size and observed.get("digest") == f"sha256:{digest}", f"uploaded asset API identity mismatch: {name}") + temporary = Path(tempfile.gettempdir()) / f"phase6-readback-{os.getpid()}-{observed['id']}" + expect(not temporary.exists(), "read-back temporary path collision") + try: + download_asset(observed["id"], temporary) + actual_digest, actual_size = sha256_file(temporary, 2**31 - 1) + expect((actual_size, actual_digest) == (size, digest), f"uploaded asset independent download mismatch: {name}") + finally: + temporary.unlink(missing_ok=True) + rows = pagination(f"/repos/{REPOSITORY}/releases/{draft['id']}/assets") + expect(sorted(row["name"] for row in rows) == claims["completeAssetNames"], "complete uploaded asset set mismatch") + policy_checks.append(immutable_release_policy("immediately-before-public-transition")) + published, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"draft": False}) + policy_checks.append(immutable_release_policy("immediately-after-publication")) + reread, _ = request(f"/repos/{REPOSITORY}/releases/{draft['id']}") + expect(published["id"] == reread["id"] and reread["draft"] is False and reread["prerelease"] is False and reread.get("immutable") is True, "published release is not immutable") + mutation_rejected = False + try: + request(f"/repos/{REPOSITORY}/releases/{draft['id']}", "PATCH", {"name": reread["name"]}) + except ForgeError: + mutation_rejected = True + expect(mutation_rejected, "immutable published release accepted a no-op metadata mutation probe") + value = {"schemaVersion": "phase6-published-candidate-v1", "releaseId": reread["id"], "tag": reread["tag_name"], "targetCommitish": reread["target_commitish"], "immutable": True, "assetCount": len(rows), "completeAssetNameListSha256": claims["completeAssetNameListSha256"], "immutableReleasePolicyChecks": policy_checks, "mutationRejected": True} + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def capture_live(envelope_path: Path, bundle_path: Path, run_id: int, output: Path) -> None: + envelope = canonical_json.load_json(envelope_path) + canonical_json.verify_envelope(envelope) + claims = envelope["claims"] + expect(claims["staging"]["runId"] == run_id, "candidate run ID differs from signed staging run") + repository, date = request(f"/repos/{REPOSITORY}") + branch, _ = request(f"/repos/{REPOSITORY}/branches/main") + workflow, _ = request(f"/repos/{REPOSITORY}/contents/{WORKFLOW_PATH}?ref={claims['issuer']['commitSha']}") + run, _ = request(f"/repos/{REPOSITORY}/actions/runs/{run_id}") + artifact, _ = request(f"/repos/{REPOSITORY}/actions/artifacts/{claims['staging']['artifactId']}") + release, _ = request(f"/repos/{REPOSITORY}/releases/{claims['candidateDraft']['releaseId']}") + immutable_release_policy("live-verifier-readback") + assets = pagination(f"/repos/{REPOSITORY}/releases/{release['id']}/assets") + asset_rows = [] + for asset in sorted(assets, key=lambda row: row["name"]): + temporary = Path(tempfile.gettempdir()) / f"phase6-live-{os.getpid()}-{asset['id']}" + expect(not temporary.exists(), "live-evidence temporary path collision") + try: + download_asset(asset["id"], temporary) + digest, size = sha256_file(temporary, 2**31 - 1) + expect(asset.get("state") == "uploaded" and asset.get("size") == size and asset.get("digest") == f"sha256:{digest}", f"live release asset API/download mismatch: {asset['name']}") + asset_rows.append({"name": asset["name"], "size": size, "sha256": digest}) + finally: + temporary.unlink(missing_ok=True) + value = { + "schemaVersion": "promotion-live-evidence-v1", + "capturedAt": api_time(date), + "repository": {"fullName": repository["full_name"], "id": repository["id"], "nodeId": repository["node_id"]}, + "protectedRef": {"ref": MAIN_REF, "commitSha": branch["commit"]["sha"], "protected": branch["protected"]}, + "workflowFile": {"path": WORKFLOW_PATH, "commitSha": claims["issuer"]["commitSha"], "blobSha": workflow["sha"]}, + "run": {"id": run["id"], "attempt": run["run_attempt"], "repository": run["repository"]["full_name"], "workflowPath": run["path"], "event": run["event"], "headSha": run["head_sha"], "headRef": run["head_branch"], "status": run["status"], "conclusion": run["conclusion"]}, + "stagingArtifact": {"id": artifact["id"], "runId": artifact["workflow_run"]["id"], "runAttempt": claims["staging"]["runAttempt"], "name": artifact["name"], "archiveSha256": artifact["digest"].removeprefix("sha256:"), "expired": artifact["expired"], "expiresAt": artifact["expires_at"]}, + "release": {"id": release["id"], "nodeId": release["node_id"], "repository": REPOSITORY, "tag": release["tag_name"], "targetCommitish": release["target_commitish"], "url": release["html_url"], "draft": release["draft"], "prerelease": release["prerelease"], "immutable": release.get("immutable")}, + "releaseAssets": asset_rows, + } + canonical_json.verify_live_evidence(envelope, value, envelope_path, bundle_path, allow_expired_staging=False) + create_file_atomic(output, canonical_bytes(value), 0o600) + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + capture = sub.add_parser("capture-inputs") + capture.add_argument("--build-set", type=Path, required=True); capture.add_argument("--output", type=Path, required=True) + source = sub.add_parser("verify-source") + source.add_argument("--expected-sha", required=True); source.add_argument("--output", type=Path, required=True) + staging = sub.add_parser("capture-staging") + staging.add_argument("--artifact-id", type=int, required=True); staging.add_argument("--expected-name", required=True); staging.add_argument("--run-id", type=int, required=True); staging.add_argument("--run-attempt", type=int, required=True); staging.add_argument("--output", type=Path, required=True) + draft = sub.add_parser("allocate-draft") + draft.add_argument("--expected-sha", required=True); draft.add_argument("--source", type=Path, required=True); draft.add_argument("--output", type=Path, required=True) + publish_parser = sub.add_parser("publish") + publish_parser.add_argument("--claims", type=Path, required=True); publish_parser.add_argument("--content", type=Path, required=True); publish_parser.add_argument("--bundle", type=Path, required=True); publish_parser.add_argument("--envelope", type=Path, required=True); publish_parser.add_argument("--draft", type=Path, required=True); publish_parser.add_argument("--output", type=Path, required=True) + live = sub.add_parser("capture-live") + live.add_argument("--envelope", type=Path, required=True); live.add_argument("--bundle", type=Path, required=True); live.add_argument("--run-id", type=int, required=True); live.add_argument("--output", type=Path, required=True) + recover = sub.add_parser("recover-publication") + recover.add_argument("--run-id", type=int, required=True); recover.add_argument("--expected-head", required=True); recover.add_argument("--build-set-id", required=True); recover.add_argument("--output-dir", type=Path, required=True) + verify_recovery = sub.add_parser("verify-recovery") + verify_recovery.add_argument("--claims", type=Path, required=True); verify_recovery.add_argument("--draft", type=Path, required=True); verify_recovery.add_argument("--envelope", type=Path, required=True); verify_recovery.add_argument("--bundle", type=Path, required=True); verify_recovery.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "capture-inputs": capture_inputs(args.build_set, args.output) + elif args.command == "verify-source": verify_source(args.expected_sha, args.output) + elif args.command == "capture-staging": capture_staging(args.artifact_id, args.expected_name, args.run_id, args.run_attempt, args.output) + elif args.command == "allocate-draft": allocate_draft(args.expected_sha, args.source, args.output) + elif args.command == "publish": publish(args.claims, args.content, args.bundle, args.envelope, args.draft, args.output) + elif args.command == "capture-live": capture_live(args.envelope, args.bundle, args.run_id, args.output) + elif args.command == "recover-publication": recover_publication(args.run_id, args.expected_head, args.build_set_id, args.output_dir) + else: + verify_recovery_evidence(load_json(args.evidence), load_json(args.claims), load_json(args.draft), canonical_json.load_json(args.envelope), args.bundle) + print(f"OK Phase-6 GitHub boundary {args.command}") + return 0 + except (ForgeError, canonical_json.ProtocolError, OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase6_candidate.py b/scripts/phase6_candidate.py new file mode 100644 index 0000000..900887a --- /dev/null +++ b/scripts/phase6_candidate.py @@ -0,0 +1,653 @@ +#!/usr/bin/env python3 +"""Assemble and non-executingly verify the exact initial 31-payload candidate.""" + +from __future__ import annotations + +import argparse +import copy +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import sys +from pathlib import Path, PurePosixPath +from typing import Any + +import canonical_json +import compare_phase5_indexer_builds +import consolidate_phase4_macos +import proof_data_pipeline +import validate_archive +import validate_catalog +from forge_io import ForgeError, canonical_bytes, create_file_atomic, expect, load_json, safe_basename, sha256_file, sha256_stream, validate_regular_file, validate_unique_names + + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = "acedward/midnight-binary-forge" +REPOSITORY_ID = 1349127482 +REVIEWED_BASE_SHA = "19de8be5f434225dbf17126e86b6c3cc6aacc4fe" +WARNING = "DEVELOPMENT ONLY — NOT FOR PRODUCTION USE. Release 0.3.120 is mutable; verify every downloaded SHA-256 against committed metadata before installation or execution." +EXPECTED_INPUTS = { + "phase3p-proof-data": (33170546601, 9685464135, "508aab47ab266344aedd6359fe971928bed309b5", "proof-data-q8b-163729d8422b431af7551ee6c47392d10d6943a1", "b17dbb1883b12c5c98c39dd46e8db29b273aaf67c202aa1aaa0353772b1fe40f"), + "phase4-celestia-appd-linux-arm64": (33177534764, 9688244894, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-celestia-appd-linux-arm64", "3dae16d1cef7ec52a48b0d6b09a3505afc37755e909d4d841acc7f94363d5e56"), + "phase4-celestia-node-linux-arm64": (33177534764, 9688243729, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-celestia-node-linux-arm64", "b9c024854fcb198100eec2f95e5aa6fbafd9b230cc8740b269bee621ab27e1ba"), + "phase4-node-linux-arm64": (33177534764, 9688330126, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-node-linux-arm64", "ba8d8082c278aa7c0615b989d0c2acf0872465eccecff79b90f9d060259d090e"), + "phase4-toolkit-linux-amd64": (33177534764, 9688263793, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-linux-amd64", "9024830935e22337414d3d33fceaf5051734820d52c0c20c9253d1a9af8db93b"), + "phase4-toolkit-linux-arm64": (33177534764, 9688255774, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-linux-arm64", "e88cff76b7b687dd359daf06766773b712bf0775e66ba1c42923c5d38dd76afd"), + "phase4-toolkit-macos-arm64": (33177534764, 9689647047, "656cd9664d23bda4ef0578d62c9e27392bff063e", "phase4-toolkit-macos-arm64", "ed2bbaa44a86a6931dd8ab19fca5920701ce25080b05193af8578024a8e4df9e"), + "phase5-indexer": (33176004154, 9690093579, "e581add8952bae5ffeac39fb07e6b5c6f482862d", "phase5-indexer-verified-candidate-5b78f001926340626a93485f9f60f23d5c2a070a", "eccdbef40775259ba53eefeb624e2379c2d8091cc2be44ea0645d8998bcb57d9"), +} +EXPECTED_PAYLOAD_NAMES = { + *{f"bls_midnight_2p{k}" for k in range(20)}, + "celestia-appd-linux-arm64-v6.4.10.tar.gz", + "celestia-node-linux-arm64-v0.28.4.tar.gz", + "indexer-standalone-linux-amd64-v4.4.0-rc.3.zip", + "indexer-standalone-linux-arm64-v4.4.0-rc.3.zip", + "indexer-standalone-macos-amd64-v4.4.0-rc.3.zip", + "indexer-standalone-macos-arm64-v4.4.0-rc.3.zip", + "midnight-ledger-static-noarch-9.0.0.zip", + "midnight-node-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip", + "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip", + "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip", +} +EXPECTED_SOFTWARE_IDENTITIES = { + "celestia-appd-linux-arm64-v6.4.10.tar.gz": (180685852, "52cc9d59f9db5e3d2b7de91008c808f46ba319922db4a39404735b0a5dd6a76b", "phase4-celestia-appd-linux-arm64", "payloads/celestia-appd-linux-arm64-v6.4.10.tar.gz"), + "celestia-node-linux-arm64-v0.28.4.tar.gz": (71184641, "09eb0505c5265bb08dfd09f14aa397516efd89d7b8f120e06f133d9e387ad50c", "phase4-celestia-node-linux-arm64", "payloads/celestia-node-linux-arm64-v0.28.4.tar.gz"), + "indexer-standalone-linux-amd64-v4.4.0-rc.3.zip": (31479027, "4b5df2ae3ed01f378adfb64d1c0d20d306470f8fba23a36638f937a4486a9434", "phase5-indexer", "payload/indexer-standalone-linux-amd64-v4.4.0-rc.3.zip"), + "indexer-standalone-linux-arm64-v4.4.0-rc.3.zip": (29782570, "eb44e8493df141d552334399dc25277e76cd500e937bedd5c6ff42a068fb15d0", "phase5-indexer", "payload/indexer-standalone-linux-arm64-v4.4.0-rc.3.zip"), + "indexer-standalone-macos-amd64-v4.4.0-rc.3.zip": (30713420, "28590ac9c35ed464cabdf121ac745ec7aff5c7fd6af2165bf46e4ab018fbe1cc", "phase5-indexer", "payload/indexer-standalone-macos-amd64-v4.4.0-rc.3.zip"), + "indexer-standalone-macos-arm64-v4.4.0-rc.3.zip": (29072181, "b75e96c088b705722d561c6b46997759ed73b494dde0de72964851b5eda09ad2", "phase5-indexer", "payload/indexer-standalone-macos-arm64-v4.4.0-rc.3.zip"), + "midnight-node-linux-arm64-2.0.0-rc.4.zip": (82544614, "490ef12ddf58a2a188f70edbfce974fd8d6cfa392e131232aa04e28557dbc55c", "phase4-node-linux-arm64", "payloads/midnight-node-linux-arm64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip": (50017428, "92836fa7e301ec153fbeeb18ffc113eea4503732ff335f88c2823ad3e527524c", "phase4-toolkit-linux-amd64", "payloads/midnight-node-toolkit-linux-amd64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip": (48585850, "4887874e114dafac8807e524b9d7694e1debd098a8d06ede0831ed7fec576528", "phase4-toolkit-linux-arm64", "payloads/midnight-node-toolkit-linux-arm64-2.0.0-rc.4.zip"), + "midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip": (45553847, "8df786b56f80bd4c2ea4226240a9855481f7c3d56e5794d939d4391dcfb9a02c", "phase4-toolkit-macos-arm64", "payloads/midnight-node-toolkit-macos-arm64-2.0.0-rc.4.zip"), +} +EXPECTED_IDENTITY_MIRROR_TAR_HEADERS = { + "celestia-appd-linux-arm64-v6.4.10.tar.gz": { + "componentId": "celestia-appd-6.4.10-linux-arm64", + "headers": [ + {"path": "LICENSE", "uid": 1001, "gid": 1001, "uname": "", "gname": "", "mtime": 1770110835}, + {"path": "README.md", "uid": 1001, "gid": 1001, "uname": "", "gname": "", "mtime": 1770110835}, + {"path": "celestia-appd", "uid": 0, "gid": 0, "uname": "root", "gname": "root", "mtime": 1770111631}, + ], + }, + "celestia-node-linux-arm64-v0.28.4.tar.gz": { + "componentId": "celestia-node-0.28.4-linux-arm64", + "headers": [ + {"path": "LICENSE", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256280}, + {"path": "README.md", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256280}, + {"path": "celestia", "uid": 1001, "gid": 1001, "uname": "runner", "gname": "runner", "mtime": 1764256973}, + ], + }, +} +EVIDENCE_ROLE_BY_PREFIX = { + "LICENSE-": "license", + "NOTICE-": "notice", + "license-evidence-": "license", + "provenance-": "provenance", + "signing-evidence-": "provenance", + "software-sbom-": "sbom-spdx", + "software-member-manifests-": "member-manifest", + "proof-data-lineage-": "lineage-manifest", + "proof-cache-content-manifest-": "lineage-manifest", + "ledger-static-member-manifest-": "member-manifest", + "source-manifest-": "source-manifest", + "sha256sums-": "checksums", +} + + +def identity(path: Path, ceiling: int = 2**31 - 1) -> dict[str, Any]: + validate_regular_file(path) + digest, size = sha256_file(path, ceiling) + return {"name": path.name, "size": size, "sha256": digest} + + +def strict_relative(value: str) -> PurePosixPath: + path = PurePosixPath(value) + expect(value == path.as_posix() and not path.is_absolute() and all(part not in {"", ".", ".."} for part in path.parts), f"unsafe relative path: {value!r}") + return path + + +def repository_file(path: Path, root: Path = ROOT, label: str = "repository file") -> tuple[Path, PurePosixPath]: + """Resolve a CLI path from the caller's CWD without permitting escape or symlinks.""" + repository = root.resolve(strict=True) + expect(repository.is_dir() and not root.is_symlink(), "repository root is missing or unsafe") + lexical = Path(os.path.abspath(os.fspath(path))) + try: + lexical_relative = lexical.relative_to(repository) + except ValueError as exc: + raise ForgeError(f"{label} is outside the repository root: {path}") from exc + cursor = repository + for part in lexical_relative.parts: + cursor = cursor / part + expect(not cursor.is_symlink(), f"{label} traverses a symlink: {path}") + resolved = lexical.resolve(strict=True) + try: + relative = resolved.relative_to(repository) + except ValueError as exc: + raise ForgeError(f"{label} resolves outside the repository root: {path}") from exc + validate_regular_file(resolved) + return resolved, PurePosixPath(relative.as_posix()) + + +def media_type(name: str) -> str: + if name.endswith(".spdx.json"): + return "application/spdx+json" + if name.endswith(".json"): + return "application/json" + if name.endswith(".zip"): + return "application/zip" + if name.endswith(".tar.gz"): + return "application/gzip" + if name.endswith(".txt"): + return "text/plain" + return "application/octet-stream" + + +def evidence_role(name: str) -> str: + matches = [role for prefix, role in EVIDENCE_ROLE_BY_PREFIX.items() if name.startswith(prefix)] + expect(len(matches) == 1, f"candidate evidence name has no unique typed role: {name}") + return matches[0] + + +def expected_evidence_names(build_id: str) -> set[str]: + return { + "LICENSE-Apache-2.0.txt", + "NOTICE-DEVELOPMENT-ONLY.txt", + f"license-evidence-{build_id}.json", + f"signing-evidence-{build_id}.json", + f"provenance-{build_id}.json", + f"software-member-manifests-{build_id}.json", + f"proof-data-lineage-{build_id}.json", + f"proof-cache-content-manifest-{build_id}.json", + f"ledger-static-member-manifest-{build_id}.json", + f"source-manifest-{build_id}.json", + f"sha256sums-{build_id}.txt", + *{f"software-sbom-{name}.spdx.json" for name in EXPECTED_SOFTWARE_IDENTITIES}, + } + + +def validate_buildset(buildset_path: Path, root: Path = ROOT, require_git: bool = False) -> tuple[dict[str, Any], dict[str, Any]]: + buildset = load_json(buildset_path) + report = validate_catalog.validate_build_set(buildset, root, require_source_head=require_git) + expect(buildset["buildSetId"] == "initial-warehouse-v1", "unexpected Phase-6 build-set ID") + expect(buildset["sourceFullSha"] == REVIEWED_BASE_SHA, "unexpected reviewed Phase-6 input baseline") + inputs = {row["key"]: row for row in buildset["inputArtifacts"]} + expect(set(inputs) == set(EXPECTED_INPUTS), "Phase-6 input artifact set differs from exact audited allowlist") + for key, expected in EXPECTED_INPUTS.items(): + row = inputs[key] + actual = (row["runId"], row["artifactId"], row["sourceHeadSha"], row["artifactName"], row["archiveSha256"]) + expect(actual == expected, f"Phase-6 input identity differs: {key}") + expect(row["repository"] == REPOSITORY and row["repositoryId"] == REPOSITORY_ID, f"Phase-6 input repository differs: {key}") + payloads = buildset["payloads"] + names = [row["name"] for row in payloads] + expect(len(payloads) == 31 and set(names) == EXPECTED_PAYLOAD_NAMES, "Phase-6 payload allowlist/count differs from exact 31") + expect(sum(row["artifactKind"] == "software" for row in payloads) == 10, "Phase-6 binary payload count must be ten") + expect(sum(row["artifactKind"] == "proof-data" for row in payloads) == 21, "Phase-6 proof-data payload count must be 21") + expect(not any("compact" in row["name"].casefold() or "compact" in row["componentId"].casefold() for row in payloads), "Compact compiler payload forbidden") + proof = [row for row in payloads if row["artifactKind"] == "proof-data"] + software = {row["name"]: row for row in payloads if row["artifactKind"] == "software"} + expect(set(software) == set(EXPECTED_SOFTWARE_IDENTITIES), "Phase-6 software payload allowlist differs") + for name, expected in EXPECTED_SOFTWARE_IDENTITIES.items(): + row = software[name] + expect((row["size"], row["sha256"], row["sourceArtifactKey"], row["sourcePath"]) == expected, f"Phase-6 software payload identity differs: {name}") + q8b = load_json(root / "catalog/proof-data/q8b-v1.json") + expected_proof = {row["releaseName"]: (row["size"], row["sha256"], row["componentId"], row["mode"]) for row in q8b["srs"]} + ledger = q8b["ledgerStatic"] + expected_proof[ledger["releaseName"]] = (ledger["archiveSize"], ledger["archiveSha256"], ledger["componentId"], "0644") + expect({row["name"] for row in proof} == set(expected_proof), "Phase-6 proof payload allowlist differs") + for row in proof: + expected = expected_proof[row["name"]] + expect((row["size"], row["sha256"], row["componentId"], row["installMode"]) == expected, f"Phase-6 proof payload identity differs: {row['name']}") + expect(row["sourceArtifactKey"] == "phase3p-proof-data" and row["sourcePath"] == f"payloads/{row['name']}", f"Phase-6 proof payload source differs: {row['name']}") + expect(not any("linux" in row["name"] or "macos" in row["name"] or "rc.5" in row["name"] for row in proof), "proof data cannot be duplicated by platform or proof-server release") + expect({row["sourceArtifactKey"] for row in payloads} == set(inputs), "every exact audited input must contribute an approved payload") + return buildset, report + + +def verify_live_metadata(buildset: dict[str, Any], metadata: dict[str, Any], require_live: bool = True) -> None: + expect(metadata.get("schemaVersion") == "phase6-input-live-metadata-v1", "wrong Phase-6 input metadata schema") + expect(metadata.get("repository") == {"fullName": REPOSITORY, "id": REPOSITORY_ID}, "input metadata repository mismatch") + runs = {row.get("id"): row for row in metadata.get("runs", []) if isinstance(row, dict)} + artifacts = {row.get("id"): row for row in metadata.get("artifacts", []) if isinstance(row, dict)} + expect(len(runs) == 3 and len(artifacts) == 8, "input live metadata count mismatch") + now = canonical_json.parse_time(metadata["capturedAt"], "input metadata capturedAt") + for expected in buildset["inputArtifacts"]: + run = runs.get(expected["runId"]) + artifact = artifacts.get(expected["artifactId"]) + expect(run is not None and artifact is not None, f"input live metadata missing: {expected['key']}") + expect(run.get("run_attempt") == expected["runAttempt"] and run.get("event") == expected["runEvent"], f"input run attempt/event mismatch: {expected['key']}") + expect(run.get("status") == "completed" and run.get("conclusion") == expected["runConclusion"], f"input run is not successful: {expected['key']}") + expect(run.get("head_sha") == expected["sourceHeadSha"] and run.get("head_branch") == expected["sourceRef"], f"input run source identity mismatch: {expected['key']}") + expect(run.get("path") == expected["workflowPath"], f"input run workflow mismatch: {expected['key']}") + repo = run.get("repository", {}) + expect(repo.get("full_name") == REPOSITORY and repo.get("id") == REPOSITORY_ID, f"input run repository mismatch: {expected['key']}") + expect(artifact.get("name") == expected["artifactName"] and artifact.get("size_in_bytes") == expected["artifactSize"], f"input artifact name/size mismatch: {expected['key']}") + expect(artifact.get("digest") == f"sha256:{expected['archiveSha256']}", f"input artifact archive digest mismatch: {expected['key']}") + expect(artifact.get("expired") is False and artifact.get("expires_at") == expected["expiresAt"], f"input artifact expiry/state mismatch: {expected['key']}") + workflow = artifact.get("workflow_run", {}) + expect(workflow.get("id") == expected["runId"] and workflow.get("head_sha") == expected["sourceHeadSha"] and workflow.get("repository_id") == REPOSITORY_ID, f"input artifact/run relation mismatch: {expected['key']}") + if require_live: + expect(now < canonical_json.parse_time(expected["expiresAt"], f"{expected['key']} expiresAt"), f"input artifact expired before assembly: {expected['key']}") + + +def git_ancestry(buildset: dict[str, Any], root: Path) -> None: + validate_catalog.require_complete_git_history(root) + resolved_head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10) + expect(resolved_head.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}\n", resolved_head.stdout) is not None and resolved_head.stderr == "", "cannot resolve candidate full source HEAD") + head = resolved_head.stdout.rstrip("\n") + for source in [buildset["sourceFullSha"], *[row["sourceHeadSha"] for row in buildset["inputArtifacts"]]]: + result = subprocess.run(["git", "-C", str(root), "merge-base", "--is-ancestor", source, head], check=False, capture_output=True, timeout=10) + expect(result.returncode == 0, f"reviewed input SHA is not reachable from candidate source HEAD: {source}") + + +def copy_inert(source: Path, destination: Path) -> dict[str, Any]: + validate_regular_file(source) + expect(not destination.exists() and destination.parent.is_dir(), f"unsafe/duplicate candidate destination: {destination}") + shutil.copyfile(source, destination) + os.chmod(destination, 0o644) + return identity(destination) + + +def phase4_record(root: Path, payload_name: str) -> tuple[dict[str, Any], dict[str, Any], Path]: + record = load_json(root / "evidence/payload-evidence.json") + expect(record.get("schemaVersion") == "phase4-payload-evidence-v1" and record.get("payload", {}).get("name") == payload_name, f"Phase-4 payload evidence mismatch: {payload_name}") + observed = identity(root / "payloads" / payload_name) + expect(record["payload"] == observed, f"Phase-4 payload bytes differ from evidence: {payload_name}") + member_manifest = load_json(root / "evidence/member-manifest.json") + sbom = next((path for path in (root / "sbom").iterdir() if path.name.endswith(".spdx.json")), None) + expect(sbom is not None, f"Phase-4 SPDX SBOM missing: {payload_name}") + return record, member_manifest, sbom + + +def checksum_manifest(root: Path) -> None: + compare_phase5_indexer_builds.validate_checksum_manifest(root) + + +def validate_input_layout(buildset: dict[str, Any], input_root: Path) -> dict[str, Path]: + expect(input_root.is_dir() and not input_root.is_symlink(), "input root is missing or unsafe") + expected_keys = {row["key"] for row in buildset["inputArtifacts"]} + expect({path.name for path in input_root.iterdir()} == expected_keys, "downloaded input directory set differs from build set") + inputs = {key: input_root / key for key in expected_keys} + expected_top = { + "phase3p-proof-data": {"payloads", "evidence"}, + "phase4-celestia-appd-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-celestia-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-amd64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-macos-arm64": {"SHA256SUMS", "payloads", "evidence", "sbom", "independent-builds"}, + "phase5-indexer": {"SHA256SUMS", "payload", "evidence"}, + } + expect(set(expected_top) == expected_keys, "Phase-6 input-layout policy differs from pinned inputs") + for key, root in inputs.items(): + expect(root.is_dir() and not root.is_symlink(), f"downloaded input artifact root is unsafe: {key}") + children = {path.name: path for path in root.iterdir()} + expect(set(children) == expected_top[key], f"downloaded input top-level layout differs: {key}") + for name, path in children.items(): + expect(not path.is_symlink(), f"downloaded input top-level symlink forbidden: {key}/{name}") + expected_file = name == "SHA256SUMS" + expect(path.is_file() if expected_file else path.is_dir(), f"downloaded input top-level type differs: {key}/{name}") + return inputs + + +def assemble(buildset_path: Path, input_root: Path, output: Path, root: Path = ROOT) -> dict[str, Any]: + buildset_path, buildset_relative = repository_file(buildset_path, root, "build-set path") + buildset, coverage = validate_buildset(buildset_path, root) + expect(not output.exists() and output.parent.is_dir(), "candidate output must not already exist") + inputs = validate_input_layout(buildset, input_root) + proof_data_pipeline.verify_output(root / "catalog/proof-data/q8b-v1.json", inputs["phase3p-proof-data"]) + checksum_manifest(inputs["phase5-indexer"]) + consolidate_phase4_macos.verify(inputs["phase4-toolkit-macos-arm64"]) + + temporary = output.parent / f".{output.name}.tmp-{os.getpid()}" + expect(not temporary.exists(), "candidate temporary output collision") + temporary.mkdir(mode=0o700) + payload_rows: list[dict[str, Any]] = [] + archive_rows: list[dict[str, Any]] = [] + signing_rows: list[dict[str, Any]] = [] + sbom_sources: dict[str, Path] = {} + phase_records: dict[str, Any] = {} + try: + components = {row["componentId"]: load_json(root / row["manifestPath"]) for row in buildset["components"]} + for payload in buildset["payloads"]: + source = inputs[payload["sourceArtifactKey"]] / strict_relative(payload["sourcePath"]) + observed = identity(source) + expect(observed["name"] == payload["name"] and observed["size"] == payload["size"] and observed["sha256"] == payload["sha256"], f"payload input identity mismatch: {payload['name']}") + copied = copy_inert(source, temporary / payload["name"]) + row = copy.deepcopy(payload) + row.update({"size": copied["size"], "sha256": copied["sha256"]}) + payload_rows.append(row) + component = components[payload["componentId"]] + if payload["artifactKind"] == "software" and payload["sourceArtifactKey"].startswith("phase4-"): + record, members, sbom = phase4_record(inputs[payload["sourceArtifactKey"]], payload["name"]) + phase_records[payload["name"]] = record + mirror = EXPECTED_IDENTITY_MIRROR_TAR_HEADERS.get(payload["name"]) + archive_rows.append({"name": payload["name"], "componentId": payload["componentId"], "operation": component["operation"], "container": payload["container"], "wholeArchive": {"size": copied["size"], "sha256": copied["sha256"]}, "limits": component["naming"]["limits"], "members": members["members"], "tarHeaders": mirror["headers"] if mirror is not None else []}) + sbom_sources[payload["name"]] = sbom + signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": record["signing"]}) + elif payload["artifactKind"] == "software": + os_name, arch = payload["os"], payload["arch"] + evidence = inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/build1/evidence" + reproduction = load_json(inputs["phase5-indexer"] / f"evidence/indexer-standalone/{os_name}-{arch}/reproducibility.json") + binary = reproduction["binary"] + archive_rows.append({"name": payload["name"], "componentId": payload["componentId"], "operation": component["operation"], "container": "zip", "wholeArchive": {"size": copied["size"], "sha256": copied["sha256"]}, "limits": component["naming"]["limits"], "members": [{"path": binary["name"], "type": "file", "mode": "0755", "size": binary["size"], "sha256": binary["sha256"]}], "tarHeaders": []}) + sbom_sources[payload["name"]] = evidence / "sbom-indexer-standalone.spdx.json" + signing_rows.append({"name": payload["name"], "componentId": payload["componentId"], "signing": load_json(evidence / "signing-evidence.json")}) + + build_id = buildset["buildSetId"] + sbom_evidence: list[dict[str, Any]] = [] + for payload_name, source in sorted(sbom_sources.items()): + evidence_name = f"software-sbom-{payload_name}.spdx.json" + record = copy_inert(source, temporary / evidence_name) + record["role"] = "sbom-spdx" + record["payloadName"] = payload_name + sbom_evidence.append(record) + expect(len(sbom_evidence) == 10, "candidate must retain exactly one SPDX SBOM per software payload") + + member_name = f"software-member-manifests-{build_id}.json" + create_file_atomic(temporary / member_name, canonical_bytes({"schemaVersion": "phase6-software-member-manifests-v1", "archives": sorted(archive_rows, key=lambda row: row["name"])}) ) + proof_lineage_name = f"proof-data-lineage-{build_id}.json" + copy_inert(inputs["phase3p-proof-data"] / "evidence/proof-data-lineage-v1.json", temporary / proof_lineage_name) + proof_content_name = f"proof-cache-content-manifest-{build_id}.json" + copy_inert(inputs["phase3p-proof-data"] / "evidence/proof-cache-content-manifest-v1.json", temporary / proof_content_name) + ledger_member_name = f"ledger-static-member-manifest-{build_id}.json" + source_ledger = root / "catalog/proof-data/ledger-static-9-member-manifest.json" + copy_inert(source_ledger, temporary / ledger_member_name) + + license_name = "LICENSE-Apache-2.0.txt" + copy_inert(root / "LICENSE", temporary / license_name) + license_evidence_name = f"license-evidence-{build_id}.json" + license_rows = [{"componentId": key, "license": components[key]["license"]} for key in sorted(components)] + create_file_atomic(temporary / license_evidence_name, canonical_bytes({"schemaVersion": "phase6-license-evidence-v1", "licenses": license_rows, "proofDataOwnerAcceptanceRequiredBeforeWarehouseUpload": True, "proofDataOwnerAcceptanceStatus": "pending"})) + signing_name = f"signing-evidence-{build_id}.json" + create_file_atomic(temporary / signing_name, canonical_bytes({"schemaVersion": "phase6-signing-evidence-v1", "payloads": sorted(signing_rows, key=lambda row: row["name"]), "macosPolicy": "UNSIGNED_DEVELOPMENT_ONLY; actual codeSignatureKind is recorded per payload; later Developer ID bytes require a distinct name/version"})) + notice_name = "NOTICE-DEVELOPMENT-ONLY.txt" + create_file_atomic(temporary / notice_name, (WARNING + "\nCompact 0.34 is consumed directly from official LFDT-Minokawa assets and is not a candidate payload.\n").encode("utf-8")) + provenance_name = f"provenance-{build_id}.json" + provenance = { + "_type": "https://in-toto.io/Statement/v1", + "predicateType": "https://github.com/acedward/midnight-binary-forge/predicates/phase6-candidate/v1", + "subject": [{"name": row["name"], "digest": {"sha256": row["sha256"]}} for row in payload_rows], + "predicate": {"buildSetId": build_id, "reviewedInputBaselineSha": buildset["sourceFullSha"], "inputArtifacts": buildset["inputArtifacts"], "phase4PayloadEvidence": phase_records, "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, + } + create_file_atomic(temporary / provenance_name, canonical_bytes(provenance)) + + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + evidence_names = sorted([path.name for path in temporary.iterdir() if path.name not in EXPECTED_PAYLOAD_NAMES] + [source_name, checksums_name]) + source_manifest = { + "schemaVersion": "phase6-source-manifest-v1", + "buildSetId": build_id, + "buildSet": {"path": buildset_relative.as_posix(), "size": buildset_path.stat().st_size, "sha256": sha256_file(buildset_path)[0]}, + "reviewedInputBaselineSha": buildset["sourceFullSha"], + "inputArtifacts": buildset["inputArtifacts"], + "destination": buildset["destination"], + "distributionTier": "development-only", + "releaseMutability": "mutable-warehouse", + "warning": WARNING, + "payloadCount": 31, + "binaryPayloadCount": 10, + "proofDataPayloadCount": 21, + "payloadNameListSha256": hashlib.sha256(canonical_bytes(sorted(EXPECTED_PAYLOAD_NAMES))).hexdigest(), + "payloads": payload_rows, + "coverage": coverage, + "evidenceAssetNames": evidence_names, + "evidenceCount": len(evidence_names), + "compactCompilerPayloadCount": 0, + "proofDataScope": {"platform": "noarch", "k": list(range(20)), "ledgerStaticSemver": "9.0.0", "customProvingKeys": False}, + } + create_file_atomic(temporary / source_name, canonical_bytes(source_manifest)) + rows = [] + for path in sorted(temporary.iterdir(), key=lambda item: item.name): + if path.name != checksums_name: + digest, _ = sha256_file(path, 2**31 - 1) + rows.append(f"{digest} {path.name}\n") + create_file_atomic(temporary / checksums_name, "".join(rows).encode("utf-8")) + os.chmod(temporary, 0o755) + os.replace(temporary, output) + result = verify_candidate(buildset_path, output, root) + return result + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def verify_checksums(content: Path, checksums_name: str) -> None: + rows: dict[str, str] = {} + previous = "" + for line in (content / checksums_name).read_text(encoding="utf-8").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9][A-Za-z0-9._-]{0,255})", line) + expect(match is not None, f"malformed candidate checksum row: {line!r}") + digest, name = match.groups() + expect(name > previous and name not in rows and name != checksums_name, "candidate checksum names must be unique and sorted") + previous = name + rows[name] = digest + files = {path.name: path for path in content.iterdir() if path.is_file() and not path.is_symlink() and path.name != checksums_name} + expect(set(rows) == set(files), "candidate checksum closure differs from exact content") + for name, path in files.items(): + expect(sha256_file(path, 2**31 - 1)[0] == rows[name], f"candidate checksum mismatch: {name}") + + +def verify_one_archive(path: Path, policy: dict[str, Any], expected_payload: dict[str, Any] | None = None, component: dict[str, Any] | None = None) -> None: + required = {"name", "componentId", "operation", "container", "wholeArchive", "limits", "members", "tarHeaders"} + expect(set(policy) == required and policy["name"] == path.name, f"archive evidence fields/name differ: {path.name}") + whole = identity(path) + expect(policy["wholeArchive"] == {"size": whole["size"], "sha256": whole["sha256"]}, f"whole-archive identity differs: {path.name}") + if expected_payload is not None: + expect(policy["componentId"] == expected_payload["componentId"], f"archive component differs: {path.name}") + expect(policy["container"] == expected_payload["container"], f"archive container differs: {path.name}") + expect(policy["wholeArchive"] == {"size": expected_payload["size"], "sha256": expected_payload["sha256"]}, f"archive differs from exact build-set payload: {path.name}") + if component is not None: + expect(policy["componentId"] == component["componentId"] and policy["operation"] == component["operation"], f"archive operation/component differs from manifest: {path.name}") + expect(policy["container"] == component["naming"]["container"] and policy["limits"] == component["naming"]["limits"], f"archive container/limits differ from component: {path.name}") + member_contract = [{key: row[key] for key in ("path", "type", "mode")} for row in policy["members"]] + component_contract = [{key: row[key] for key in ("path", "type", "mode")} for row in component["naming"]["members"]] + component_by_path = {row["path"]: row for row in component_contract} + member_by_path = {row["path"]: row for row in member_contract} + expect(all(member_by_path.get(name) == row for name, row in component_by_path.items()), f"archive misses/differs from component member contract: {path.name}") + for row in member_contract: + if row["path"] not in component_by_path: + expect(any(parent["type"] == "directory" and row["path"].startswith(parent["path"] + "/") for parent in component_contract), f"archive member is outside component contract: {path.name}:{row['path']}") + compressed = path.stat().st_size + limits = policy["limits"] + expect(compressed <= limits["maxCompressedBytes"], f"archive exceeds compressed bound: {path.name}") + mirror = EXPECTED_IDENTITY_MIRROR_TAR_HEADERS.get(path.name) + if mirror is None: + expect(policy["tarHeaders"] == [], f"tar-header exception forbidden for build/repackage archive: {path.name}") + iterator = validate_archive.zip_members if policy["container"] == "zip" else validate_archive.tar_members + members = list(iterator(path)) + else: + expect(policy["container"] == "tar.gz" and policy["operation"] in {"identity-mirror", "rename-only"}, f"upstream tar-header policy requires identity-mirror/rename-only: {path.name}") + expect(policy["componentId"] == mirror["componentId"] and policy["tarHeaders"] == mirror["headers"], f"upstream tar-header allowlist differs: {path.name}") + members = list(validate_archive.tar_members(path, require_canonical_owner=False)) + headers = [{"path": member.path, "uid": member.uid, "gid": member.gid, "uname": member.uname, "gname": member.gname, "mtime": member.mtime} for member in members] + expect(headers == mirror["headers"], f"pinned upstream tar headers differ: {path.name}") + expect(len(members) <= limits["maxMembers"], f"archive exceeds member bound: {path.name}") + validate_unique_names(member.path for member in members) + expected = {row["path"]: row for row in policy["members"]} + expect(len(expected) == len(policy["members"]) and set(expected) == {member.path for member in members}, f"archive member set mismatch: {path.name}") + expanded = sum(member.size for member in members) + expect(expanded <= limits["maxExpandedBytes"] and expanded / max(compressed, 1) <= limits["maxExpansionRatio"], f"archive expansion bound exceeded: {path.name}") + for member in members: + row = expected[member.path] + expect(member.type == row["type"] and member.mode == row["mode"], f"archive member type/mode mismatch: {path.name}:{member.path}") + if member.type == "file": + expect(member.size == row["size"], f"archive member size mismatch: {path.name}:{member.path}") + context = member.opener() + expect(context is not None, f"archive member cannot be streamed: {path.name}:{member.path}") + with context as stream: + digest, size = sha256_stream(stream, row["size"]) + expect(size == row["size"] and digest == row["sha256"], f"archive member digest mismatch: {path.name}:{member.path}") + + +def content_assets(buildset: dict[str, Any], content: Path) -> list[dict[str, Any]]: + payload_by_name = {row["name"]: row for row in buildset["payloads"]} + rows = [] + for path in sorted(content.iterdir(), key=lambda item: item.name): + safe_basename(path.name, "candidate asset name") + observed = identity(path) + row: dict[str, Any] = {**observed, "mediaType": media_type(path.name)} + if path.name in payload_by_name: + payload = payload_by_name[path.name] + row.update({"role": "payload", "artifactKind": payload["artifactKind"], "componentId": payload["componentId"]}) + else: + row["role"] = evidence_role(path.name) + rows.append(row) + return rows + + +def validate_staged_content_layout(content: Path, build_id: str) -> None: + expect(content.is_dir() and not content.is_symlink(), "candidate content root is unsafe") + expected = EXPECTED_PAYLOAD_NAMES | expected_evidence_names(build_id) + children = {path.name: path for path in content.iterdir()} + expect(len(expected) == 52 and set(children) == expected, "staged content must be the exact flat 31-payload plus 21-evidence closure") + for name, path in children.items(): + validate_regular_file(path, "0644") + safe_basename(name, "candidate content name") + + +def verify_candidate(buildset_path: Path, content: Path, root: Path = ROOT) -> dict[str, Any]: + buildset, _ = validate_buildset(buildset_path, root) + build_id = buildset["buildSetId"] + validate_staged_content_layout(content, build_id) + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + verify_checksums(content, checksums_name) + source = load_json(content / source_name) + expect(source.get("schemaVersion") == "phase6-source-manifest-v1" and source.get("payloadCount") == 31 and source.get("binaryPayloadCount") == 10 and source.get("proofDataPayloadCount") == 21, "candidate source-manifest counts differ") + expect(source.get("distributionTier") == "development-only" and source.get("releaseMutability") == "mutable-warehouse" and source.get("warning") == WARNING, "candidate distribution warning/policy differs") + expected_evidence = expected_evidence_names(build_id) + expect(len(expected_evidence) == 21 and source.get("evidenceCount") == 21 and source.get("evidenceAssetNames") == sorted(expected_evidence), "candidate exact evidence allowlist/count differs") + payloads = source.get("payloads") + expect(isinstance(payloads, list) and {row.get("name") for row in payloads if isinstance(row, dict)} == EXPECTED_PAYLOAD_NAMES, "candidate source-manifest payload allowlist differs") + for row in payloads: + observed = identity(content / row["name"]) + expect(observed["size"] == row["size"] and observed["sha256"] == row["sha256"], f"candidate payload identity mismatch: {row['name']}") + expect(source.get("compactCompilerPayloadCount") == 0 and not any("compact" in name.casefold() for name in EXPECTED_PAYLOAD_NAMES), "Compact compiler leaked into candidate") + raw_names = {f"bls_midnight_2p{k}" for k in range(20)} + q8b = load_json(root / "catalog/proof-data/q8b-v1.json") + srs = {row["releaseName"]: row for row in q8b["srs"]} + for name in raw_names: + observed = identity(content / name) + expect(observed["size"] == srs[name]["size"] and observed["sha256"] == srs[name]["sha256"], f"raw proof payload differs: {name}") + policies = load_json(content / f"software-member-manifests-{build_id}.json") + expect(policies.get("schemaVersion") == "phase6-software-member-manifests-v1" and len(policies.get("archives", [])) == 10, "software archive policy set differs") + component_by_id = {row["componentId"]: load_json(root / row["manifestPath"]) for row in buildset["components"]} + for policy in policies["archives"]: + expected = next((row for row in buildset["payloads"] if row["name"] == policy["name"]), None) + expect(expected is not None, f"archive evidence names an unapproved payload: {policy['name']}") + verify_one_archive(content / policy["name"], policy, expected, component_by_id[expected["componentId"]]) + ledger = content / "midnight-ledger-static-noarch-9.0.0.zip" + ledger_component = load_json(root / "catalog/components/midnight-ledger-static-9.0.0.json") + ledger_payload = next(row for row in buildset["payloads"] if row["name"] == ledger.name) + ledger_policy = {"name": ledger.name, "componentId": ledger_payload["componentId"], "operation": ledger_component["operation"], "container": "zip", "wholeArchive": {"size": ledger_payload["size"], "sha256": ledger_payload["sha256"]}, "limits": ledger_component["naming"]["limits"], "members": load_json(root / "catalog/proof-data/ledger-static-9-zip-layout-manifest.json")["members"], "tarHeaders": []} + verify_one_archive(ledger, ledger_policy, ledger_payload, ledger_component) + lineage = load_json(content / f"proof-data-lineage-{build_id}.json") + expect(lineage.get("payloadCount") == 21 and len(lineage.get("payloads", [])) == 21 and lineage.get("softwareSbom") == "not-applicable", "proof-data lineage differs") + signing = load_json(content / f"signing-evidence-{build_id}.json") + expect(len(signing.get("payloads", [])) == 10, "candidate signing evidence count differs") + for row in signing["payloads"]: + state = row["signing"].get("distributionSigningState") + expect(state in {"NOT_APPLICABLE", "UNSIGNED_DEVELOPMENT_ONLY"}, f"candidate signing state forbidden: {row['name']}") + if row["name"].startswith("indexer-standalone-macos-") or row["name"].startswith("midnight-node-toolkit-macos-"): + expect(state == "UNSIGNED_DEVELOPMENT_ONLY" and row["signing"].get("codeSignatureKind") in {"none", "linker-adhoc"}, f"macOS signature metadata incomplete: {row['name']}") + assets = content_assets(buildset, content) + payload_count = sum(row["role"] == "payload" for row in assets) + evidence_count = len(assets) - payload_count + expect(payload_count == 31 and evidence_count == 21 and set(row["name"] for row in assets if row["role"] != "payload") == expected_evidence and len(assets) == 52, "candidate typed asset allowlist/count differs") + result = {"schemaVersion": "phase6-candidate-verification-v1", "payloadCount": payload_count, "evidenceCount": evidence_count, "contentAssetCount": len(assets), "contentAssetListSha256": canonical_json.digest(assets), "payloadNameListSha256": canonical_json.digest(sorted(EXPECTED_PAYLOAD_NAMES))} + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return result + + +def make_claims(buildset_path: Path, content: Path, draft: dict[str, Any], staging: dict[str, Any], commit_sha: str, workflow_sha: str, run_id: int, run_attempt: int, output: Path, root: Path = ROOT) -> None: + buildset, _ = validate_buildset(buildset_path, root) + verification = verify_candidate(buildset_path, content, root) + assets = content_assets(buildset, content) + build_id = buildset["buildSetId"] + by_name = {row["name"]: row for row in assets} + source_name = f"source-manifest-{build_id}.json" + checksums_name = f"sha256sums-{build_id}.txt" + complete_names = sorted([row["name"] for row in assets] + [f"promotion-envelope-{build_id}.json", f"attestation-{build_id}.sigstore.json"]) + claims = { + "issuer": {"repository": REPOSITORY, "repositoryId": REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "workflowPath": canonical_json.WORKFLOW_PATH, "workflowSha": workflow_sha, "ref": canonical_json.MAIN_REF, "commitSha": commit_sha}, + "staging": {"provider": "github-actions-artifact", "runId": run_id, "runAttempt": run_attempt, "artifactId": staging["artifactId"], "artifactName": staging["artifactName"], "archiveSha256": staging["archiveSha256"], "expiresAt": staging["expiresAt"]}, + "candidateDraft": {"repository": REPOSITORY, "repositoryId": REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "tag": draft["tag_name"], "targetCommitish": commit_sha, "releaseId": draft["id"], "releaseNodeId": draft["node_id"], "releaseUrl": draft["html_url"], "liveImmutableVerificationRequired": True}, + "buildSet": {"id": build_id, "manifestName": source_name, "manifestSha256": by_name[source_name]["sha256"], "checksumsName": checksums_name, "checksumsSha256": by_name[checksums_name]["sha256"]}, + "transport": {"envelopeName": f"promotion-envelope-{build_id}.json", "attestationBundleName": f"attestation-{build_id}.sigstore.json"}, + "contentAssets": assets, + "contentAssetListSha256": verification["contentAssetListSha256"], + "completeAssetNames": complete_names, + "completeAssetNameListSha256": canonical_json.digest(complete_names), + "payloadCount": 31, + "contentEvidenceCount": verification["evidenceCount"], + "transportAssetCount": 2, + "totalAssetCount": len(complete_names), + } + expect(len(complete_names) == 54 and verification["evidenceCount"] == 21, "Phase-6 complete candidate must contain 31 payload, 21 evidence and two transport assets") + canonical_json.verify_claims(claims) + create_file_atomic(output, canonical_json.canonical_bytes(claims)) + + +def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + validate = sub.add_parser("validate-buildset") + validate.add_argument("--build-set", type=Path, required=True) + validate.add_argument("--root", type=Path, default=ROOT) + validate.add_argument("--require-git-ancestry", action="store_true") + metadata = sub.add_parser("verify-live-inputs") + metadata.add_argument("--build-set", type=Path, required=True) + metadata.add_argument("--metadata", type=Path, required=True) + metadata.add_argument("--allow-expired", action="store_true") + assemble_parser = sub.add_parser("assemble") + assemble_parser.add_argument("--build-set", type=Path, required=True) + assemble_parser.add_argument("--input-root", type=Path, required=True) + assemble_parser.add_argument("--output", type=Path, required=True) + assemble_parser.add_argument("--result-output", type=Path) + verify_parser = sub.add_parser("verify") + verify_parser.add_argument("--build-set", type=Path, required=True) + verify_parser.add_argument("--content", type=Path, required=True) + claims = sub.add_parser("make-claims") + claims.add_argument("--build-set", type=Path, required=True) + claims.add_argument("--content", type=Path, required=True) + claims.add_argument("--draft", type=Path, required=True) + claims.add_argument("--staging", type=Path, required=True) + claims.add_argument("--commit-sha", required=True) + claims.add_argument("--workflow-sha", required=True) + claims.add_argument("--run-id", type=int, required=True) + claims.add_argument("--run-attempt", type=int, required=True) + claims.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "validate-buildset": + buildset, report = validate_buildset(args.build_set, args.root, args.require_git_ancestry) + if args.require_git_ancestry: + git_ancestry(buildset, args.root) + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + elif args.command == "verify-live-inputs": + buildset, _ = validate_buildset(args.build_set) + verify_live_metadata(buildset, load_json(args.metadata), not args.allow_expired) + print("OK exact audited Phase-6 input metadata") + elif args.command == "assemble": + result = assemble(args.build_set, args.input_root, args.output) + if args.result_output is not None: + create_file_atomic(args.result_output, canonical_bytes(result), 0o644) + elif args.command == "verify": + verify_candidate(args.build_set, args.content) + else: + make_claims(args.build_set, args.content, load_json(args.draft), load_json(args.staging), args.commit_sha, args.workflow_sha, args.run_id, args.run_attempt, args.output) + return 0 + except (ForgeError, canonical_json.ProtocolError, OSError, KeyError, TypeError, ValueError, json.JSONDecodeError, subprocess.SubprocessError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_archive.py b/scripts/validate_archive.py index e6018e0..a474803 100644 --- a/scripts/validate_archive.py +++ b/scripts/validate_archive.py @@ -36,6 +36,11 @@ class Member: size: int compressed_size: int opener: callable + uid: int | None = None + gid: int | None = None + uname: str | None = None + gname: str | None = None + mtime: int | float | None = None @contextlib.contextmanager @@ -79,7 +84,7 @@ def zip_members(path: Path) -> Iterator[Member]: archive.close() -def tar_members(path: Path) -> Iterator[Member]: +def tar_members(path: Path, require_canonical_owner: bool = True) -> Iterator[Member]: archive = tarfile.open(path, "r:gz") try: expect(not archive.pax_headers, "global PAX metadata forbidden") @@ -88,12 +93,13 @@ def tar_members(path: Path) -> Iterator[Member]: safe_member_name(name) expect(not info.pax_headers, f"PAX metadata forbidden: {name}") expect(not info.issym() and not info.islnk(), f"archive links forbidden: {name}") - expect(info.uid in {0} and info.gid in {0}, f"non-canonical tar owner forbidden: {name}") + if require_canonical_owner: + expect(info.uid in {0} and info.gid in {0}, f"non-canonical tar owner forbidden: {name}") if info.isdir(): - yield Member(name, "directory", f"{info.mode:04o}", 0, 0, lambda: None) + yield Member(name, "directory", f"{info.mode:04o}", 0, 0, lambda: None, info.uid, info.gid, info.uname, info.gname, info.mtime) else: expect(info.isfile(), f"unsafe tar member type: {name}") - yield Member(name, "file", f"{info.mode:04o}", info.size, info.size, lambda member_name=info.name: open_tar_member(path, member_name)) + yield Member(name, "file", f"{info.mode:04o}", info.size, info.size, lambda member_name=info.name: open_tar_member(path, member_name), info.uid, info.gid, info.uname, info.gname, info.mtime) finally: archive.close() diff --git a/scripts/validate_catalog.py b/scripts/validate_catalog.py index 7dc4fa2..3ec13d8 100755 --- a/scripts/validate_catalog.py +++ b/scripts/validate_catalog.py @@ -7,8 +7,9 @@ import hashlib import json import re +import subprocess import sys -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any from forge_io import ForgeError, expect, load_json, safe_basename, sha256_file @@ -257,20 +258,63 @@ def load_components(root: Path, build_set: dict[str, Any]) -> dict[str, dict[str return components +def require_complete_git_history(root: Path) -> None: + """Reject ancestry claims unless Git proves this exact checkout is non-shallow.""" + expect(root.is_dir(), "source-head verification requires a repository directory") + git_metadata = root / ".git" + expect(git_metadata.exists() and not git_metadata.is_symlink(), "source-head verification requires an exact Git checkout") + try: + shallow = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--is-shallow-repository"], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ForgeError("cannot prove complete Git history") from exc + expect(shallow.returncode == 0, "cannot prove complete Git history") + expect(shallow.stdout == "false\n" and shallow.stderr == "", "source-head verification requires exact non-shallow Git history") + + def validate_build_set(build_set: dict[str, Any], root: Path, require_source_head: bool = False) -> dict[str, Any]: schema_validate(build_set, "build-set-v1.schema.json") if require_source_head: - head_file = root / ".git" - expect(head_file.exists(), "source-head verification requires a Git checkout") - import subprocess - + require_complete_git_history(root) result = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], check=False, capture_output=True, text=True, timeout=10) - expect(result.returncode == 0 and result.stdout.strip() == build_set["sourceFullSha"], "build set is not bound to current full source HEAD") + expect(result.returncode == 0 and len(result.stdout.strip()) == 40, "cannot resolve current full source HEAD") + ancestor = subprocess.run( + ["git", "-C", str(root), "merge-base", "--is-ancestor", build_set["sourceFullSha"], result.stdout.strip()], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + expect(ancestor.returncode == 0, "build-set reviewed input baseline is not reachable from current full source HEAD") components = load_components(root, build_set) + input_artifacts = build_set["inputArtifacts"] + input_keys = [row["key"] for row in input_artifacts] + expect(input_keys == sorted(input_keys), "input artifact keys must be lexically sorted") + expect(len(input_keys) == len(set(input_keys)), "duplicate input artifact key") + expect(len({row["artifactId"] for row in input_artifacts}) == len(input_artifacts), "duplicate input artifact ID") + for row in input_artifacts: + expect(row["workflowPath"] in { + ".github/workflows/proof-data-q8b.yml", + ".github/workflows/phase4-payloads.yml", + ".github/workflows/phase5-indexer.yml", + }, "unapproved input workflow") + expect(row["sourceHeadSha"] != "0" * 40, "input source head cannot be a sentinel SHA") + input_key_set = set(input_keys) declared_semantics: dict[tuple[str, str, str, str], str] = {} + family_templates: dict[tuple[str, str], str] = {} for component in components.values(): if component["artifactKind"] != "software": continue + family_key = (component["family"], component["version"]) + template = component["naming"]["outerTemplate"] + prior_template = family_templates.get(family_key) + expect(prior_template is None or prior_template == template, f"software family/version has conflicting public name templates: {family_key}") + family_templates[family_key] = template for target in component["targets"]: key = (component["family"], component["version"], target["os"], target["arch"]) prior = declared_semantics.get(key) @@ -284,9 +328,19 @@ def validate_build_set(build_set: dict[str, Any], root: Path, require_source_hea candidate_coverage: dict[tuple[str, str, str, str], tuple[str, str]] = {} for payload in payloads: safe_basename(payload["name"], "payload name") + expect(payload["sourceArtifactKey"] in input_key_set, "payload references unknown input artifact") + source_path = PurePosixPath(payload["sourcePath"]) + expect( + payload["sourcePath"] == source_path.as_posix() + and not source_path.is_absolute() + and all(part not in {"", ".", ".."} for part in source_path.parts), + "unsafe payload source path", + ) expect(payload["componentId"] in components, f"payload references unknown component: {payload['componentId']}") component = components[payload["componentId"]] expect(payload["artifactKind"] == component["artifactKind"], "payload/component artifactKind mismatch") + expect(payload["container"] == component["naming"]["container"], "payload/component archive container mismatch") + expect(payload["installMode"] == component["install"]["mode"], "payload/component install mode mismatch") if payload["artifactKind"] == "software": pair = (payload["os"], payload["arch"]) target = next((row for row in component["targets"] if (row["os"], row["arch"]) == pair), None) @@ -328,19 +382,12 @@ def validate_build_set(build_set: dict[str, Any], root: Path, require_source_hea expect(asset.get("digest") == f"sha256:{row['sha256']}", "existing coverage asset digest mismatch") key = (row["family"], row["version"], row["os"], row["arch"]) expect(key not in all_coverage, f"duplicate software semantic tuple across candidate/existing coverage: {key}") - component = next( - ( - value - for value in components.values() - if value["artifactKind"] == "software" - and value["family"] == row["family"] - and value["version"] == row["version"] - and any((target["os"], target["arch"]) == (row["os"], row["arch"]) for target in value["targets"]) - ), - None, - ) - expect(component is not None, "existing coverage tuple is not declared by a pinned component") - expected_name = render_name(component["naming"]["outerTemplate"], version=row["version"], os_name=row["os"], arch=row["arch"]) + template = family_templates.get((row["family"], row["version"])) + expect(template is not None, "existing coverage family/version is not declared by a pinned component") + # A legacy exact warehouse asset may prove coverage even when no new-build component + # truthfully claims that old target. The reviewed family/version naming contract, + # pinned snapshot identity and exact asset digest remain mandatory. + expected_name = render_name(template, version=row["version"], os_name=row["os"], arch=row["arch"]) expect(row["name"] == expected_name, "existing coverage asset name is not the component's canonical rendered name") all_coverage.add(key) diff --git a/tests/test_phase6_candidate.py b/tests/test_phase6_candidate.py new file mode 100644 index 0000000..57db01a --- /dev/null +++ b/tests/test_phase6_candidate.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import copy +import hashlib +import io +import json +import os +import stat +import subprocess +import sys +import tempfile +import tarfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import phase6_candidate # noqa: E402 +import github_phase6 # noqa: E402 +import validate_catalog # noqa: E402 +from forge_io import ForgeError, load_json # noqa: E402 + + +BUILD_SET = ROOT / "catalog/buildsets/initial-warehouse-v1.json" + + +def run_git(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +class Phase6BuildSetTest(unittest.TestCase): + def test_exact_31_payload_build_set_and_generator_are_closed(self) -> None: + buildset, report = phase6_candidate.validate_buildset(BUILD_SET) + self.assertEqual(buildset["payloadCount"], 31) + self.assertEqual(len(buildset["inputArtifacts"]), 8) + self.assertEqual(len(buildset["existingCoverage"]), 6) + self.assertEqual(len(report["families"]), 5) + self.assertEqual(len(phase6_candidate.expected_evidence_names(buildset["buildSetId"])), 21) + self.assertFalse(any("compact" in row["name"].casefold() for row in buildset["payloads"])) + + def test_input_payload_and_compact_substitutions_fail(self) -> None: + base = load_json(BUILD_SET) + mutations = [] + wrong_input = copy.deepcopy(base) + wrong_input["inputArtifacts"][0]["artifactId"] += 1 + mutations.append(wrong_input) + wrong_payload = copy.deepcopy(base) + wrong_payload["payloads"][0]["sha256"] = "0" * 64 + mutations.append(wrong_payload) + extra_compact = copy.deepcopy(base) + compact = copy.deepcopy(extra_compact["payloads"][0]) + compact["name"] = "compactc-linux-amd64-v0.34.0.zip" + extra_compact["payloads"].append(compact) + extra_compact["payloads"].sort(key=lambda row: row["name"]) + extra_compact["payloadCount"] += 1 + mutations.append(extra_compact) + platform_proof = copy.deepcopy(base) + proof = next(row for row in platform_proof["payloads"] if row["artifactKind"] == "proof-data") + proof["platform"] = "linux-amd64" + mutations.append(platform_proof) + for index, value in enumerate(mutations): + with self.subTest(index=index), tempfile.TemporaryDirectory() as text: + path = Path(text) / "buildset.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaises(ForgeError): + phase6_candidate.validate_buildset(path) + + def test_live_metadata_exactness_and_mutations(self) -> None: + buildset = load_json(BUILD_SET) + run_by_id = {} + artifacts = [] + for row in buildset["inputArtifacts"]: + run_by_id[row["runId"]] = { + "id": row["runId"], "run_attempt": row["runAttempt"], "event": row["runEvent"], + "status": "completed", "conclusion": row["runConclusion"], "head_sha": row["sourceHeadSha"], + "head_branch": row["sourceRef"], "path": row["workflowPath"], + "repository": {"full_name": row["repository"], "id": row["repositoryId"]}, + } + artifacts.append({ + "id": row["artifactId"], "name": row["artifactName"], "size_in_bytes": row["artifactSize"], + "digest": f"sha256:{row['archiveSha256']}", "expired": False, "expires_at": row["expiresAt"], + "workflow_run": {"id": row["runId"], "head_sha": row["sourceHeadSha"], "repository_id": row["repositoryId"]}, + }) + live = {"schemaVersion": "phase6-input-live-metadata-v1", "capturedAt": "2026-08-28T00:00:00Z", "repository": {"fullName": phase6_candidate.REPOSITORY, "id": phase6_candidate.REPOSITORY_ID}, "runs": sorted(run_by_id.values(), key=lambda row: row["id"]), "artifacts": sorted(artifacts, key=lambda row: row["id"])} + phase6_candidate.verify_live_metadata(buildset, live) + mutations = [ + ("runs", 0, "event", "push"), + ("runs", 0, "path", ".github/workflows/candidate.yml"), + ("runs", 0, "head_sha", "0" * 40), + ("artifacts", 0, "digest", "sha256:" + "0" * 64), + ("artifacts", 0, "expired", True), + ] + for section, index, field, value in mutations: + adversarial = copy.deepcopy(live) + adversarial[section][index][field] = value + with self.subTest(field=field), self.assertRaises(ForgeError): + phase6_candidate.verify_live_metadata(buildset, adversarial) + + def test_unexpected_actions_repository_branch_and_event_fail(self) -> None: + valid = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": phase6_candidate.REPOSITORY, "GITHUB_REF": "refs/heads/main", "GITHUB_EVENT_NAME": "workflow_dispatch"} + with mock.patch.dict("os.environ", valid, clear=True): + github_phase6.require_actions_context() + for field, value in (("GITHUB_REPOSITORY", "attacker/fork"), ("GITHUB_REF", "refs/heads/topic"), ("GITHUB_EVENT_NAME", "pull_request")): + adversarial = {**valid, field: value} + with self.subTest(field=field), mock.patch.dict("os.environ", adversarial, clear=True), self.assertRaises(ForgeError): + github_phase6.require_actions_context() + + def test_staging_artifact_identity_is_api_bound(self) -> None: + artifact = {"id": 7, "name": "verified-content-fixture", "expired": False, "digest": "sha256:" + "a" * 64, "expires_at": "2026-09-01T00:00:00Z", "workflow_run": {"id": 11, "repository_id": phase6_candidate.REPOSITORY_ID}} + with tempfile.TemporaryDirectory() as text: + root = Path(text) + with mock.patch("github_phase6.request", return_value=(artifact, "")): + github_phase6.capture_staging(7, artifact["name"], 11, 1, root / "staging.json") + for field, value in (("name", "substituted"), ("digest", "sha256:malformed"), ("expired", True)): + adversarial = copy.deepcopy(artifact) + adversarial[field] = value + with self.subTest(field=field), mock.patch("github_phase6.request", return_value=(adversarial, "")), self.assertRaises(ForgeError): + github_phase6.capture_staging(7, artifact["name"], 11, 1, root / f"failed-{field}.json") + + def test_downloaded_input_layout_is_exact_and_rejects_artifact_name_nesting(self) -> None: + buildset = load_json(BUILD_SET) + top = { + "phase3p-proof-data": {"payloads", "evidence"}, + "phase4-celestia-appd-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-celestia-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-node-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-amd64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-linux-arm64": {"payloads", "evidence", "sbom"}, + "phase4-toolkit-macos-arm64": {"SHA256SUMS", "payloads", "evidence", "sbom", "independent-builds"}, + "phase5-indexer": {"SHA256SUMS", "payload", "evidence"}, + } + with tempfile.TemporaryDirectory() as text: + root = Path(text) + for key, children in top.items(): + directory = root / key + directory.mkdir() + for name in children: + if name == "SHA256SUMS": + (directory / name).write_text("fixture\n", encoding="utf-8") + else: + (directory / name).mkdir() + phase6_candidate.validate_input_layout(buildset, root) + proof = root / "phase3p-proof-data" + (proof / "payloads").rename(proof / "proof-data-q8b-wrapper") + with self.assertRaisesRegex(ForgeError, "top-level layout"): + phase6_candidate.validate_input_layout(buildset, root) + + def test_staged_content_layout_is_exact_and_rejects_artifact_name_nesting(self) -> None: + build_id = "initial-warehouse-v1" + names = phase6_candidate.EXPECTED_PAYLOAD_NAMES | phase6_candidate.expected_evidence_names(build_id) + self.assertEqual(len(names), 52) + with tempfile.TemporaryDirectory() as text: + root = Path(text) + for name in names: + (root / name).write_bytes(b"") + (root / name).chmod(0o644) + phase6_candidate.validate_staged_content_layout(root, build_id) + wrapper = root / "phase6-pr-verified-content-wrapper" + wrapper.mkdir() + for name in names: + (root / name).rename(wrapper / name) + with self.assertRaisesRegex(ForgeError, "exact flat"): + phase6_candidate.validate_staged_content_layout(root, build_id) + + def test_relative_cwd_buildset_path_resolves_inside_repository(self) -> None: + previous = Path.cwd() + try: + os.chdir(ROOT) + resolved, relative = phase6_candidate.repository_file(Path("catalog/buildsets/initial-warehouse-v1.json")) + finally: + os.chdir(previous) + self.assertEqual(resolved, BUILD_SET) + self.assertEqual(relative.as_posix(), "catalog/buildsets/initial-warehouse-v1.json") + + def test_out_of_root_and_symlink_buildset_paths_fail(self) -> None: + with tempfile.TemporaryDirectory() as repository_text, tempfile.TemporaryDirectory() as outside_text: + repository = Path(repository_text) + inside = repository / "buildset.json" + inside.write_text("{}\n", encoding="utf-8") + outside = Path(outside_text) / "buildset.json" + outside.write_text("{}\n", encoding="utf-8") + with self.assertRaisesRegex(ForgeError, "outside the repository root"): + phase6_candidate.repository_file(outside, repository) + link = repository / "buildset-link.json" + link.symlink_to(inside) + with self.assertRaisesRegex(ForgeError, "traverses a symlink"): + phase6_candidate.repository_file(link, repository) + + +class FullHistoryAncestryTest(unittest.TestCase): + def _repository_with_three_commits(self, root: Path) -> tuple[Path, list[str]]: + repository = root / "origin" + repository.mkdir() + run_git(repository, "init", "--initial-branch=main") + run_git(repository, "config", "user.email", "phase6-test@example.invalid") + run_git(repository, "config", "user.name", "Phase 6 test") + commits = [] + for index in range(3): + (repository / "value.txt").write_text(f"{index}\n", encoding="utf-8") + run_git(repository, "add", "value.txt") + run_git(repository, "commit", "-m", f"fixture {index}") + commits.append(run_git(repository, "rev-parse", "HEAD")) + return repository, commits + + def test_full_history_positive_and_real_shallow_clone_negative(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + complete, commits = self._repository_with_three_commits(root) + buildset = {"sourceFullSha": commits[0], "inputArtifacts": [{"sourceHeadSha": commits[1]}]} + validate_catalog.require_complete_git_history(complete) + phase6_candidate.git_ancestry(buildset, complete) + + shallow = root / "shallow" + subprocess.run( + ["git", "clone", "--depth=2", complete.as_uri(), str(shallow)], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(run_git(shallow, "rev-parse", "--is-shallow-repository"), "true") + self.assertEqual(run_git(shallow, "merge-base", "--is-ancestor", commits[1], commits[2]), "") + self.assertTrue((shallow / ".git/shallow").is_file()) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(shallow) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + phase6_candidate.git_ancestry(buildset, shallow) + with self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.validate_build_set(load_json(BUILD_SET), shallow, require_source_head=True) + + def test_malformed_failed_and_missing_git_context_are_rejected(self) -> None: + completed = subprocess.CompletedProcess([], 0, stdout="false\n", stderr="") + with mock.patch("validate_catalog.subprocess.run", return_value=completed): + validate_catalog.require_complete_git_history(ROOT) + for output in ("", "true\n", "false", "false \n", "false\nextra\n"): + malformed = subprocess.CompletedProcess([], 0, stdout=output, stderr="") + with self.subTest(output=repr(output)), mock.patch("validate_catalog.subprocess.run", return_value=malformed), self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(ROOT) + noisy = subprocess.CompletedProcess([], 0, stdout="false\n", stderr="warning\n") + with mock.patch("validate_catalog.subprocess.run", return_value=noisy), self.assertRaisesRegex(ForgeError, "non-shallow"): + validate_catalog.require_complete_git_history(ROOT) + failed = subprocess.CompletedProcess([], 128, stdout="", stderr="fatal\n") + with mock.patch("validate_catalog.subprocess.run", return_value=failed), self.assertRaisesRegex(ForgeError, "cannot prove"): + validate_catalog.require_complete_git_history(ROOT) + with mock.patch("validate_catalog.subprocess.run", side_effect=subprocess.TimeoutExpired(["git"], 10)), self.assertRaisesRegex(ForgeError, "cannot prove"): + validate_catalog.require_complete_git_history(ROOT) + with tempfile.TemporaryDirectory() as text, self.assertRaisesRegex(ForgeError, "exact Git checkout"): + validate_catalog.require_complete_git_history(Path(text)) + + +class Phase6StreamingVerifierTest(unittest.TestCase): + def _archive(self, root: Path, value: bytes, mode: int = 0o755) -> tuple[Path, dict]: + path = root / "fixture.zip" + info = zipfile.ZipInfo("fixture") + info.create_system = 3 + info.external_attr = (stat.S_IFREG | mode) << 16 + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(info, value) + policy = {"name": path.name, "componentId": "fixture-component", "operation": "build", "container": "zip", "wholeArchive": {"size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}, "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 2, "maxExpansionRatio": 100}, "members": [{"path": "fixture", "type": "file", "mode": f"{mode:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()}], "tarHeaders": []} + return path, policy + + def test_streamed_member_identity_and_substitution(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + archive, policy = self._archive(root, b"reviewed bytes") + phase6_candidate.verify_one_archive(archive, policy) + policy["members"][0]["sha256"] = "0" * 64 + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(archive, policy) + + def test_pinned_upstream_tar_headers_and_owner_header_digest_mutations(self) -> None: + name = "celestia-appd-linux-arm64-v6.4.10.tar.gz" + expected = phase6_candidate.EXPECTED_IDENTITY_MIRROR_TAR_HEADERS[name] + values = {"LICENSE": b"license", "README.md": b"readme", "celestia-appd": b"binary"} + modes = {"LICENSE": 0o644, "README.md": 0o644, "celestia-appd": 0o755} + with tempfile.TemporaryDirectory() as text: + path = Path(text) / name + with tarfile.open(path, "w:gz", format=tarfile.GNU_FORMAT) as archive: + for header in expected["headers"]: + value = values[header["path"]] + info = tarfile.TarInfo(header["path"]) + info.size = len(value) + info.mode = modes[header["path"]] + info.uid, info.gid = header["uid"], header["gid"] + info.uname, info.gname = header["uname"], header["gname"] + info.mtime = header["mtime"] + archive.addfile(info, io.BytesIO(value)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + members = [{"path": item, "type": "file", "mode": f"{modes[item]:04o}", "size": len(value), "sha256": hashlib.sha256(value).hexdigest()} for item, value in values.items()] + members.sort(key=lambda row: row["path"]) + policy = {"name": name, "componentId": expected["componentId"], "operation": "rename-only", "container": "tar.gz", "wholeArchive": {"size": path.stat().st_size, "sha256": digest}, "limits": {"maxCompressedBytes": 10000, "maxExpandedBytes": 10000, "maxMembers": 3, "maxExpansionRatio": 100}, "members": members, "tarHeaders": expected["headers"]} + payload = {"componentId": expected["componentId"], "container": "tar.gz", "size": path.stat().st_size, "sha256": digest} + component = {"componentId": expected["componentId"], "operation": "rename-only", "naming": {"container": "tar.gz", "limits": policy["limits"], "members": [{key: row[key] for key in ("path", "type", "mode")} for row in members]}} + phase6_candidate.verify_one_archive(path, policy, payload, component) + for field, value in (("uid", 0), ("mtime", 0)): + mutation = copy.deepcopy(policy) + mutation["tarHeaders"][0][field] = value + with self.subTest(field=field), self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, mutation, payload, component) + mutation = copy.deepcopy(policy) + mutation["wholeArchive"]["sha256"] = "0" * 64 + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, mutation, payload, component) + mutation = copy.deepcopy(component) + mutation["operation"] = "build" + with self.assertRaises(ForgeError): + phase6_candidate.verify_one_archive(path, policy, payload, mutation) + canonical_only = copy.deepcopy(policy) + canonical_only.update({"name": "built.tar.gz", "componentId": "built-component", "operation": "build", "tarHeaders": []}) + built = path.with_name("built.tar.gz") + built.write_bytes(path.read_bytes()) + with self.assertRaisesRegex(ForgeError, "non-canonical tar owner"): + phase6_candidate.verify_one_archive(built, canonical_only) + + def test_all_evidence_names_are_typed(self) -> None: + self.assertEqual(phase6_candidate.evidence_role("provenance-initial-warehouse-v1.json"), "provenance") + with self.assertRaises(ForgeError): + phase6_candidate.evidence_role("untyped-evidence.json") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase6_release_boundaries.py b/tests/test_phase6_release_boundaries.py new file mode 100644 index 0000000..4adb035 --- /dev/null +++ b/tests/test_phase6_release_boundaries.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import canonical_json # noqa: E402 +import github_phase6 # noqa: E402 +from forge_io import ForgeError, canonical_bytes # noqa: E402 + + +DATE = "Fri, 28 Aug 2026 17:00:00 GMT" +HEAD = "2" * 40 +WORKFLOW_SHA = "1" * 40 + + +def policy(boundary: str) -> dict: + return { + "boundary": boundary, + "endpoint": github_phase6.IMMUTABLE_RELEASES_PATH, + "apiVersion": github_phase6.IMMUTABLE_RELEASES_API_VERSION, + "capturedAt": "2026-08-28T17:00:00Z", + "enabled": True, + "enforcedByOwner": False, + } + + +def fixture_transport() -> tuple[dict, dict, bytes, dict[str, bytes]]: + content: dict[str, bytes] = {} + rows = [] + for index in range(31): + name = f"fixture-payload-{index:02d}.bin" + value = f"payload-{index}\n".encode() + content[name] = value + rows.append({"name": name, "role": "payload", "size": len(value), "sha256": hashlib.sha256(value).hexdigest(), "mediaType": "application/octet-stream", "artifactKind": "software", "componentId": f"fixture-{index:02d}"}) + evidence = [ + ("source-manifest-fixture-1.json", "source-manifest"), + ("sha256sums-fixture-1.txt", "checksums"), + *[(f"provenance-fixture-{index:02d}.json", "provenance") for index in range(19)], + ] + for name, role in evidence: + value = f"{role}:{name}\n".encode() + content[name] = value + rows.append({"name": name, "role": role, "size": len(value), "sha256": hashlib.sha256(value).hexdigest(), "mediaType": "text/plain" if role == "checksums" else "application/json"}) + rows.sort(key=lambda row: row["name"]) + content_list_digest = canonical_json.digest(rows) + bundle_name = "attestation-fixture-1.sigstore.json" + envelope_name = "promotion-envelope-fixture-1.json" + claims = { + "issuer": {"repository": github_phase6.REPOSITORY, "repositoryId": github_phase6.REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "workflowPath": github_phase6.WORKFLOW_PATH, "workflowSha": WORKFLOW_SHA, "ref": github_phase6.MAIN_REF, "commitSha": HEAD}, + "staging": {"provider": "github-actions-artifact", "runId": 100, "runAttempt": 1, "artifactId": 200, "artifactName": f"verified-content-fixture-1-{content_list_digest}", "archiveSha256": "3" * 64, "expiresAt": "2026-09-03T00:00:00Z"}, + "candidateDraft": {"repository": github_phase6.REPOSITORY, "repositoryId": github_phase6.REPOSITORY_ID, "repositoryNodeId": canonical_json.REPOSITORY_NODE_ID, "tag": "forge-2026.08.28.1", "targetCommitish": HEAD, "releaseId": 300, "releaseNodeId": "RE_fixture_node", "releaseUrl": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "liveImmutableVerificationRequired": True}, + "buildSet": {"id": "fixture-1", "manifestName": "source-manifest-fixture-1.json", "manifestSha256": next(row["sha256"] for row in rows if row["name"] == "source-manifest-fixture-1.json"), "checksumsName": "sha256sums-fixture-1.txt", "checksumsSha256": next(row["sha256"] for row in rows if row["name"] == "sha256sums-fixture-1.txt")}, + "transport": {"envelopeName": envelope_name, "attestationBundleName": bundle_name}, + "contentAssets": rows, + "contentAssetListSha256": content_list_digest, + "completeAssetNames": sorted([*content, envelope_name, bundle_name]), + "completeAssetNameListSha256": "", + "payloadCount": 31, + "contentEvidenceCount": 21, + "transportAssetCount": 2, + "totalAssetCount": 54, + } + claims["completeAssetNameListSha256"] = canonical_json.digest(claims["completeAssetNames"]) + claims_digest = canonical_json.verify_claims(claims) + bundle = b"fixture attestation bundle\n" + envelope = { + "schemaVersion": "promotion-envelope-v1", + "canonicalization": "forge-canonical-json-v1", + "claims": claims, + "claimsDigest": f"sha256:{claims_digest}", + "attestation": {"kind": "github-artifact-attestation", "predicateType": canonical_json.PREDICATE_TYPE, "predicateCanonicalization": "forge-canonical-json-v1", "predicateSha256": claims_digest, "subjectName": "promotion-claims-fixture-1", "bundleName": bundle_name, "bundleSha256": hashlib.sha256(bundle).hexdigest(), "subjectDigest": f"sha256:{claims_digest}", "issuer": "https://token.actions.githubusercontent.com", "identity": canonical_json.ATTESTATION_IDENTITY}, + } + canonical_json.verify_envelope(envelope) + released = {**content, envelope_name: canonical_json.canonical_bytes(envelope), bundle_name: bundle} + draft = {"id": 300, "node_id": "RE_fixture_node", "html_url": claims["candidateDraft"]["releaseUrl"], "tag_name": claims["candidateDraft"]["tag"], "target_commitish": HEAD, "immutableReleasePolicyBeforeCreate": policy("immediately-before-draft-create")} + return claims, envelope, bundle, released | {"__draft__": canonical_json.canonical_bytes(draft)} + + +class ImmutableReleaseSettingTest(unittest.TestCase): + def test_dedicated_exact_api_and_typed_response(self) -> None: + with mock.patch("github_phase6.request", return_value=({"enabled": True, "enforced_by_owner": False}, DATE)) as request: + observed = github_phase6.immutable_release_policy("fixture") + request.assert_called_once_with(github_phase6.IMMUTABLE_RELEASES_PATH, api_version="2026-03-10") + self.assertEqual(observed, policy("fixture")) + invalid = [ + {}, {"enabled": False, "enforced_by_owner": False}, {"enabled": None, "enforced_by_owner": False}, + {"enabled": 1, "enforced_by_owner": False}, {"enabled": True}, + {"enabled": True, "enforced_by_owner": None}, {"enabled": True, "enforced_by_owner": False, "extra": False}, [], None, + ] + for value in invalid: + with self.subTest(value=value), mock.patch("github_phase6.request", return_value=(value, DATE)), self.assertRaises(ForgeError): + github_phase6.immutable_release_policy("fixture") + with mock.patch("github_phase6.request", side_effect=ForgeError("HTTP/API-version failure")), self.assertRaises(ForgeError): + github_phase6.immutable_release_policy("fixture") + + def test_protected_source_ignores_absent_general_field_and_uses_dedicated_endpoint(self) -> None: + def api(path: str, method: str = "GET", body=None, accept="application/vnd.github+json", api_version=github_phase6.DEFAULT_API_VERSION): + if path == f"/repos/{github_phase6.REPOSITORY}": + return ({"id": github_phase6.REPOSITORY_ID, "full_name": github_phase6.REPOSITORY, "node_id": canonical_json.REPOSITORY_NODE_ID, "default_branch": "main"}, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + self.assertEqual(api_version, "2026-03-10") + return ({"enabled": True, "enforced_by_owner": False}, DATE) + if path.endswith("/branches/main"): + return ({"protected": True, "commit": {"sha": HEAD}}, DATE) + if "/rulesets/7" in path: + return ({"id": 7, "rules": [{"type": "pull_request"}], "bypass_actors": []}, DATE) + if path.endswith("/environments/candidate-publish"): + return ({"deployment_branch_policy": {"protected_branches": True, "custom_branch_policies": False}}, DATE) + if "/contents/" in path: + return ({"type": "file", "sha": WORKFLOW_SHA}, DATE) + raise AssertionError((path, method, body, accept, api_version)) + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_REF": github_phase6.MAIN_REF, "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_SHA": HEAD} + with tempfile.TemporaryDirectory() as text, mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", return_value=[{"id": 7, "enforcement": "active"}]): + output = Path(text) / "source.json" + github_phase6.verify_source(HEAD, output) + value = json.loads(output.read_text()) + self.assertEqual(value["immutableReleasePolicy"], policy("protected-source-admission")) + self.assertNotIn("immutable_releases_enabled", value["repository"]) + + +class WriteBoundaryStateFlipTest(unittest.TestCase): + def _source(self, root: Path) -> Path: + path = root / "source.json" + path.write_bytes(canonical_bytes({"commitSha": HEAD, "immutableReleasePolicy": policy("protected-source-admission")})) + return path + + def _publisher_fixture(self, root: Path) -> tuple[Path, Path, Path, Path, Path]: + content = root / "content" + content.mkdir() + bundle = root / "attestation-fixture.sigstore.json" + envelope = root / "promotion-envelope-fixture.json" + bundle.write_bytes(b"bundle") + claims = {"issuer": {"commitSha": HEAD}, "candidateDraft": {"releaseId": 7}, "contentAssets": [], "completeAssetNames": sorted([bundle.name, envelope.name]), "completeAssetNameListSha256": "4" * 64} + claims_path = root / "claims.json" + claims_path.write_bytes(canonical_bytes(claims)) + envelope.write_bytes(canonical_bytes({"claims": claims})) + draft = root / "draft.json" + draft.write_bytes(canonical_bytes({"id": 7, "tag_name": "forge-2026.08.28.1", "immutableReleasePolicyBeforeCreate": policy("immediately-before-draft-create")})) + return content, bundle, envelope, claims_path, draft + + @mock.patch("github_phase6.require_actions_context") + def test_setting_flip_before_draft_post_prevents_create(self, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + source = self._source(root) + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method)) + if path == "/rate_limit": + return ({}, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + return ({"enabled": False, "enforced_by_owner": False}, DATE) + raise AssertionError((path, method)) + + with mock.patch("github_phase6.pagination", return_value=[]), mock.patch("github_phase6.request", side_effect=api), self.assertRaisesRegex(ForgeError, "not enabled"): + github_phase6.allocate_draft(HEAD, source, root / "draft.json") + self.assertFalse(any(method == "POST" for _, method in calls)) + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_flip_before_upload_prevents_first_write(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + with mock.patch("github_phase6.request", return_value=({"draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE)), mock.patch("github_phase6.pagination", return_value=[]), mock.patch("github_phase6.immutable_release_policy", side_effect=ForgeError("disabled before upload")), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run") as upload, self.assertRaisesRegex(ForgeError, "disabled"): + github_phase6.publish(claims, content, bundle, envelope, draft, root / "published.json") + upload.assert_not_called() + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_flip_after_upload_prevents_public_patch(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + values = {bundle.name: bundle.read_bytes(), envelope.name: envelope.read_bytes()} + rows = [{"id": index + 1, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(values.items()))] + pages = [[], rows[:1], rows, rows] + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method)) + return ({"draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + + def download(asset_id: int, output: Path): + output.write_bytes(values[rows[asset_id - 1]["name"]]) + + with mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", side_effect=pages), mock.patch("github_phase6.immutable_release_policy", side_effect=[policy("immediately-before-draft-asset-upload"), ForgeError("disabled before public transition")]), mock.patch("github_phase6.download_asset", side_effect=download), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run"), self.assertRaisesRegex(ForgeError, "disabled"): + github_phase6.publish(claims, content, bundle, envelope, draft, root / "published.json") + self.assertFalse(any(method == "PATCH" for _, method in calls)) + + @mock.patch("github_phase6.require_actions_context") + @mock.patch("github_phase6.publisher_guard.verify_transport") + def test_setting_rechecked_after_publication_and_immutable_readback(self, _transport, _context) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + content, bundle, envelope, claims, draft = self._publisher_fixture(root) + values = {bundle.name: bundle.read_bytes(), envelope.name: envelope.read_bytes()} + rows = [{"id": index + 1, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(values.items()))] + pages = [[], rows[:1], rows, rows] + calls = [] + + def api(path: str, method: str = "GET", body=None, **kwargs): + calls.append((path, method, body)) + if method == "GET": + if len([call for call in calls if call[1] == "PATCH"]) == 0: + return ({"id": 7, "draft": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + return ({"id": 7, "name": "forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True, "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD}, DATE) + if body == {"draft": False}: + return ({"id": 7}, DATE) + raise ForgeError("immutable mutation rejected") + + def download(asset_id: int, output: Path): + output.write_bytes(values[rows[asset_id - 1]["name"]]) + + checks = [policy("immediately-before-draft-asset-upload"), policy("immediately-before-public-transition"), policy("immediately-after-publication")] + output = root / "published.json" + with mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.pagination", side_effect=pages), mock.patch("github_phase6.immutable_release_policy", side_effect=checks) as setting, mock.patch("github_phase6.download_asset", side_effect=download), mock.patch("github_phase6.token", return_value="fixture-token"), mock.patch("github_phase6.subprocess.run"): + github_phase6.publish(claims, content, bundle, envelope, draft, output) + self.assertEqual(setting.call_count, 3) + self.assertEqual(json.loads(output.read_text())["immutableReleasePolicyChecks"], checks) + self.assertEqual(sum(method == "PATCH" and body == {"draft": False} for _, method, body in calls), 1) + self.assertEqual(sum(method == "GET" for _, method, _ in calls), 2) + + +class ReadOnlyRecoveryTest(unittest.TestCase): + def _fixture(self, root: Path): + claims, envelope, bundle, released = fixture_transport() + draft = canonical_json.load_json(self._write(root / "draft-source.json", released.pop("__draft__"))) + claims_raw = canonical_json.canonical_bytes(claims) + draft_raw = canonical_json.canonical_bytes(draft) + retained = { + "claims": {"id": 501, "name": f"phase6-claims-{HEAD}", "archiveSize": 100, "archiveSha256": "a" * 64, "expiresAt": "2026-09-03T00:00:00Z", "innerName": "promotion-claims-fixture-1.json", "innerSize": len(claims_raw), "innerSha256": hashlib.sha256(claims_raw).hexdigest()}, + "draft": {"id": 502, "name": f"phase6-draft-{HEAD}", "archiveSize": 100, "archiveSha256": "b" * 64, "expiresAt": "2026-09-03T00:00:00Z", "innerName": "draft.json", "innerSize": len(draft_raw), "innerSha256": hashlib.sha256(draft_raw).hexdigest()}, + "staging": {"id": 200, "name": claims["staging"]["artifactName"], "archiveSha256": claims["staging"]["archiveSha256"], "expiresAt": claims["staging"]["expiresAt"], "runId": 100}, + } + release_assets = [{"id": index + 1000, "name": name, "size": len(value), "sha256": hashlib.sha256(value).hexdigest()} for index, (name, value) in enumerate(sorted(released.items()))] + value = { + "schemaVersion": "phase6-recovered-publication-v1", "recoveryMode": "read-only-post-publication-handoff-loss", "capturedAt": "2026-08-28T17:00:00Z", + "repository": {"fullName": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID, "nodeId": canonical_json.REPOSITORY_NODE_ID}, + "protectedRef": {"ref": github_phase6.MAIN_REF, "commitSha": HEAD, "protected": True}, + "workflowFile": {"path": github_phase6.WORKFLOW_PATH, "commitSha": HEAD, "blobSha": WORKFLOW_SHA}, + "originalRun": {"id": 100, "attempt": 1, "repository": github_phase6.REPOSITORY, "workflowPath": github_phase6.WORKFLOW_PATH, "event": "workflow_dispatch", "headSha": HEAD, "headRef": "main", "status": "completed", "conclusion": "failure"}, + "retainedArtifacts": retained, "missingPublishedHandoff": {"name": "published-candidate-fixture-1", "confirmedAbsent": True}, "immutableReleasePolicy": policy("recovery-live-readback"), + "release": {"id": 300, "nodeId": "RE_fixture_node", "repository": github_phase6.REPOSITORY, "tag": "forge-2026.08.28.1", "targetCommitish": HEAD, "url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True}, + "releaseAssets": release_assets, "claimsSha256": canonical_json.digest(claims), "draftSha256": canonical_json.digest(draft), "envelopeSha256": canonical_json.digest(envelope), "bundleSha256": hashlib.sha256(bundle).hexdigest(), + } + bundle_path = self._write(root / claims["transport"]["attestationBundleName"], bundle) + return claims, draft, envelope, bundle_path, value, released + + @staticmethod + def _write(path: Path, value: bytes) -> Path: + path.write_bytes(value) + return path + + def test_exact_recovery_evidence_and_identity_mutations(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, envelope, bundle, value, _ = self._fixture(root) + github_phase6.verify_recovery_evidence(value, claims, draft, envelope, bundle) + mutations = [] + for section, field, replacement in ( + ("originalRun", "id", 101), ("release", "id", 301), ("release", "tag", "forge-2026.08.28.2"), + ("protectedRef", "commitSha", "f" * 40), ("missingPublishedHandoff", "confirmedAbsent", False), + ): + changed = copy.deepcopy(value) + changed[section][field] = replacement + mutations.append(changed) + changed = copy.deepcopy(value) + changed["releaseAssets"][0]["sha256"] = "f" * 64 + mutations.append(changed) + changed_claims = copy.deepcopy(claims) + changed_claims["candidateDraft"]["releaseId"] = 301 + mutations.append((value, changed_claims)) + for index, mutation in enumerate(mutations): + candidate_value, candidate_claims = mutation if isinstance(mutation, tuple) else (mutation, claims) + with self.subTest(index=index), self.assertRaises((ForgeError, canonical_json.ProtocolError)): + github_phase6.verify_recovery_evidence(candidate_value, candidate_claims, draft, envelope, bundle) + changed_draft = copy.deepcopy(draft) + changed_draft["id"] = 301 + with self.assertRaises(ForgeError): + github_phase6.verify_recovery_evidence(value, claims, changed_draft, envelope, bundle) + + def test_post_publication_handoff_loss_recovers_with_gets_only(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, envelope, _, _, released = self._fixture(root) + workflow_relation = {"id": 100, "repository_id": github_phase6.REPOSITORY_ID} + artifacts = [ + {"id": 501, "name": f"phase6-claims-{HEAD}", "expired": False, "digest": "sha256:" + "a" * 64, "size_in_bytes": 100, "expires_at": "2026-09-03T00:00:00Z", "workflow_run": workflow_relation}, + {"id": 502, "name": f"phase6-draft-{HEAD}", "expired": False, "digest": "sha256:" + "b" * 64, "size_in_bytes": 100, "expires_at": "2026-09-03T00:00:00Z", "workflow_run": workflow_relation}, + {"id": 200, "name": claims["staging"]["artifactName"], "expired": False, "digest": "sha256:" + claims["staging"]["archiveSha256"], "size_in_bytes": 100, "expires_at": claims["staging"]["expiresAt"], "workflow_run": workflow_relation}, + ] + release_rows = [{"id": index + 1000, "name": name, "state": "uploaded", "size": len(value), "digest": f"sha256:{hashlib.sha256(value).hexdigest()}"} for index, (name, value) in enumerate(sorted(released.items()))] + by_id = {row["id"]: released[row["name"]] for row in release_rows} + run = {"id": 100, "run_attempt": 1, "event": "workflow_dispatch", "status": "completed", "conclusion": "failure", "path": github_phase6.WORKFLOW_PATH, "head_sha": HEAD, "head_branch": "main", "repository": {"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID}} + release = {"id": 300, "node_id": "RE_fixture_node", "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD, "html_url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": False, "prerelease": False, "immutable": True} + calls = [] + + def api(path: str, method: str = "GET", body=None, api_version=github_phase6.DEFAULT_API_VERSION, **kwargs): + calls.append((path, method, api_version)) + if path.endswith("/actions/runs/100"): + return (run, DATE) + if path.endswith("/releases/300"): + return (release, DATE) + if path == github_phase6.IMMUTABLE_RELEASES_PATH: + return ({"enabled": True, "enforced_by_owner": False}, DATE) + if path == f"/repos/{github_phase6.REPOSITORY}": + return ({"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID, "node_id": canonical_json.REPOSITORY_NODE_ID}, DATE) + if path.endswith("/branches/main"): + return ({"protected": True, "commit": {"sha": HEAD}}, DATE) + if "/contents/" in path: + return ({"sha": WORKFLOW_SHA}, DATE) + raise AssertionError(path) + + def retained(row, inner, output): + value = claims if inner.startswith("promotion-claims") else draft + output.write_bytes(canonical_json.canonical_bytes(value)) + raw = output.read_bytes() + return value, {"id": row["id"], "name": row["name"], "archiveSize": 100, "archiveSha256": row["digest"].removeprefix("sha256:"), "expiresAt": row["expires_at"], "innerName": inner, "innerSize": len(raw), "innerSha256": hashlib.sha256(raw).hexdigest()} + + def release_download(asset_id: int, output: Path): + output.write_bytes(by_id[asset_id]) + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_EVENT_NAME": "workflow_run"} + output = root / "recovered" + with mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.run_artifacts", return_value=artifacts), mock.patch("github_phase6.retained_json_artifact", side_effect=retained), mock.patch("github_phase6.pagination", return_value=release_rows), mock.patch("github_phase6.download_asset", side_effect=release_download): + github_phase6.recover_publication(100, HEAD, "fixture-1", output) + self.assertTrue((output / "recovered-publication.json").is_file()) + self.assertTrue(all(method == "GET" for _, method, _ in calls)) + recovered = json.loads((output / "recovered-publication.json").read_text()) + self.assertEqual(recovered["release"]["id"], 300) + self.assertEqual(len(recovered["releaseAssets"]), 54) + + def test_before_publication_failure_is_abandoned_without_release_write(self) -> None: + with tempfile.TemporaryDirectory() as text: + root = Path(text) + claims, draft, _, _, _, _ = self._fixture(root) + relation = {"id": 100, "repository_id": github_phase6.REPOSITORY_ID} + artifacts = [ + {"id": 501, "name": f"phase6-claims-{HEAD}", "workflow_run": relation}, + {"id": 502, "name": f"phase6-draft-{HEAD}", "workflow_run": relation}, + {"id": 200, "name": claims["staging"]["artifactName"], "digest": "sha256:" + claims["staging"]["archiveSha256"], "expires_at": claims["staging"]["expiresAt"], "workflow_run": relation}, + ] + run = {"id": 100, "run_attempt": 1, "event": "workflow_dispatch", "status": "completed", "conclusion": "failure", "path": github_phase6.WORKFLOW_PATH, "head_sha": HEAD, "head_branch": "main", "repository": {"full_name": github_phase6.REPOSITORY, "id": github_phase6.REPOSITORY_ID}} + release = {"id": 300, "node_id": "RE_fixture_node", "tag_name": "forge-2026.08.28.1", "target_commitish": HEAD, "html_url": f"https://github.com/{github_phase6.REPOSITORY}/releases/tag/forge-2026.08.28.1", "draft": True, "prerelease": False, "immutable": False} + + def api(path: str, method: str = "GET", body=None, **kwargs): + self.assertEqual(method, "GET") + return (run if path.endswith("/actions/runs/100") else release, DATE) + + def retained(row, inner, output): + value = claims if inner.startswith("promotion-claims") else draft + output.write_bytes(canonical_json.canonical_bytes(value)) + return value, {} + + actions = {"GITHUB_ACTIONS": "true", "GITHUB_REPOSITORY": github_phase6.REPOSITORY, "GITHUB_EVENT_NAME": "workflow_run"} + with mock.patch.dict(os.environ, actions, clear=True), mock.patch("github_phase6.request", side_effect=api), mock.patch("github_phase6.run_artifacts", return_value=artifacts), mock.patch("github_phase6.retained_json_artifact", side_effect=retained), mock.patch("github_phase6.immutable_release_policy") as setting, mock.patch("github_phase6.download_asset") as download, self.assertRaisesRegex(ForgeError, "failed before"): + github_phase6.recover_publication(100, HEAD, "fixture-1", root / "abandoned") + setting.assert_not_called() + download.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_policy.py b/tests/test_policy.py index 346b03c..64555f2 100644 --- a/tests/test_policy.py +++ b/tests/test_policy.py @@ -191,12 +191,13 @@ def _make_build_root(self, directory: Path) -> tuple[Path, dict]: target.write_bytes((POLICY_FIXTURES / name).read_bytes()) components[value["componentId"]] = value references.append({"componentId": value["componentId"], "manifestPath": f"catalog/components/{name}", "manifestSha256": sha256(target)}) + common = {"size": 1, "sha256": "b" * 64, "sourceArtifactKey": "fixture-input", "installMode": "0755"} payloads = [ - {"name": "bls_midnight_2p0", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-srs-k0", "tier": "noarch", "platform": "noarch", "k": 0}, - {"name": "fixture-tool-linux-amd64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "linux", "arch": "amd64"}, - {"name": "fixture-tool-linux-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "desired", "os": "linux", "arch": "arm64"}, - {"name": "fixture-tool-macos-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "macos", "arch": "arm64"}, - {"name": "midnight-ledger-static-noarch-9.0.0.zip", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-ledger-static-9.0.0", "tier": "noarch", "platform": "noarch", "ledgerStaticSemver": "9.0.0"}, + {"name": "bls_midnight_2p0", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-srs-k0", "tier": "noarch", "platform": "noarch", "k": 0, **common, "container": "raw", "sourcePath": "payloads/bls_midnight_2p0", "installMode": "0644"}, + {"name": "fixture-tool-linux-amd64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "linux", "arch": "amd64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-linux-amd64-v1.0.0.zip"}, + {"name": "fixture-tool-linux-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "desired", "os": "linux", "arch": "arm64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-linux-arm64-v1.0.0.zip"}, + {"name": "fixture-tool-macos-arm64-v1.0.0.zip", "role": "payload", "artifactKind": "software", "componentId": "fixture-tool-1.0.0", "tier": "required", "os": "macos", "arch": "arm64", **common, "container": "zip", "sourcePath": "payloads/fixture-tool-macos-arm64-v1.0.0.zip"}, + {"name": "midnight-ledger-static-noarch-9.0.0.zip", "role": "payload", "artifactKind": "proof-data", "componentId": "midnight-ledger-static-9.0.0", "tier": "noarch", "platform": "noarch", "ledgerStaticSemver": "9.0.0", **common, "container": "zip", "sourcePath": "payloads/midnight-ledger-static-noarch-9.0.0.zip", "installMode": "0644"}, ] build_set = { "schemaVersion": "build-set-v1", @@ -204,11 +205,18 @@ def _make_build_root(self, directory: Path) -> tuple[Path, dict]: "sourceFullSha": "a" * 40, "destination": {"repository": "effectstream/binaries", "tag": "0.3.120", "distributionTier": "development-only", "releaseMutability": "mutable-warehouse"}, "components": references, + "inputArtifacts": [{ + "key": "fixture-input", "repository": "acedward/midnight-binary-forge", "repositoryId": 1349127482, + "workflowPath": ".github/workflows/phase4-payloads.yml", "runId": 1, "runAttempt": 1, + "runEvent": "pull_request", "runConclusion": "success", "sourceRef": "fixture", + "sourceHeadSha": "c" * 40, "artifactId": 1, "artifactName": "fixture-input", + "artifactSize": 1, "archiveSha256": "d" * 64, "expiresAt": "2026-09-27T00:00:00Z", + }], "existingCoverage": [], "payloads": sorted(payloads, key=lambda row: row["name"]), "payloadCount": len(payloads), "coveragePolicy": {"required": ["linux/amd64", "macos/arm64"], "desired": ["linux/arm64"], "optional": ["macos/amd64"], "proofDataPlatform": "noarch"}, - "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "destinationCredentialAllowed": False}, + "candidatePolicy": {"immutableReleaseRequired": True, "protectedDefaultBranchRequired": True, "typedAssetListRequired": True, "sourceManifestTemplate": "source-manifest-.json", "checksumsTemplate": "sha256sums-.txt", "inputArtifactPinningRequired": True, "destinationCredentialAllowed": False}, } return build_dir, build_set @@ -249,6 +257,12 @@ def test_existing_coverage_is_bound_to_pinned_warehouse_asset(self) -> None: "tier": "required", "os": "linux", "arch": "amd64", + "container": "tar.gz", + "size": 1, + "sha256": "e" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": "payloads/celestia-appd-linux-amd64-v6.4.10.tar.gz", + "installMode": "0755", } build_set["payloads"].append(payload) build_set["payloads"].sort(key=lambda row: row["name"]) @@ -270,6 +284,51 @@ def test_existing_coverage_is_bound_to_pinned_warehouse_asset(self) -> None: with self.subTest(case=mutation["name"]), self.assertRaises(ForgeError): validate_catalog.validate_build_set(adversarial, root) + def test_existing_coverage_can_complete_a_reviewed_family_without_fabricated_target_component(self) -> None: + """Legacy exact bytes prove coverage; they are not retroactively declared as new native builds.""" + coverage_fixture = fixture("adversarial-existing-coverage.json") + with tempfile.TemporaryDirectory() as text: + root = Path(text) + _, build_set = self._make_build_root(root) + component = fixture("valid-existing-coverage-software.json") + component["targets"] = [{"os": "linux", "arch": "amd64", "tier": "required", "runner": "ubuntu-24.04", "native": True}] + component["signing"] = { + "applicability": "not-applicable", "distributionSigningState": "NOT_APPLICABLE", + "codeSignatureKind": "none", "cdHash": None, "authorities": [], + "teamId": None, "hardenedRuntime": None, "strictVerification": False, + } + component_path = root / "catalog/components/valid-existing-coverage-software.json" + write_json(component_path, component) + build_set["components"].append({ + "componentId": component["componentId"], + "manifestPath": "catalog/components/valid-existing-coverage-software.json", + "manifestSha256": sha256(component_path), + }) + candidate_name = "celestia-appd-linux-amd64-v6.4.10.tar.gz" + build_set["payloads"].append({ + "name": candidate_name, + "role": "payload", + "artifactKind": "software", + "componentId": component["componentId"], + "tier": "required", + "os": "linux", + "arch": "amd64", + "container": "tar.gz", + "size": 1, + "sha256": "e" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": f"payloads/{candidate_name}", + "installMode": "0755", + }) + build_set["payloads"].sort(key=lambda row: row["name"]) + build_set["payloadCount"] += 1 + coverage = copy.deepcopy(coverage_fixture["positive"]) + build_set["existingCoverage"] = [coverage] + report = validate_catalog.validate_build_set(build_set, root) + celestia = next(row for row in report["families"] if row["family"] == "celestia-appd") + self.assertEqual(celestia["required"]["present"], ["linux/amd64", "macos/arm64"]) + self.assertEqual(celestia["required"]["missing"], []) + def test_duplicate_software_semantic_tuple_across_names_and_components(self) -> None: adversarial = fixture("adversarial-duplicate-software-tuple.json") with tempfile.TemporaryDirectory() as text: @@ -285,10 +344,19 @@ def test_duplicate_software_semantic_tuple_across_names_and_components(self) -> "manifestPath": "catalog/components/duplicate-software-semantic-tuple.json", "manifestSha256": sha256(path), }) - build_set["payloads"].append(adversarial["duplicatePayload"]) + duplicate_payload = copy.deepcopy(adversarial["duplicatePayload"]) + duplicate_payload.update({ + "container": "zip", + "size": 1, + "sha256": "f" * 64, + "sourceArtifactKey": "fixture-input", + "sourcePath": f"payloads/{duplicate_payload['name']}", + "installMode": "0755", + }) + build_set["payloads"].append(duplicate_payload) build_set["payloads"].sort(key=lambda row: row["name"]) build_set["payloadCount"] += 1 - with self.assertRaisesRegex(ForgeError, "duplicate software semantic tuple"): + with self.assertRaisesRegex(ForgeError, "duplicate software semantic tuple|conflicting public name"): validate_catalog.validate_build_set(build_set, root)