diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index e52371d4..85d1ad8d 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -9,7 +9,7 @@ run-name: >- on: pull_request_target: - types: [opened, synchronize, reopened, ready_for_review, closed] + types: [opened, synchronize, reopened, ready_for_review, edited, closed] workflow_run: workflows: ["Required OpenCode Review", "Strix Security Scan"] types: [completed] @@ -66,6 +66,96 @@ jobs: run: | echo "::notice::Noema review skipped: no pull request number is associated with this event." + - name: Bind Noema inputs to the live organization pull request + if: env.PR_NUMBER != '' + id: live_pr + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + DISPATCH_ACTOR: ${{ github.triggering_actor }} + DISPATCH_SENDER: ${{ github.event.sender.login || '' }} + ALLOWED_DISPATCH_ACTOR: ${{ vars.NOEMA_REPOSITORY_DISPATCH_ACTOR || '' }} + ALLOWED_DISPATCH_TARGETS: ${{ vars.NOEMA_REPOSITORY_DISPATCH_TARGETS || '' }} + SUPPLIED_BASE_REF: ${{ github.event.client_payload.pr_base_ref || '' }} + SUPPLIED_BASE_SHA: ${{ github.event.client_payload.pr_base_sha || '' }} + SUPPLIED_HEAD_REF: ${{ github.event.client_payload.pr_head_ref || '' }} + SUPPLIED_HEAD_SHA: ${{ github.event.client_payload.pr_head_sha || '' }} + run: | + set -euo pipefail + + fail_validation() { + echo "::error::$1" + exit 1 + } + + if [[ ! "$TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]]; then + fail_validation "Noema target repository must be an exact ContextualWisdomLab repository name." + fi + if [[ ! "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]]; then + fail_validation "Noema pull request number must be a positive integer." + fi + + if [ "$EVENT_NAME" = "repository_dispatch" ]; then + if [ -z "$ALLOWED_DISPATCH_ACTOR" ]; then + fail_validation "NOEMA_REPOSITORY_DISPATCH_ACTOR must be configured before repository_dispatch can mint reviewer credentials." + fi + if [ "$DISPATCH_ACTOR" != "$ALLOWED_DISPATCH_ACTOR" ] || [ "$DISPATCH_SENDER" != "$ALLOWED_DISPATCH_ACTOR" ]; then + fail_validation "Noema repository_dispatch actor and sender must exactly match the configured identity." + fi + target_allowed=false + IFS=',' read -ra allowed_targets <<<"$ALLOWED_DISPATCH_TARGETS" + for allowed_target in "${allowed_targets[@]}"; do + if [ "$TARGET_REPOSITORY" = "${allowed_target//[[:space:]]/}" ]; then + target_allowed=true + break + fi + done + if [ "$target_allowed" != true ]; then + fail_validation "Noema repository_dispatch target is not present in NOEMA_REPOSITORY_DISPATCH_TARGETS." + fi + fi + + if ! live_pr="$(gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}")"; then + fail_validation "Noema could not load the requested live pull request." + fi + state="$(jq -r '.state // empty' <<<"$live_pr")" + base_repository="$(jq -r '.base.repo.full_name // empty' <<<"$live_pr")" + head_repository="$(jq -r '.head.repo.full_name // empty' <<<"$live_pr")" + base_ref="$(jq -r '.base.ref // empty' <<<"$live_pr")" + base_sha="$(jq -r '.base.sha // empty' <<<"$live_pr")" + head_ref="$(jq -r '.head.ref // empty' <<<"$live_pr")" + head_sha="$(jq -r '.head.sha // empty' <<<"$live_pr")" + + if [ "$state" != "open" ]; then + fail_validation "Noema only reviews open pull requests." + fi + if [ "$base_repository" != "$TARGET_REPOSITORY" ] || [ "$head_repository" != "$TARGET_REPOSITORY" ]; then + fail_validation "Noema reviewer credentials are restricted to same-repository pull requests." + fi + if [[ ! "$base_ref" =~ ^[A-Za-z0-9._/-]+$ ]] || [[ ! "$head_ref" =~ ^[A-Za-z0-9._/-]+$ ]]; then + fail_validation "Noema live pull request refs are invalid." + fi + if [[ ! "$base_sha" =~ ^[0-9a-fA-F]{40}$ ]] || [[ ! "$head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + fail_validation "Noema live pull request SHAs are invalid." + fi + + if [ "$EVENT_NAME" = "repository_dispatch" ]; then + if [ "$SUPPLIED_BASE_REF" != "$base_ref" ] || [ "$SUPPLIED_BASE_SHA" != "$base_sha" ] || \ + [ "$SUPPLIED_HEAD_REF" != "$head_ref" ] || [ "$SUPPLIED_HEAD_SHA" != "$head_sha" ]; then + fail_validation "Noema repository_dispatch payload does not match the live base/head identity." + fi + fi + + { + echo "target_repository=$TARGET_REPOSITORY" + echo "repository_name=${TARGET_REPOSITORY#*/}" + echo "pr_number=$PR_NUMBER" + echo "base_ref=$base_ref" + echo "base_sha=$base_sha" + echo "head_ref=$head_ref" + echo "head_sha=$head_sha" + } >>"$GITHUB_OUTPUT" + - name: Resolve trusted Noema review source ref if: env.PR_NUMBER != '' id: trusted_source @@ -139,6 +229,7 @@ jobs: if: env.PR_NUMBER != '' id: noema_credential env: + TARGET_REPOSITORY: ${{ steps.live_pr.outputs.target_repository }} NOEMA_GITHUB_APP_CLIENT_ID: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID || '' }} NOEMA_GITHUB_APP_PRIVATE_KEY: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY || '' }} TOKEN_EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} @@ -182,7 +273,7 @@ jobs: client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }} private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }} owner: ContextualWisdomLab - repositories: ${{ steps.noema_credential.outputs.repository }} + repositories: ${{ steps.live_pr.outputs.repository_name }} permission-actions: read permission-checks: read permission-contents: read @@ -196,6 +287,7 @@ jobs: if: env.PR_NUMBER != '' && steps.noema_credential.outputs.source == 'oidc' id: noema_oidc_token env: + TARGET_REPOSITORY: ${{ steps.live_pr.outputs.target_repository }} OIDC_AUDIENCE: ${{ vars.NOEMA_OIDC_AUDIENCE || 'cwl-noema-review' }} TOKEN_EXCHANGE_URL: ${{ vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || '' }} run: | @@ -250,14 +342,102 @@ jobs: echo "::add-mask::$app_token" echo "token=$app_token" >>"$GITHUB_OUTPUT" + - name: Materialize exact OpenCode approval evidence + if: env.PR_NUMBER != '' + id: primary_approval_evidence + env: + GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} + GH_REPOSITORY: ${{ steps.live_pr.outputs.target_repository }} + PR_BASE_REF: ${{ steps.live_pr.outputs.base_ref }} + PR_BASE_SHA: ${{ steps.live_pr.outputs.base_sha }} + PR_HEAD_REF: ${{ steps.live_pr.outputs.head_ref }} + PR_HEAD_SHA: ${{ steps.live_pr.outputs.head_sha }} + OPENCODE_SOURCE_GIT_DIR: ${{ runner.temp }}/noema-pr-objects.git + OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/noema-pr-head + OPENCODE_SOURCE_MANIFEST: ${{ runner.temp }}/noema-pr-source-manifest.json + OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt + OPENCODE_ARTIFACT_MANIFEST_FILE: ${{ runner.temp }}/opencode-artifact-manifest.json + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "::error::Noema reviewer token is unavailable for exact approval evidence." + exit 1 + fi + git check-ref-format "refs/heads/$PR_BASE_REF" + git check-ref-format "refs/heads/$PR_HEAD_REF" + gh auth setup-git + rm -rf "$OPENCODE_SOURCE_GIT_DIR" "$OPENCODE_SOURCE_WORKDIR" + rm -f "$OPENCODE_SOURCE_MANIFEST" "$OPENCODE_CHANGED_FILES_FILE" \ + "$OPENCODE_ARTIFACT_MANIFEST_FILE" + git init --bare "$OPENCODE_SOURCE_GIT_DIR" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" remote add pr-source \ + "$GITHUB_SERVER_URL/$GH_REPOSITORY.git" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --no-recurse-submodules \ + pr-source "+refs/heads/${PR_BASE_REF}:refs/remotes/pr-source/base" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --no-recurse-submodules \ + pr-source "+refs/heads/${PR_HEAD_REF}:refs/remotes/pr-source/head" + [ "$(git --git-dir="$OPENCODE_SOURCE_GIT_DIR" rev-parse refs/remotes/pr-source/base)" = "$PR_BASE_SHA" ] + [ "$(git --git-dir="$OPENCODE_SOURCE_GIT_DIR" rev-parse refs/remotes/pr-source/head)" = "$PR_HEAD_SHA" ] + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" update-ref refs/heads/noema-review "$PR_HEAD_SHA" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" symbolic-ref HEAD refs/heads/noema-review + python3 scripts/ci/materialize_pr_review_source.py \ + --git-dir "$OPENCODE_SOURCE_GIT_DIR" \ + --head-sha "$PR_HEAD_SHA" \ + --output-dir "$OPENCODE_SOURCE_WORKDIR" \ + --manifest "$OPENCODE_SOURCE_MANIFEST" + merge_base="$(git -C "$OPENCODE_SOURCE_WORKDIR" merge-base "$PR_BASE_SHA" "$PR_HEAD_SHA")" + git -C "$OPENCODE_SOURCE_WORKDIR" diff --name-only --find-renames \ + "$merge_base" "$PR_HEAD_SHA" >"$OPENCODE_CHANGED_FILES_FILE" + if [ ! -s "$OPENCODE_CHANGED_FILES_FILE" ]; then + echo "::error::Noema cannot validate an approval without a non-empty exact-head changed-file manifest." + exit 1 + fi + python3 <<'PY' + import hashlib + import json + import os + from pathlib import Path + + runner_temp = Path(os.environ["RUNNER_TEMP"]).resolve(strict=True) + changed_files = Path(os.environ["OPENCODE_CHANGED_FILES_FILE"]).resolve(strict=True) + if changed_files != runner_temp / "opencode-changed-files.txt": + raise SystemExit("changed-file evidence escaped runner temp") + changed_files.chmod(0o600) + manifest = Path(os.environ["OPENCODE_ARTIFACT_MANIFEST_FILE"]) + manifest.write_text( + json.dumps( + { + "schema": 1, + "artifacts": { + changed_files.name: hashlib.sha256( + changed_files.read_bytes() + ).hexdigest() + }, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + manifest.chmod(0o600) + digest = hashlib.sha256(manifest.read_bytes()).hexdigest() + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: + output.write(f"manifest_sha256={digest}\n") + PY + - name: Run Noema LLM review and submit verdict if: env.PR_NUMBER != '' env: + TARGET_REPOSITORY: ${{ steps.live_pr.outputs.target_repository }} + PR_NUMBER: ${{ steps.live_pr.outputs.pr_number }} GH_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN || steps.noema_github_app_token.outputs.token || steps.noema_oidc_token.outputs.token }} NOEMA_REVIEW_TOKEN_SOURCE: ${{ steps.noema_credential.outputs.source == 'pat' && 'noema-review-pat' || steps.noema_credential.outputs.source == 'github-app' && 'noema-review-github-app' || 'noema-review-app-oidc' }} NOEMA_LLM_API_URL: ${{ vars.NOEMA_LLM_API_URL || '' }} NOEMA_LLM_MODEL: ${{ vars.NOEMA_LLM_MODEL || '' }} NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }} + OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION: "true" + OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/noema-pr-head + OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt + OPENCODE_ARTIFACT_MANIFEST_SHA256: ${{ steps.primary_approval_evidence.outputs.manifest_sha256 }} run: | set -euo pipefail if [ -z "${PR_NUMBER:-}" ]; then diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 7010b262..db93345d 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -10,7 +10,7 @@ on: # pull_request workflow is evaluated from the PR merge ref and therefore lets # an in-repository branch rewrite steps before repository secrets are bound. pull_request_target: - types: [opened, synchronize, reopened, ready_for_review, closed] + types: [opened, synchronize, reopened, ready_for_review, edited, closed] # repository_dispatch is evaluated only from the default branch. This keeps # privileged retries from loading workflow code from a caller-selected ref. repository_dispatch: @@ -384,6 +384,13 @@ jobs: with: name: opencode-coverage-source path: ${{ runner.temp }}/opencode-coverage-artifact + # Force the REST-backed download path so "re-run failed jobs" can + # consume the artifact uploaded by a successful prior run attempt. + # The token is scoped to this trusted download step; the later + # untrusted test step explicitly clears every GitHub credential. + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} - name: Prepare pull request merge tree for coverage measurement env: @@ -543,9 +550,10 @@ jobs: # Build the coverage tool image before the pull-request tree is # mounted anywhere. The networked build context contains only this - # trusted Dockerfile plus the reviewed, hash-pinned CI requirements - # from the default-branch checkout; it never contains PR-controlled - # source, dependency manifests, credentials, or runner command files. + # trusted Dockerfile, reviewed CI requirements, and strictly parsed + # hash-pinned dependency locks read from the exact target base SHA; + # it never contains PR-controlled source, dependency manifests, + # credentials, or runner command files. coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" coverage_build_dir="${RUNNER_TEMP}/opencode-coverage-tool-build" trusted_ci_requirements="${GITHUB_WORKSPACE}/requirements-opencode-review-ci-hashes.txt" @@ -558,6 +566,10 @@ jobs: chmod 0700 "$coverage_build_dir" install -m 0644 "$trusted_ci_requirements" \ "$coverage_build_dir/requirements-opencode-review-ci-hashes.txt" + python3 -I "${GITHUB_WORKSPACE}/scripts/ci/materialize_base_python_locks.py" \ + --repo-root "$COVERAGE_SOURCE_WORKDIR" \ + --base-sha "$PR_BASE_SHA" \ + --output-dir "$coverage_build_dir/base-python-locks" cat >"$coverage_build_dir/Dockerfile" <<'DOCKERFILE' FROM docker.io/library/ubuntu@sha256:52df9b1ee71626e0088f7d400d5c6b5f7bb916f8f0c82b474289a4ece6cf3faf ENV DEBIAN_FRONTEND=noninteractive @@ -573,8 +585,6 @@ jobs: libxml2-dev \ mesa-vulkan-drivers \ libvulkan1 \ - nodejs \ - npm \ pkg-config \ python3 \ python3-pip \ @@ -585,7 +595,25 @@ jobs: rustc \ util-linux \ vulkan-tools \ + xz-utils \ && rm -rf /var/lib/apt/lists/* + RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/node.tar.xz \ + https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz \ + && echo '69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec /tmp/node.tar.xz' | sha256sum -c - \ + && tar -xJf /tmp/node.tar.xz --strip-components=1 -C /usr/local \ + && rm -f /tmp/node.tar.xz \ + && test "$(node --version)" = "v22.14.0" \ + && test "$(npm --version)" = "10.9.2" + RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/pnpm.tgz \ + https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz \ + && echo '238d639a47712278bb72e8b6db2c297ac1ccd80dd7642f7c933b73aebde7b51f /tmp/pnpm.tgz' | sha256sum -c - \ + && mkdir -p /opt/pnpm-11.5.3 \ + && tar -xzf /tmp/pnpm.tgz --strip-components=1 -C /opt/pnpm-11.5.3 \ + && chmod 0755 /opt/pnpm-11.5.3/bin/pnpm.mjs /opt/pnpm-11.5.3/bin/pnpx.mjs \ + && ln -s /opt/pnpm-11.5.3/bin/pnpm.mjs /usr/local/bin/pnpm \ + && ln -s /opt/pnpm-11.5.3/bin/pnpx.mjs /usr/local/bin/pnpx \ + && rm -f /tmp/pnpm.tgz \ + && test "$(pnpm --version)" = "11.5.3" RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/cargo-llvm-cov.tar.gz \ https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.8.7/cargo-llvm-cov-x86_64-unknown-linux-musl.tar.gz \ && echo '967b5cc996c29d8baa52bbb4595ef1f53af35255af8e2036ddbc6468d7b523c7 /tmp/cargo-llvm-cov.tar.gz' | sha256sum -c - \ @@ -600,6 +628,21 @@ jobs: --only-binary=:all: \ -r /tmp/requirements-opencode-review-ci-hashes.txt \ && rm -f /tmp/requirements-opencode-review-ci-hashes.txt + COPY base-python-locks /tmp/base-python-locks + RUN set -eux; \ + mkdir -p /opt/base-python-envs; \ + while IFS="$(printf '\t')" read -r project_dir environment lock_file; do \ + [ -n "$project_dir" ] && [ -n "$environment" ] && [ -n "$lock_file" ]; \ + python3 -m venv --system-site-packages "/opt/base-python-envs/${environment}"; \ + "/opt/base-python-envs/${environment}/bin/python" -m pip install \ + --disable-pip-version-check \ + --require-hashes \ + --only-binary=:all: \ + -r "/tmp/base-python-locks/${lock_file}"; \ + done &2 return 1 fi - if command -v "$runner" >/dev/null 2>&1; then - return 0 + if ! command -v "$runner" >/dev/null 2>&1; then + printf 'Coverage package runner %s at exact specification %s is not preinstalled in the pinned sandbox image; central review will not activate PR-selected package-manager code outside an untrusted command boundary or fall back to npm.\n' "$runner" "$spec" >&2 + return 1 fi - printf 'Coverage package runner %s at exact specification %s is not preinstalled in the pinned sandbox image; central review will not activate PR-selected package-manager code outside an untrusted command boundary or fall back to npm.\n' "$runner" "$spec" >&2 - return 1 + requested_version="${spec#${runner}@}" + # Corepack permits an integrity/build suffix (for example + # +sha512.), while the executable reports only semver. + requested_runtime_version="${requested_version%%+*}" + actual_version="$("$runner" --version 2>/dev/null || true)" + if [ "$actual_version" != "$requested_runtime_version" ]; then + printf 'Coverage package runner %s version mismatch: requested %s but pinned sandbox provides %s; refusing mutable activation or npm fallback.\n' "$runner" "$requested_runtime_version" "${actual_version:-}" >&2 + return 1 + fi + return 0 } select_package_runner() { @@ -989,24 +1069,24 @@ jobs: declared_spec="$(declared_package_manager_spec)" case "$declared_runner" in pnpm) - ensure_corepack_runner pnpm "$declared_spec" && printf '%s\n' "pnpm" + ensure_preinstalled_package_runner pnpm "$declared_spec" && printf '%s\n' "pnpm" return ;; yarn) - ensure_corepack_runner yarn "$declared_spec" && printf '%s\n' "yarn" + ensure_preinstalled_package_runner yarn "$declared_spec" && printf '%s\n' "yarn" return ;; npm) - command -v npm >/dev/null 2>&1 && printf '%s\n' "npm" + ensure_preinstalled_package_runner npm "$declared_spec" && printf '%s\n' "npm" return ;; esac if [ -f pnpm-lock.yaml ]; then - ensure_corepack_runner pnpm "$declared_spec" && printf '%s\n' "pnpm" + ensure_preinstalled_package_runner pnpm "$declared_spec" && printf '%s\n' "pnpm" return elif [ -f yarn.lock ]; then - ensure_corepack_runner yarn "$declared_spec" && printf '%s\n' "yarn" + ensure_preinstalled_package_runner yarn "$declared_spec" && printf '%s\n' "yarn" return elif command -v npm >/dev/null 2>&1; then printf '%s\n' "npm" @@ -1018,8 +1098,9 @@ jobs: while IFS= read -r project_dir; do if [ -f "${project_dir}/tests/test_docstrings.py" ]; then measured_projects=1 + project_python="$(python_for_project "$project_dir")" run_and_capture "Python docstring coverage (${project_dir})" \ - bash -c 'cd "$1" && PYTHONPATH=. python3 -m pytest tests/test_docstrings.py' bash "$project_dir" + bash -c 'cd "$1" && PYTHONPATH=. "$2" -m pytest tests/test_docstrings.py' bash "$project_dir" "$project_python" fi done < <(tracked_python_projects_with_tests) [ "$measured_projects" -eq 1 ] @@ -1460,7 +1541,8 @@ jobs: measured_any=1 # PR-selected dependency manifests are never resolved in the # networkless execution phase. The trusted image supplies the - # pinned review toolchain; missing project imports fail in pytest + # pinned review toolchain plus exact hash-locked dependencies from + # the trusted base commit; missing imports still fail in pytest # with the exact dependency name instead of reaching the network. verify_trusted_python_test_toolchain run_python_test_coverage @@ -1818,7 +1900,7 @@ jobs: echo "token=$app_token" } >>"$GITHUB_OUTPUT" - - name: Materialize pull request head for OpenCode review data + - name: Materialize inert pull request blobs for OpenCode review data env: GH_TOKEN: ${{ steps.review_read_app_token.outputs.token || secrets.OPENCODE_APPROVE_TOKEN || github.token }} GH_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }} @@ -1826,24 +1908,35 @@ jobs: PR_BASE_REF: ${{ needs.validate-pr-metadata.outputs.base_ref }} PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} + OPENCODE_SOURCE_GIT_DIR: ${{ runner.temp }}/opencode-pr-objects.git OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-pr-head + OPENCODE_SOURCE_MANIFEST: ${{ runner.temp }}/opencode-pr-source-manifest.json run: | set -euo pipefail gh auth setup-git - git remote remove pr-source 2>/dev/null || true - git remote add pr-source "$GITHUB_SERVER_URL/$GH_REPOSITORY.git" - git fetch --no-tags pr-source \ + rm -rf "$OPENCODE_SOURCE_GIT_DIR" "$OPENCODE_SOURCE_WORKDIR" + rm -f "$OPENCODE_SOURCE_MANIFEST" + git init --bare "$OPENCODE_SOURCE_GIT_DIR" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" remote add pr-source \ + "$GITHUB_SERVER_URL/$GH_REPOSITORY.git" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --no-recurse-submodules pr-source \ "+refs/heads/${PR_BASE_REF}:refs/remotes/pr-source/${PR_BASE_REF}" - if ! git cat-file -e "${PR_BASE_SHA}^{commit}" >/dev/null 2>&1; then - git fetch --no-tags pr-source "$PR_BASE_SHA" + if ! git --git-dir="$OPENCODE_SOURCE_GIT_DIR" cat-file -e "${PR_BASE_SHA}^{commit}" >/dev/null 2>&1; then + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --no-recurse-submodules \ + pr-source "$PR_BASE_SHA" fi - if ! git cat-file -e "${PR_HEAD_SHA}^{commit}" >/dev/null 2>&1; then - git fetch --no-tags pr-source "$PR_HEAD_SHA" || true + if ! git --git-dir="$OPENCODE_SOURCE_GIT_DIR" cat-file -e "${PR_HEAD_SHA}^{commit}" >/dev/null 2>&1; then + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --no-recurse-submodules \ + pr-source "$PR_HEAD_SHA" || true fi - if ! git cat-file -e "${PR_HEAD_SHA}^{commit}" >/dev/null 2>&1; then + if ! git --git-dir="$OPENCODE_SOURCE_GIT_DIR" cat-file -e "${PR_HEAD_SHA}^{commit}" >/dev/null 2>&1; then for pr_head_fetch_attempt in 1 2 3 4 5 6; do - git fetch --no-tags --prune pr-source "+refs/pull/${PR_NUMBER}/head:refs/remotes/pr-source/pull/${PR_NUMBER}/head" - fetched_head_sha="$(git rev-parse "refs/remotes/pr-source/pull/${PR_NUMBER}/head")" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch --no-tags --prune --no-recurse-submodules \ + pr-source "+refs/pull/${PR_NUMBER}/head:refs/remotes/pr-source/pull/${PR_NUMBER}/head" + fetched_head_sha="$( + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" rev-parse \ + "refs/remotes/pr-source/pull/${PR_NUMBER}/head" + )" if [ "$fetched_head_sha" = "$PR_HEAD_SHA" ]; then break fi @@ -1853,11 +1946,26 @@ jobs: fi done fi - git cat-file -e "${PR_BASE_SHA}^{commit}" - git cat-file -e "${PR_HEAD_SHA}^{commit}" - rm -rf "$OPENCODE_SOURCE_WORKDIR" - git worktree add --detach "$OPENCODE_SOURCE_WORKDIR" "$PR_HEAD_SHA" - git -C "$OPENCODE_SOURCE_WORKDIR" status --short + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" cat-file -e "${PR_BASE_SHA}^{commit}" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" cat-file -e "${PR_HEAD_SHA}^{commit}" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" update-ref refs/heads/opencode-review "$PR_HEAD_SHA" + git --git-dir="$OPENCODE_SOURCE_GIT_DIR" symbolic-ref HEAD refs/heads/opencode-review + python3 scripts/ci/materialize_pr_review_source.py \ + --git-dir "$OPENCODE_SOURCE_GIT_DIR" \ + --head-sha "$PR_HEAD_SHA" \ + --output-dir "$OPENCODE_SOURCE_WORKDIR" \ + --manifest "$OPENCODE_SOURCE_MANIFEST" + [ "$(git -C "$OPENCODE_SOURCE_WORKDIR" rev-parse HEAD)" = "$PR_HEAD_SHA" ] + if find "$OPENCODE_SOURCE_WORKDIR" -type l -print -quit | grep -q .; then + echo "::error::Inert OpenCode source unexpectedly contains a symbolic link." + exit 1 + fi + if find "$OPENCODE_SOURCE_WORKDIR" -type f -perm /111 -print -quit | grep -q .; then + echo "::error::Inert OpenCode source unexpectedly contains an executable file." + exit 1 + fi + jq '{head_sha, tree_entries, written_files, tree_bytes, skipped, special_representations}' \ + "$OPENCODE_SOURCE_MANIFEST" - name: Configure git identity for OpenCode action run: | @@ -2744,7 +2852,7 @@ jobs: printf '\n```\n' printf '\n## Review inspection contract\n\n' - printf 'Use the local checkout for exact source and diff inspection.\n' + printf 'Use the inert local source tree and isolated read-only Git objects for exact source and diff inspection.\n' printf 'Do not run a broad full-diff read into the model context; inspect changed files and focused hunks only.\n' printf 'If direct file reads fail but focused changed hunks are present above, review those hunks; do not return file-inaccessible findings for paths shown in this evidence.\n' } >"$OPENCODE_EVIDENCE_FILE" @@ -3516,56 +3624,54 @@ jobs: SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" - # High-sensitivity review candidates only. DeepSeek V3 has been the - # most reliable first-pass reviewer in the org queue, then the pool - # falls through to the direct GPT-5.6 Luna slot, then the full-size - # GPT-4.1 long-context endpoint and provider-specific GPT/o3 fallbacks. + # High-sensitivity review candidates only. The live #579 review run + # showed GPT-4.1 returning an almost-valid control block in 21s while + # two DeepSeek candidates consumed 90m each without provider output. + # Lead with the responsive long-context endpoint, then fall through + # to direct OpenAI and bounded provider-specific fallbacks. # The direct-OpenAI slot runs GPT-5.6 Luna: the newest family's # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" - # One attempt per model, then fall through to the next model. Retrying - # the SAME model 5x let a rate-limited/hung leader consume the whole - # step, so the pool never reached a healthy fallback model. - OPENCODE_MODEL_ATTEMPTS: "1" - # Preserve reviews that legitimately need tens of minutes to inspect a - # large repository. Changed-file count is not a repository-complexity - # proxy, so every cadence class gets 90 minutes per candidate while the - # bounded provider-pool watchdog remains the outer guard. - OPENCODE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-4.1 openai/gpt-5.6-luna github-models/deepseek/deepseek-v3-0324 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + # A responsive model gets one bounded format retry: #579 produced a + # substantive control block that failed only one adversarial receipt + # field. Fatal provider errors still skip the second attempt. + OPENCODE_MODEL_ATTEMPTS: "2" + OPENCODE_RUN_TIMEOUT_SECONDS: "1800" OPENCODE_EXPORT_TIMEOUT_SECONDS: "180" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700" - OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000" - # Keep cycling through the high-sensitivity candidate catalog until - # the retry budget or step timeout is exhausted; a single invalid - # cycle can be all provider formatting noise rather than review - # evidence. - OPENCODE_POOL_MAX_CYCLES: "0" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600" + OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3900" + # Two attempts across the full catalog are the bounded run. A later + # scheduler heartbeat owns a new same-head retry after exhaustion. + OPENCODE_POOL_MAX_CYCLES: "1" OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" - OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "5400" - OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700" - OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "5400" - OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700" - OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400" - OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700" - OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400" - OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700" - OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400" - OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700" - OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0" + OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "1800" + OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "3600" + OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is # larger. Keep the exact runtime failure visible without spending a # full medium/large cadence slot after the long-context candidate. OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45" - OPENCODE_DYNAMIC_MAX_CYCLES: "0" + # GitHub Models DeepSeek endpoints returned no provider detail for + # 90m twice in #579. Preserve them as fallbacks without queue capture. + OPENCODE_GITHUB_DEEPSEEK_RUN_TIMEOUT_SECONDS: "600" + OPENCODE_DYNAMIC_MAX_CYCLES: "1" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400" - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600" OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" @@ -3683,6 +3789,7 @@ jobs: GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }} GH_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }} PR_NUMBER: ${{ needs.validate-pr-metadata.outputs.pr_number }} + PR_BASE_REF: ${{ needs.validate-pr-metadata.outputs.base_ref }} HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} @@ -4228,6 +4335,8 @@ jobs: "- Reason: ${model_reason}" \ "- Scope: \`${CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL:-unknown}\`" \ "- Changed files: \`${CENTRAL_REVIEW_PROCESS_FALLBACK_CHANGED_COUNT:-unknown}\`" \ + "- Base ref: \`${PR_BASE_REF}\`" \ + "- Base SHA: \`${PR_BASE_SHA}\`" \ "- Head SHA: \`${HEAD_SHA}\`" \ "- Workflow run: ${RUN_ID}" \ "- Workflow attempt: ${RUN_ATTEMPT}" \ @@ -4244,9 +4353,11 @@ jobs: echo "::warning::CENTRAL_FAST_APPROVAL_LIVE_HEAD_UNAVAILABLE: could not re-check the live pull request head immediately before publishing an approval for ${HEAD_SHA}; skipping this GitHub side effect." exit 0 fi + live_base_ref="$(jq -r '.base.ref // empty' "$live_head_file")" + live_base_sha="$(jq -r '.base.sha // empty' "$live_head_file")" live_head="$(jq -r '.head.sha // empty' "$live_head_file")" - if [ "$live_head" != "$HEAD_SHA" ]; then - echo "::notice::CENTRAL_FAST_APPROVAL_STALE_HEAD: expected ${HEAD_SHA}, observed ${live_head:-missing}; skipping review publication." + if [ "$live_base_ref" != "$PR_BASE_REF" ] || [ "$live_base_sha" != "$PR_BASE_SHA" ] || [ "$live_head" != "$HEAD_SHA" ]; then + echo "::notice::CENTRAL_FAST_APPROVAL_STALE_IDENTITY: expected ${PR_BASE_REF}@${PR_BASE_SHA}..${HEAD_SHA}, observed ${live_base_ref:-missing}@${live_base_sha:-missing}..${live_head:-missing}; skipping review publication." exit 0 fi if ! curl_api_write -X POST --data-binary "@${payload_file}" \ @@ -4321,6 +4432,7 @@ jobs: NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" PR_NUMBER: ${{ needs.validate-pr-metadata.outputs.pr_number }} + PR_BASE_REF: ${{ needs.validate-pr-metadata.outputs.base_ref }} HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} @@ -4430,10 +4542,11 @@ jobs: --input "$review_payload_file" >"$response_file" 2>"$error_file" } - review_live_head_sha() { + review_live_pr_identity() { timeout "${REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS:-120}s" \ env GH_TOKEN="$review_head_guard_token" \ - gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha // empty' + gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" \ + --jq '[.base.ref // "", .base.sha // "", .head.sha // ""] | @tsv' } dismiss_stale_published_review() { @@ -4463,19 +4576,20 @@ jobs: validate_published_review_head() { local token_value="$1" response_file="$2" error_file="$3" - local live_head + local live_identity live_base_ref live_base_sha live_head - if ! live_head="$(review_live_head_sha 2>>"$error_file")"; then + if ! live_identity="$(review_live_pr_identity 2>>"$error_file")"; then REVIEW_PUBLICATION_STALE_HEAD=1 - printf 'OPENCODE_REVIEW_STALE_HEAD: live PR head could not be verified after publication for expected head %s.\n' "$HEAD_SHA" >>"$error_file" + printf 'OPENCODE_REVIEW_STALE_IDENTITY: live PR identity could not be verified after publication for expected base %s@%s and head %s.\n' "$PR_BASE_REF" "$PR_BASE_SHA" "$HEAD_SHA" >>"$error_file" return 1 fi - if [ "$live_head" = "$HEAD_SHA" ]; then + IFS=$'\t' read -r live_base_ref live_base_sha live_head <<<"$live_identity" + if [ "$live_base_ref" = "$PR_BASE_REF" ] && [ "$live_base_sha" = "$PR_BASE_SHA" ] && [ "$live_head" = "$HEAD_SHA" ]; then return 0 fi REVIEW_PUBLICATION_STALE_HEAD=1 - printf 'OPENCODE_REVIEW_STALE_HEAD: publication raced with a head update; expected %s, observed %s.\n' "$HEAD_SHA" "${live_head:-missing}" >>"$error_file" + printf 'OPENCODE_REVIEW_STALE_IDENTITY: publication raced with a PR identity update; expected %s@%s..%s, observed %s@%s..%s.\n' "$PR_BASE_REF" "$PR_BASE_SHA" "$HEAD_SHA" "${live_base_ref:-missing}" "${live_base_sha:-missing}" "${live_head:-missing}" >>"$error_file" dismiss_stale_published_review "$token_value" "$response_file" "$live_head" "$error_file" || true return 1 } @@ -4511,7 +4625,7 @@ jobs: post_pull_review_with_retry() { local token_label="$1" token_value="$2" review_payload_file="$3" error_file="$4" response_file="$5" - local attempts default_sleep attempt sleep_seconds api_timeout publish_status live_head + local attempts default_sleep attempt sleep_seconds api_timeout publish_status live_identity live_base_ref live_base_sha live_head attempts="${REVIEW_PUBLISH_RETRY_ATTEMPTS:-3}" default_sleep="${REVIEW_PUBLISH_RETRY_SLEEP_SECONDS:-30}" @@ -4520,9 +4634,15 @@ jobs: while :; do : >"$error_file" : >"$response_file" - if ! live_head="$(review_live_head_sha 2>>"$error_file")" || [ "$live_head" != "$HEAD_SHA" ]; then + if ! live_identity="$(review_live_pr_identity 2>>"$error_file")"; then REVIEW_PUBLICATION_STALE_HEAD=1 - printf 'OPENCODE_REVIEW_STALE_HEAD: refusing publication because expected head %s no longer matches live head %s.\n' "$HEAD_SHA" "${live_head:-missing}" >>"$error_file" + printf 'OPENCODE_REVIEW_STALE_IDENTITY: refusing publication because live PR identity could not be read for expected %s@%s..%s.\n' "$PR_BASE_REF" "$PR_BASE_SHA" "$HEAD_SHA" >>"$error_file" + return 1 + fi + IFS=$'\t' read -r live_base_ref live_base_sha live_head <<<"$live_identity" + if [ "$live_base_ref" != "$PR_BASE_REF" ] || [ "$live_base_sha" != "$PR_BASE_SHA" ] || [ "$live_head" != "$HEAD_SHA" ]; then + REVIEW_PUBLICATION_STALE_HEAD=1 + printf 'OPENCODE_REVIEW_STALE_IDENTITY: refusing publication because expected %s@%s..%s no longer matches live %s@%s..%s.\n' "$PR_BASE_REF" "$PR_BASE_SHA" "$HEAD_SHA" "${live_base_ref:-missing}" "${live_base_sha:-missing}" "${live_head:-missing}" >>"$error_file" return 1 fi printf 'OpenCode publishing pull review with %s token (attempt %s/%s, timeout %ss).\n' "$token_label" "$attempt" "$attempts" "$api_timeout" >&2 @@ -4614,6 +4734,8 @@ jobs: printf 'OpenCode current-head approval bridge\n\n' printf 'OpenCode approved current head `%s` with the primary review token, but legacy OpenCode `REQUEST_CHANGES` reviews published by `github-actions[bot]` can still determine GitHub `reviewDecision`. This same-head bridge approval supersedes only those stale OpenCode workflow reviews.\n\n' "$HEAD_SHA" printf -- '- Result: `APPROVE`\n' + printf -- '- Base ref: `%s`\n' "$PR_BASE_REF" + printf -- '- Base SHA: `%s`\n' "$PR_BASE_SHA" printf -- '- Head SHA: `%s`\n' "$HEAD_SHA" printf -- '- Workflow run: %s\n' "$RUN_ID" printf -- '- Workflow attempt: %s\n' "$RUN_ATTEMPT" @@ -6903,6 +7025,8 @@ jobs: printf '%s\n' "$reviews_json" | python3 scripts/ci/opencode_existing_approval_gate.py \ --head "$HEAD_SHA" \ + --base-ref "$PR_BASE_REF" \ + --base-sha "$PR_BASE_SHA" \ --require-opencode-app } @@ -7248,6 +7372,8 @@ jobs: "" \ "- Result: APPROVE" \ "- Reason: ${reason}" \ + "- Base ref: \`${PR_BASE_REF}\`" \ + "- Base SHA: \`${PR_BASE_SHA}\`" \ "- Head SHA: \`${HEAD_SHA}\`" \ "- Workflow run: ${RUN_ID}" \ "- Workflow attempt: ${RUN_ATTEMPT}" @@ -7366,6 +7492,8 @@ jobs: GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} GH_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }} PR_NUMBER: ${{ needs.validate-pr-metadata.outputs.pr_number }} + PR_BASE_REF: ${{ needs.validate-pr-metadata.outputs.base_ref }} + PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} OPENCODE_MODEL_POOL_OUTCOME: ${{ steps.opencode_review_model_pool.outputs.review_status }} @@ -7400,6 +7528,8 @@ jobs: --model-outcome "${OPENCODE_MODEL_POOL_OUTCOME:-missing}" \ --coverage-result "${COVERAGE_EVIDENCE_RESULT:-missing}" \ --expected-head "$PR_HEAD_SHA" \ + --expected-base-ref "$PR_BASE_REF" \ + --expected-base-sha "$PR_BASE_SHA" \ --pull-request-file "$pull_request_file" \ --reviews-file "$reviews_file" )" @@ -7425,6 +7555,7 @@ jobs: SCHEDULER_MUTATION_TOKEN_SOURCE: ${{ secrets.PR_REVIEW_MERGE_TOKEN != '' && 'PR_REVIEW_MERGE_TOKEN' || secrets.OPENCODE_APPROVE_TOKEN != '' && 'OPENCODE_APPROVE_TOKEN' || steps.opencode_app_token.outputs.available == 'true' && 'opencode-app' || 'github-token' }} GH_REPOSITORY: ${{ needs.validate-pr-metadata.outputs.target_repository }} PR_BASE_REF: ${{ needs.validate-pr-metadata.outputs.base_ref }} + PR_BASE_SHA: ${{ needs.validate-pr-metadata.outputs.base_sha }} PR_NUMBER: ${{ needs.validate-pr-metadata.outputs.pr_number }} PR_HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }} run: | @@ -7454,6 +7585,8 @@ jobs: if printf '%s\n' "$reviews_json" | python3 scripts/ci/opencode_existing_approval_gate.py \ --head "$PR_HEAD_SHA" \ + --base-ref "$PR_BASE_REF" \ + --base-sha "$PR_BASE_SHA" \ --require-opencode-app \ 2>"$gate_error_file"; then approval_visible=1 @@ -7481,6 +7614,24 @@ jobs: exit 0 fi + live_pr_file="$(mktemp)" + live_pr_error_file="$(mktemp)" + if ! GH_TOKEN="$approval_read_token" timeout 30s \ + gh api "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" >"$live_pr_file" 2>"$live_pr_error_file"; then + live_reason="$(tail -n 1 "$live_pr_error_file" 2>/dev/null || true)" + rm -f "$live_pr_file" "$live_pr_error_file" + printf '::warning::Merge scheduler follow-up skipped because live PR identity could not be revalidated after approval: %s.\n' "${live_reason:-GitHub API lookup failed}" + exit 0 + fi + live_base_ref="$(jq -r '.base.ref // empty' "$live_pr_file")" + live_base_sha="$(jq -r '.base.sha // empty' "$live_pr_file")" + live_head_sha="$(jq -r '.head.sha // empty' "$live_pr_file")" + rm -f "$live_pr_file" "$live_pr_error_file" + if [ "$live_base_ref" != "$PR_BASE_REF" ] || [ "$live_base_sha" != "$PR_BASE_SHA" ] || [ "$live_head_sha" != "$PR_HEAD_SHA" ]; then + printf '::warning::Merge scheduler follow-up skipped because PR identity changed after approval validation. Expected=%s@%s..%s observed=%s@%s..%s.\n' "$PR_BASE_REF" "$PR_BASE_SHA" "$PR_HEAD_SHA" "${live_base_ref:-missing}" "${live_base_sha:-missing}" "${live_head_sha:-missing}" + exit 0 + fi + default_branch="$( gh api "repos/${GH_REPOSITORY}" --jq '.default_branch // empty' 2>/dev/null || true )" diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 4ab0c1b8..a4d692c1 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -136,7 +136,11 @@ jobs: ) && ( github.event_name != 'repository_dispatch' || - github.event.client_payload.org_sweep != true + ( + github.event.client_payload.org_sweep != true && + github.actor == 'github-actions[bot]' && + github.event.sender.login == 'github-actions[bot]' + ) ) runs-on: ubuntu-latest permissions: @@ -148,7 +152,7 @@ jobs: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true GH_TOKEN: ${{ github.token }} - DEFAULT_BRANCH: ${{ github.event.client_payload.base_branch || inputs.base_branch || github.event.repository.default_branch }} + DEFAULT_BRANCH: ${{ github.event_name == 'workflow_call' && inputs.base_branch || github.event.repository.default_branch }} DRY_RUN: ${{ github.event.client_payload.dry_run == true || inputs.dry_run == true }} MAX_PRS: ${{ github.event.client_payload.max_prs || inputs.max_prs || '100' }} PROJECT_FLOW_INPUT: ${{ github.event.client_payload.project_flow || inputs.project_flow || vars.PROJECT_FLOW || '' }} @@ -483,7 +487,12 @@ jobs: github.repository == 'ContextualWisdomLab/.github' && ( (github.event_name == 'schedule' && github.event.schedule == '*/15 * * * *') || - (github.event_name == 'repository_dispatch' && github.event.client_payload.org_sweep == true) + ( + github.event_name == 'repository_dispatch' && + github.event.client_payload.org_sweep == true && + github.actor == 'github-actions[bot]' && + github.event.sender.login == 'github-actions[bot]' + ) ) runs-on: ubuntu-latest timeout-minutes: 30 @@ -498,6 +507,9 @@ jobs: GH_TOKEN: ${{ github.token }} DRY_RUN: ${{ github.event.client_payload.dry_run == true || inputs.dry_run == true }} ORG_SWEEP_OWNER: ContextualWisdomLab + # A repository_dispatch caller can bound a repair sweep to one exact + # organization repository instead of waking every open PR in the org. + ORG_SWEEP_TARGET_REPOSITORY: ${{ github.event.client_payload.target_repository || '' }} # Inspect the complete practical queue for every repository. The previous # default of 30 silently omitted older PRs whenever a repository had a # larger queue (BandScope had 34 during the incident that established @@ -705,6 +717,25 @@ jobs: | "\(.full_name)\t\(.default_branch)" ' <<<"$repositories_json" ) + if [ -n "$ORG_SWEEP_TARGET_REPOSITORY" ]; then + if [[ ! "$ORG_SWEEP_TARGET_REPOSITORY" =~ ^ContextualWisdomLab/[A-Za-z0-9_.-]+$ ]]; then + echo "::error::Scoped organization sweep target must be an exact ContextualWisdomLab repository name." + exit 1 + fi + mapfile -t sweep_targets < <( + jq -r --arg target "$ORG_SWEEP_TARGET_REPOSITORY" ' + .[] + | select(.archived == false and .disabled == false) + | select(.full_name == $target) + | select(.full_name != "ContextualWisdomLab/.github") + | "\(.full_name)\t\(.default_branch)" + ' <<<"$repositories_json" + ) + if [ "${#sweep_targets[@]}" -ne 1 ]; then + echo "::error::Scoped organization sweep target was not found as one active non-central repository." + exit 1 + fi + fi echo "Sweeping ${#sweep_targets[@]} repositories." failures=0 diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index efbfb361..379f1b5a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -406,18 +406,37 @@ jobs: && github.repository == 'ContextualWisdomLab/.github' && github.event.pull_request.base.repo.full_name == 'ContextualWisdomLab/.github' && github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github' + && github.event.pull_request.user.login == 'dependabot[bot]' + && startsWith(github.event.pull_request.head.ref, 'dependabot/pip/') env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | set -euo pipefail + if ! [[ "$PR_BASE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::PR base SHA must be a 40-character git SHA." + exit 1 + fi if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::PR head SHA must be a 40-character git SHA." exit 1 fi - if git -C "$TRUSTED_WORKSPACE" cat-file -e "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" 2>/dev/null; then - git -C "$TRUSTED_WORKSPACE" show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" > "$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt" - printf 'Materialized central Strix dependency lock from same-repository PR head.\n' + mapfile -d '' -t changed_files < <( + git -C "$TRUSTED_WORKSPACE" diff --name-only -z "$PR_BASE_SHA...$PR_HEAD_SHA" + ) + if [ "${#changed_files[@]}" -ne 1 ] || [ "${changed_files[0]}" != "requirements-strix-ci-hashes.txt" ]; then + echo "::error::Dependabot Strix lock validation requires the hashed lock to be the only changed file." + exit 1 + fi + read -r lock_mode lock_type _ lock_path < <( + git -C "$TRUSTED_WORKSPACE" ls-tree "$PR_HEAD_SHA" -- requirements-strix-ci-hashes.txt + ) + if [ "$lock_mode" != "100644" ] || [ "$lock_type" != "blob" ] || [ "$lock_path" != "requirements-strix-ci-hashes.txt" ]; then + echo "::error::Dependabot Strix lock must be a regular Git blob at the exact trusted path." + exit 1 fi + git -C "$TRUSTED_WORKSPACE" show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" > "$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt" + printf 'Materialized central Strix dependency lock from the exact Dependabot PR head.\n' - name: Gate Strix secrets id: gate @@ -490,7 +509,20 @@ jobs: # private install umask before creating the credential-bearing Strix # entry point; the runtime gate still rejects any later relaxation. umask 022 - python3 -m pip install --disable-pip-version-check --no-cache-dir --require-hashes -r requirements-strix-ci-hashes.txt + trusted_lock="$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt" + if [ ! -f "$trusted_lock" ] || [ -L "$trusted_lock" ]; then + echo "::error::Trusted Strix dependency lock is missing or is a symbolic link." + exit 1 + fi + resolved_trusted_lock="$(realpath "$trusted_lock")" + case "$resolved_trusted_lock" in + "$TRUSTED_STRIX_SOURCE"/*) ;; + *) + echo "::error::Trusted Strix dependency lock resolved outside the trusted source checkout." + exit 1 + ;; + esac + python3 -m pip install --disable-pip-version-check --no-cache-dir --require-hashes -r "$resolved_trusted_lock" strix_executable="$(command -v strix || true)" if [ -z "$strix_executable" ] || [[ "$strix_executable" != /* ]] \ || [ ! -f "$strix_executable" ] || [ -L "$strix_executable" ] \ @@ -752,50 +784,16 @@ jobs: export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds" export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" - # Capture the gate exit code plus its console output. The gate returns - # exit 1 both for genuine blocking vulnerabilities AND for - # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI - # infrastructure noise, not a security finding, so it must not fail - # the required check and block merges. + # Capture the gate exit code plus its console output, then preserve it + # exactly. A provider outage is not a vulnerability, but it also is not + # current-head security evidence: a required security check must fail + # closed whenever Strix cannot produce and validate a report. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" strix_rc=0 set +e bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log" strix_rc="${PIPESTATUS[0]}" set -e - - if [ "$strix_rc" -eq 0 ]; then - exit 0 - fi - - # Preserve configuration failures (exit 2) and any unexpected exit - # code as hard failures — only the scan-failure code (1) can be an - # infrastructure/backend-unavailability outcome. - if [ "$strix_rc" -ne 1 ]; then - exit "$strix_rc" - fi - - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - - # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported anywhere. This preserves - # real security gating while keeping uncontrollable provider outages - # from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ - && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." - exit 0 - fi - - echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2 exit "$strix_rc" - name: Collect Strix reports for artifact upload diff --git a/requirements-strix-ci-hashes.txt b/requirements-strix-ci-hashes.txt index 4cf45203..655e090c 100644 --- a/requirements-strix-ci-hashes.txt +++ b/requirements-strix-ci-hashes.txt @@ -1501,9 +1501,9 @@ protobuf==6.33.6 \ # grpc-google-iam-v1 # grpcio-status # proto-plus -pyasn1==0.6.3 \ - --hash=sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf \ - --hash=sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b # via pyasn1-modules pyasn1-modules==0.4.2 \ --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ diff --git a/scripts/ci/adversarial_evidence.py b/scripts/ci/adversarial_evidence.py index 4fbc4ce5..1c2e5f69 100644 --- a/scripts/ci/adversarial_evidence.py +++ b/scripts/ci/adversarial_evidence.py @@ -41,6 +41,8 @@ def adversarial_evidence_rejection_reason( evidence: str, path: str, line: int | None = None, + *, + require_location_citation: bool = True, ) -> str | None: """Return why probe evidence is circular, unbound, or lacks proof.""" cleaned = evidence.strip() @@ -49,23 +51,24 @@ def adversarial_evidence_rejection_reason( return "explicitly denies execution or an observed result" if any(phrase in lowered for phrase in CIRCULAR_EVIDENCE_PHRASES): return "repeats the implementation claim instead of citing independent proof" - escaped_path = rf"(? str: + """Return a trusted Git binary selected without the inherited runner PATH.""" + candidate = Path(GIT_EXECUTABLE or "") + if not GIT_EXECUTABLE or not candidate.is_absolute(): + raise RuntimeError("an absolute Git executable path is required") + try: + resolved = candidate.resolve(strict=True) + candidate_parent = candidate.parent.resolve(strict=True) + resolved_parent = resolved.parent.resolve(strict=True) + resolved_stat = resolved.stat() + parent_stats = (candidate_parent.stat(), resolved_parent.stat()) + except OSError as exc: + raise RuntimeError("the configured Git executable is unavailable") from exc + if ( + not resolved.is_file() + or not os.access(resolved, os.X_OK) + or resolved_stat.st_uid != TRUSTED_GIT_OWNER_UID + or resolved_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + or any( + parent_stat.st_uid != TRUSTED_GIT_OWNER_UID + or parent_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + for parent_stat in parent_stats + ) + ): + raise RuntimeError("the configured Git executable failed trust validation") + return str(resolved) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args(argv) + if not SHA_RE.fullmatch(args.base_sha): + parser.error("--base-sha must be a 40-character hexadecimal commit SHA") + return args + + +def git_bytes(repo_root: Path, *args: str) -> bytes: + """Run a bounded read-only Git command without shell interpretation.""" + completed = subprocess.run( # nosec B603 + [validated_git_executable(), "-C", str(repo_root), *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(detail or f"git {' '.join(args)} failed") + if len(completed.stdout) > MAX_TREE_OUTPUT_BYTES: + raise ValueError("Git output exceeded the bounded materialization limit") + return completed.stdout + + +def validate_repo_root(repo_root: Path, base_sha: str) -> Path: + """Resolve a repository containing the exact requested base commit.""" + resolved = repo_root.resolve(strict=True) + if not resolved.is_dir(): + raise ValueError("--repo-root must resolve to a directory") + commit = git_bytes(resolved, "rev-parse", f"{base_sha}^{{commit}}").decode().strip() + if commit.lower() != base_sha.lower(): + raise ValueError("--base-sha did not resolve to the exact requested commit") + return resolved + + +def safe_output_dir(output_dir: Path) -> Path: + """Return a fresh output path without symbolic-link ancestors.""" + absolute = output_dir.absolute() + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current /= part + try: + mode = current.lstat().st_mode + except FileNotFoundError: + continue + if stat.S_ISLNK(mode): + raise ValueError("--output-dir contains a symbolic-link component") + if absolute.exists() or absolute.is_symlink(): + raise ValueError("--output-dir must not already exist") + return absolute + + +def logical_requirement_lines(text: str) -> list[str]: + """Return non-comment requirement records with continuations joined.""" + logical: list[str] = [] + pending = "" + for raw_line in text.splitlines(): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + continue + continuation = stripped.endswith("\\") + fragment = stripped[:-1].rstrip() if continuation else stripped + pending = f"{pending} {fragment}".strip() + if not continuation: + logical.append(pending) + pending = "" + if pending: + raise ValueError("hashed requirements ended with an incomplete continuation") + return logical + + +def validate_lock_content(data: bytes) -> str: + """Accept only pinned package records with one or more SHA-256 hashes.""" + if len(data) > MAX_LOCK_BYTES: + raise ValueError("hashed requirements file exceeds the per-lock size limit") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("hashed requirements file must be UTF-8") from exc + if "\x00" in text: + raise ValueError("hashed requirements file contains a NUL byte") + records = logical_requirement_lines(text) + if not records: + raise ValueError("hashed requirements file contains no package records") + for record in records: + hashes = HASH_RE.findall(record) + requirement = HASH_RE.sub("", record).strip() + if not hashes or not PINNED_REQUIREMENT_RE.fullmatch(requirement): + raise ValueError( + "hashed requirements must contain only pinned package records " + "and sha256 hashes" + ) + return text + + +def parse_lock_entries(tree: bytes) -> list[tuple[str, str, int]]: + """Select bounded regular hashed-lock blobs from NUL-delimited ls-tree output.""" + selected: list[tuple[str, str, int]] = [] + total_bytes = 0 + for record in tree.split(b"\0"): + if not record: + continue + try: + metadata, raw_path = record.split(b"\t", 1) + mode, kind, oid, size_text = metadata.decode("ascii").split() + path = raw_path.decode("utf-8") + except (UnicodeDecodeError, ValueError) as exc: + raise ValueError("could not parse Git tree entry") from exc + pure_path = PurePosixPath(path) + if pure_path.name != "requirements-hashes.txt": + continue + if ( + pure_path.is_absolute() + or ".." in pure_path.parts + or pure_path.as_posix() != path + ): + raise ValueError(f"hashed requirements path or size is unsafe: {path}") + if mode != "100644" or kind != "blob" or not SHA_RE.fullmatch(oid): + raise ValueError(f"hashed requirements must be a regular file: {path}") + if not size_text.isdigit() or not SAFE_PATH_RE.fullmatch(path): + raise ValueError(f"hashed requirements path or size is unsafe: {path}") + size = int(size_text) + if size > MAX_LOCK_BYTES: + raise ValueError(f"hashed requirements file is too large: {path}") + total_bytes += size + if total_bytes > MAX_TOTAL_BYTES: + raise ValueError("hashed requirements exceed the aggregate size limit") + selected.append((path, oid, size)) + if len(selected) > MAX_LOCKS: + raise ValueError("too many hashed requirements files in the base commit") + return selected + + +def write_exclusive(path: Path, data: bytes) -> None: + """Write a non-executable file without following a final symlink.""" + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + handle = os.fdopen(descriptor, "wb") + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + with handle: + handle.write(data) + path.chmod(0o444) + + +def materialize(repo_root: Path, base_sha: str, output_dir: Path) -> dict[str, object]: + """Materialize validated base locks and a project-to-environment manifest.""" + repo = validate_repo_root(repo_root, base_sha) + output = safe_output_dir(output_dir) + tree = git_bytes(repo, "ls-tree", "-r", "-z", "-l", "--full-tree", base_sha) + entries = parse_lock_entries(tree) + output.mkdir(mode=0o755) + locks: list[dict[str, object]] = [] + manifest_lines: list[str] = [] + for index, (path, oid, expected_size) in enumerate(entries): + data = git_bytes(repo, "cat-file", "blob", oid) + if len(data) != expected_size: + raise RuntimeError(f"Git blob size changed after tree validation: {path}") + validate_lock_content(data) + slug = f"lock-{index:03d}" + filename = f"{slug}.txt" + project_dir = PurePosixPath(path).parent.as_posix() + if project_dir == ".": + project_dir = "." + write_exclusive(output / filename, data) + manifest_lines.append(f"{project_dir}\t{slug}\t{filename}\n") + locks.append( + { + "project_dir": project_dir, + "path": path, + "oid": oid, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "environment": slug, + "file": filename, + } + ) + write_exclusive(output / "manifest.tsv", "".join(manifest_lines).encode()) + metadata: dict[str, object] = { + "schema": 1, + "base_sha": base_sha.lower(), + "locks": locks, + } + write_exclusive( + output / "manifest.json", + (json.dumps(metadata, sort_keys=True, indent=2) + "\n").encode(), + ) + return metadata + + +def main(argv: list[str] | None = None) -> int: + """Run base-lock materialization.""" + args = parse_args(sys.argv[1:] if argv is None else argv) + try: + metadata = materialize(args.repo_root, args.base_sha, args.output_dir) + except (OSError, RuntimeError, ValueError) as exc: + print(f"base Python lock materialization failed: {exc}", file=sys.stderr) + return 1 + print(json.dumps(metadata, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through main() + raise SystemExit(main()) diff --git a/scripts/ci/materialize_pr_review_source.py b/scripts/ci/materialize_pr_review_source.py new file mode 100644 index 00000000..f431f381 --- /dev/null +++ b/scripts/ci/materialize_pr_review_source.py @@ -0,0 +1,519 @@ +#!/usr/bin/env python3 +"""Materialize an inert PR source tree from an isolated bare Git repository. + +The privileged OpenCode review job must inspect pull-request content without +checking an untrusted commit out into the trusted workflow repository. This +helper copies only validated Git blobs into a fresh directory, strips every +executable bit, represents symbolic links as inert regular files, and connects +read-only Git queries to the separate bare object store through a trusted +``.git`` pointer. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path, PurePosixPath +import re +import selectors +import shutil +import stat + +# Git is invoked through an absolute executable path, a fixed argv, and no shell. +import subprocess # nosec B404 +import sys +import time +from typing import BinaryIO + + +SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") +REGULAR_MODES = {"100644", "100755"} +SYMLINK_MODE = "120000" +GITLINK_MODE = "160000" +RESERVED_ROOTS = {".codegraph", ".git"} +DEFAULT_MAX_FILES = 100_000 +DEFAULT_MAX_BYTES = 1_073_741_824 +DEFAULT_MAX_TREE_METADATA_BYTES = 67_108_864 +DEFAULT_TREE_TIMEOUT_SECONDS = 60 +TREE_READ_CHUNK_BYTES = 65_536 +MAX_TREE_RECORD_BYTES = 1_048_576 +TREE_ENTRY_METADATA_OVERHEAD_BYTES = 128 +GIT_EXECUTABLE = shutil.which("git", path=os.defpath) +TRUSTED_GIT_OWNER_UID = 0 +if not GIT_EXECUTABLE or not Path(GIT_EXECUTABLE).is_absolute(): + raise RuntimeError("an absolute Git executable path is required") + + +def validated_git_executable() -> str: + """Return a trusted absolute Git binary for privileged materialization.""" + candidate = Path(GIT_EXECUTABLE or "") + if not GIT_EXECUTABLE or not candidate.is_absolute(): + raise RuntimeError("an absolute Git executable path is required") + try: + resolved = candidate.resolve(strict=True) + candidate_parent = candidate.parent.resolve(strict=True) + resolved_parent = resolved.parent.resolve(strict=True) + resolved_stat = resolved.stat() + parent_stats = (candidate_parent.stat(), resolved_parent.stat()) + except OSError as exc: + raise RuntimeError("the configured Git executable is unavailable") from exc + if ( + not resolved.is_file() + or not os.access(resolved, os.X_OK) + or resolved_stat.st_uid != TRUSTED_GIT_OWNER_UID + or resolved_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + or any( + parent_stat.st_uid != TRUSTED_GIT_OWNER_UID + or parent_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + for parent_stat in parent_stats + ) + ): + raise RuntimeError("the configured Git executable failed trust validation") + return str(resolved) + + +def positive_int(value: str) -> int: + """Parse a positive integer command-line limit.""" + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be positive") + return parsed + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + epilog="Precondition: the immediate parent of --output-dir must already exist.", + ) + parser.add_argument("--git-dir", type=Path, required=True) + parser.add_argument("--head-sha", required=True) + parser.add_argument( + "--output-dir", + type=Path, + required=True, + help="fresh output path whose immediate parent already exists", + ) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--max-files", type=positive_int, default=DEFAULT_MAX_FILES) + parser.add_argument("--max-bytes", type=positive_int, default=DEFAULT_MAX_BYTES) + parser.add_argument( + "--max-tree-metadata-bytes", + type=positive_int, + default=DEFAULT_MAX_TREE_METADATA_BYTES, + ) + parser.add_argument( + "--tree-timeout-seconds", + type=positive_int, + default=DEFAULT_TREE_TIMEOUT_SECONDS, + ) + args = parser.parse_args(argv) + if not SHA_RE.fullmatch(args.head_sha): + parser.error("--head-sha must be a 40-character hexadecimal commit SHA") + return args + + +def git_bytes(git_dir: Path, *args: str) -> bytes: + """Run a read-only Git command against the isolated object store.""" + # argv only; the Git directory and commit inputs are validated before use. + completed = subprocess.run( # nosec B603 + [validated_git_executable(), f"--git-dir={git_dir}", *args], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + if completed.returncode != 0: + detail = completed.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(detail or f"git {' '.join(args)} failed") + return completed.stdout + + +def validate_git_dir(git_dir: Path, head_sha: str) -> Path: + """Return a resolved, bare Git directory containing the requested commit.""" + resolved = git_dir.resolve(strict=True) + if not resolved.is_dir(): + raise ValueError("--git-dir must resolve to a directory") + bare = git_bytes(resolved, "rev-parse", "--is-bare-repository").decode().strip() + if bare != "true": + raise ValueError("--git-dir must be an isolated bare repository") + commit = git_bytes(resolved, "rev-parse", f"{head_sha}^{{commit}}").decode().strip() + if commit.lower() != head_sha.lower(): + raise ValueError("--head-sha did not resolve to the exact requested commit") + return resolved + + +def reject_symlink_components(path: Path, option: str) -> Path: + """Return an absolute path only when no existing component is a symlink.""" + absolute = path.absolute() + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current /= part + try: + mode = current.lstat().st_mode + except FileNotFoundError: + continue + if stat.S_ISLNK(mode): + raise ValueError(f"{option} contains a symbolic-link path component") + return absolute + + +def validate_output_path(output_dir: Path, git_dir: Path) -> Path: + """Require a fresh output path that cannot overlap the Git object store.""" + output = reject_symlink_components(output_dir, "--output-dir") + if output.exists() or output.is_symlink(): + raise ValueError("--output-dir must not already exist") + output_parent = output.parent.resolve(strict=True) + output = output_parent / output.name + if output == git_dir or output in git_dir.parents or git_dir in output.parents: + raise ValueError("--output-dir and --git-dir must not overlap") + return output + + +def safe_relative_path(raw_path: bytes) -> PurePosixPath: + """Decode and validate a Git tree path without accepting traversal.""" + try: + decoded = raw_path.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ValueError("Git tree contains a non-UTF-8 path") from exc + path = PurePosixPath(decoded) + if not decoded or decoded.startswith("/") or path.is_absolute(): + raise ValueError(f"unsafe absolute or empty Git tree path: {decoded!r}") + if ( + "\\" in decoded + or decoded != path.as_posix() + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"unsafe Git tree path component: {decoded!r}") + return path + + +def parse_tree_entry( + record: bytes, +) -> tuple[str, str, str, int, PurePosixPath]: + """Parse one bounded NUL-delimited ``git ls-tree`` record.""" + try: + metadata, raw_path = record.split(b"\t", 1) + mode_raw, kind_raw, oid_raw, size_raw = metadata.split(maxsplit=3) + mode = mode_raw.decode("ascii") + kind = kind_raw.decode("ascii") + oid = oid_raw.decode("ascii") + size_text = size_raw.decode("ascii") + except (UnicodeDecodeError, ValueError) as exc: + raise ValueError("could not parse Git tree entry") from exc + if not SHA_RE.fullmatch(oid): + raise ValueError("Git tree entry has an invalid object id") + if mode == GITLINK_MODE and kind == "commit" and size_text == "-": + size = len(f"Submodule commit {oid}\n".encode()) + elif kind == "blob" and size_text.isdigit(): + size = int(size_text) + else: + raise ValueError(f"unsupported Git tree entry type/mode: {kind}/{mode}") + return mode, kind, oid, size, safe_relative_path(raw_path) + + +def open_tree_reader(git_dir: Path, head_sha: str) -> subprocess.Popen[bytes]: + """Start bounded streaming enumeration of one exact commit tree.""" + return subprocess.Popen( # nosec B603 + [ + validated_git_executable(), + f"--git-dir={git_dir}", + "ls-tree", + "-r", + "-z", + "-l", + "--full-tree", + head_sha, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + ) + + +def terminate_process(process: subprocess.Popen[bytes]) -> None: + """Stop a tree producer promptly after a limit or parser failure.""" + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def parse_tree( + git_dir: Path, + head_sha: str, + *, + max_files: int, + max_bytes: int, + timeout_seconds: int, + max_tree_metadata_bytes: int = DEFAULT_MAX_TREE_METADATA_BYTES, +) -> tuple[list[tuple[str, str, str, int, PurePosixPath]], int]: + """Stream and bound validated recursive Git tree entries.""" + process = open_tree_reader(git_dir, head_sha) + stdout = process.stdout + if stdout is None: + terminate_process(process) + raise RuntimeError("Git ls-tree stdout pipe is unavailable") + + selector = selectors.DefaultSelector() + selector.register(stdout, selectors.EVENT_READ) + deadline = time.monotonic() + timeout_seconds + pending = bytearray() + entries: list[tuple[str, str, str, int, PurePosixPath]] = [] + total_bytes = 0 + total_metadata_bytes = 0 + try: + reached_eof = False + while not reached_eof: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise ValueError( + f"Git tree enumeration exceeded --tree-timeout-seconds ({timeout_seconds})" + ) + events = selector.select(timeout=min(1.0, remaining)) + if not events: + if process.poll() is not None: + reached_eof = True + continue + chunk = os.read(stdout.fileno(), TREE_READ_CHUNK_BYTES) + if not chunk: + reached_eof = True + continue + pending.extend(chunk) + while True: + separator = pending.find(0) + if separator < 0: + if len(pending) > MAX_TREE_RECORD_BYTES: + raise ValueError( + "Git tree entry exceeds the bounded record-size limit" + ) + break + if separator > MAX_TREE_RECORD_BYTES: + raise ValueError( + "Git tree entry exceeds the bounded record-size limit" + ) + record = bytes(pending[:separator]) + del pending[: separator + 1] + if not record: + continue + next_metadata_bytes = ( + total_metadata_bytes + + len(record) + + TREE_ENTRY_METADATA_OVERHEAD_BYTES + ) + if next_metadata_bytes > max_tree_metadata_bytes: + raise ValueError( + "Git tree exceeds --max-tree-metadata-bytes " + f"({next_metadata_bytes} > {max_tree_metadata_bytes})" + ) + entry = parse_tree_entry(record) + next_file_count = len(entries) + 1 + if next_file_count > max_files: + raise ValueError( + f"Git tree exceeds --max-files ({next_file_count} > {max_files})" + ) + next_total_bytes = total_bytes + entry[3] + if next_total_bytes > max_bytes: + raise ValueError( + f"Git tree exceeds --max-bytes ({next_total_bytes} > {max_bytes})" + ) + entries.append(entry) + total_bytes = next_total_bytes + total_metadata_bytes = next_metadata_bytes + if pending: + raise ValueError("Git tree output ended with an unterminated record") + except BaseException: + terminate_process(process) + raise + finally: + selector.close() + + try: + return_code = process.wait(timeout=5) + except subprocess.TimeoutExpired as exc: + terminate_process(process) + raise RuntimeError("Git ls-tree did not exit after output completed") from exc + stderr = ( + process.stderr.read().decode("utf-8", errors="replace").strip() + if process.stderr + else "" + ) + if return_code != 0: + raise RuntimeError(stderr or "Git ls-tree failed") + return entries, total_bytes + + +def open_batch_reader(git_dir: Path) -> subprocess.Popen[bytes]: + """Start one Git batch process for bounded blob reads.""" + # Fixed Git subcommand and no shell evaluation. + return subprocess.Popen( # nosec B603 + [validated_git_executable(), f"--git-dir={git_dir}", "cat-file", "--batch"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def read_blob(process: subprocess.Popen[bytes], oid: str, expected_size: int) -> bytes: + """Read one exact blob through the long-lived Git batch process.""" + stdin: BinaryIO | None = process.stdin + stdout: BinaryIO | None = process.stdout + if stdin is None or stdout is None: + raise RuntimeError("Git cat-file batch pipes are unavailable") + stdin.write(f"{oid}\n".encode("ascii")) + stdin.flush() + header = stdout.readline().rstrip(b"\n") + fields = header.split() + if len(fields) != 3 or fields[0].decode("ascii", errors="replace") != oid: + raise RuntimeError("Git cat-file returned an unexpected object header") + if fields[1] != b"blob" or not fields[2].isdigit(): + raise RuntimeError("Git cat-file object is not a blob") + actual_size = int(fields[2]) + if actual_size != expected_size: + raise RuntimeError("Git blob size changed after tree validation") + data = stdout.read(actual_size) + delimiter = stdout.read(1) + if len(data) != actual_size or delimiter != b"\n": + raise RuntimeError("Git cat-file returned a truncated blob") + return data + + +def write_inert_file(path: Path, data: bytes) -> None: + """Create one non-executable regular file without following links.""" + path.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + handle = os.fdopen(descriptor, "wb") + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + raise + with handle: + handle.write(data) + path.chmod(0o444) + + +def materialize(args: argparse.Namespace) -> dict[str, object]: + """Materialize validated inert files and return provenance metadata.""" + git_dir = validate_git_dir(args.git_dir, args.head_sha) + output_dir = validate_output_path(args.output_dir, git_dir) + manifest_path = reject_symlink_components(args.manifest, "--manifest") + if manifest_path.exists() or manifest_path.is_symlink(): + raise ValueError("--manifest must not already exist") + if manifest_path == output_dir or output_dir in manifest_path.parents: + raise ValueError("--manifest must be outside --output-dir") + + entries, total_bytes = parse_tree( + git_dir, + args.head_sha, + max_files=args.max_files, + max_bytes=args.max_bytes, + timeout_seconds=args.tree_timeout_seconds, + max_tree_metadata_bytes=args.max_tree_metadata_bytes, + ) + + output_dir.mkdir(mode=0o755) + skipped: list[dict[str, str]] = [] + special: list[dict[str, str]] = [] + written = 0 + process = open_batch_reader(git_dir) + try: + for mode, kind, oid, size, relative in entries: + rendered = relative.as_posix() + if relative.parts[0] in RESERVED_ROOTS: + skipped.append({"path": rendered, "reason": "reserved-review-metadata"}) + continue + destination = output_dir.joinpath(*relative.parts) + if mode == GITLINK_MODE and kind == "commit": + data = f"Submodule commit {oid}\n".encode() + special.append( + { + "path": rendered, + "original_mode": mode, + "representation": "gitlink-marker", + } + ) + elif kind == "blob" and mode in REGULAR_MODES | {SYMLINK_MODE}: + data = read_blob(process, oid, size) + if mode == SYMLINK_MODE: + special.append( + { + "path": rendered, + "original_mode": mode, + "representation": "inert-regular-file", + } + ) + elif mode == "100755": + special.append( + { + "path": rendered, + "original_mode": mode, + "representation": "non-executable-regular-file", + } + ) + else: + raise ValueError(f"unsupported Git entry {kind}/{mode} at {rendered}") + write_inert_file(destination, data) + written += 1 + finally: + if process.stdin is not None: + process.stdin.close() + stderr = ( + process.stderr.read().decode("utf-8", errors="replace") + if process.stderr + else "" + ) + return_code = process.wait() + if return_code != 0 and sys.exc_info()[0] is None: + raise RuntimeError(stderr.strip() or "Git cat-file batch failed") + + git_pointer = output_dir / ".git" + write_inert_file(git_pointer, f"gitdir: {git_dir}\n".encode()) + metadata: dict[str, object] = { + "schema": 1, + "head_sha": args.head_sha.lower(), + "git_dir": str(git_dir), + "source_dir": str(output_dir), + "tree_entries": len(entries), + "written_files": written, + "tree_bytes": total_bytes, + "skipped": skipped, + "special_representations": special, + } + # Recheck after materialization so an ancestor swapped to a link cannot + # redirect the final provenance write. + reject_symlink_components(manifest_path, "--manifest") + manifest_path.parent.mkdir(parents=True, exist_ok=True) + write_inert_file( + manifest_path, (json.dumps(metadata, sort_keys=True, indent=2) + "\n").encode() + ) + return metadata + + +def main(argv: list[str] | None = None) -> int: + """Run the inert PR source materializer.""" + args = parse_args(sys.argv[1:] if argv is None else argv) + try: + metadata = materialize(args) + except (OSError, RuntimeError, ValueError) as exc: + print(f"materialize_pr_review_source: {exc}", file=sys.stderr) + return 1 + print( + "Materialized inert PR source blobs: " + f"head={metadata['head_sha']} files={metadata['written_files']} bytes={metadata['tree_bytes']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 5b024c84..be3283e0 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -5,6 +5,7 @@ import argparse import base64 +import http.client import ipaddress import json import os @@ -16,19 +17,32 @@ import urllib.parse import urllib.request from collections.abc import Sequence +from pathlib import Path from typing import Any +TRUSTED_REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +if str(TRUSTED_REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(TRUSTED_REPOSITORY_ROOT)) + +from scripts.ci.opencode_existing_approval_gate import review_rejection_reason # noqa: E402 + PRIMARY_REVIEW_AUTHORS = { "opencode-agent[bot]", "opencode-agent", } -PRIMARY_REVIEW_MARKERS = ( - "OpenCode reviewed the current-head bounded evidence and found no blocking issues.", - "Result: APPROVE", - "opencode-review-control-v1", +REVIEW_BODY_HEAD_SHA_RE = re.compile( + r"^[ \t]*-[ \t]+Head SHA:[ \t]*`([0-9a-fA-F]{40})`[ \t]*$", + re.MULTILINE, +) +REVIEW_BODY_BASE_REF_RE = re.compile( + r"^[ \t]*-[ \t]+Base ref:[ \t]*`([A-Za-z0-9._/-]+)`[ \t]*$", + re.MULTILINE, +) +REVIEW_BODY_BASE_SHA_RE = re.compile( + r"^[ \t]*-[ \t]+Base SHA:[ \t]*`([0-9a-fA-F]{40})`[ \t]*$", + re.MULTILINE, ) -REVIEW_BODY_HEAD_SHA_RE = re.compile(r"Head SHA:\s*`([0-9a-fA-F]{40})`") IGNORED_RUNNING_CHECKS = { "approve-after-primary-review", "noema-review", @@ -41,6 +55,7 @@ MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 MAX_THREAD_BODY_CHARS = 1200 +MAX_LLM_RESPONSE_BYTES = 1_048_576 # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. @@ -64,6 +79,22 @@ def scrub_sensitive_data(text: str | None) -> str | None: return text +def read_bounded_llm_response(response: http.client.HTTPResponse) -> bytes: + """Read one LLM response within a strict byte budget.""" + raw_content_length = response.getheader("Content-Length") + if raw_content_length is not None: + try: + content_length = int(raw_content_length) + except ValueError as exc: + raise RuntimeError("Noema LLM endpoint returned an invalid Content-Length") from exc + if content_length < 0 or content_length > MAX_LLM_RESPONSE_BYTES: + raise RuntimeError("Noema LLM endpoint response exceeded the size limit") + raw = response.read(MAX_LLM_RESPONSE_BYTES + 1) + if len(raw) > MAX_LLM_RESPONSE_BYTES: + raise RuntimeError("Noema LLM endpoint response exceeded the size limit") + return raw + + def run(args: Sequence[str], *, stdin: str | None = None) -> str: """Run a command without invoking a shell and return stdout.""" if isinstance(args, str): @@ -109,6 +140,8 @@ def graphql(query: str, **fields: str | int) -> dict[str, Any]: title body isDraft + baseRefName + baseRefOid headRefOid reviewDecision reviewThreads(first: 100) { @@ -186,62 +219,111 @@ def review_body_head_sha(review: dict[str, Any]) -> str | None: return matches[-1] if matches else None -def review_matches_current_head(review: dict[str, Any], head_sha: str) -> bool: - """Return whether commit and explicit review-body evidence match the live head.""" +def review_matches_current_head( + review: dict[str, Any], + pr: dict[str, Any], + *, + require_base_identity: bool = False, +) -> bool: + """Return whether review evidence matches the live pull-request identity.""" + head_sha = str(pr.get("headRefOid") or "") if not head_sha or review_commit(review) != head_sha: return False body_head = review_body_head_sha(review) - return body_head is None or body_head.lower() == head_sha.lower() + if body_head is not None and body_head.lower() != head_sha.lower(): + return False + if not require_base_identity: + return True + body = str(review.get("body") or "") + base_refs = REVIEW_BODY_BASE_REF_RE.findall(body) + base_shas = REVIEW_BODY_BASE_SHA_RE.findall(body) + base_ref = str(pr.get("baseRefName") or "") + base_sha = str(pr.get("baseRefOid") or "") + return bool( + body_head + and base_ref + and base_sha + and base_refs + and base_refs[-1] == base_ref + and base_shas + and base_shas[-1].lower() == base_sha.lower() + ) def current_primary_approval(pr: dict[str, Any]) -> dict[str, Any] | None: """Return the current-head OpenCode approval when it matches the contract.""" - head_sha = str(pr.get("headRefOid") or "") - reviews = (((pr.get("reviews") or {}).get("nodes")) or []) + reviews = ((pr.get("reviews") or {}).get("nodes")) or [] for review in reversed(reviews): - if not review_matches_current_head(review, head_sha): + if not review_matches_current_head(review, pr, require_base_identity=True): continue if str(review.get("state") or "").upper() != "APPROVED": continue - body = str(review.get("body") or "") author = review_author(review) - if author in PRIMARY_REVIEW_AUTHORS and any(marker in body for marker in PRIMARY_REVIEW_MARKERS): + if author not in PRIMARY_REVIEW_AUTHORS: + continue + rest_review = { + "state": review.get("state"), + "commit_id": review_commit(review), + "user": {"login": author}, + "body": str(review.get("body") or ""), + } + if ( + review_rejection_reason( + rest_review, + str(pr.get("headRefOid") or ""), + str(pr.get("baseRefName") or ""), + str(pr.get("baseRefOid") or ""), + ) + is None + ): return review return None def has_current_changes_requested(pr: dict[str, Any]) -> bool: """Return whether the current head has any changes-requested review.""" - head_sha = str(pr.get("headRefOid") or "") - reviews = (((pr.get("reviews") or {}).get("nodes")) or []) + reviews = ((pr.get("reviews") or {}).get("nodes")) or [] for review in reversed(reviews): - if review_matches_current_head(review, head_sha) and str(review.get("state") or "").upper() == "CHANGES_REQUESTED": + if ( + review_matches_current_head(review, pr) + and str(review.get("state") or "").upper() == "CHANGES_REQUESTED" + ): return True return False def has_unresolved_threads(pr: dict[str, Any]) -> bool: """Return whether any non-outdated review thread is unresolved.""" - threads = (((pr.get("reviewThreads") or {}).get("nodes")) or []) - return any(not thread.get("isResolved") and not thread.get("isOutdated") for thread in threads) + threads = ((pr.get("reviewThreads") or {}).get("nodes")) or [] + return any( + not thread.get("isResolved") and not thread.get("isOutdated") + for thread in threads + ) def check_label(node: dict[str, Any]) -> str: """Return a human-readable label for a status context or check run.""" if node.get("__typename") == "StatusContext": return str(node.get("context") or "") - workflow = ((((node.get("checkSuite") or {}).get("workflowRun") or {}).get("workflow") or {}).get("name") or "") + workflow = ( + ((node.get("checkSuite") or {}).get("workflowRun") or {}).get("workflow") or {} + ).get("name") or "" name = str(node.get("name") or "") return f"{workflow} / {name}" if workflow else name def blocking_checks(pr: dict[str, Any]) -> list[str]: """Return check contexts that should block Noema review.""" - contexts = ((((pr.get("statusCheckRollup") or {}).get("contexts") or {}).get("nodes")) or []) + contexts = ( + ((pr.get("statusCheckRollup") or {}).get("contexts") or {}).get("nodes") + ) or [] blockers: list[str] = [] for node in contexts: label = check_label(node) - if label in IGNORED_RUNNING_CHECKS or str(node.get("name") or "") in IGNORED_RUNNING_CHECKS: + if ( + label in IGNORED_RUNNING_CHECKS + or str(node.get("name") or "") in IGNORED_RUNNING_CHECKS + ): continue if node.get("__typename") == "StatusContext": state = str(node.get("state") or "").upper() @@ -259,14 +341,22 @@ def blocking_checks(pr: dict[str, Any]) -> list[str]: def existing_noema_review(pr: dict[str, Any], actor: str) -> bool: """Return whether Noema already reviewed the current head.""" - head_sha = str(pr.get("headRefOid") or "") marker = "", + f"", ] ) payload = { @@ -581,33 +758,31 @@ def inspect_and_review(repo: str, number: int) -> int: """Inspect PR state and submit Noema's LLM review when gates are clean.""" pr = fetch_pr(repo, number) actor = current_actor() + if not actor: + raise RuntimeError( + "Noema review cannot verify the active reviewer credential identity" + ) if actor in PRIMARY_REVIEW_AUTHORS: - print( + raise RuntimeError( f"Current token actor {actor!r} is already a primary review actor; " - "Noema review skipped so GitHub receives an independent reviewer." + "an independent Noema reviewer credential is required" ) - return 0 if pr.get("isDraft"): - print("PR is draft; Noema review skipped.") - return 0 + raise RuntimeError("Noema review cannot approve a draft pull request") if existing_noema_review(pr, actor): print("Current head already has a Noema review; nothing to do.") return 0 if not current_primary_approval(pr): - print("Current head does not have a primary OpenCode approval; Noema review skipped.") - return 0 + raise RuntimeError( + "Current head does not have a base-bound primary OpenCode approval" + ) if has_current_changes_requested(pr): - print("Current head has requested changes; Noema review skipped.") - return 0 + raise RuntimeError("Current head has a request-changes review") if has_unresolved_threads(pr): - print("PR has unresolved review threads; Noema review skipped.") - return 0 + raise RuntimeError("Pull request has unresolved review threads") blockers = blocking_checks(pr) if blockers: - print("Blocking checks remain; Noema review skipped:") - for blocker in blockers: - print(f"- {blocker}") - return 0 + raise RuntimeError("Blocking checks remain: " + "; ".join(blockers)) diff, truncated = fetch_diff(repo, number) review_context = build_review_context(repo, number, pr) verdict = call_llm(repo, number, pr, diff, truncated, review_context) diff --git a/scripts/ci/opencode_dispatch_status.py b/scripts/ci/opencode_dispatch_status.py index e413fb00..b14a06d1 100644 --- a/scripts/ci/opencode_dispatch_status.py +++ b/scripts/ci/opencode_dispatch_status.py @@ -10,11 +10,24 @@ from typing import Any, Sequence APPROVAL_AUTHORS = frozenset({"opencode-agent", "opencode-agent[bot]"}) -HEAD_SHA_RE = re.compile(r"Head SHA:\s*`?([0-9a-fA-F]{40})`?", re.IGNORECASE) +HEAD_SHA_RE = re.compile( + r"^- Head SHA:\s*`([0-9a-fA-F]{40})`\s*$", re.IGNORECASE | re.MULTILINE +) +BASE_SHA_RE = re.compile( + r"^- Base SHA:\s*`([0-9a-fA-F]{40})`\s*$", re.IGNORECASE | re.MULTILINE +) +BASE_REF_RE = re.compile( + r"^- Base ref:\s*`([A-Za-z0-9._/-]+)`\s*$", re.IGNORECASE | re.MULTILINE +) -def _has_current_approval(reviews: Sequence[dict[str, Any]], head_sha: str) -> bool: - """Return whether the latest OpenCode decision explicitly approves the exact head.""" +def _has_current_approval( + reviews: Sequence[dict[str, Any]], + head_sha: str, + base_ref: str, + base_sha: str, +) -> bool: + """Return whether the latest OpenCode decision approves the exact PR identity.""" for review in reversed(reviews): author = str((review.get("user") or {}).get("login") or "").casefold() if author not in APPROVAL_AUTHORS: @@ -24,6 +37,16 @@ def _has_current_approval(reviews: Sequence[dict[str, Any]], head_sha: str) -> b body_heads = HEAD_SHA_RE.findall(str(review.get("body") or "")) if not body_heads or body_heads[-1].lower() != head_sha.lower(): continue + body = str(review.get("body") or "") + body_base_refs = BASE_REF_RE.findall(body) + body_base_shas = BASE_SHA_RE.findall(body) + if ( + not body_base_refs + or body_base_refs[-1] != base_ref + or not body_base_shas + or body_base_shas[-1].lower() != base_sha.lower() + ): + return False return str(review.get("state") or "").upper() == "APPROVED" return False @@ -33,23 +56,36 @@ def decide_status( model_outcome: str, coverage_result: str, expected_head: str, + expected_base_ref: str, + expected_base_sha: str, pull_request: dict[str, Any], reviews: Sequence[dict[str, Any]], ) -> dict[str, str]: """Return a fail-closed GitHub commit-status decision.""" live_head = str((pull_request.get("head") or {}).get("sha") or "") + live_base_ref = str((pull_request.get("base") or {}).get("ref") or "") + live_base_sha = str((pull_request.get("base") or {}).get("sha") or "") if model_outcome != "success": reason = "OpenCode model review did not produce approval evidence." elif coverage_result != "success": reason = "OpenCode coverage evidence did not pass for the current head." elif not expected_head or live_head.lower() != expected_head.lower(): reason = "OpenCode status target is stale or the live PR head is unavailable." - elif not _has_current_approval(reviews, expected_head): - reason = "No validated exact-current-head OpenCode approval was published." + elif ( + not expected_base_ref + or live_base_ref != expected_base_ref + or not expected_base_sha + or live_base_sha.lower() != expected_base_sha.lower() + ): + reason = "OpenCode status target base changed or is unavailable." + elif not _has_current_approval( + reviews, expected_head, expected_base_ref, expected_base_sha + ): + reason = "No validated exact-current-PR OpenCode approval was published." else: return { "state": "success", - "description": "Validated current-head OpenCode approval and coverage passed.", + "description": "Validated base-bound current-head OpenCode approval and coverage.", } return {"state": "failure", "description": reason} @@ -60,6 +96,8 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--model-outcome", required=True) parser.add_argument("--coverage-result", required=True) parser.add_argument("--expected-head", required=True) + parser.add_argument("--expected-base-ref", required=True) + parser.add_argument("--expected-base-sha", required=True) parser.add_argument("--pull-request-file", required=True, type=Path) parser.add_argument("--reviews-file", required=True, type=Path) return parser.parse_args(argv) @@ -78,6 +116,8 @@ def main(argv: Sequence[str] | None = None) -> int: model_outcome=args.model_outcome, coverage_result=args.coverage_result, expected_head=args.expected_head, + expected_base_ref=args.expected_base_ref, + expected_base_sha=args.expected_base_sha, pull_request=pull_request, reviews=reviews, ), diff --git a/scripts/ci/opencode_existing_approval_gate.py b/scripts/ci/opencode_existing_approval_gate.py index 6712c022..9ab099da 100644 --- a/scripts/ci/opencode_existing_approval_gate.py +++ b/scripts/ci/opencode_existing_approval_gate.py @@ -35,8 +35,17 @@ re.IGNORECASE | re.DOTALL, ) SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") -WORKFLOW_RUN_RE = re.compile(r"(?m)^- Workflow run: [1-9][0-9]*\s*$") -WORKFLOW_ATTEMPT_RE = re.compile(r"(?m)^- Workflow attempt: [1-9][0-9]*\s*$") +BASE_REF_RE = re.compile(r"^(?!-)[A-Za-z0-9._/-]+$") +WORKFLOW_RUN_RE = re.compile(r"(?m)^- Workflow run: ([1-9][0-9]*)\s*$") +WORKFLOW_ATTEMPT_RE = re.compile(r"(?m)^- Workflow attempt: ([1-9][0-9]*)\s*$") +RESULT_LINE_RE = re.compile(r"(?m)^- Result: ([A-Z_]+)\s*$") +HEAD_SHA_LINE_RE = re.compile(r"(?m)^- Head SHA: `([0-9a-fA-F]{40})`\s*$") +BASE_REF_LINE_RE = re.compile(r"(?m)^- Base ref: `([A-Za-z0-9._/-]+)`\s*$") +BASE_SHA_LINE_RE = re.compile(r"(?m)^- Base SHA: `([0-9a-fA-F]{40})`\s*$") +CONTROL_BLOCK_RE = re.compile( + r"", + re.DOTALL, +) REQUIRED_PROBE_FIELDS = ( "path", "hypothesis", @@ -74,6 +83,20 @@ def extract_adversarial_evidence(body: str) -> dict[str, Any] | None: return evidence +def extract_control_payload(body: str) -> tuple[dict[str, Any] | None, str | None]: + """Return one unambiguous structured OpenCode control payload.""" + matches = list(CONTROL_BLOCK_RE.finditer(body)) + if len(matches) != 1: + return None, "review body must contain exactly one OpenCode control block" + try: + payload = json.loads(matches[0].group("payload")) + except json.JSONDecodeError: + return None, "OpenCode control block is not parseable JSON" + if not isinstance(payload, dict): + return None, "OpenCode control block must be a JSON object" + return payload, None + + def adversarial_rejection_reason(body: str) -> str | None: """Explain why structured adversarial evidence is not reusable.""" evidence = extract_adversarial_evidence(body) @@ -103,6 +126,7 @@ def adversarial_rejection_reason(body: str) -> str | None: str(probe["evidence"]), str(probe["path"]), probe.get("line") if isinstance(probe.get("line"), int) else None, + require_location_citation=False, ) if evidence_error: return f"adversarial-validation probe evidence {evidence_error}" @@ -119,10 +143,12 @@ def adversarial_rejection_reason(body: str) -> str | None: def review_rejection_reason( review: dict[str, Any], head_sha: str, + base_ref: str, + base_sha: str, *, approval_authors: frozenset[str] = APPROVAL_AUTHORS, ) -> str | None: - """Explain why a review cannot prove a real current-head model approval.""" + """Explain why a review cannot prove a real current-PR model approval.""" if str(review.get("state") or "").upper() != "APPROVED": return "review state is not APPROVED" if str(review.get("commit_id") or "").lower() != head_sha.lower(): @@ -138,20 +164,44 @@ def review_rejection_reason( return "review body is deterministic or model-unavailable fallback evidence" if PRIMARY_APPROVAL_MARKER not in body: return "review body lacks the real-model approval marker" - if "- Result: APPROVE" not in body: - return "review body lacks an APPROVE result" - if f"- Head SHA: `{head_sha}`" not in body: + if RESULT_LINE_RE.findall(body) != ["APPROVE"]: + return "review body must contain exactly one unambiguous APPROVE result" + if head_sha.lower() not in { + candidate.lower() for candidate in HEAD_SHA_LINE_RE.findall(body) + }: return "review body lacks the exact current-head SHA" - if not WORKFLOW_RUN_RE.search(body): + if BASE_REF_LINE_RE.findall(body)[-1:] != [base_ref]: + return "review body lacks the exact current base ref" + if base_sha.lower() not in { + candidate.lower() for candidate in BASE_SHA_LINE_RE.findall(body) + }: + return "review body lacks the exact current base SHA" + workflow_run = WORKFLOW_RUN_RE.search(body) + if not workflow_run: return "review body lacks a workflow run id" - if not WORKFLOW_ATTEMPT_RE.search(body): + workflow_attempt = WORKFLOW_ATTEMPT_RE.search(body) + if not workflow_attempt: return "review body lacks a workflow attempt" + control, control_error = extract_control_payload(body) + if control_error: + return control_error + assert control is not None + if str(control.get("result") or "").upper() != "APPROVE": + return "OpenCode control result is not APPROVE" + if str(control.get("head_sha") or "").lower() != head_sha.lower(): + return "OpenCode control head does not match current head" + if str(control.get("run_id") or "") != workflow_run.group(1): + return "OpenCode control workflow run does not match review metadata" + if str(control.get("run_attempt") or "") != workflow_attempt.group(1): + return "OpenCode control workflow attempt does not match review metadata" return adversarial_rejection_reason(body) def has_reusable_real_model_approval( reviews: list[dict[str, Any]], head_sha: str, + base_ref: str, + base_sha: str, *, log: TextIO, approval_authors: frozenset[str] = APPROVAL_AUTHORS, @@ -170,13 +220,15 @@ def has_reusable_real_model_approval( reason = review_rejection_reason( review, head_sha, + base_ref, + base_sha, approval_authors=approval_authors, ) review_id = review.get("id", "unknown") if reason is None: print( "existing-approval gate accepted real-model review " - f"id={review_id} author={login} head={head_sha}", + f"id={review_id} author={login} base={base_ref}@{base_sha} head={head_sha}", file=log, ) return True @@ -188,7 +240,7 @@ def has_reusable_real_model_approval( print( "existing-approval gate found no reusable real-model approval " - f"for head={head_sha}; same-head candidates={candidate_count}", + f"for base={base_ref}@{base_sha} head={head_sha}; same-head candidates={candidate_count}", file=log, ) return False @@ -198,6 +250,8 @@ def parse_args(argv: list[str]) -> argparse.Namespace: """Parse existing-approval gate command-line arguments.""" parser = argparse.ArgumentParser() parser.add_argument("--head", required=True) + parser.add_argument("--base-ref", required=True) + parser.add_argument("--base-sha", required=True) parser.add_argument( "--require-opencode-app", action="store_true", @@ -214,6 +268,14 @@ def main(argv: list[str]) -> int: "existing-approval gate requires a 40-character head SHA", file=sys.stderr ) return 2 + if not BASE_REF_RE.fullmatch(args.base_ref): + print("existing-approval gate requires a valid base ref", file=sys.stderr) + return 2 + if not SHA_RE.fullmatch(args.base_sha): + print( + "existing-approval gate requires a 40-character base SHA", file=sys.stderr + ) + return 2 try: reviews = flatten_reviews(json.load(sys.stdin)) except (json.JSONDecodeError, ValueError) as exc: @@ -227,6 +289,8 @@ def main(argv: list[str]) -> int: if has_reusable_real_model_approval( reviews, args.head, + args.base_ref, + args.base_sha, log=sys.stderr, approval_authors=approval_authors, ) diff --git a/scripts/ci/opencode_review_normalize_output.py b/scripts/ci/opencode_review_normalize_output.py index 4045d457..9d130dad 100755 --- a/scripts/ci/opencode_review_normalize_output.py +++ b/scripts/ci/opencode_review_normalize_output.py @@ -729,6 +729,7 @@ def adversarial_validation_error( probe_evidence, path, line, + require_location_citation=False, ) if evidence_error: return f"adversarial probe {index} evidence {evidence_error}" diff --git a/scripts/ci/opencode_review_prompt_template.md b/scripts/ci/opencode_review_prompt_template.md index 23592a87..284c393b 100644 --- a/scripts/ci/opencode_review_prompt_template.md +++ b/scripts/ci/opencode_review_prompt_template.md @@ -8,7 +8,7 @@ Read ./bounded-review-evidence.md first, especially Current-head authority order Use peer reviewer comments as adversarial seeds, not as authority. For every unresolved current-head comment from another review bot, independently verify the claim from source, tests, runtime/library documentation, or a scratch repro before deciding. Do not merely quote, summarize, or defer to the peer reviewer. If you would otherwise APPROVE but cannot source-back either a fix or a false-positive dismissal for each plausible peer finding, return REQUEST_CHANGES with your own line-specific finding and verification direction. -Adversarial validation is mandatory before every verdict. Begin from the hypothesis that the patch is wrong and try to falsify its safety and correctness claims. For each materially changed surface, construct concrete attacks or counterexamples from the most relevant classes: malformed or boundary input, authorization or tenant crossover, stale or concurrent state, dependency/runtime mismatch, error/rollback behavior, numerical extremes, and mobile/accessibility behavior. Execute a focused test, trace, source proof, or current-head check for each probe. Each evidence field must name the exact command, test/assertion, log/check/SARIF receipt, source trace, diff, CodeGraph path, or changed file and the observed result. It must also include exactly one `source-line-sha256=<64 lowercase hex>` receipt computed from the exact cited current-head line bytes without the line ending (for example with `hashlib.sha256(path.read_bytes().splitlines()[line - 1]).hexdigest()`). The trusted normalizer recomputes that digest; free-form prose, a digest for another line, or repeated receipts fail closed. Generic claims such as "source inspection and test coverage verify it" are invalid unless the evidence also states the concrete observed pass, failure, rejection, return value, exit code, or trace outcome. An implementation restatement such as "handles this case", "properly handles all cases", "works as expected", or "is safe" is circular and invalid. Do not count green checks, a repeated PR claim, or the absence of an observed failure as a probe. APPROVE requires at least two falsified probes for source, workflow, config, package, or test changes and at least one for non-code changes. REQUEST_CHANGES requires at least one confirmed probe anchored to a published finding. Record this evidence in `adversarial_validation`; every probe path must be an exact current-head changed file and every line must be a positive current-head line. +Adversarial validation is mandatory before every verdict. Begin from the hypothesis that the patch is wrong and try to falsify its safety and correctness claims. For each materially changed surface, construct concrete attacks or counterexamples from the most relevant classes: malformed or boundary input, authorization or tenant crossover, stale or concurrent state, dependency/runtime mismatch, error/rollback behavior, numerical extremes, and mobile/accessibility behavior. Execute a focused test, trace, source proof, or current-head check for each probe. Each evidence field must name the exact command, test/assertion, log/check/SARIF receipt, source trace, diff, or CodeGraph path and the observed result. The structured `path` and `line` fields identify the changed-file location, so the evidence prose does not need to repeat them. Evidence must include exactly one `source-line-sha256=<64 lowercase hex>` receipt computed from the structured current-head path and line bytes without the line ending (for example with `hashlib.sha256(path.read_bytes().splitlines()[line - 1]).hexdigest()`). The trusted normalizer verifies the structured path and line against the changed-file manifest and recomputes that digest; free-form prose, a digest for another line, or repeated receipts fail closed. Generic claims such as "source inspection and test coverage verify it" are invalid unless the evidence also states the concrete observed pass, failure, rejection, return value, exit code, or trace outcome. An implementation restatement such as "handles this case", "properly handles all cases", "works as expected", or "is safe" is circular and invalid. Do not count green checks, a repeated PR claim, or the absence of an observed failure as a probe. APPROVE requires at least two falsified probes for source, workflow, config, package, or test changes and at least one for non-code changes. REQUEST_CHANGES requires at least one confirmed probe anchored to a published finding. Record this evidence in `adversarial_validation`; every probe path must be an exact current-head changed file and every line must be a positive current-head line. Execution provenance is mandatory. Never claim that React DevTools, Chrome DevTools, browser DevTools, Playwright, Cypress, or Selenium ran, passed, confirmed, verified, or observed behavior unless bounded evidence contains a trusted `OPENCODE_EXECUTION_RECEIPT tool= status=passed|observed` line produced by the workflow. Source inspection and green checks are not runtime-tool receipts. When no receipt exists, describe only the source trace or explicit execution limitation; fabricating browser or DevTools evidence invalidates the entire control block. diff --git a/scripts/ci/pr_review_fix_scheduler.py b/scripts/ci/pr_review_fix_scheduler.py index 5ffc1368..a62f7843 100755 --- a/scripts/ci/pr_review_fix_scheduler.py +++ b/scripts/ci/pr_review_fix_scheduler.py @@ -72,18 +72,41 @@ def issue_comments(repo: str, number: int) -> list[dict[str, Any]]: return [comment for page in pages for comment in page] +def current_token_actor() -> str: + """Return the exact login represented by the active mutation credential.""" + try: + actor = run_json(["api", "user"]) + except (RuntimeError, json.JSONDecodeError): + return "" + if not isinstance(actor, dict): + return "" + return str(actor.get("login") or "").strip() + + def recent_fix_marker_exists( comments: list[dict[str, Any]], head_sha: str, min_interval_seconds: int, + trusted_author: str, ) -> bool: - """Return whether this head was already dispatched recently.""" + """Return whether this credential already dispatched the head recently.""" + trusted_author = trusted_author.strip().casefold() + if not trusted_author: + return False now = int(time.time()) for comment in reversed(comments): + author = str(((comment.get("user") or {}).get("login")) or "").casefold() + if author != trusted_author: + continue match = FIX_MARKER_RE.search(str(comment.get("body") or "")) if not match or match.group(1).lower() != head_sha.lower(): continue - return now - int(match.group(2)) < min_interval_seconds + try: + marker_epoch = int(match.group(2)) + except ValueError: + continue + age_seconds = now - marker_epoch + return 0 <= age_seconds < min_interval_seconds return False @@ -234,6 +257,7 @@ def inspect_pr( args: argparse.Namespace, *, comments: list[dict[str, Any]] | None = None, + trusted_actor: str | None = None, ) -> tuple[str, tuple[str, ...]]: """Inspect one PR and optionally dispatch autofix.""" number = int(pr["number"]) @@ -258,7 +282,12 @@ def inspect_pr( if comments is None: comments = issue_comments(repo, number) - if recent_fix_marker_exists(comments, str(pr["headRefOid"]), args.retry_hours * 3600): + if recent_fix_marker_exists( + comments, + str(pr["headRefOid"]), + args.retry_hours * 3600, + current_token_actor() if trusted_actor is None else trusted_actor, + ): return "wait", ("recent autofix marker exists for this head",) dispatch_autofix( @@ -294,6 +323,12 @@ def process_queue(args: argparse.Namespace) -> int: prs_needing_comments.append(pr) comments_by_pr: dict[int, list[dict[str, Any]]] = {} + trusted_actor = current_token_actor() if prs_needing_comments else "" + if prs_needing_comments and not trusted_actor: + raise RuntimeError( + "autofix dispatch blocked: active mutation credential actor " + "could not be resolved" + ) if len(prs_needing_comments) <= 1: # Fast path for single items for pr in prs_needing_comments: @@ -328,6 +363,7 @@ def fetch_comments(pr_number: int) -> tuple[int, list[dict[str, Any]]]: pr, args, comments=comments_by_pr.get(pr_number), + trusted_actor=trusted_actor, ) except RuntimeError as exc: action, reasons = "error", (str(exc),) @@ -343,9 +379,19 @@ def fetch_comments(pr_number: int) -> tuple[int, list[dict[str, Any]]]: def self_test() -> int: """Run cheap contract checks.""" head = "a" * 40 - comments = [{"body": f"{FIX_MARKER} head_sha={head} epoch={int(time.time())} -->"}] - assert recent_fix_marker_exists(comments, head, 24 * 3600) - assert not recent_fix_marker_exists(comments, "b" * 40, 24 * 3600) + base = "b" * 40 + comments = [ + { + "body": f"{FIX_MARKER} head_sha={head} epoch={int(time.time())} -->", + "user": {"login": "github-actions[bot]"}, + } + ] + assert recent_fix_marker_exists( + comments, head, 24 * 3600, "github-actions[bot]" + ) + assert not recent_fix_marker_exists( + comments, "b" * 40, 24 * 3600, "github-actions[bot]" + ) pr = { "reviews": { "nodes": [ @@ -371,11 +417,18 @@ def self_test() -> int: "state": "APPROVED", "author": {"login": "opencode-agent"}, "commit": {"oid": head}, - "body": "Approved.", + "body": ( + "Approved.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{base}`\n" + f"- Head SHA: `{head}`" + ), } ] }, "reviewThreads": {"nodes": []}, + "baseRefName": "main", + "baseRefOid": base, "headRefOid": head, "mergeStateStatus": "DIRTY", } diff --git a/scripts/ci/pr_review_merge_scheduler.py b/scripts/ci/pr_review_merge_scheduler.py index c13adfe9..eb389a9f 100644 --- a/scripts/ci/pr_review_merge_scheduler.py +++ b/scripts/ci/pr_review_merge_scheduler.py @@ -125,7 +125,18 @@ GIT_REF_RE = re.compile(r"^(?!-)[A-Za-z0-9._/-]+$") GIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") GITHUB_REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") -REVIEW_BODY_HEAD_SHA_RE = re.compile(r"Head SHA:\s*`([0-9a-fA-F]{40})`") +REVIEW_BODY_HEAD_SHA_RE = re.compile( + r"^[ \t]*-[ \t]+Head SHA:[ \t]*`([0-9a-fA-F]{40})`[ \t]*$", + re.MULTILINE, +) +REVIEW_BODY_BASE_REF_RE = re.compile( + r"^[ \t]*-[ \t]+Base ref:[ \t]*`([A-Za-z0-9._/-]+)`[ \t]*$", + re.MULTILINE, +) +REVIEW_BODY_BASE_SHA_RE = re.compile( + r"^[ \t]*-[ \t]+Base SHA:[ \t]*`([0-9a-fA-F]{40})`[ \t]*$", + re.MULTILINE, +) ACTIONS_JOB_DETAILS_URL_RE = re.compile(r"/actions/runs/\d+/job/(\d+)(?:[/?#]|$)") DIRECT_MERGE_AUTO_FALLBACK_MARKERS = ( "base branch policy prohibits the merge", @@ -987,17 +998,38 @@ def parse_github_datetime(value: str | None) -> datetime | None: def review_matches_current_head(review: dict[str, Any], pr: dict[str, Any]) -> bool: - """Return whether a review is valid evidence for the current head commit.""" + """Return whether a review is valid evidence for the current PR identity.""" head = pr.get("headRefOid") commit = (review.get("commit") or {}).get("oid") if not head: return False body_head = review_body_head_sha(review) + head_matches = False if commit == head: - return body_head is None or body_head.lower() == head.lower() - if not commit and body_head is not None: - return body_head.lower() == head.lower() - return False + head_matches = body_head is None or body_head.lower() == head.lower() + elif not commit and body_head is not None: + head_matches = body_head.lower() == head.lower() + if not head_matches: + return False + if str(review.get("state") or "").upper() != "APPROVED": + return True + + body = str(review.get("body") or "") + if any(marker in body.lower() for marker in DETERMINISTIC_APPROVAL_MARKERS): + return True + base_refs = REVIEW_BODY_BASE_REF_RE.findall(body) + base_shas = REVIEW_BODY_BASE_SHA_RE.findall(body) + base_ref = str(pr.get("baseRefName") or "") + base_sha = str(pr.get("baseRefOid") or "") + return bool( + (body_head or commit == head) + and base_ref + and base_sha + and base_refs + and base_refs[-1] == base_ref + and base_shas + and base_shas[-1].lower() == base_sha.lower() + ) def review_body_head_sha(review: dict[str, Any]) -> str | None: @@ -1396,18 +1428,9 @@ def failed_status_checks(pr: dict[str, Any]) -> list[str]: ): latest_check_runs[key] = (started_at, index, node) - successful_status_contexts = { - node.get("context") - for node in status_contexts - if (node.get("state") or "").upper() == "SUCCESS" - } for _, _, node in sorted(latest_check_runs.values(), key=lambda item: item[1]): conclusion = (node.get("conclusion") or "").upper() if conclusion in FAILED_CHECK_CONCLUSIONS: - if is_strix_context(node) and "strix" in successful_status_contexts: - continue - if is_opencode_context(node) and "opencode-review" in successful_status_contexts: - continue failed.append(node.get("name") or "check-run") for node in status_contexts: state = (node.get("state") or "").upper() @@ -3133,6 +3156,8 @@ def summarize_action_error(exc: RuntimeError) -> str: def self_test() -> None: """Exercise scheduler invariants without GitHub network access.""" + sample_head = "a" * 40 + sample_base = "b" * 40 assert split_repo("owner/name") == ("owner", "name") assert split_repo("owner/name/extra") == ("owner", "name/extra") try: @@ -3152,9 +3177,9 @@ def self_test() -> None: pass sample = { "number": 1, - "headRefOid": "abc", + "headRefOid": sample_head, "baseRefName": "main", - "baseRefOid": "base", + "baseRefOid": sample_base, "headRefName": "feature", "mergeStateStatus": "CLEAN", "restMergeableState": "CLEAN", @@ -3167,7 +3192,7 @@ def self_test() -> None: "nodes": [ { "commit": { - "oid": "abc", + "oid": sample_head, "committedDate": "2026-06-25T16:38:22Z", "messageHeadline": "feat: sample", } @@ -3180,9 +3205,14 @@ def self_test() -> None: { "state": "APPROVED", "author": {"login": "opencode-agent"}, - "body": "OpenCode Agent approved this head.", + "body": ( + "OpenCode Agent approved this head.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{sample_base}`\n" + f"- Head SHA: `{sample_head}`" + ), "submittedAt": "2026-06-25T15:42:19Z", - "commit": {"oid": "abc"}, + "commit": {"oid": sample_head}, } ] }, @@ -3284,7 +3314,7 @@ def self_test() -> None: "state": "APPROVED", "author": {"login": "not-opencode-agent"}, "body": "OpenCode Agent approved this head.", - "commit": {"oid": "abc"}, + "commit": {"oid": sample_head}, } ) assert has_current_head_approval(sample) @@ -3302,7 +3332,7 @@ def self_test() -> None: { "state": "CHANGES_REQUESTED", "author": {"login": "opencode-agent"}, - "commit": {"oid": "abc"}, + "commit": {"oid": sample_head}, } ] sample["autoMergeRequest"] = {"enabledAt": "2026-01-01T00:02:00Z"} @@ -3370,7 +3400,13 @@ def self_test() -> None: ) assert decision.action == "update_branch" assert "branch is outdated before review dispatch" in decision.reason - sample["reviews"]["nodes"][0]["commit"]["oid"] = "abc" + sample["reviews"]["nodes"][0]["commit"]["oid"] = sample_head + sample["reviews"]["nodes"][0]["body"] = ( + "OpenCode approved the current PR identity.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{sample_base}`\n" + f"- Head SHA: `{sample_head}`" + ) decision = inspect_pr( "owner/repo", sample, @@ -3521,9 +3557,9 @@ def self_test() -> None: assert "git status --short" in conflict_guidance["commands"] blocked_sample = { "number": 2, - "headRefOid": "abc", + "headRefOid": sample_head, "baseRefName": "main", - "baseRefOid": "base", + "baseRefOid": sample_base, "headRefName": "feature", "mergeStateStatus": "BLOCKED", "restMergeableState": "BLOCKED", @@ -3539,7 +3575,7 @@ def self_test() -> None: "nodes": [ { "commit": { - "oid": "abc", + "oid": sample_head, "committedDate": "2026-06-25T16:38:22Z", "messageHeadline": "ci: exercise blocked approval path", } @@ -3552,9 +3588,14 @@ def self_test() -> None: { "state": "APPROVED", "author": {"login": "opencode-agent"}, - "body": "OpenCode Agent approved this head.", + "body": ( + "OpenCode Agent approved this head.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{sample_base}`\n" + f"- Head SHA: `{sample_head}`" + ), "submittedAt": "2026-06-25T15:42:19Z", - "commit": {"oid": "abc"}, + "commit": {"oid": sample_head}, } ] }, diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 85e122ab..b2fbdf23 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -185,7 +185,7 @@ write_prompt() { fi printf 'Do not request changes solely because your tool call, MCP call, or full-file read was not executed. Treat that as a review source limitation unless current-head evidence explicitly reports a materialization failure; any such finding must be tied to that evidence, not a generic model-exhaustion message. REQUEST_CHANGES findings must cite a positive source/evidence line; never use line 0.\n' printf 'Always return a final control block instead of a progress summary. Return only the final review body.\n\n' - printf 'Adversarial evidence must state a concrete observed pass, failure, rejection, return value, exit code, or trace outcome and exactly one source-line-sha256=<64 lowercase hex> digest computed from the cited current-head line bytes without its line ending; generic source-inspection or coverage-verification claims are invalid.\n' + printf 'Adversarial evidence must state a concrete observed pass, failure, rejection, return value, exit code, or trace outcome and exactly one source-line-sha256=<64 lowercase hex> digest computed from the structured current-head path and line bytes without its line ending; path and line prose repetition is unnecessary, while generic source-inspection or coverage-verification claims are invalid.\n' printf 'Required control block shape:\n' printf '```json\n' printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"executed command or source-backed trace, observed outcome, and source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" @@ -346,6 +346,9 @@ cap_model_run_timeout() { github-models/openai/gpt-5 | github-models/openai/gpt-5-chat) cap_seconds="$(env_integer_or_default OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS 45)" ;; + github-models/deepseek/*) + cap_seconds="$(env_integer_or_default OPENCODE_GITHUB_DEEPSEEK_RUN_TIMEOUT_SECONDS 600)" + ;; *) printf '%s\n' "$run_timeout_seconds" return 0 @@ -413,7 +416,8 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" if [ "$opencode_status" -eq 124 ] || [ "$opencode_status" -eq 137 ]; then - printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" + printf 'OpenCode %s attempt %s/%s timed out after %ss; skipping remaining attempts for this model and falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" + return 2 fi if is_fatal_provider_failure "$opencode_json_file"; then printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" @@ -548,7 +552,7 @@ main() { uncapped_run_timeout="$OPENCODE_RUN_TIMEOUT_SECONDS" OPENCODE_RUN_TIMEOUT_SECONDS="$(cap_model_run_timeout "$model_candidate" "$OPENCODE_RUN_TIMEOUT_SECONDS")" if [ "$OPENCODE_RUN_TIMEOUT_SECONDS" -lt "$uncapped_run_timeout" ]; then - printf 'OpenCode %s runtime cap selected %ss instead of %ss because this installation has returned a constrained request-body limit for that endpoint.\n' \ + printf 'OpenCode %s provider-specific queue cap selected %ss instead of %ss so a constrained or non-responsive endpoint cannot monopolize the organization review queue.\n' \ "$model_candidate" "$OPENCODE_RUN_TIMEOUT_SECONDS" "$uncapped_run_timeout" fi export OPENCODE_RUN_TIMEOUT_SECONDS diff --git a/scripts/ci/safe_pytest_command.py b/scripts/ci/safe_pytest_command.py index a5b1c655..e6059520 100644 --- a/scripts/ci/safe_pytest_command.py +++ b/scripts/ci/safe_pytest_command.py @@ -9,24 +9,64 @@ import pathlib import re import shlex +import shutil import subprocess +import stat +import sys +import sysconfig from collections.abc import Sequence RUN_LINE_RE = re.compile(r"\s*(?:-\s*)?run:\s*(.+?)\s*$") PYTEST_EXECUTABLES = frozenset({"pytest", "py.test"}) PYTHON_EXECUTABLES = frozenset({"python", "python3"}) +TRUSTED_PYTHON_ENV_ROOT = pathlib.Path("/opt/base-python-envs") +TRUSTED_PYTHON_ENV_OWNER_UID = 0 +TRUSTED_INHERITED_EXECUTABLE_OWNER_UID = 0 +TRUSTED_INHERITED_EXECUTABLE_DIRS = tuple( + dict.fromkeys( + pathlib.Path(entry) + for entry in ( + *os.defpath.split(os.pathsep), + sysconfig.get_path("scripts"), + str(pathlib.Path(sys.executable).resolve().parent), + ) + if entry + ) +) + +def _is_within(path: pathlib.Path, root: pathlib.Path) -> bool: + """Return whether path is root or one of its descendants.""" + return path == root or root in path.parents -def _basename(value: str) -> str: - """Return a command token's POSIX basename.""" - return pathlib.PurePosixPath(value).name + +def _trusted_inherited_search_dirs() -> list[pathlib.Path]: + """Resolve the fixed runtime/system executable allowlist.""" + resolved_dirs: list[pathlib.Path] = [] + for configured_dir in TRUSTED_INHERITED_EXECUTABLE_DIRS: + try: + resolved_dir = configured_dir.resolve(strict=True) + resolved_stat = resolved_dir.stat() + except OSError: + continue + if ( + not resolved_dir.is_dir() + or resolved_stat.st_uid != TRUSTED_INHERITED_EXECUTABLE_OWNER_UID + or resolved_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + or resolved_dir in resolved_dirs + ): + continue + resolved_dirs.append(resolved_dir) + return resolved_dirs def _is_pytest_argv(argv: Sequence[str]) -> bool: """Return whether argv is a supported direct pytest invocation.""" if not argv: return False - executable = _basename(argv[0]) + executable = argv[0] + if "/" in executable or "\\" in executable: + return False if executable in PYTEST_EXECUTABLES: return True if executable in PYTHON_EXECUTABLES: @@ -47,7 +87,9 @@ def parse_safe_pytest_command(command: str) -> list[str] | None: argv = shlex.split(command) except ValueError: return None - if not argv or any("\n" in arg or "\x00" in arg or _has_shell_control(arg) for arg in argv): + if not argv or any( + "\n" in arg or "\x00" in arg or _has_shell_control(arg) for arg in argv + ): return None return argv if _is_pytest_argv(argv) else None @@ -78,16 +120,99 @@ def execute_command(project_dir: pathlib.Path, argv: Sequence[str]) -> int: raise ValueError("configured command is not a safe direct pytest invocation") env = os.environ.copy() env["PYTHONPATH"] = "." - virtualenv_bin = project_dir.resolve() / ".venv" / "bin" - if virtualenv_bin.is_dir(): + supplied_env_bin = os.environ.get("OPENCODE_PYTHON_ENV_BIN", "").strip() + trusted_bin: pathlib.Path | None = None + trusted_root: pathlib.Path | None = None + if supplied_env_bin: + candidate = pathlib.Path(supplied_env_bin) + try: + resolved = candidate.resolve(strict=True) + candidate_stat = resolved.stat() + trusted_root = TRUSTED_PYTHON_ENV_ROOT.resolve(strict=True) + except OSError as exc: + raise ValueError("trusted Python environment path is unavailable") from exc + if ( + not resolved.is_dir() + or trusted_root not in resolved.parents + or candidate_stat.st_uid != TRUSTED_PYTHON_ENV_OWNER_UID + or candidate_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ): + raise ValueError("trusted Python environment path failed validation") + trusted_bin = resolved + + if trusted_bin is not None: + search_path = str(trusted_bin) + trusted_resolution_roots = [trusted_root, *_trusted_inherited_search_dirs()] + else: inherited_path = env.get("PATH") - env["PATH"] = ( - os.pathsep.join((str(virtualenv_bin), inherited_path)) - if inherited_path - else str(virtualenv_bin) + if not inherited_path or any( + not entry or not pathlib.Path(entry).is_absolute() + for entry in inherited_path.split(os.pathsep) + ): + raise ValueError("inherited trusted PATH is missing or unsafe") + trusted_resolution_roots = _trusted_inherited_search_dirs() + if not trusted_resolution_roots: + raise ValueError("trusted executable directory allowlist is unavailable") + search_path = os.pathsep.join(str(path) for path in trusted_resolution_roots) + trusted_resolution_roots = [ + root for root in trusted_resolution_roots if root is not None + ] + executable_path = shutil.which(argv[0], path=search_path) + if not executable_path: + raise ValueError("configured pytest executable is unavailable from the trusted PATH") + candidate_executable = pathlib.Path(executable_path) + if not candidate_executable.is_absolute(): + raise ValueError("configured pytest executable must resolve to an absolute path") + try: + candidate_absolute = candidate_executable.absolute() + resolved_executable = candidate_executable.resolve(strict=True) + candidate_stat = candidate_executable.lstat() + resolved_stat = resolved_executable.stat() + project_root = project_dir.resolve(strict=True) + except OSError as exc: + raise ValueError("configured pytest executable path is unavailable") from exc + if ( + not resolved_executable.is_file() + or not os.access(resolved_executable, os.X_OK) + or ( + not stat.S_ISLNK(candidate_stat.st_mode) + and candidate_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + ) + or ( + trusted_bin is None + and resolved_stat.st_uid != TRUSTED_INHERITED_EXECUTABLE_OWNER_UID ) + or resolved_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH) + or candidate_absolute == project_root + or project_root in candidate_absolute.parents + or resolved_executable == project_root + or project_root in resolved_executable.parents + or not any( + _is_within(resolved_executable, root) + for root in trusted_resolution_roots + ) + ): + raise ValueError("configured pytest executable failed trust validation") + if trusted_bin is not None: + if ( + candidate_absolute.parent != trusted_bin + or candidate_stat.st_uid != TRUSTED_PYTHON_ENV_OWNER_UID + or resolved_stat.st_uid != TRUSTED_PYTHON_ENV_OWNER_UID + ): + raise ValueError("trusted Python environment executable failed validation") + else: + if not any( + candidate_absolute.parent == root for root in trusted_resolution_roots + ) or ( + not stat.S_ISLNK(candidate_stat.st_mode) + and candidate_stat.st_uid != TRUSTED_INHERITED_EXECUTABLE_OWNER_UID + ): + raise ValueError("inherited executable is outside the trusted allowlist") + env["PATH"] = search_path + + command = [str(resolved_executable), *argv[1:]] completed = subprocess.run( - list(argv), + command, cwd=project_dir, env=env, shell=False, @@ -119,11 +244,13 @@ def main(argv: Sequence[str] | None = None) -> int: command = json.loads(args.command_json) except json.JSONDecodeError as exc: raise SystemExit(f"invalid --command-json: {exc}") from exc - if not isinstance(command, list) or not all(isinstance(arg, str) for arg in command): + if not isinstance(command, list) or not all( + isinstance(arg, str) for arg in command + ): raise SystemExit("--command-json must be an array of strings") print(f"Executing configured pytest argv: {shlex.join(command)}") return execute_command(args.project_dir, command) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover - exercised through main() raise SystemExit(main()) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 8a8f8b31..a2e3f7c5 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -199,6 +199,11 @@ PY } has_strix_report_failure_signal() { + local report_dirs_snapshot="" + if [ "${1:-}" = "--since-snapshot" ]; then + report_dirs_snapshot="$2" + shift 2 + fi local report_root local report_log for report_root in "$@"; do @@ -206,14 +211,56 @@ has_strix_report_failure_signal() { continue fi while IFS= read -r -d '' report_log; do + if [ -n "$report_dirs_snapshot" ] && report_path_predates_snapshot "$report_log" "$report_dirs_snapshot"; then + continue + fi if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then return 0 fi - done < <(find "$report_root" -type f -name '*.log' -print0) + # gate-attempts contains this wrapper's own captured console output. It is + # evidence for reviewers, not a Strix-emitted report: scanning it would + # turn benign phrases such as "timeout disabled" into false failures. + done < <( + find "$report_root" \ + -type d -name gate-attempts -prune -o \ + -type f -name '*.log' -print0 + ) done return 1 } +capture_report_dirs_snapshot() { + local output_file="$1" + shift + : >"$output_file" + local report_root report_dir + for report_root in "$@"; do + if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then + continue + fi + for report_dir in "$report_root"/*; do + if [ -d "$report_dir" ] && [ ! -L "$report_dir" ]; then + printf '%s\n' "$report_dir" >>"$output_file" + fi + done + done +} + +report_path_predates_snapshot() { + local candidate="$1" + local snapshot_file="$2" + if [ -z "$snapshot_file" ] || [ ! -f "$snapshot_file" ] || [ -L "$snapshot_file" ]; then + return 1 + fi + local existing + while IFS= read -r existing; do + if [[ "$candidate" == "$existing" || "$candidate" == "$existing/"* ]]; then + return 0 + fi + done <"$snapshot_file" + return 1 +} + # shellcheck disable=SC2317,SC2329 # invoked from EXIT/INT/TERM trap cleanup_runtime() { publish_artifact_reports || true @@ -1179,7 +1226,14 @@ pull_request_scope_context_files() { local needs_deployment_context=0 local changed_file normalized_changed_file for changed_file in "$@"; do - normalized_changed_file="$(normalize_changed_file_path "$changed_file")" || return 2 + # Callers should pass only the cache produced by + # normalize_changed_files_cache. Revalidate the boundary so a future raw + # call fails closed instead of silently changing the context allowlist. + if ! normalized_changed_file="$(normalize_changed_file_path "$changed_file")" || + [ "$normalized_changed_file" != "$changed_file" ]; then + echo "ERROR: pull_request_scope_context_files received a non-normalized changed path" >&2 + return 2 + fi case "$normalized_changed_file" in backend/*) if [[ "$normalized_changed_file" =~ ^backend/.+\.py$ ]]; then @@ -1291,6 +1345,17 @@ changed_file_list_contains() { return 1 } +normalized_changed_file_list_contains() { + local candidate="$1" + local normalized_changed_file + for normalized_changed_file in "${NORMALIZED_CHANGED_FILES[@]}"; do + if [ "$normalized_changed_file" = "$candidate" ]; then + return 0 + fi + done + return 1 +} + build_pull_request_scope_dir() { local scope_dir scope_dir="$(mktemp -d "${TMPDIR:-/tmp}/strix-pr-scope.XXXXXX")" @@ -1300,22 +1365,27 @@ build_pull_request_scope_dir() { copy_changed_file_into_scope() { local changed_file="$1" local relative_path - relative_path="$(normalize_changed_file_path "$changed_file")" || { - echo "ERROR: pull request changed file path is unsafe: $changed_file" >&2 + if ! normalized_changed_file_list_contains "$changed_file"; then + echo "ERROR: pull request changed file path was not validated: $changed_file" >&2 return 2 - } + fi + relative_path="$changed_file" + case "$relative_path" in + "" | /* | . | .. | ./* | ../* | */./* | */../* | */. | */..) + echo "ERROR: pull request changed file escaped the generated scope: $changed_file" >&2 + return 2 + ;; + esac local dst_path - dst_path="$( - python3 - "$scope_dir" "$relative_path" <<'PY' -from pathlib import Path -import sys - -scope_root = Path(sys.argv[1]).resolve(strict=True) -relative_path = Path(sys.argv[2]) -dst_path = scope_root / relative_path -print(dst_path) -PY - )" + dst_path="$scope_dir/$relative_path" + case "$dst_path" in + "$scope_dir"/*) + ;; + *) + echo "ERROR: pull request changed file escaped the generated scope: $changed_file" >&2 + return 2 + ;; + esac mkdir -p -- "$(dirname -- "$dst_path")" local copy_rc=1 local head_sha_for_copy @@ -1342,27 +1412,16 @@ PY copy_trusted_context_file_into_scope() { local context_file="$1" local relative_path - relative_path="$(normalize_changed_file_path "$context_file")" || { - echo "ERROR: pull request context file path is unsafe: $context_file" >&2 - return 2 - } + # context_file comes exclusively from the central, literal allowlist in + # pull_request_scope_context_files; it is never PR-controlled input. + relative_path="$context_file" local dst_path - dst_path="$( - python3 - "$scope_dir" "$relative_path" <<'PY' -from pathlib import Path -import sys - -scope_root = Path(sys.argv[1]).resolve(strict=True) -relative_path = Path(sys.argv[2]) -dst_path = scope_root / relative_path -print(dst_path) -PY - )" + dst_path="$scope_dir/$relative_path" if [ -e "$dst_path" ]; then return 0 fi local changed_context_rc=0 - changed_file_list_contains "$relative_path" || changed_context_rc=$? + normalized_changed_file_list_contains "$relative_path" || changed_context_rc=$? case "$changed_context_rc" in 0) mkdir -p -- "$(dirname -- "$dst_path")" @@ -1400,17 +1459,7 @@ PY copy_scope_support_file() { local relative_path="$1" local dst_path - dst_path="$( - python3 - "$scope_dir" "$relative_path" <<'PY' -from pathlib import Path -import sys - -scope_root = Path(sys.argv[1]).resolve(strict=True) -relative_path = Path(sys.argv[2]) -dst_path = scope_root / relative_path -print(dst_path) -PY - )" + dst_path="$scope_dir/$relative_path" if [ -e "$dst_path" ]; then return 0 fi @@ -1428,7 +1477,10 @@ PY local include_opencode_normalizer=0 local changed_file relative_path for changed_file in "$@"; do - relative_path="$(normalize_changed_file_path "$changed_file")" || return 2 + if ! normalized_changed_file_list_contains "$changed_file"; then + return 2 + fi + relative_path="$changed_file" case "$relative_path" in scripts/ci/strix_quick_gate.sh | scripts/ci/test_strix_quick_gate.sh) include_strix_model_utils=1 @@ -1674,7 +1726,7 @@ PY printf "Using narrowed target path %s for pull request Strix scan with %s scannable changed file(s).\n" "$narrowed_target" "$total_files" >&2 else local build_scope_rc=0 - build_pull_request_scope_dir "${CHANGED_FILES[@]}" || build_scope_rc=$? + build_pull_request_scope_dir "${NORMALIZED_CHANGED_FILES[@]}" || build_scope_rc=$? if [ "$build_scope_rc" -eq 0 ]; then TARGET_PATH="$LAST_PULL_REQUEST_SCOPE_DIR" TARGET_PATH_IS_INTERNAL_PR_SCOPE=1 @@ -1688,7 +1740,7 @@ PY return 0 fi local build_scope_rc=0 - build_pull_request_scope_dir "${CHANGED_FILES[@]}" || build_scope_rc=$? + build_pull_request_scope_dir "${NORMALIZED_CHANGED_FILES[@]}" || build_scope_rc=$? if [ "$build_scope_rc" -ne 0 ]; then return 2 fi @@ -1972,6 +2024,9 @@ evaluate_pull_request_findings() { if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then continue fi + if [ "$(basename -- "$run_dir")" = "gate-attempts" ]; then + continue + fi if is_preexisting_report_dir "$run_dir"; then continue fi @@ -2274,7 +2329,11 @@ child_model_for_api_base() { if [ -n "$llm_api_base_value" ] && is_github_models_api_base "$llm_api_base_value"; then case "$model" in github_models/openai/*) - printf '%s\n' "${model#github_models/}" + # LiteLLM consumes the first openai/ segment as its provider + # selector. Preserve the second one so the GitHub Models request + # body receives the catalog ID openai/ instead of the + # invalid publisher-less value. + printf 'openai/%s\n' "${model#github_models/}" return 0 ;; github_models/*) @@ -2335,6 +2394,12 @@ run_strix_once() { if ! resolved_target_path="$(resolve_current_target_path "$TARGET_PATH")"; then return 1 fi + local attempt_report_dirs_snapshot + attempt_report_dirs_snapshot="$(mktemp "$STRIX_RUNTIME_DIR/report-dirs-before.XXXXXX")" + capture_report_dirs_snapshot \ + "$attempt_report_dirs_snapshot" \ + "$ACTIVE_REPORTS_DIR" \ + "${resolved_target_path%/}/strix_runs" local start_epoch start_epoch="$(date +%s)" local child_llm_api_key="" @@ -2566,7 +2631,9 @@ PY sanitize_known_strix_report_warnings "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs" local report_failure_signal=0 - if has_strix_report_failure_signal "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs"; then + if has_strix_report_failure_signal \ + --since-snapshot "$attempt_report_dirs_snapshot" \ + "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs"; then report_failure_signal=1 echo "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." | tee -a "$STRIX_LOG" >&2 fi @@ -2580,7 +2647,7 @@ PY fi if [ "$rc" -eq 0 ]; then - if has_blocking_vulnerability_reports; then + if has_blocking_vulnerability_reports "$attempt_report_dirs_snapshot"; then echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 return 1 fi @@ -2959,6 +3026,9 @@ latest_strix_report_dir() { if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then continue fi + if [ "$(basename -- "$run_dir")" = "gate-attempts" ]; then + continue + fi if is_preexisting_report_dir "$run_dir"; then continue @@ -3068,6 +3138,7 @@ has_only_below_threshold_vulnerabilities() { } has_blocking_vulnerability_reports() { + local report_dirs_snapshot="${1:-}" local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" @@ -3076,9 +3147,15 @@ has_blocking_vulnerability_reports() { if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then continue fi + if [ "$(basename -- "$run_dir")" = "gate-attempts" ]; then + continue + fi if is_preexisting_report_dir "$run_dir"; then continue fi + if [ -n "$report_dirs_snapshot" ] && report_path_predates_snapshot "$run_dir" "$report_dirs_snapshot"; then + continue + fi vulnerabilities_dir="$run_dir/vulnerabilities" if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 57df964a..416f0a46 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -136,6 +136,10 @@ assert_file_contains "$workflow_file" 'repository: ${{ steps.trusted_source.outp assert_file_contains "$workflow_file" 'ref: ${{ steps.trusted_source.outputs.ref }}' "Strix workflow checks out resolved central ref" assert_file_contains "$workflow_file" "Materialize central Strix dependency lock from PR head" "Strix workflow validates same-repo central lock-file PRs against the PR head lock" assert_file_contains "$workflow_file" "requirements-strix-ci-hashes.txt" "Strix workflow can materialize the central Strix hashed requirements lock" +assert_file_contains "$workflow_file" "github.event.pull_request.user.login == 'dependabot[bot]'" "Strix PR-head lock validation is limited to Dependabot" +assert_file_contains "$workflow_file" "startsWith(github.event.pull_request.head.ref, 'dependabot/pip/')" "Strix PR-head lock validation is limited to Dependabot pip branches" +assert_file_contains "$workflow_file" '"${#changed_files[@]}" -ne 1' "Strix Dependabot validation requires exactly one changed file" +assert_file_contains "$workflow_file" '"$lock_mode" != "100644"' "Strix Dependabot validation requires a regular Git blob" assert_file_contains "$workflow_file" "Materialize target workspace" "Strix workflow separates target workspace from trusted source" assert_file_contains "$workflow_file" 'STRIX_REPO_ROOT:' "Strix workflow passes target root explicitly" assert_file_contains "$workflow_file" 'bash "$TRUSTED_STRIX_GATE"' "Strix workflow executes central Strix gate" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index aae63c5d..f16a7aca 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -193,9 +193,19 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "Checkout trusted Strix source" "strix workflow checks out the central Strix source" assert_file_contains "$workflow_file" 'repository: ${{ steps.trusted_source.outputs.repository }}' "strix workflow checks out central Strix scripts instead of target-repo copies" assert_file_contains "$workflow_file" 'ref: ${{ steps.trusted_source.outputs.ref }}' "strix workflow checks out the exact trusted Strix source ref" - assert_file_contains "$workflow_file" "Materialize central Strix dependency lock from PR head" "strix workflow validates central same-repo lock-file PRs against the PR head lock" - assert_file_contains "$workflow_file" "github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github'" "strix workflow limits central lock materialization to same-repository PR heads" - assert_file_contains "$workflow_file" 'git -C "$TRUSTED_WORKSPACE" show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt"' "strix workflow copies only the hashed requirements lock from the PR head" + assert_file_contains "$workflow_file" "Materialize central Strix dependency lock from PR head" "strix workflow validates same-repository central dependency-lock PRs" + assert_file_contains "$workflow_file" "github.event.pull_request.base.repo.full_name == 'ContextualWisdomLab/.github'" "strix PR-head lock override requires the central base repository" + assert_file_contains "$workflow_file" "github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github'" "strix PR-head lock override requires the central head repository" + assert_file_contains "$workflow_file" "github.event.pull_request.user.login == 'dependabot[bot]'" "strix PR-head lock override is limited to Dependabot" + assert_file_contains "$workflow_file" "startsWith(github.event.pull_request.head.ref, 'dependabot/pip/')" "strix PR-head lock override is limited to Dependabot pip branches" + assert_file_contains "$workflow_file" 'diff --name-only -z "$PR_BASE_SHA...$PR_HEAD_SHA"' "strix Dependabot lock validation uses merge-base PR diff semantics" + assert_file_contains "$workflow_file" '[[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]' "strix PR-head lock override requires an exact commit SHA" + assert_file_contains "$workflow_file" 'PR_HEAD_SHA:requirements-strix-ci-hashes.txt' "strix workflow reads only the central hashed dependency lock from the exact PR head" + assert_file_contains "$workflow_file" '"${#changed_files[@]}" -ne 1' "strix Dependabot lock validation requires exactly one changed file" + assert_file_contains "$workflow_file" '"$lock_mode" != "100644"' "strix Dependabot lock validation requires a regular Git blob" + assert_file_contains "$workflow_file" 'trusted_lock="$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt"' "strix workflow selects the hashed dependency lock from trusted source" + assert_file_contains "$workflow_file" 'resolved_trusted_lock="$(realpath "$trusted_lock")"' "strix workflow resolves the trusted lock before installation" + assert_file_contains "$workflow_file" '--require-hashes -r "$resolved_trusted_lock"' "strix workflow installs only the resolved trusted lock" assert_file_contains "$workflow_file" 'TRUSTED_STRIX_SOURCE=$trusted_strix_source' "strix workflow exports the central Strix source path" assert_file_contains "$workflow_file" 'TRUSTED_STRIX_GATE=$trusted_strix_source/scripts/ci/strix_quick_gate.sh' "strix workflow executes the central Strix gate script" assert_file_contains "$workflow_file" "Materialize target workspace" "strix workflow materializes target repository data separately from trusted scripts" @@ -295,6 +305,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$GATE_SCRIPT" 'child_env["YARN_ENABLE_SCRIPTS"] = "false"' "strix gate child process disables yarn lifecycle scripts" assert_file_contains "$GATE_SCRIPT" 'child_env["PYTHONWARNINGS"] = "ignore:Pydantic serializer warnings:UserWarning:pydantic.main"' "strix gate child env narrowly filters the known third-party Pydantic serializer warning" assert_file_contains "$GATE_SCRIPT" '[[ "$normalized_changed_file" =~ ^backend/.+\.py$ ]]' "strix gate detects nested backend Python files for PR-scoped import context" + assert_file_contains "$GATE_SCRIPT" "pull_request_scope_context_files received a non-normalized changed path" "strix context allowlist rejects raw changed-file paths at its boundary" assert_file_contains "$GATE_SCRIPT" '[[ "$normalized_changed_file" == scripts/ci/test_*.sh || "$normalized_changed_file" == scripts/ci/*_test.sh ]]' "strix gate excludes large CI test harness scripts from model scan input" assert_file_contains "$GATE_SCRIPT" "Materialized PR-head changed-file scope for Strix scan" "strix gate avoids copying the full PR head tree into privileged scan targets by default" assert_file_contains "$GATE_SCRIPT" "sanitize_known_strix_report_warnings" "strix gate sanitizes only known internal Strix report warnings" @@ -302,6 +313,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$GATE_SCRIPT" "iter_report_logs" "strix gate enumerates report logs through a safe walker" assert_file_contains "$GATE_SCRIPT" "os.walk(root, topdown=True, followlinks=False)" "strix gate does not recurse into symlinked report directories" assert_file_not_contains "$GATE_SCRIPT" 'root.rglob("*.log")' "strix gate avoids recursive pathlib glob traversal for report logs" + assert_file_contains "$GATE_SCRIPT" '[[ "$candidate" == "$existing" || "$candidate" == "$existing/"* ]]' "strix snapshot comparison keeps captured report paths literal" + assert_file_not_contains "$GATE_SCRIPT" 'case "$candidate" in' "strix snapshot comparison does not interpret captured paths as case patterns" assert_file_contains "$GATE_SCRIPT" "has_strix_report_failure_signal" "strix gate fails closed on warning-class Strix report artifacts" assert_file_not_contains "$workflow_file" "ignore::UserWarning" "strix workflow must not blanket-suppress all UserWarning output" assert_file_contains "$GATE_SCRIPT" "vulnerability_file_reports_generic_github_actions_workflow_insecurity" "strix gate fact-checks generic GitHub Actions workflow security reports before accepting whole-file claims" @@ -436,6 +449,7 @@ assert_changed_file_membership_uses_cached_normalized_paths() { assert_file_contains "$GATE_SCRIPT" "NORMALIZED_CHANGED_FILES=()" "strix gate caches normalized PR changed paths" assert_file_contains "$GATE_SCRIPT" 'NORMALIZED_CHANGED_FILES+=("$normalized_changed_file")' "strix gate populates cached normalized PR changed paths" assert_file_contains "$GATE_SCRIPT" "for normalized_changed_file in \"\${NORMALIZED_CHANGED_FILES[@]}\"" "strix gate uses cached normalized paths for membership checks" + assert_file_contains "$GATE_SCRIPT" "pull request changed file escaped the generated scope" "strix gate revalidates changed-file containment at the copy sink" } assert_absent_endpoint_search_uses_canonical_target_path() { @@ -461,7 +475,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { local opencode_config="$REPO_ROOT/opencode.jsonc" assert_file_contains "$workflow_file" "pull_request_target:" "opencode review workflow loads privileged review logic from the protected base ref" - assert_file_contains "$workflow_file" "types: [opened, synchronize, reopened, ready_for_review, closed]" "opencode required workflow reacts to current PR head changes and closed-PR cleanup" + assert_file_contains "$workflow_file" "types: [opened, synchronize, reopened, ready_for_review, edited, closed]" "opencode required workflow reacts to current PR/base changes and closed-PR cleanup" assert_file_contains "$workflow_file" "repository_dispatch:" "opencode review supports default-branch scheduler current-head dispatch" assert_file_contains "$workflow_file" "types: [opencode-review]" "opencode repository dispatch accepts only its dedicated event type" assert_file_not_contains "$workflow_file" "workflow_dispatch:" "privileged opencode retries cannot load a caller-selected workflow ref" @@ -562,13 +576,17 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'status publication failed because pr_head_sha was empty' "opencode repository_dispatch status fails closed when current-head identity is unavailable" assert_file_not_contains "$workflow_file" "actions/cache@" "opencode coverage does not restore PR-writable static R caches" assert_file_not_contains "$workflow_file" 'ref: ${{ github.event.client_payload.pr_head_sha }}' "opencode review must not checkout PR head into the trusted workflow workspace" - assert_file_contains "$workflow_file" "Materialize pull request head for OpenCode review data" "opencode review materializes PR-head source as read-only review data" - assert_file_contains "$workflow_file" 'git remote add pr-source "$GITHUB_SERVER_URL/$GH_REPOSITORY.git"' "opencode review fetches target PR commits through a separate PR-source remote" + assert_file_contains "$workflow_file" "Materialize inert pull request blobs for OpenCode review data" "opencode review materializes PR-head blobs as inert read-only review data" + assert_file_contains "$workflow_file" 'remote add pr-source' "opencode review fetches target PR commits through a separate PR-source remote" assert_file_contains "$workflow_file" 'refs/pull/${PR_NUMBER}/head' "opencode review can fetch fork PR heads without local workflow copies" - assert_file_contains "$workflow_file" 'git worktree add --detach "$OPENCODE_SOURCE_WORKDIR" "$PR_HEAD_SHA"' "opencode review materializes the PR head without actions/checkout credentials" - assert_file_contains "$workflow_file" 'cd "$OPENCODE_SOURCE_WORKDIR"' "opencode CodeGraph indexing runs against the PR-head source worktree" - assert_file_contains "$workflow_file" 'PR_MERGE_BASE="$(git -C "$OPENCODE_SOURCE_WORKDIR" merge-base "$PR_BASE_SHA" "$PR_HEAD_SHA")"' "opencode review evidence diffs use the PR-head worktree merge base" - assert_file_contains "$workflow_file" 'git -C "$OPENCODE_SOURCE_WORKDIR" diff' "opencode review builds changed-file evidence from the PR-head worktree" + assert_file_contains "$workflow_file" 'git init --bare "$OPENCODE_SOURCE_GIT_DIR"' "opencode review isolates PR objects from the trusted workflow checkout" + assert_file_contains "$workflow_file" 'python3 scripts/ci/materialize_pr_review_source.py' "opencode review materializes only validated inert PR blobs" + assert_file_not_contains "$workflow_file" 'git worktree add --detach "$OPENCODE_SOURCE_WORKDIR"' "opencode review never creates an untrusted PR worktree" + assert_file_contains "$workflow_file" 'find "$OPENCODE_SOURCE_WORKDIR" -type l' "opencode review rejects materialized symbolic links" + assert_file_contains "$workflow_file" 'find "$OPENCODE_SOURCE_WORKDIR" -type f -perm /111' "opencode review rejects materialized executable files" + assert_file_contains "$workflow_file" 'cd "$OPENCODE_SOURCE_WORKDIR"' "opencode CodeGraph indexing runs against inert PR-head blobs" + assert_file_contains "$workflow_file" 'PR_MERGE_BASE="$(git -C "$OPENCODE_SOURCE_WORKDIR" merge-base "$PR_BASE_SHA" "$PR_HEAD_SHA")"' "opencode review evidence diffs use the isolated bare object store merge base" + assert_file_contains "$workflow_file" 'git -C "$OPENCODE_SOURCE_WORKDIR" diff' "opencode review builds changed-file evidence from the isolated bare object store" assert_file_not_contains "$workflow_file" 'ref: ${{ github.event.pull_request.base.sha' "opencode trusted checkout avoids dynamic pull_request refs that Scorecard flags" assert_file_not_contains "$workflow_file" 'ref: ${{ github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha || github.sha }}' "opencode review must not checkout PR head into the trusted workflow workspace" assert_file_not_contains "$workflow_file" 'secrets.GITHUB_TOKEN' "opencode review uses github.token instead of a nonexistent GITHUB_TOKEN secret" @@ -673,26 +691,27 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "not a generic model-exhaustion message" "opencode review tells models to return concrete missing-evidence findings instead of progress-only output" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "a required security check must fail" "strix wrapper fails closed when provider outages prevent current-head security evidence" + assert_file_not_contains "$REPO_ROOT/.github/workflows/strix.yml" "Treating as a neutral skip" "strix wrapper must not convert provider failure into a successful required check" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$workflow_file" 'timeout-minutes: 300' "opencode review target contains evidence, the bounded long-review pool, publication, and cleanup overhead" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" - assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model pool preserves full-hour candidates within a bounded provider-pool window" + assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model stage keeps an outer safety timeout above the bounded provider-pool budget" assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review preserves legitimate full-hour provider sessions" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "0"' "opencode model pool keeps cycling until the bounded retry budget or step timeout is exhausted" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode primary review bounds each responsive provider session" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" + assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "1"' "opencode model pool performs one bounded catalog cycle before scheduler-owned retry" assert_file_not_contains "$workflow_file" 'opencode-exhausted-retry:' "opencode model exhaustion retries stay owned by the least-privilege central scheduler" assert_file_not_contains "$workflow_file" 'RETRY_DISPATCH_TOKEN' "opencode does not retain a recursive write-token dispatch path" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" assert_file_contains "$workflow_file" "always()" "opencode fallback chain uses always() so failed model steps cannot skip every fallback" - assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode fallback tries the catalog promptly instead of spending the entire review on one model" + assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "2"' "opencode fallback permits one format retry while timeout and fatal failures skip the second attempt" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review includes a broad catalog fallback pool" assert_file_not_contains "$workflow_file" "steps.opencode_review_model_pool.outcome == 'success'" "opencode approval gate still runs after model pool failure to publish a reason" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" - assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against the PR-head worktree" + assert_file_contains "$workflow_file" "github-models/openai/gpt-4.1 openai/gpt-5.6-luna github-models/deepseek/deepseek-v3-0324 github-models/openai/gpt-5" "opencode review starts with the responsive GPT-4.1 endpoint before bounded fallbacks" + assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against inert PR-head blobs" assert_file_contains "$workflow_file" '"openai/o3"' "opencode config declares OpenAI o3 fallback" assert_file_contains "$workflow_file" '"openai/o4-mini"' "opencode config declares OpenAI o4-mini fallback" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OpenCode %s attempt %s/%s failed with exit %s.' "opencode review logs per-model retry attempts" @@ -833,12 +852,12 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "configured max cycle count" "opencode model pool exits before the job timeout after configured cycles" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500' "opencode model pool keeps a bounded default retry budget unless the workflow explicitly disables it" assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" - assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" - assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode catalog fallback preserves legitimate full-hour provider sessions" + assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "2"' "opencode primary and fallback paths permit one format retry without retrying timeout-class failures" + assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "2"' "opencode catalog fallback retries only responsive invalid-control output before moving on" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode catalog fallback bounds provider sessions within the queue budget" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" + assert_file_contains "$workflow_file" "github-models/openai/gpt-4.1 openai/gpt-5.6-luna github-models/deepseek/deepseek-v3-0324 github-models/openai/gpt-5" "opencode review tries responsive GPT-4.1 before direct and DeepSeek fallbacks" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" "opencode review keeps DeepSeek reasoning fallback coverage after OpenAI candidates" assert_file_contains "$workflow_file" "coverage-source-tree:" "opencode workflow materializes coverage source before running PR-head tests" assert_file_contains "$workflow_file" "coverage-evidence:" "opencode workflow measures coverage before review" @@ -846,6 +865,9 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "Exchange OpenCode app token for target repository coverage reads" "coverage source materialization can read private target repositories during central manual dispatch" assert_file_contains "$workflow_file" "Upload materialized pull request merge tree" "coverage source materialization passes only a prepared merge tree artifact to the PR-head coverage job" assert_file_contains "$workflow_file" "Download materialized pull request merge tree" "coverage evidence consumes the prepared merge tree artifact without target-repository credentials" + assert_file_contains "$workflow_file" 'github-token: ${{ github.token }}' "coverage artifact download uses the REST-backed token path across failed-job rerun attempts" + assert_file_contains "$workflow_file" 'repository: ${{ github.repository }}' "coverage artifact download binds the artifact source repository explicitly" + assert_file_contains "$workflow_file" 'run-id: ${{ github.run_id }}' "coverage artifact download binds the original workflow run across attempts" assert_file_contains "$workflow_file" "Report coverage source materialization failure" "coverage evidence logs source materialization failures as the coverage blocker" local coverage_merge_tree_step coverage_merge_tree_step="$( @@ -886,7 +908,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "tail -n 180" "coverage evidence keeps the tail of long failed logs where compiler and test errors usually appear" assert_file_not_contains "$workflow_file" 'sed -n '\''1,220p'\'' "$log_file"' "coverage evidence must not hide failed-command reasons by keeping only the first lines" assert_file_contains "$workflow_file" "declared_package_manager()" "coverage evidence reads packageManager before selecting a JavaScript package runner" - assert_file_contains "$workflow_file" "ensure_corepack_runner pnpm" "coverage evidence activates pnpm through corepack for pnpm workspaces" + assert_file_contains "$workflow_file" "ensure_preinstalled_package_runner pnpm" "coverage evidence uses only the exact preinstalled pnpm version for pnpm workspaces" + assert_file_contains "$workflow_file" "node-v22.14.0-linux-x64.tar.xz" "coverage evidence pins a Node version compatible with the pinned pnpm release" + assert_file_contains "$workflow_file" "69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec" "coverage evidence verifies the pinned Node archive digest" + assert_file_contains "$workflow_file" "pnpm/-/pnpm-11.5.3.tgz" "coverage evidence downloads the reviewed exact pnpm package" + assert_file_contains "$workflow_file" "238d639a47712278bb72e8b6db2c297ac1ccd80dd7642f7c933b73aebde7b51f" "coverage evidence verifies the pinned pnpm archive digest" assert_file_contains "$workflow_file" "or fall back to npm" "coverage evidence logs package-runner activation failures instead of silently using npm" assert_file_not_contains "$workflow_file" '@latest' "coverage evidence refuses mutable package-manager toolchains" assert_file_contains "$workflow_file" "npm ci --ignore-scripts" "coverage dependency installation suppresses npm lifecycle hooks" @@ -957,10 +983,10 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'safe_pytest_command.py" discover' "opencode coverage evidence discovers default CI workflow pytest commands through the trusted shell-free parser" assert_file_not_contains "$REPO_ROOT/scripts/ci/safe_pytest_command.py" "RUNNER_EXECUTABLES" "configured pytest evidence cannot invoke uv, poetry, or pipenv dependency resolution" assert_file_contains "$workflow_file" "Python configured CI test suite" "opencode coverage evidence labels repository-configured pytest evidence separately" - assert_file_contains "$workflow_file" 'cd "$1" && PYTHONPATH=. python3 -m coverage run -m pytest tests' "opencode coverage runs Python tests with the trusted preinstalled toolchain" - assert_file_contains "$workflow_file" 'python3 -m coverage report --show-missing' "opencode coverage preserves the missing-line report with the trusted toolchain" - assert_file_contains "$workflow_file" 'cd "$1" && PYTHONPATH=. python3 -m pytest tests/test_docstrings.py' "opencode docstring tests use the trusted preinstalled pytest" - assert_file_contains "$workflow_file" "missing project imports fail in pytest" "unavailable project dependencies fail closed with their import error" + assert_file_contains "$workflow_file" '"$2" -m coverage run -m pytest tests' "opencode coverage runs Python tests with the base hash-locked environment" + assert_file_contains "$workflow_file" '"$2" -m coverage report --show-missing' "opencode coverage preserves the missing-line report with the selected trusted environment" + assert_file_contains "$workflow_file" '"$2" -m pytest tests/test_docstrings.py' "opencode docstring tests use the selected trusted environment" + assert_file_contains "$workflow_file" "trusted base commit; missing imports still fail in pytest" "unavailable base dependencies fail closed with their import error" assert_file_contains "$workflow_file" "JavaScript/TypeScript dependencies (npm ci, lifecycle hooks disabled)" "opencode coverage evidence installs npm workspace dependencies without lifecycle hooks before JS coverage" assert_file_contains "$workflow_file" "coverage/coverage-summary.json" "opencode coverage evidence reads JS coverage summaries instead of trusting test exit codes" assert_file_contains "$workflow_file" "coverage/coverage-final.json" "opencode coverage evidence supports Vitest Istanbul final coverage files" @@ -1155,7 +1181,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'GH_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || steps.review_read_app_token.outputs.token || github.token }}' "opencode manual dispatch uses the cross-repo approval token for target PR evidence lookups with app-token fallback" assert_file_contains "$workflow_file" 'repos/${GH_REPOSITORY}' "opencode review workflow uses env-backed repository context in shell commands" assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review starts the central model pool" - assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review starts with DeepSeek V3 before full-size GPT fallbacks" + assert_file_contains "$workflow_file" "github-models/openai/gpt-4.1 openai/gpt-5.6-luna github-models/deepseek/deepseek-v3-0324 github-models/openai/gpt-5" "opencode review starts with responsive GPT-4.1 before bounded full-size fallbacks" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-r1-0528" "opencode review keeps a reachable DeepSeek R1 reasoning fallback model" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324" "opencode review has a reachable DeepSeek V3 fallback model" assert_file_contains "$workflow_file" "github-models/openai/gpt-5" "opencode review still has a bounded GPT-5 fallback model" @@ -1319,8 +1345,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "get_validated_pr_diff_range" "failed-check fallback validates PR diff range before comparing trusted Strix inputs" assert_file_contains "$workflow_file" ".github/workflows/strix.yml" "opencode inline fallback watches Strix workflow changes" assert_file_contains "$workflow_file" "self_modifying_strix_base_failure" "opencode approval detects trusted-base Strix failures for self-modifying workflow PRs" - assert_file_contains "$workflow_file" 'local source_root="${OPENCODE_SOURCE_WORKDIR:-${GITHUB_WORKSPACE:-$PWD}}"' "opencode trusted-base Strix lag detection inspects the PR-head worktree" - assert_file_contains "$workflow_file" 'git -C "$source_root" diff --quiet' "opencode trusted-base Strix lag detection compares trusted-input changes in the PR-head worktree" + assert_file_contains "$workflow_file" 'local source_root="${OPENCODE_SOURCE_WORKDIR:-${GITHUB_WORKSPACE:-$PWD}}"' "opencode trusted-base Strix lag detection inspects inert PR-head blobs" + assert_file_contains "$workflow_file" 'git -C "$source_root" diff --quiet' "opencode trusted-base Strix lag detection compares trusted-input changes in the isolated bare object store" assert_file_contains "$workflow_file" "opencode.jsonc: No such file or directory" "opencode approval recognizes base-workflow Strix self-test evidence that cannot see PR-head OpenCode config" assert_file_contains "$workflow_file" "latest_current_head_manual_strix_run" "opencode approval inspects same-head manual Strix repository_dispatch runs before suppressing trusted-base Strix failures" assert_file_contains "$workflow_file" 'wait_for_peer_github_checks "$pending_checks_file"' "opencode approval waits for pending same-head manual Strix evidence before failing self-modifying workflow PRs" @@ -1852,7 +1878,7 @@ EOF assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" "never say no source files changed, no test files changed, or no executable changes when exact changed-file evidence lists workflow, script, source, or test files" "opencode prompt rejects contradictory changed-file kind claims" assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" "Never approve material workflow, script, source, config, package, or test changes with a reason or summary that says simple typo fix" "opencode prompt rejects trivial approval claims for material changes" assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" "OPENCODE_CHANGED_FILES_FILE" "opencode workflow exports exact current-head changed files" - assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" 'git -C "$OPENCODE_SOURCE_WORKDIR" diff --name-only --find-renames "$PR_MERGE_BASE" "$PR_HEAD_SHA" |' "opencode workflow derives exact changed files from the PR-head worktree" + assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" 'git -C "$OPENCODE_SOURCE_WORKDIR" diff --name-only --find-renames "$PR_MERGE_BASE" "$PR_HEAD_SHA" |' "opencode workflow derives exact changed files from the isolated bare object store" assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" 'awk '\''NF > 0 && $0 !~ /^\// && $0 !~ /(^|\/)\.\.($|\/)/ { print }'\'' >"$OPENCODE_CHANGED_FILES_FILE"' "opencode workflow writes path-safe exact changed files for the normalizer" assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" "changed-files.txt" "opencode workflow copies exact changed-file evidence into the isolated review workspace" assert_file_contains "$REPO_ROOT/.github/workflows/opencode-review.yml" 'A["text"]' "opencode prompt requires quoted Mermaid labels" @@ -3262,7 +3288,7 @@ REPORT echo "openai.RateLimitError: Error code: 429" exit 1 ;; - openai/o3) + openai/openai/o3) if [ "${LLM_API_KEY:-}" != "github-models-fallback-token" ]; then echo "unexpected GitHub Models key for fallback (${LLM_API_KEY:-})" >&2 exit 16 @@ -5687,7 +5713,7 @@ run_filtered_gate_case_if_requested() { "vertex_ai/hallucination-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/hallucination-primary" \ "" @@ -5697,7 +5723,7 @@ run_filtered_gate_case_if_requested() { "vertex_ai/hallucination-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/hallucination-primary" \ "" \ @@ -5765,7 +5791,7 @@ run_filtered_gate_case_if_requested() { "0" \ "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ "2" \ - "openai/gpt-5.6-luna|openai/o3" \ + "openai/gpt-5.6-luna|openai/openai/o3" \ "|https://models.github.ai/inference" \ "vertex_ai" \ "" \ @@ -6035,6 +6061,68 @@ run_filtered_gate_case_if_requested() { "ls-tree" \ "1" ;; + timeout-disabled-success) + run_gate_case "timeout-disabled-success" \ + "vertex_ai/timeout-disabled-primary" \ + "" \ + "0" \ + "scan ok with timeout disabled" \ + "1" \ + "vertex_ai/timeout-disabled-primary" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "0" + ;; + nonrecoverable) + run_gate_case "nonrecoverable" \ + "openai/gpt-4o-mini" \ + "vertex_ai/fallback-one" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" + ;; + pr-large-scope-full-set) + local large_pr_changed_files="" + local large_pr_index large_pr_path + for large_pr_index in $(seq 1 38); do + large_pr_path="backend/large-scope/file-$large_pr_index.py" + if [ -n "$large_pr_changed_files" ]; then + large_pr_changed_files+=$'\n' + fi + large_pr_changed_files+="$large_pr_path" + done + run_gate_case "pr-large-scope-full-set" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with large full PR scope" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "$large_pr_changed_files" \ + "" \ + "12" + ;; pull-request-target-changed-file-list-diff-failure) run_pull_request_target_aborts_on_pr_head_blob_failure_case \ "pull-request-target-changed-file-list-diff-failure" \ @@ -8909,11 +8997,11 @@ run_gate_case "provider-prefix-required-resource-path-primary-implicit-default-p run_gate_case "provider-prefix-required-resource-path-primary-explicit-empty-default-provider" \ "projects/p1/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "vertex_ai/fallback-one" \ + "2" \ + "Vertex resource paths require an explicit vertex_ai or vertex_ai_beta provider." \ "0" \ - "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/gemini-2.5-pro'." \ - "1" \ - "vertex_ai/gemini-2.5-pro" \ - "" \ + "" \ + "" \ "" run_gate_case "provider-prefix-resource-path-primary-notfound-fallback-success" \ @@ -9729,7 +9817,7 @@ run_gate_case "vertex-primary-hallucinated-endpoint-fallback-success" \ "vertex_ai/hallucination-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/hallucination-primary" \ "" @@ -9903,7 +9991,7 @@ run_gate_case "multi-severity-low-then-critical" \ "vertex_ai/multi-severity-primary" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/multi-severity-primary" \ "" @@ -9921,7 +10009,7 @@ run_gate_case "medium-vuln-default-threshold" \ "openai/gpt-4o-mini" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "openai/gpt-4o-mini" \ "https://example.invalid" \ @@ -10020,7 +10108,7 @@ run_gate_case "critical-vuln-at-threshold" \ "vertex_ai/critical-vuln-primary" \ "" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/critical-vuln-primary" \ "" @@ -10036,17 +10124,16 @@ run_gate_case "malformed-severity-marker-nonrecoverable" \ # Bug 7: Model disagreement — primary produces CRITICAL, fallback produces LOW. # The CRITICAL from the earlier report must NOT be ignored. -# Both models produce NOT_FOUND errors, so the gate exhausts fallbacks and -# reports "Configured Vertex model and fallback models were unavailable." -# The key assertion is exit 1: the CRITICAL finding is NOT downgraded to pass. +# The first model's CRITICAL report is authoritative, so the gate must block +# immediately without spending provider capacity on a fallback model. run_gate_case "model-disagreement-critical-in-earlier-report" \ "vertex_ai/model-a" \ "vertex_ai/model-b" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ - "2" \ - "vertex_ai/model-a|vertex_ai/model-b" \ - "|" + "Unable to map Strix findings to changed files; failing closed for pull request." \ + "1" \ + "vertex_ai/model-a" \ + "" # Bug 4: deepseek/models/deepseek-r1 must NOT be rewritten to vertex_ai/deepseek-r1 run_gate_case "nonvertex-slash-model-not-rewritten" \ @@ -10067,7 +10154,7 @@ run_gate_case "target-path-src-default-source-dirs" \ "vertex_ai/hallucination-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ "1" \ - "Strix quick scan failed with a non-recoverable error." \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "vertex_ai/hallucination-primary" \ "" \ @@ -11643,7 +11730,7 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "0" \ "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ "2" \ - "openai/gpt-5.6-luna|openai/o3" \ + "openai/gpt-5.6-luna|openai/openai/o3" \ "|https://models.github.ai/inference" \ "vertex_ai" \ "" \ diff --git a/tests/test_adversarial_evidence.py b/tests/test_adversarial_evidence.py index d5849012..a7ba0680 100644 --- a/tests/test_adversarial_evidence.py +++ b/tests/test_adversarial_evidence.py @@ -86,6 +86,19 @@ def test_requires_the_exact_probe_path_and_line_when_line_is_supplied(): ) +def test_structurally_bound_location_does_not_require_prose_duplication(): + """Trusted structured callers may bind path/line outside the prose field.""" + assert ( + evidence.adversarial_evidence_rejection_reason( + f"Focused regression command passed with exit code 0. {SOURCE_RECEIPT}", + ".github/workflows/review.yml", + 42, + require_location_citation=False, + ) + is None + ) + + def test_path_only_citation_rejects_longer_path_substrings(): """A filename embedded inside another path is not an exact citation.""" assert "exact probe path" in evidence.adversarial_evidence_rejection_reason( diff --git a/tests/test_materialize_base_python_locks.py b/tests/test_materialize_base_python_locks.py new file mode 100644 index 00000000..f14c9e96 --- /dev/null +++ b/tests/test_materialize_base_python_locks.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import json +import os +import runpy +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.ci import materialize_base_python_locks as locks + + +VALID_LOCK = b"""\ +# Generated lock +fastapi==0.139.0 \\ + --hash=sha256:cf15e1e9e667ddb0ad63811e60bd11390d1aac838ca4a7a23f421807b2308189 +pytest==9.1.1 \\ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c +""" + + +def git(repo: Path, *args: str) -> str: + """Run a test-only Git command.""" + return subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + text=True, + capture_output=True, + ).stdout.strip() + + +def committed_repo(tmp_path: Path) -> tuple[Path, str]: + """Create a repository containing one base-controlled hashed lock.""" + repo = tmp_path / "repo" + repo.mkdir() + git(repo, "init") + git(repo, "config", "user.name", "Test") + git(repo, "config", "user.email", "test@example.com") + backend = repo / "backend" + backend.mkdir() + (backend / "requirements-hashes.txt").write_bytes(VALID_LOCK) + git(repo, "add", ".") + git(repo, "commit", "-m", "base lock") + return repo, git(repo, "rev-parse", "HEAD") + + +def test_materializes_only_exact_base_lock_with_provenance(tmp_path: Path): + """The current worktree cannot replace dependency evidence from the base SHA.""" + repo, base_sha = committed_repo(tmp_path) + (repo / "backend" / "requirements-hashes.txt").write_text( + "malware @ https://example.invalid/malware.whl\n", + encoding="utf-8", + ) + + output = tmp_path / "locks" + metadata = locks.materialize(repo, base_sha, output) + + assert metadata["base_sha"] == base_sha + assert (output / "lock-000.txt").read_bytes() == VALID_LOCK + assert (output / "manifest.tsv").read_text() == ( + "backend\tlock-000\tlock-000.txt\n" + ) + assert metadata["locks"][0]["path"] == "backend/requirements-hashes.txt" + + +@pytest.mark.parametrize( + "content", + [ + b"fastapi>=0.139.0 --hash=sha256:" + b"a" * 64 + b"\n", + b"fastapi==0.139.0\n", + b"--index-url https://example.invalid/simple\n", + b"pkg @ https://example.invalid/pkg.whl --hash=sha256:" + b"a" * 64 + b"\n", + b"-r nested.txt\n", + ], +) +def test_rejects_mutable_or_redirecting_requirement_records(content: bytes): + """Only exact pins with hashes may enter the networked image build.""" + with pytest.raises(ValueError, match="only pinned package records"): + locks.validate_lock_content(content) + + +def test_rejects_symlink_lock_entry(): + """A base-tree symlink cannot redirect dependency materialization.""" + tree = ( + b"120000 blob 0123456789012345678901234567890123456789 9\t" + b"backend/requirements-hashes.txt\0" + ) + with pytest.raises(ValueError, match="must be a regular file"): + locks.parse_lock_entries(tree) + + +def test_parse_args_accepts_exact_sha_and_rejects_malformed_sha(tmp_path: Path): + """CLI parsing binds materialization to one exact hexadecimal commit.""" + sha = "a" * 40 + args = locks.parse_args( + [ + "--repo-root", + str(tmp_path), + "--base-sha", + sha, + "--output-dir", + str(tmp_path / "output"), + ] + ) + assert args.base_sha == sha + + with pytest.raises(SystemExit): + locks.parse_args( + [ + "--repo-root", + str(tmp_path), + "--base-sha", + "main", + "--output-dir", + str(tmp_path / "output"), + ] + ) + + +def test_git_bytes_reports_failures_and_bounds_output(monkeypatch, tmp_path: Path): + """Git reads expose stderr and refuse unexpectedly large output.""" + monkeypatch.setattr( + locks.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 1, stdout=b"", stderr=b"missing commit" + ), + ) + with pytest.raises(RuntimeError, match="missing commit"): + locks.git_bytes(tmp_path, "rev-parse", "HEAD") + + monkeypatch.setattr(locks, "MAX_TREE_OUTPUT_BYTES", 1) + monkeypatch.setattr( + locks.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 0, stdout=b"ab", stderr=b"" + ), + ) + with pytest.raises(ValueError, match="bounded materialization"): + locks.git_bytes(tmp_path, "ls-tree", "HEAD") + + +def test_git_bytes_requires_an_absolute_verified_executable(monkeypatch, tmp_path: Path): + """Dependency materialization cannot resolve Git through a mutable relative PATH.""" + monkeypatch.setattr(locks, "GIT_EXECUTABLE", "git") + with pytest.raises(RuntimeError, match="absolute Git executable"): + locks.git_bytes(tmp_path, "rev-parse", "HEAD") + + monkeypatch.setattr(locks, "GIT_EXECUTABLE", str(tmp_path / "missing" / "git")) + with pytest.raises(RuntimeError, match="configured Git executable is unavailable"): + locks.validated_git_executable() + + +def test_git_executable_requires_trusted_owner_and_permissions( + monkeypatch, tmp_path: Path +) -> None: + """Dependency-lock extraction rejects a user-controlled executable path.""" + candidate = tmp_path / "bin" / "git" + candidate.parent.mkdir() + candidate.write_text("#!/bin/sh\n", encoding="utf-8") + candidate.chmod(0o755) + monkeypatch.setattr(locks, "GIT_EXECUTABLE", str(candidate)) + monkeypatch.setattr(locks, "TRUSTED_GIT_OWNER_UID", os.getuid() + 1) + + with pytest.raises(RuntimeError, match="failed trust validation"): + locks.validated_git_executable() + + monkeypatch.setattr(locks, "TRUSTED_GIT_OWNER_UID", os.getuid()) + assert locks.validated_git_executable() == str(candidate.resolve()) + + +def test_write_exclusive_closes_descriptor_when_fdopen_fails( + monkeypatch, + tmp_path: Path, +): + """A failed stream wrapper cannot leak the exclusive output descriptor.""" + closed: list[int] = [] + real_close = os.close + + def recording_close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(locks.os, "close", recording_close) + monkeypatch.setattr( + locks.os, + "fdopen", + lambda *args, **kwargs: (_ for _ in ()).throw(OSError("fdopen failed")), + ) + + with pytest.raises(OSError, match="fdopen failed"): + locks.write_exclusive(tmp_path / "lock.txt", b"data") + assert len(closed) == 1 + + def close_then_fail(descriptor: int) -> None: + real_close(descriptor) + raise OSError("close failed") + + monkeypatch.setattr(locks.os, "close", close_then_fail) + with pytest.raises(OSError, match="fdopen failed"): + locks.write_exclusive(tmp_path / "lock-close-failure.txt", b"data") + + +def test_write_exclusive_does_not_double_close_after_write_failure( + monkeypatch, + tmp_path: Path, +): + """A wrapped descriptor is closed only by its owning stream.""" + closed: list[int] = [] + real_close = os.close + + class FailingWriter: + def __init__(self, descriptor: int) -> None: + self.descriptor = descriptor + + def __enter__(self): + return self + + def __exit__(self, *unused) -> None: + locks.os.close(self.descriptor) + + def write(self, unused: bytes) -> None: + raise RuntimeError("write failed") + + def recording_close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(locks.os, "close", recording_close) + monkeypatch.setattr( + locks.os, + "fdopen", + lambda descriptor, *args, **kwargs: FailingWriter(descriptor), + ) + + with pytest.raises(RuntimeError, match="write failed"): + locks.write_exclusive(tmp_path / "lock.txt", b"data") + assert len(closed) == 1 + + +def test_repository_and_output_paths_fail_closed(monkeypatch, tmp_path: Path): + """Repository identity, existing outputs, and symlink ancestors are rejected.""" + not_directory = tmp_path / "repo-file" + not_directory.write_text("not a repository", encoding="utf-8") + with pytest.raises(ValueError, match="resolve to a directory"): + locks.validate_repo_root(not_directory, "a" * 40) + + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr(locks, "git_bytes", lambda *args: ("b" * 40).encode()) + with pytest.raises(ValueError, match="exact requested commit"): + locks.validate_repo_root(repo, "a" * 40) + + existing = tmp_path / "existing" + existing.mkdir() + with pytest.raises(ValueError, match="must not already exist"): + locks.safe_output_dir(existing) + + target = tmp_path / "target" + target.mkdir() + link = tmp_path / "link" + link.symlink_to(target, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic-link component"): + locks.safe_output_dir(link / "locks") + + +@pytest.mark.parametrize( + ("content", "message"), + [ + (b"pkg==1.0 \\\n", "incomplete continuation"), + (b"\xff", "must be UTF-8"), + (b"pkg==1.0\x00 --hash=sha256:" + b"a" * 64, "contains a NUL"), + (b"# comments only\n", "no package records"), + ], +) +def test_lock_content_rejects_encoding_and_record_boundaries( + content: bytes, message: str +): + """Malformed, empty, and unterminated lock records cannot reach pip.""" + with pytest.raises(ValueError, match=message): + locks.validate_lock_content(content) + + +def test_lock_content_rejects_per_file_size_limit(monkeypatch): + """A single base lock cannot exceed its explicit byte budget.""" + monkeypatch.setattr(locks, "MAX_LOCK_BYTES", 1) + with pytest.raises(ValueError, match="per-lock size limit"): + locks.validate_lock_content(b"ab") + + +def tree_record(path: str, *, size: str = "1", mode: str = "100644") -> bytes: + """Build one NUL-delimited ls-tree fixture record.""" + oid = "1" * 40 + return f"{mode} blob {oid} {size}\t{path}\0".encode() + + +def test_tree_parser_rejects_malformed_and_unsafe_entries(monkeypatch): + """Only bounded regular requirement locks at safe paths are selected.""" + with pytest.raises(ValueError, match="could not parse"): + locks.parse_lock_entries(b"malformed\0") + assert locks.parse_lock_entries(tree_record("README.md")) == [] + + with pytest.raises(ValueError, match="path or size is unsafe"): + locks.parse_lock_entries(tree_record("../requirements-hashes.txt")) + with pytest.raises(ValueError, match="path or size is unsafe"): + locks.parse_lock_entries(tree_record("requirements-hashes.txt", size="nan")) + + monkeypatch.setattr(locks, "MAX_LOCK_BYTES", 0) + with pytest.raises(ValueError, match="file is too large"): + locks.parse_lock_entries(tree_record("requirements-hashes.txt")) + + +def test_tree_parser_enforces_aggregate_and_count_limits(monkeypatch): + """Multiple individually valid locks remain bounded as one materialization.""" + tree = tree_record("a/requirements-hashes.txt", size="2") + tree_record( + "b/requirements-hashes.txt", size="2" + ) + monkeypatch.setattr(locks, "MAX_TOTAL_BYTES", 3) + with pytest.raises(ValueError, match="aggregate size limit"): + locks.parse_lock_entries(tree) + + monkeypatch.setattr(locks, "MAX_TOTAL_BYTES", 100) + monkeypatch.setattr(locks, "MAX_LOCKS", 0) + with pytest.raises(ValueError, match="too many"): + locks.parse_lock_entries(tree_record("requirements-hashes.txt")) + + +def test_materialize_rejects_blob_size_change(monkeypatch, tmp_path: Path): + """A blob differing from its validated tree size fails before any install.""" + output = tmp_path / "output" + monkeypatch.setattr(locks, "validate_repo_root", lambda repo, sha: tmp_path) + monkeypatch.setattr(locks, "safe_output_dir", lambda path: output) + + def fake_git_bytes(repo, *args): + if args[0] == "ls-tree": + return tree_record("requirements-hashes.txt", size="2") + return b"x" + + monkeypatch.setattr(locks, "git_bytes", fake_git_bytes) + with pytest.raises(RuntimeError, match="blob size changed"): + locks.materialize(tmp_path, "a" * 40, output) + + +def test_materializes_repository_root_lock(tmp_path: Path): + """A lock at repository root is represented by the manifest dot project.""" + repo, _ = committed_repo(tmp_path) + (repo / "backend" / "requirements-hashes.txt").unlink() + (repo / "requirements-hashes.txt").write_bytes(VALID_LOCK) + git(repo, "add", "-A") + git(repo, "commit", "-m", "move lock to root") + base_sha = git(repo, "rev-parse", "HEAD") + output = tmp_path / "root-locks" + + locks.materialize(repo, base_sha, output) + + assert (output / "manifest.tsv").read_text() == ".\tlock-000\tlock-000.txt\n" + + +def test_main_reports_success_and_materialization_failure( + monkeypatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +): + """The CLI emits machine-readable metadata and a visible failure reason.""" + sha = "a" * 40 + argv = [ + "--repo-root", + str(tmp_path), + "--base-sha", + sha, + "--output-dir", + str(tmp_path / "output"), + ] + monkeypatch.setattr(locks, "materialize", lambda *args: {"schema": 1}) + assert locks.main(argv) == 0 + assert json.loads(capsys.readouterr().out) == {"schema": 1} + + monkeypatch.setattr( + locks, + "materialize", + lambda *args: (_ for _ in ()).throw(ValueError("unsafe lock")), + ) + assert locks.main(argv) == 1 + assert "unsafe lock" in capsys.readouterr().err + + monkeypatch.setattr(sys, "argv", ["materialize_base_python_locks.py", *argv]) + assert locks.main() == 1 + + +def test_materializer_dunder_entrypoint(tmp_path: Path, monkeypatch): + """Executing the helper as a script exits with the main return code.""" + repo, base_sha = committed_repo(tmp_path) + output = tmp_path / "script-locks" + monkeypatch.setattr( + sys, + "argv", + [ + "materialize_base_python_locks.py", + "--repo-root", + str(repo), + "--base-sha", + base_sha, + "--output-dir", + str(output), + ], + ) + with pytest.raises(SystemExit) as exc: + runpy.run_path("scripts/ci/materialize_base_python_locks.py", run_name="__main__") + assert exc.value.code == 0 diff --git a/tests/test_materialize_pr_review_source.py b/tests/test_materialize_pr_review_source.py new file mode 100644 index 00000000..be9bbe55 --- /dev/null +++ b/tests/test_materialize_pr_review_source.py @@ -0,0 +1,845 @@ +"""Tests for inert pull-request source materialization.""" + +from __future__ import annotations + +import argparse +import io +import json +import os +from pathlib import Path +from pathlib import PurePosixPath +import runpy +import stat +import subprocess +import sys +import time + +import pytest + +from scripts.ci import materialize_pr_review_source as materializer + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT = REPO_ROOT / "scripts" / "ci" / "materialize_pr_review_source.py" + + +def git(repo: Path, *args: str) -> str: + """Run Git in a test repository and return stdout.""" + completed = subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + +def build_repository(tmp_path: Path) -> tuple[Path, Path, str, str]: + """Create a work repository and an isolated bare clone.""" + work = tmp_path / "work" + work.mkdir() + git(work, "init", "--quiet") + git(work, "config", "user.name", "test") + git(work, "config", "user.email", "test@example.com") + (work / "README.md").write_text("base\n", encoding="utf-8") + git(work, "add", "README.md") + git(work, "commit", "--quiet", "-m", "base fixture") + base_sha = git(work, "rev-parse", "HEAD") + (work / "src").mkdir() + (work / "src" / "app.py").write_text("print('review data')\n", encoding="utf-8") + executable = work / "run.sh" + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o755) + os.symlink("src/app.py", work / "app-link") + (work / ".codegraph").mkdir() + (work / ".codegraph" / "config.json").write_text( + '{"untrusted": true}\n', encoding="utf-8" + ) + git(work, "add", ".") + git(work, "commit", "--quiet", "-m", "fixture") + head_sha = git(work, "rev-parse", "HEAD") + bare = tmp_path / "objects.git" + subprocess.run( + ["git", "clone", "--quiet", "--bare", str(work), str(bare)], + check=True, + ) + return work, bare, base_sha, head_sha + + +def test_materializes_only_inert_validated_blobs(tmp_path: Path) -> None: + """Executable, symlink, and CodeGraph-controlled paths stay inert.""" + _work, bare, base_sha, head_sha = build_repository(tmp_path) + source = tmp_path / "source" + manifest = tmp_path / "manifest.json" + + completed = subprocess.run( + [ + "python3", + str(SCRIPT), + "--git-dir", + str(bare), + "--head-sha", + head_sha, + "--output-dir", + str(source), + "--manifest", + str(manifest), + ], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + assert (source / "src" / "app.py").read_text( + encoding="utf-8" + ) == "print('review data')\n" + assert stat.S_IMODE((source / "run.sh").stat().st_mode) == 0o444 + assert not (source / "app-link").is_symlink() + assert (source / "app-link").read_text(encoding="utf-8") == "src/app.py" + assert not (source / ".codegraph").exists() + assert (source / ".git").is_file() + assert git(source, "rev-parse", "HEAD") == head_sha + assert git(source, "merge-base", base_sha, head_sha) == base_sha + assert set(git(source, "diff", "--name-only", base_sha, head_sha).splitlines()) == { + ".codegraph/config.json", + "app-link", + "run.sh", + "src/app.py", + } + + evidence = json.loads(manifest.read_text(encoding="utf-8")) + assert evidence["head_sha"] == head_sha + assert evidence["written_files"] == 4 + assert {entry["path"] for entry in evidence["skipped"]} == { + ".codegraph/config.json" + } + representations = { + entry["path"]: entry["representation"] + for entry in evidence["special_representations"] + } + assert representations == { + "app-link": "inert-regular-file", + "run.sh": "non-executable-regular-file", + } + + +def test_rejects_non_bare_git_directory(tmp_path: Path) -> None: + """A normal work repository cannot be confused with the object store.""" + work, _bare, _base_sha, head_sha = build_repository(tmp_path) + args = materializer.parse_args( + [ + "--git-dir", + str(work / ".git"), + "--head-sha", + head_sha, + "--output-dir", + str(tmp_path / "source"), + "--manifest", + str(tmp_path / "manifest.json"), + ] + ) + + try: + materializer.materialize(args) + except ValueError as exc: + assert "isolated bare repository" in str(exc) + else: + raise AssertionError("non-bare repository was accepted") + + +def test_rejects_unsafe_tree_paths() -> None: + """Traversal and absolute paths fail closed before filesystem writes.""" + for unsafe in ( + b"../escape", + b"/absolute", + b"a/../escape", + b"directory\\file", + b"directory//file", + b"./file", + ): + try: + materializer.safe_relative_path(unsafe) + except ValueError: + continue + raise AssertionError(f"unsafe path was accepted: {unsafe!r}") + + +def test_rejects_symlink_ancestors_for_manifest_and_output(tmp_path: Path) -> None: + """Existing parent links cannot redirect source or provenance writes.""" + outside = tmp_path / "outside" + intended = tmp_path / "intended" + outside.mkdir() + intended.mkdir() + os.symlink(outside, intended / "linked-parent") + + for option_path, option in ( + (intended / "linked-parent" / "manifest.json", "--manifest"), + (intended / "linked-parent" / "source", "--output-dir"), + ): + try: + materializer.reject_symlink_components(option_path, option) + except ValueError as exc: + assert option in str(exc) + assert "symbolic-link path component" in str(exc) + else: + raise AssertionError(f"{option} accepted a symlink ancestor") + assert not (outside / "manifest.json").exists() + + +def test_tree_file_limit_stops_streaming_producer_early( + monkeypatch, tmp_path: Path +) -> None: + """The parser terminates Git as soon as the next entry exceeds the limit.""" + oid = "a" * 40 + + def record(name: str) -> bytes: + return f"100644 blob {oid} 1\t{name}\0".encode() + + producer = ( + "import os,time; " + f"os.write(1, {record('one')!r}); " + f"os.write(1, {record('two')!r}); " + "time.sleep(30)" + ) + + def open_producer(_git_dir: Path, _head_sha: str): + return subprocess.Popen( + [sys.executable, "-c", producer], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + monkeypatch.setattr(materializer, "open_tree_reader", open_producer) + started = time.monotonic() + try: + materializer.parse_tree( + tmp_path, + oid, + max_files=1, + max_bytes=10, + timeout_seconds=10, + ) + except ValueError as exc: + assert "--max-files (2 > 1)" in str(exc) + else: + raise AssertionError("oversized streaming tree was accepted") + assert time.monotonic() - started < 5 + + +def test_tree_byte_limit_fails_before_materializing_output(tmp_path: Path) -> None: + """Accumulated blob sizes are rejected before the output directory exists.""" + _work, bare, _base_sha, head_sha = build_repository(tmp_path) + output = tmp_path / "bounded-source" + args = materializer.parse_args( + [ + "--git-dir", + str(bare), + "--head-sha", + head_sha, + "--output-dir", + str(output), + "--manifest", + str(tmp_path / "bounded-manifest.json"), + "--max-bytes", + "1", + ] + ) + + try: + materializer.materialize(args) + except ValueError as exc: + assert "--max-bytes" in str(exc) + else: + raise AssertionError("oversized tree bytes were accepted") + assert not output.exists() + + +def test_tree_metadata_budget_stops_large_paths_before_append( + monkeypatch, tmp_path: Path +) -> None: + """Aggregate path metadata is bounded independently of blob payload bytes.""" + oid = "a" * 40 + record = f"100644 blob {oid} 1\t{'x' * 256}\0".encode() + producer = f"import os; os.write(1, {record!r})" + + def open_producer(_git_dir: Path, _head_sha: str): + return subprocess.Popen( + [sys.executable, "-c", producer], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + monkeypatch.setattr(materializer, "open_tree_reader", open_producer) + try: + materializer.parse_tree( + tmp_path, + oid, + max_files=10, + max_bytes=10, + max_tree_metadata_bytes=128, + timeout_seconds=10, + ) + except ValueError as exc: + assert "--max-tree-metadata-bytes" in str(exc) + else: + raise AssertionError("oversized tree path metadata was accepted") + + +def test_argument_git_and_path_validation_edges(monkeypatch, tmp_path: Path, capsys) -> None: + """Malformed limits, Git identities, and output paths fail before writes.""" + with pytest.raises(argparse.ArgumentTypeError, match="positive"): + materializer.positive_int("0") + + with pytest.raises(SystemExit): + materializer.parse_args(["--help"]) + assert "immediate parent of --output-dir must already exist" in capsys.readouterr().out + with pytest.raises(SystemExit): + materializer.parse_args( + [ + "--git-dir", + str(tmp_path), + "--head-sha", + "short", + "--output-dir", + str(tmp_path / "out"), + "--manifest", + str(tmp_path / "manifest"), + ] + ) + + monkeypatch.setattr( + materializer.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args[0], 1, stdout=b"", stderr=b"git detail" + ), + ) + with pytest.raises(RuntimeError, match="git detail"): + materializer.git_bytes(tmp_path, "rev-parse", "HEAD") + + not_directory = tmp_path / "not-directory" + not_directory.write_text("x", encoding="utf-8") + with pytest.raises(ValueError, match="resolve to a directory"): + materializer.validate_git_dir(not_directory, "a" * 40) + + responses = iter((b"true\n", b"b" * 40 + b"\n")) + monkeypatch.setattr(materializer, "git_bytes", lambda *args: next(responses)) + with pytest.raises(ValueError, match="exact requested commit"): + materializer.validate_git_dir(tmp_path, "a" * 40) + + existing = tmp_path / "existing" + existing.mkdir() + with pytest.raises(ValueError, match="must not already exist"): + materializer.validate_output_path(existing, tmp_path) + with pytest.raises(ValueError, match="must not overlap"): + materializer.validate_output_path(tmp_path / "nested", tmp_path) + + with pytest.raises(ValueError, match="non-UTF-8"): + materializer.safe_relative_path(b"bad-\xff") + + +def test_git_executable_requires_trusted_owner_and_permissions( + monkeypatch, tmp_path: Path +) -> None: + """A user-controlled absolute Git path is rejected before subprocess execution.""" + monkeypatch.setattr(materializer, "GIT_EXECUTABLE", None) + with pytest.raises(RuntimeError, match="absolute Git executable"): + materializer.validated_git_executable() + + candidate = tmp_path / "bin" / "git" + candidate.parent.mkdir() + candidate.write_text("#!/bin/sh\n", encoding="utf-8") + candidate.chmod(0o755) + monkeypatch.setattr(materializer, "GIT_EXECUTABLE", str(candidate)) + monkeypatch.setattr(materializer, "TRUSTED_GIT_OWNER_UID", os.getuid() + 1) + + with pytest.raises(RuntimeError, match="failed trust validation"): + materializer.validated_git_executable() + + monkeypatch.setattr(materializer, "TRUSTED_GIT_OWNER_UID", os.getuid()) + assert materializer.validated_git_executable() == str(candidate.resolve()) + + monkeypatch.setattr( + materializer, "GIT_EXECUTABLE", str(tmp_path / "missing" / "git") + ) + with pytest.raises(RuntimeError, match="configured Git executable is unavailable"): + materializer.validated_git_executable() + + +def test_streaming_git_readers_revalidate_the_executable(monkeypatch, tmp_path: Path) -> None: + """Every long-lived Git process crosses the executable trust boundary.""" + popen_calls: list[list[str]] = [] + + def fake_popen(argv, **unused): + popen_calls.append(argv) + return object() + + monkeypatch.setattr( + materializer, "validated_git_executable", lambda: "/trusted/bin/git" + ) + monkeypatch.setattr(materializer.subprocess, "Popen", fake_popen) + + materializer.open_tree_reader(tmp_path, "a" * 40) + materializer.open_batch_reader(tmp_path) + + assert [call[0] for call in popen_calls] == [ + "/trusted/bin/git", + "/trusted/bin/git", + ] + + +def test_tree_entry_parser_covers_gitlink_and_malformed_records() -> None: + """Tree metadata accepts only exact blob and gitlink shapes.""" + oid = "a" * 40 + mode, kind, parsed_oid, size, path = materializer.parse_tree_entry( + f"160000 commit {oid} -\tvendor/submodule".encode() + ) + assert (mode, kind, parsed_oid, path.as_posix()) == ( + "160000", + "commit", + oid, + "vendor/submodule", + ) + assert size == len(f"Submodule commit {oid}\n".encode()) + + for record, reason in ( + (b"not-a-tree-entry", "could not parse"), + (b"100644 blob short 1\tfile", "invalid object id"), + (f"100644 tree {oid} 1\tfile".encode(), "unsupported"), + ): + with pytest.raises(ValueError, match=reason): + materializer.parse_tree_entry(record) + + +def test_process_termination_handles_finished_and_stubborn_producers() -> None: + """Producer cleanup returns early or escalates from terminate to kill.""" + + class Finished: + def poll(self): + return 0 + + materializer.terminate_process(Finished()) + + class Stubborn: + terminated = False + killed = False + waits = 0 + + def poll(self): + return None + + def terminate(self): + self.terminated = True + + def kill(self): + self.killed = True + + def wait(self, timeout=None): + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired("git", timeout) + return 0 + + process = Stubborn() + materializer.terminate_process(process) + assert process.terminated and process.killed and process.waits == 2 + + +def test_tree_reader_missing_stdout_and_timeout(monkeypatch, tmp_path: Path) -> None: + """Missing pipes and stalled enumeration terminate without materialization.""" + terminated = [] + + class Missing: + stdout = None + + missing = Missing() + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: missing) + monkeypatch.setattr(materializer, "terminate_process", terminated.append) + with pytest.raises(RuntimeError, match="stdout pipe"): + materializer.parse_tree( + tmp_path, + "a" * 40, + max_files=1, + max_bytes=1, + timeout_seconds=1, + ) + assert terminated == [missing] + + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: process) + monkeypatch.setattr( + materializer, + "terminate_process", + lambda candidate: (candidate.kill(), candidate.wait()), + ) + with pytest.raises(ValueError, match="tree-timeout-seconds"): + materializer.parse_tree( + tmp_path, + "a" * 40, + max_files=1, + max_bytes=1, + timeout_seconds=0, + ) + + +def test_tree_stream_record_and_exit_edge_cases(monkeypatch, tmp_path: Path) -> None: + """EOF, bounded records, trailing bytes, wait timeout, and Git errors fail closed.""" + oid = "a" * 40 + + def producer(payload: bytes, *, linger: bool = False): + program = f"import os; os.write(1, {payload!r})" + if linger: + program += "; import time; time.sleep(30)" + return subprocess.Popen( + [sys.executable, "-c", program], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + monkeypatch.setattr(materializer, "MAX_TREE_RECORD_BYTES", 3) + for payload in (b"abcd", b"abcd\0"): + process = producer(payload, linger=True) + monkeypatch.setattr( + materializer, "open_tree_reader", lambda *args, p=process: p + ) + with pytest.raises(ValueError, match="record-size"): + materializer.parse_tree( + tmp_path, + oid, + max_files=10, + max_bytes=10, + timeout_seconds=10, + ) + + monkeypatch.setattr(materializer, "MAX_TREE_RECORD_BYTES", 1024) + process = producer(b"partial") + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: process) + with pytest.raises(ValueError, match="unterminated"): + materializer.parse_tree( + tmp_path, + oid, + max_files=10, + max_bytes=10, + timeout_seconds=10, + ) + + valid = f"100644 blob {oid} 1\tone".encode() + process = producer(b"\0" + valid + b"\0") + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: process) + entries, total = materializer.parse_tree( + tmp_path, + oid, + max_files=10, + max_bytes=10, + timeout_seconds=10, + ) + assert len(entries) == 1 and total == 1 + + class Selector: + def register(self, *args): + return None + + def select(self, timeout=None): + return [] + + def close(self): + return None + + class Pipe(io.BytesIO): + def fileno(self): + return 1 + + class Exited: + stdout = Pipe() + stderr = None + + def poll(self): + return 0 + + def wait(self, timeout=None): + return 0 + + monkeypatch.setattr(materializer.selectors, "DefaultSelector", Selector) + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: Exited()) + assert materializer.parse_tree( + tmp_path, + oid, + max_files=1, + max_bytes=1, + timeout_seconds=1, + ) == ([], 0) + + class ReadySelector(Selector): + def select(self, timeout=None): + return [object()] + + class WaitFailure(Exited): + stderr = io.BytesIO(b"tree failed") + + def __init__(self, result): + self.result = result + + def wait(self, timeout=None): + if self.result == "timeout": + raise subprocess.TimeoutExpired("git", timeout) + return self.result + + monkeypatch.setattr(materializer.selectors, "DefaultSelector", ReadySelector) + monkeypatch.setattr(materializer.os, "read", lambda *args: b"") + stopped = [] + monkeypatch.setattr(materializer, "terminate_process", stopped.append) + timeout_process = WaitFailure("timeout") + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: timeout_process) + with pytest.raises(RuntimeError, match="did not exit"): + materializer.parse_tree( + tmp_path, + oid, + max_files=1, + max_bytes=1, + timeout_seconds=1, + ) + assert stopped == [timeout_process] + + failed_process = WaitFailure(1) + monkeypatch.setattr(materializer, "open_tree_reader", lambda *args: failed_process) + with pytest.raises(RuntimeError, match="tree failed"): + materializer.parse_tree( + tmp_path, + oid, + max_files=1, + max_bytes=1, + timeout_seconds=1, + ) + + +def test_batch_blob_reader_rejects_pipe_header_size_and_truncation() -> None: + """The batch protocol binds every blob header, type, size, and delimiter.""" + oid = "a" * 40 + + class Batch: + def __init__(self, payload: bytes, *, stdin=True): + self.stdin = io.BytesIO() if stdin else None + self.stdout = io.BytesIO(payload) + + with pytest.raises(RuntimeError, match="pipes"): + materializer.read_blob(Batch(b"", stdin=False), oid, 1) + for payload, size, reason in ( + (b"bad\n", 1, "unexpected object header"), + (f"{oid} tree 1\n".encode(), 1, "not a blob"), + (f"{oid} blob 2\n".encode(), 1, "size changed"), + (f"{oid} blob 2\nx".encode(), 2, "truncated blob"), + ): + with pytest.raises(RuntimeError, match=reason): + materializer.read_blob(Batch(payload), oid, size) + + +def test_inert_writer_closes_descriptor_after_fdopen_failure( + monkeypatch, tmp_path: Path +) -> None: + """A wrapper failure cannot leave a writable descriptor behind.""" + real_close = os.close + + def broken_fdopen(descriptor, *args, **kwargs): + real_close(descriptor) + raise RuntimeError("fdopen failed") + + monkeypatch.setattr(materializer.os, "fdopen", broken_fdopen) + with pytest.raises(RuntimeError, match="fdopen failed"): + materializer.write_inert_file(tmp_path / "inert", b"data") + + +def test_inert_writer_does_not_double_close_after_write_failure( + monkeypatch, tmp_path: Path +) -> None: + """Once fdopen owns the descriptor, stream cleanup is the only closer.""" + closed: list[int] = [] + real_close = os.close + + class FailingWriter: + def __init__(self, descriptor: int) -> None: + self.descriptor = descriptor + + def __enter__(self): + return self + + def __exit__(self, *unused) -> None: + materializer.os.close(self.descriptor) + + def write(self, unused: bytes) -> None: + raise RuntimeError("write failed") + + def recording_close(descriptor: int) -> None: + closed.append(descriptor) + real_close(descriptor) + + monkeypatch.setattr(materializer.os, "close", recording_close) + monkeypatch.setattr( + materializer.os, + "fdopen", + lambda descriptor, *args, **kwargs: FailingWriter(descriptor), + ) + + with pytest.raises(RuntimeError, match="write failed"): + materializer.write_inert_file(tmp_path / "inert", b"data") + assert len(closed) == 1 + + +def test_materialize_in_process_covers_gitlink_manifest_and_cli( + monkeypatch, tmp_path: Path, capsys +) -> None: + """The in-process path writes inert blobs, gitlink markers, provenance, and CLI output.""" + work, bare, _base_sha, fixture_head = build_repository(tmp_path) + git( + work, + "update-index", + "--add", + "--cacheinfo", + f"160000,{fixture_head},vendor/submodule", + ) + git(work, "commit", "--quiet", "-m", "gitlink fixture") + head_sha = git(work, "rev-parse", "HEAD") + git(work, "push", "--quiet", str(bare), f"{head_sha}:refs/heads/master") + + def argv(output: Path, manifest: Path) -> list[str]: + return [ + "--git-dir", + str(bare), + "--head-sha", + head_sha, + "--output-dir", + str(output), + "--manifest", + str(manifest), + ] + + output = tmp_path / "in-process-source" + manifest = tmp_path / "in-process-manifest.json" + metadata = materializer.materialize(materializer.parse_args(argv(output, manifest))) + assert metadata["written_files"] == 5 + assert (output / "vendor" / "submodule").read_text(encoding="utf-8") == ( + f"Submodule commit {fixture_head}\n" + ) + assert any( + entry["representation"] == "gitlink-marker" + for entry in metadata["special_representations"] + ) + assert stat.S_IMODE(manifest.stat().st_mode) == 0o444 + + cli_output = tmp_path / "cli-source" + cli_manifest = tmp_path / "cli-manifest.json" + monkeypatch.setattr(sys, "argv", [str(SCRIPT), *argv(cli_output, cli_manifest)]) + assert materializer.main() == 0 + assert "Materialized inert PR source blobs" in capsys.readouterr().out + + +def test_materialize_preconditions_unsupported_entry_and_batch_failure( + monkeypatch, tmp_path: Path +) -> None: + """Manifest overlap, unsupported modes, and batch failures remain blocking.""" + git_dir = tmp_path / "objects.git" + git_dir.mkdir() + head = "a" * 40 + + def args(output: Path, manifest: Path): + return materializer.parse_args( + [ + "--git-dir", + str(git_dir), + "--head-sha", + head, + "--output-dir", + str(output), + "--manifest", + str(manifest), + ] + ) + + monkeypatch.setattr(materializer, "validate_git_dir", lambda *unused: git_dir) + monkeypatch.setattr( + materializer, "validate_output_path", lambda output, unused: output + ) + + existing_manifest = tmp_path / "existing-manifest" + existing_manifest.write_text("x", encoding="utf-8") + with pytest.raises(ValueError, match="manifest must not already exist"): + materializer.materialize(args(tmp_path / "out-existing", existing_manifest)) + output = tmp_path / "out-overlap" + with pytest.raises(ValueError, match="outside --output-dir"): + materializer.materialize(args(output, output / "manifest")) + + class Batch: + def __init__(self, *, result=0, stderr=None, stdin=True): + self.stdin = io.BytesIO() if stdin else None + self.stdout = io.BytesIO() + self.stderr = io.BytesIO(stderr) if stderr is not None else None + self.result = result + + def wait(self): + return self.result + + unsupported_output = tmp_path / "unsupported-output" + monkeypatch.setattr( + materializer, + "parse_tree", + lambda *args, **kwargs: ( + [("999999", "blob", head, 0, PurePosixPath("bad"))], + 0, + ), + ) + monkeypatch.setattr(materializer, "open_batch_reader", lambda unused: Batch()) + with pytest.raises(ValueError, match="unsupported Git entry"): + materializer.materialize( + args(unsupported_output, tmp_path / "unsupported-manifest") + ) + + monkeypatch.setattr(materializer, "parse_tree", lambda *args, **kwargs: ([], 0)) + monkeypatch.setattr( + materializer, + "open_batch_reader", + lambda unused: Batch(result=1, stderr=b"batch boom", stdin=False), + ) + with pytest.raises(RuntimeError, match="batch boom"): + materializer.materialize( + args(tmp_path / "batch-output", tmp_path / "batch-manifest") + ) + + +def test_materializer_main_failure_dunder_and_missing_git_import( + monkeypatch, tmp_path: Path, capsys +) -> None: + """CLI failures are bounded, the dunder exits, and Git discovery fails closed.""" + monkeypatch.setattr( + materializer, + "materialize", + lambda args: (_ for _ in ()).throw(ValueError("bounded failure")), + ) + argv = [ + "--git-dir", + str(tmp_path / "missing.git"), + "--head-sha", + "a" * 40, + "--output-dir", + str(tmp_path / "out"), + "--manifest", + str(tmp_path / "manifest"), + ] + assert materializer.main(argv) == 1 + assert "bounded failure" in capsys.readouterr().err + + monkeypatch.setattr(sys, "argv", [str(SCRIPT), *argv]) + with pytest.raises(SystemExit) as exc: + runpy.run_path(str(SCRIPT), run_name="__main__") + assert exc.value.code == 1 + + monkeypatch.setattr( + materializer.shutil, + "which", + lambda *args, **kwargs: None, + ) + with pytest.raises(RuntimeError, match="absolute Git executable"): + runpy.run_path(str(SCRIPT), run_name="materializer_without_git") diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 75d40dbe..9f18a778 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,10 +1,134 @@ import base64 +import hashlib import json +from pathlib import Path +import runpy import sys import pytest from scripts.ci import noema_review_gate as noema +from scripts.ci import opencode_existing_approval_gate as approval_gate +from scripts.ci import opencode_review_normalize_output as normalizer + + +HEAD = "a" * 40 +BASE_REF = "main" +BASE_SHA = "b" * 40 +SOURCE_PATH = "src/app.py" +SOURCE_LINES = (b"def one():", b" return 1") + + +@pytest.fixture(autouse=True) +def trusted_primary_approval_artifacts(tmp_path, monkeypatch): + """Provide exact-head source and changed-file evidence to the shared gate.""" + runner_temp = tmp_path / "runner-temp" + source_root = tmp_path / "source" + source_path = source_root / SOURCE_PATH + runner_temp.mkdir() + source_path.parent.mkdir(parents=True) + source_path.write_bytes(b"\n".join(SOURCE_LINES) + b"\n") + changed_files = runner_temp / "opencode-changed-files.txt" + changed_files.write_text(f"{SOURCE_PATH}\n", encoding="utf-8") + manifest = runner_temp / "opencode-artifact-manifest.json" + manifest.write_text( + json.dumps( + { + "schema": 1, + "artifacts": { + changed_files.name: hashlib.sha256( + changed_files.read_bytes() + ).hexdigest() + }, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) + monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) + monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) + monkeypatch.setenv("OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION", "true") + monkeypatch.setenv( + "OPENCODE_ARTIFACT_MANIFEST_SHA256", + hashlib.sha256(manifest.read_bytes()).hexdigest(), + ) + cached_helpers = ( + normalizer.trusted_artifact_manifest, + normalizer.current_changed_files, + normalizer.trusted_execution_receipts, + ) + for helper in cached_helpers: + if hasattr(helper, "cache_clear"): + helper.cache_clear() + yield + for helper in cached_helpers: + if hasattr(helper, "cache_clear"): + helper.cache_clear() + + +def identity_body(marker: str = "Result: APPROVE", *, head: str = HEAD) -> str: + """Build review text bound to the default test PR identity.""" + return "\n".join( + ( + marker, + f"- Base ref: `{BASE_REF}`", + f"- Base SHA: `{BASE_SHA}`", + f"- Head SHA: `{head}`", + ) + ) + + +def valid_primary_body(*, head: str = HEAD) -> str: + """Build a primary review accepted by the shared strict approval gate.""" + evidence = { + "status": "passed", + "probes": [ + { + "path": SOURCE_PATH, + "line": line, + "hypothesis": f"Approval bypass hypothesis {line}.", + "attack_or_counterexample": f"Forged review evidence variant {line}.", + "evidence": ( + f"Source trace at {SOURCE_PATH}:{line} rejected the variant. " + "source-line-sha256=" + hashlib.sha256(source_line).hexdigest() + ), + "outcome": "falsified", + } + for line, source_line in enumerate(SOURCE_LINES, start=1) + ], + "residual_risk": "Repository branch policy remains externally enforced.", + } + control = { + "head_sha": head, + "run_id": "123", + "run_attempt": "2", + "result": "APPROVE", + "reason": "No blocking findings.", + "summary": "Exact-head evidence passed adversarial validation.", + "adversarial_validation": evidence, + "findings": [], + } + return "\n".join( + ( + approval_gate.PRIMARY_APPROVAL_MARKER, + "", + "", + "", + "## Adversarial validation", + "```json", + json.dumps(evidence), + "```", + "", + "- Result: APPROVE", + f"- Base ref: `{BASE_REF}`", + f"- Base SHA: `{BASE_SHA}`", + f"- Head SHA: `{head}`", + "- Workflow run: 123", + "- Workflow attempt: 2", + ) + ) def fake_secret(*parts: str) -> str: @@ -18,7 +142,9 @@ def make_pr(**overrides): "title": "Noema", "body": "", "isDraft": False, - "headRefOid": "head", + "baseRefName": BASE_REF, + "baseRefOid": BASE_SHA, + "headRefOid": HEAD, "reviews": {"nodes": []}, "reviewThreads": {"nodes": []}, "statusCheckRollup": {"contexts": {"nodes": []}}, @@ -27,11 +153,11 @@ def make_pr(**overrides): return value -def review(state="APPROVED", commit="head", login="opencode-agent", body="Result: APPROVE"): +def review(state="APPROVED", commit=HEAD, login="opencode-agent", body=None): """Build a minimal review node for Noema tests.""" return { "state": state, - "body": body, + "body": valid_primary_body(head=commit) if body is None else body, "author": {"login": login}, "commit": {"oid": commit}, } @@ -46,6 +172,24 @@ def test_run_split_repo_graphql_and_fetch_pr(monkeypatch): assert noema.split_repo("owner/repo") == ("owner", "repo") + +def test_module_bootstrap_restores_trusted_repository_root(monkeypatch): + """Standalone execution can import shared review helpers from the trusted root.""" + repository_root = Path(noema.__file__).resolve().parents[2] + monkeypatch.setattr( + sys, + "path", + [ + entry + for entry in sys.path + if Path(entry or ".").resolve() != repository_root + ], + ) + + runpy.run_path(noema.__file__, run_name="noema_review_gate_bootstrap_test") + + assert sys.path[0] == str(repository_root) + def test_scrub_sensitive_data(): assert noema.scrub_sensitive_data(None) is None assert noema.scrub_sensitive_data("") == "" @@ -92,30 +236,72 @@ def fake_run(args, stdin=None): def test_review_state_helpers_cover_current_head_logic(): - marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." + marker_body = valid_primary_body() current = review(body=marker_body) old = review(commit="old", body=marker_body) pr = make_pr(reviews={"nodes": [old, current]}) assert noema.review_author(current) == "opencode-agent" assert noema.review_author({}) == "" - assert noema.review_commit(current) == "head" + assert noema.review_commit(current) == HEAD assert noema.review_commit({}) == "" assert noema.current_primary_approval(pr) == current + marker_only = review( + body=identity_body(approval_gate.PRIMARY_APPROVAL_MARKER) + ) + assert ( + noema.current_primary_approval( + make_pr(reviews={"nodes": [marker_only]}) + ) + is None + ) + assert ( + noema.current_primary_approval( + make_pr(baseRefName="release", reviews={"nodes": [current]}) + ) + is None + ) + assert ( + noema.current_primary_approval( + make_pr(baseRefOid="c" * 40, reviews={"nodes": [current]}) + ) + is None + ) assert noema.current_primary_approval(make_pr(reviews={"nodes": [old]})) is None - assert noema.current_primary_approval(make_pr(reviews={"nodes": [review("COMMENTED", body=marker_body)]})) is None - assert noema.current_primary_approval(make_pr(reviews={"nodes": [review(login="human", body=marker_body)]})) is None - assert noema.current_primary_approval( - make_pr( - reviews={ - "nodes": [review(login="github-actions[bot]", body=marker_body)] - } + assert ( + noema.current_primary_approval( + make_pr(reviews={"nodes": [review("COMMENTED", body=marker_body)]}) ) - ) is None - assert noema.has_current_changes_requested(make_pr(reviews={"nodes": [review("CHANGES_REQUESTED")]})) - assert not noema.has_current_changes_requested(make_pr(reviews={"nodes": [review("CHANGES_REQUESTED", commit="old")]})) - assert noema.has_unresolved_threads(make_pr(reviewThreads={"nodes": [{"isResolved": False, "isOutdated": False}]})) - assert not noema.has_unresolved_threads(make_pr(reviewThreads={"nodes": [{"isResolved": False, "isOutdated": True}]})) + is None + ) + assert ( + noema.current_primary_approval( + make_pr(reviews={"nodes": [review(login="human", body=marker_body)]}) + ) + is None + ) + assert ( + noema.current_primary_approval( + make_pr( + reviews={ + "nodes": [review(login="github-actions[bot]", body=marker_body)] + } + ) + ) + is None + ) + assert noema.has_current_changes_requested( + make_pr(reviews={"nodes": [review("CHANGES_REQUESTED")]}) + ) + assert not noema.has_current_changes_requested( + make_pr(reviews={"nodes": [review("CHANGES_REQUESTED", commit="old")]}) + ) + assert noema.has_unresolved_threads( + make_pr(reviewThreads={"nodes": [{"isResolved": False, "isOutdated": False}]}) + ) + assert not noema.has_unresolved_threads( + make_pr(reviewThreads={"nodes": [{"isResolved": False, "isOutdated": True}]}) + ) def test_review_state_helpers_reject_explicit_previous_head_evidence(): @@ -128,7 +314,7 @@ def test_review_state_helpers_reject_explicit_previous_head_evidence(): ) exact_approval = review( commit=current_head, - body=f"{approval_marker}\n\n- Head SHA: `{current_head}`", + body=valid_primary_body(head=current_head), ) stale_change_request = review( "CHANGES_REQUESTED", @@ -147,6 +333,27 @@ def test_review_state_helpers_reject_explicit_previous_head_evidence(): ) +def test_review_identity_ignores_incidental_or_quoted_labels(): + """Only canonical identity bullets can bind a review to a pull request.""" + previous_head = "b" * 40 + body = "\n".join( + ( + valid_primary_body(), + "", + f"Diagnostic prose mentions Head SHA: `{previous_head}`.", + "> - Base ref: `quoted-branch`", + f"> - Base SHA: `{previous_head}`", + ) + ) + exact_approval = review(body=body) + + assert noema.review_matches_current_head( + exact_approval, + make_pr(), + require_base_identity=True, + ) + + def test_check_helpers_and_existing_noema_review(): status_context = {"__typename": "StatusContext", "context": "ci", "state": "FAILURE"} check_run = { @@ -173,6 +380,7 @@ def test_check_helpers_and_existing_noema_review(): assert noema.check_label(status_context) == "ci" assert noema.check_label(check_run) == "CI / build" + assert not noema.existing_noema_review(make_pr(), "") blockers = noema.blocking_checks( make_pr( statusCheckRollup={ @@ -192,31 +400,76 @@ def test_check_helpers_and_existing_noema_review(): assert "CI / lint: FAILURE" in blockers assert "CI / slow: IN_PROGRESS" in blockers assert noema.existing_noema_review( - make_pr(reviews={"nodes": [review(login="noema", body="")]}), + make_pr( + reviews={ + "nodes": [ + review( + login="noema", + body=identity_body( + "" + ), + ) + ] + } + ), + "noema", + ) + assert not noema.existing_noema_review( + make_pr( + reviews={ + "nodes": [ + review( + login="attacker", + body=identity_body(""), + ) + ] + } + ), "noema", ) - assert not noema.existing_noema_review(make_pr(reviews={"nodes": [review("DISMISSED", login="noema")]}), "noema") - assert not noema.existing_noema_review(make_pr(reviews={"nodes": [review(commit="old", login="noema")]}), "noema") + assert not noema.existing_noema_review( + make_pr(reviews={"nodes": [review(login="noema", body=identity_body())]}), + "noema", + ) + assert not noema.existing_noema_review( + make_pr(reviews={"nodes": [review("DISMISSED", login="noema")]}), "noema" + ) + assert not noema.existing_noema_review( + make_pr(reviews={"nodes": [review(commit="old", login="noema")]}), "noema" + ) def test_current_actor_fetch_diff_and_json_extraction(monkeypatch): monkeypatch.setattr(noema, "run", lambda *args, **kwargs: "noema\n") assert noema.current_actor() == "noema" - monkeypatch.setattr(noema, "run", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("no gh"))) + monkeypatch.setattr( + noema, + "run", + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("no gh")), + ) assert noema.current_actor() == "" - monkeypatch.setattr(noema, "run", lambda *args, **kwargs: "x" * (noema.MAX_DIFF_CHARS + 5)) + monkeypatch.setattr( + noema, "run", lambda *args, **kwargs: "x" * (noema.MAX_DIFF_CHARS + 5) + ) diff, truncated = noema.fetch_diff("owner/repo", 1) assert truncated assert len(diff) == noema.MAX_DIFF_CHARS - assert noema.extract_json_object('{"decision":"approve"}') == {"decision": "approve"} - assert noema.extract_json_object('prefix {"decision":"comment"} suffix') == {"decision": "comment"} + assert noema.extract_json_object('{"decision":"approve"}') == { + "decision": "approve" + } + assert noema.extract_json_object('prefix {"decision":"comment"} suffix') == { + "decision": "comment" + } with pytest.raises(RuntimeError, match="did not contain"): noema.extract_json_object("not-json") -def test_review_context_builders_include_codegraph_threads_and_files(monkeypatch, tmp_path): +def test_review_context_builders_include_codegraph_threads_and_files( + monkeypatch, tmp_path +): assert noema.truncate_text("abc", 10) == "abc" assert "truncated 2 characters" in noema.truncate_text("abcdef", 4) assert "missing PR head SHA" in noema.changed_file_context("owner/repo", 7, "") @@ -297,23 +550,22 @@ def test_review_context_reports_omitted_files_and_missing_codegraph(monkeypatch, class FakeResponse: - """Small context-manager response for urllib monkeypatches.""" + """Small response object for pinned HTTPS transport tests.""" - def __init__(self, payload): + def __init__(self, payload, status=200, content_length=None): """Store a JSON-serializable response payload.""" self.payload = payload + self.status = status + self.content_length = content_length - def __enter__(self): - """Return the response for with-statement use.""" - return self - - def __exit__(self, *args): - """Propagate exceptions from the with-statement body.""" - return False + def getheader(self, name): + """Return an optional synthetic Content-Length header.""" + return self.content_length if name.lower() == "content-length" else None - def read(self): + def read(self, amount=None): """Return the payload as encoded JSON bytes.""" - return json.dumps(self.payload).encode("utf-8") + encoded = json.dumps(self.payload).encode("utf-8") + return encoded if amount is None else encoded[:amount] def test_call_llm_handles_configuration_and_verdicts(monkeypatch): @@ -325,98 +577,180 @@ def test_call_llm_handles_configuration_and_verdicts(monkeypatch): monkeypatch.setenv("NOEMA_LLM_API_URL", "file:///etc/passwd") monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") - with pytest.raises(ValueError, match="URL scheme must be http or https"): + with pytest.raises(ValueError, match="must use https://"): noema.call_llm("owner/repo", 1, pr, "diff", False) - monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") - monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") - monkeypatch.setenv("NOEMA_LLM_MODEL", "review-model") + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://llm.example.test/chat") + with pytest.raises(ValueError, match="must use https://"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + + response = { + "value": FakeResponse( + {"choices": [{"message": {"content": '{"decision":"approve","summary":"ok","findings":[]}'}}]} + ) + } seen = {} - def fake_urlopen(request, timeout): - seen["url"] = request.full_url - seen["body"] = json.loads(request.data.decode("utf-8")) - return FakeResponse({"choices": [{"message": {"content": '{"decision":"approve","summary":"ok","findings":[]}'}}]}) + class FakeConnection: + def __init__(self, hostname, port, pinned_ip, *, timeout): + seen.update(hostname=hostname, port=port, pinned_ip=pinned_ip, timeout=timeout) + + def request(self, method, target, *, body, headers): + seen.update( + method=method, + target=target, + body=json.loads(body.decode("utf-8")), + headers=headers, + ) + + def getresponse(self): + return response["value"] + + def close(self): + seen["closed"] = True - # Since we replaced urlopen with build_opener, we mock build_opener - class FakeOpener: - def __init__(self, call_func): - self.call_func = call_func - def open(self, request, timeout=None): - return self.call_func(request, timeout) + def public_dns(host, port, *args, **kwargs): + return [ + (noema.socket.AF_INET, noema.socket.SOCK_STREAM, 6, "", ("8.8.8.8", port)), + (noema.socket.AF_INET, noema.socket.SOCK_STREAM, 6, "", ("8.8.8.8", port)), + ] - monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) + monkeypatch.setattr(noema.socket, "getaddrinfo", public_dns) + monkeypatch.setattr(noema, "PinnedHTTPSConnection", FakeConnection) + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat;v=1?mode=review") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") + monkeypatch.setenv("NOEMA_LLM_MODEL", "review-model") verdict = noema.call_llm("owner/repo", 1, pr, "diff", True, "extra review context") assert verdict["decision"] == "approve" - assert seen["url"] == "https://llm.example.test/chat" + assert seen["hostname"] == "llm.example.test" + assert seen["port"] == 443 + assert seen["pinned_ip"] == "8.8.8.8" + assert seen["target"] == "/chat;v=1?mode=review" + assert seen["headers"]["authorization"] == "Bearer secret" assert seen["body"]["model"] == "review-model" assert "extra review context" in seen["body"]["messages"][1]["content"] + assert seen["closed"] is True - def fake_urlopen_defer(request, timeout=None): - return FakeResponse({"choices": [{"message": {"content": '{"decision":"defer"}'}}]}) + original_limit = noema.MAX_LLM_RESPONSE_BYTES + monkeypatch.setattr(noema, "MAX_LLM_RESPONSE_BYTES", 8) + response["value"] = FakeResponse({"oversized": "response"}) + with pytest.raises(RuntimeError, match="response exceeded the size limit"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + response["value"] = FakeResponse({}, content_length="9") + with pytest.raises(RuntimeError, match="response exceeded the size limit"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + response["value"] = FakeResponse({}, content_length="invalid") + with pytest.raises(RuntimeError, match="invalid Content-Length"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + monkeypatch.setattr(noema, "MAX_LLM_RESPONSE_BYTES", original_limit) - monkeypatch.setattr( - noema.urllib.request, - "build_opener", - lambda *args: FakeOpener(fake_urlopen_defer) + response["value"] = FakeResponse( + {"choices": [{"message": {"content": '{"decision":"defer"}'}}]} ) with pytest.raises(RuntimeError, match="unsupported decision"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test case-insensitive valid URL + response["value"] = FakeResponse( + {"choices": [{"message": {"content": '{"decision":"approve","summary":"ok","findings":[]}'}}]} + ) monkeypatch.setenv("NOEMA_LLM_API_URL", "HTTPS://llm.example.test/chat") - monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" - # Test invalid scheme (and no original URL in error) - monkeypatch.setenv("NOEMA_LLM_API_URL", "file:///etc/passwd") - with pytest.raises(ValueError, match="URL scheme must be http or https"): + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://llm.example.test/chat") + with pytest.raises(ValueError, match="must use https://"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test localhost rejection - monkeypatch.setenv("NOEMA_LLM_API_URL", "http://localhost/chat") + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://localhost/chat") with pytest.raises(ValueError, match="URL cannot target localhost"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test missing hostname - monkeypatch.setenv("NOEMA_LLM_API_URL", "http:///chat") + monkeypatch.setenv("NOEMA_LLM_API_URL", "https:///chat") with pytest.raises(ValueError, match="URL must have a valid hostname"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test internal IP rejection - monkeypatch.setenv("NOEMA_LLM_API_URL", "http://169.254.169.254/chat") - with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://169.254.169.254/chat") + monkeypatch.setattr( + noema.socket, + "getaddrinfo", + lambda host, port, *args, **kwargs: [ + (noema.socket.AF_INET, noema.socket.SOCK_STREAM, 6, "", ("169.254.169.254", port)) + ], + ) + with pytest.raises(ValueError, match="non-public IP"): noema.call_llm("owner/repo", 1, pr, "diff", False) - import socket - original_getaddrinfo = socket.getaddrinfo - - # Test DNS resolution bypass - monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-local.example.com/chat") - def fake_getaddrinfo(host, port, *args, **kwargs): - if host == "resolved-to-local.example.com": - return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] - return original_getaddrinfo(host, port, *args, **kwargs) - monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) - with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://unresolved.example.com/chat") + monkeypatch.setattr( + noema.socket, + "getaddrinfo", + lambda *args, **kwargs: (_ for _ in ()).throw(noema.socket.gaierror("not found")), + ) + with pytest.raises(ValueError, match="could not be resolved"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test unresolved hostname does not break - monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") - def fake_getaddrinfo_error(host, port, *args, **kwargs): - raise socket.gaierror("Name or service not known") - monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_error) - monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) - assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://weird-dns.example.com/chat") + monkeypatch.setattr( + noema.socket, + "getaddrinfo", + lambda host, port, *args, **kwargs: [ + (noema.socket.AF_INET, noema.socket.SOCK_STREAM, 6, "", ("not_an_ip", port)) + ], + ) + with pytest.raises(ValueError, match="invalid IP"): + noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test invalid IP string from getaddrinfo (unlikely but theoretically possible) - monkeypatch.setenv("NOEMA_LLM_API_URL", "http://weird-dns.example.com/chat") - def fake_getaddrinfo_invalid_ip(host, port, *args, **kwargs): - if host == "weird-dns.example.com": - return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("not_an_ip", 0))] - return original_getaddrinfo(host, port, *args, **kwargs) - monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_invalid_ip) - assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + monkeypatch.setattr(noema.socket, "getaddrinfo", lambda *args, **kwargs: []) + with pytest.raises(ValueError, match="no addresses"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + + monkeypatch.setattr(noema.socket, "getaddrinfo", public_dns) + for unsafe_url, message in ( + ("https://user:password@llm.example.test/chat", "user information"), + ("https://llm.example.test/chat#fragment", "fragment"), + ("https://llm.example.test:bad/chat", "invalid port"), + ): + monkeypatch.setenv("NOEMA_LLM_API_URL", unsafe_url) + with pytest.raises(ValueError, match=message): + noema.call_llm("owner/repo", 1, pr, "diff", False) + + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") + response["value"] = FakeResponse({"error": "redirect denied"}, status=302) + with pytest.raises(RuntimeError, match="HTTP 302"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + + +def test_pinned_https_connection_uses_numeric_peer_and_original_sni(monkeypatch): + """The transport must never resolve the validated hostname a second time.""" + seen = {} + + class FakeSocket: + def setsockopt(self, *args): + seen["setsockopt"] = args + + def close(self): + seen["closed"] = True + + fake_socket = FakeSocket() + + def fake_create_connection(address, timeout, source_address): + seen.update(address=address, timeout=timeout, source_address=source_address) + return fake_socket + + class FakeContext: + def wrap_socket(self, sock, *, server_hostname): + seen.update(wrapped_socket=sock, server_hostname=server_hostname) + return sock + + monkeypatch.setattr(noema.socket, "create_connection", fake_create_connection) + connection = noema.PinnedHTTPSConnection("llm.example.test", 443, "8.8.8.8", timeout=12) + connection._context = FakeContext() + connection.connect() + connection.close() + + assert seen["address"] == ("8.8.8.8", 443) + assert seen["server_hostname"] == "llm.example.test" + assert seen["wrapped_socket"] is fake_socket + assert seen["closed"] is True def test_noema_redirect_handler_rejects_redirects(): @@ -450,7 +784,7 @@ def raise_gaierror(host, port, *args, **kwargs): raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", raise_gaierror) - with pytest.raises(ValueError, match="must start with http:// or https://"): + with pytest.raises(ValueError, match="control characters"): noema.call_llm("owner/repo", 1, pr, "diff", False) @@ -462,7 +796,7 @@ def test_call_llm_rejects_non_http_parsed_scheme(monkeypatch): parsed = noema.urllib.parse.ParseResult("file", "llm.example.test", "/chat", "", "", "") monkeypatch.setattr(noema.urllib.parse, "urlparse", lambda _: parsed) - with pytest.raises(ValueError, match="URL scheme must be http or https"): + with pytest.raises(ValueError, match="https scheme"): noema.call_llm("owner/repo", 1, pr, "diff", False) @@ -479,18 +813,29 @@ def test_format_findings_and_submit_review(monkeypatch): calls = [] monkeypatch.setenv("NOEMA_REVIEW_TOKEN_SOURCE", "oidc") - monkeypatch.setattr(noema, "run", lambda args, stdin=None: calls.append((args, json.loads(stdin))) or "") + monkeypatch.setattr( + noema, + "run", + lambda args, stdin=None: calls.append((args, json.loads(stdin))) or "", + ) + monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: make_pr()) noema.submit_review( "owner/repo", 7, make_pr(), "noema", - {"decision": "request_changes", "summary": "fix it", "findings": [{"file": "a.py", "line": 1, "message": "bad"}]}, + { + "decision": "request_changes", + "summary": "fix it", + "findings": [{"file": "a.py", "line": 1, "message": "bad"}], + }, ) payload = calls[0][1] assert payload["event"] == "REQUEST_CHANGES" - assert payload["commit_id"] == "head" + assert payload["commit_id"] == HEAD assert "Noema LLM review" in payload["body"] + assert f"- Base ref: `{BASE_REF}`" in payload["body"] + assert f"- Base SHA: `{BASE_SHA}`" in payload["body"] assert "oidc" in payload["body"] calls.clear() @@ -498,9 +843,19 @@ def test_format_findings_and_submit_review(monkeypatch): assert calls[0][1]["event"] == "COMMENT" assert "No blocking findings" in calls[0][1]["body"] + monkeypatch.setattr( + noema, + "fetch_pr", + lambda repo, number: make_pr(baseRefName="release"), + ) + with pytest.raises(RuntimeError, match="live base/head identity changed"): + noema.submit_review( + "owner/repo", 7, make_pr(), "noema", {"decision": "approve"} + ) + def test_inspect_and_review_skip_paths(monkeypatch): - marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." + marker_body = valid_primary_body() clean_pr = make_pr(reviews={"nodes": [review(body=marker_body)]}) calls = [] monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: clean_pr) @@ -513,20 +868,72 @@ def test_inspect_and_review_skip_paths(monkeypatch): assert noema.inspect_and_review("owner/repo", 7) == 0 assert calls + existing_review = ( + make_pr( + reviews={ + "nodes": [ + review( + login="noema", + body=identity_body(""), + ) + ] + } + ), + "noema", + ) + calls.clear() + monkeypatch.setattr(noema, "fetch_pr", lambda repo, number: existing_review[0]) + monkeypatch.setattr(noema, "current_actor", lambda: existing_review[1]) + assert noema.inspect_and_review("owner/repo", 7) == 0 + assert calls == [] + cases = [ - (make_pr(), "noema"), - (make_pr(isDraft=True), "noema"), - (make_pr(reviews={"nodes": [review(login="noema", body="")]}), "noema"), - (make_pr(reviews={"nodes": [review("CHANGES_REQUESTED"), review(body=marker_body)]}), "noema"), - (make_pr(reviews={"nodes": [review(body=marker_body)]}, reviewThreads={"nodes": [{"isResolved": False, "isOutdated": False}]}), "noema"), - (make_pr(reviews={"nodes": [review(body=marker_body)]}, statusCheckRollup={"contexts": {"nodes": [{"__typename": "StatusContext", "context": "ci", "state": "FAILURE"}]}}), "noema"), - (clean_pr, "opencode-agent"), + (make_pr(), "noema", "primary OpenCode approval"), + (make_pr(isDraft=True), "noema", "draft pull request"), + ( + make_pr( + reviews={"nodes": [review(body=marker_body)]}, + reviewThreads={"nodes": [{"isResolved": False, "isOutdated": False}]}, + ), + "noema", + "unresolved review threads", + ), + ( + make_pr( + reviews={ + "nodes": [review("CHANGES_REQUESTED"), review(body=marker_body)] + } + ), + "noema", + "request-changes review", + ), + ( + make_pr( + reviews={"nodes": [review(body=marker_body)]}, + statusCheckRollup={ + "contexts": { + "nodes": [ + { + "__typename": "StatusContext", + "context": "ci", + "state": "FAILURE", + } + ] + } + }, + ), + "noema", + "Blocking checks remain", + ), + (clean_pr, "opencode-agent", "independent Noema reviewer"), + (clean_pr, "", "credential identity"), ] - for pr, actor in cases: + for pr, actor, message in cases: calls.clear() monkeypatch.setattr(noema, "fetch_pr", lambda repo, number, pr=pr: pr) monkeypatch.setattr(noema, "current_actor", lambda actor=actor: actor) - assert noema.inspect_and_review("owner/repo", 7) == 0 + with pytest.raises(RuntimeError, match=message): + noema.inspect_and_review("owner/repo", 7) assert calls == [] @@ -536,7 +943,11 @@ def test_parse_args_and_main(monkeypatch): assert parsed.pr_number == 9 seen = [] - monkeypatch.setattr(noema, "inspect_and_review", lambda repo, number: seen.append((repo, number)) or 0) + monkeypatch.setattr( + noema, + "inspect_and_review", + lambda repo, number: seen.append((repo, number)) or 0, + ) assert noema.main(["--repo", "owner/repo", "--pr-number", "9"]) == 0 assert seen == [("owner/repo", 9)] diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index c8eb1d34..3cc5b1a1 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -103,9 +103,9 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ - ["github-models", "deepseek/deepseek-v3-0324"], - ["openai", "gpt-5.6-luna"], ["github-models", "openai/gpt-4.1"], + ["openai", "gpt-5.6-luna"], + ["github-models", "deepseek/deepseek-v3-0324"], ["github-models", "openai/gpt-5"], ["github-models", "openai/gpt-5-chat"], ["github-models", "openai/o3"], @@ -115,8 +115,8 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert direct_openai_models == ["gpt-5.6-luna"] assert set(github_candidate_models).issubset(set(github_models)) assert github_candidate_models == [ - "deepseek/deepseek-v3-0324", "openai/gpt-4.1", + "deepseek/deepseek-v3-0324", "openai/gpt-5", "openai/gpt-5-chat", "openai/o3", @@ -283,6 +283,15 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): "actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131" in coverage_job ) + download_start = coverage_job.index( + " - name: Download materialized pull request merge tree\n" + ) + download_end = coverage_job.index("\n - name:", download_start + 1) + download_step = coverage_job[download_start:download_end] + assert "github-token: ${{ github.token }}" in download_step + assert "repository: ${{ github.repository }}" in download_step + assert "run-id: ${{ github.run_id }}" in download_step + assert "re-run failed jobs" in download_step start = workflow.index( " - name: Materialize pull request merge tree for coverage measurement\n" @@ -330,7 +339,10 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "apt-get install --no-install-recommends -y" in measure_step assert "--require-hashes" in measure_step assert 'coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' in measure_step - assert "The networked build context contains only this" in measure_step + assert ( + "hash-pinned dependency locks read from the exact target base SHA" + in measure_step + ) assert 'install -m 0644 "$trusted_ci_requirements"' in measure_step assert "docker build --pull --no-cache --network=default" in measure_step assert '"$coverage_build_dir"' in measure_step @@ -405,7 +417,16 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "uv sync --project" not in measure_step assert "uv run --no-project" not in measure_step assert "uv run --no-build" not in measure_step - assert "python3 -m coverage run -m pytest tests" in measure_step + assert '"$2" -m coverage run -m pytest tests' in measure_step + assert "materialize_base_python_locks.py" in measure_step + assert "COPY base-python-locks /tmp/base-python-locks" in measure_step + assert "--require-hashes" in measure_step + assert "--only-binary=:all:" in measure_step + assert "python_for_project()" in measure_step + assert ( + '[[ "$project_python" = /opt/base-python-envs/*/bin/python ]]' in measure_step + ) + assert 'OPENCODE_PYTHON_ENV_BIN="$(dirname "$project_python")"' in measure_step trusted_requirements = Path( "requirements-opencode-review-ci-hashes.txt" ).read_text(encoding="utf-8") @@ -493,8 +514,8 @@ def test_opencode_model_exhaustion_retry_stays_owned_by_central_scheduler(): assert "contents: write" not in workflow -def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): - """Use only the trusted image toolchain during networkless PR execution.""" +def test_opencode_python_coverage_uses_only_base_hash_locked_dependencies(): + """Use trusted base locks and never resolve PR-selected dependencies.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") measure = workflow.split( " - name: Measure test and docstring evidence\n", 1 @@ -502,17 +523,17 @@ def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): assert "verify_trusted_python_test_toolchain()" in measure assert "PR-selected dependency manifests are never resolved" in measure - assert "missing project imports fail in pytest" in measure + assert "trusted base commit; missing imports still fail in pytest" in measure assert "uv sync --project" not in measure assert "uv run --no-project" not in measure assert "uv run --no-build" not in measure - assert "python3 -m coverage run -m pytest tests" in measure - assert "python3 -m coverage report --show-missing" in measure - assert "python3 -m pytest tests/test_docstrings.py" in measure + assert '"$2" -m coverage run -m pytest tests' in measure + assert '"$2" -m coverage report --show-missing' in measure + assert '"$2" -m pytest tests/test_docstrings.py' in measure -def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): - """pnpm workspaces must not activate PR-selected tooling or fall back to npm.""" +def test_opencode_coverage_prefers_exact_preinstalled_declared_pnpm_before_npm(): + """pnpm workspaces must use the exact pinned runner and never fall back to npm.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") measure_start = workflow.index( " - name: Measure test and docstring evidence\n" @@ -530,8 +551,11 @@ def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): assert 'corepack prepare "$spec" --activate' not in measure_step assert "not preinstalled in the pinned sandbox image" in measure_step assert "or fall back to npm" in measure_step - assert "ensure_corepack_runner pnpm" in select_function - assert "ensure_corepack_runner yarn" in select_function + assert "ensure_preinstalled_package_runner pnpm" in select_function + assert "ensure_preinstalled_package_runner yarn" in select_function + assert 'actual_version="$("$runner" --version 2>/dev/null || true)"' in measure_step + assert 'requested_runtime_version="${requested_version%%+*}"' in measure_step + assert 'if [ "$actual_version" != "$requested_runtime_version" ]; then' in measure_step assert select_function.index("[ -f pnpm-lock.yaml ]") < select_function.rindex( "elif command -v npm" ) @@ -545,6 +569,18 @@ def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): assert "return" in declared_pnpm_block +def test_opencode_coverage_image_pins_node_and_pnpm_artifacts(): + """The trusted sandbox must provide the exact Node and pnpm versions Naruon declares.""" + workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") + assert "node-v22.14.0-linux-x64.tar.xz" in workflow + assert "69b09dba5c8dcb05c4e4273a4340db1005abeafe3927efda2bc5b249e80437ec" in workflow + assert "pnpm/-/pnpm-11.5.3.tgz" in workflow + assert "238d639a47712278bb72e8b6db2c297ac1ccd80dd7642f7c933b73aebde7b51f" in workflow + assert 'test "$(node --version)" = "v22.14.0"' in workflow + assert 'test "$(pnpm --version)" = "11.5.3"' in workflow + assert "corepack prepare" not in workflow + + def test_opencode_coverage_does_not_duplicate_existing_javascript_coverage(): """An existing coverage flag/tool must run once instead of receiving a duplicate flag.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") @@ -900,7 +936,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "skipping remaining attempts for this model" in model_pool_runner assert "using %ss run timeout with %ss retry budget remaining" in model_pool_runner assert ( - "timed out after %ss; falling through within the remaining retry budget" + "timed out after %ss; skipping remaining attempts for this model and " + "falling through within the remaining retry budget" in model_pool_runner ) assert "emit_sanitized_opencode_failure_detail" in model_pool_runner @@ -973,11 +1010,11 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL" in workflow assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "1800"' in workflow ) assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600"' in workflow ) assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1"' in workflow @@ -1051,19 +1088,19 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert re.search( r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 205", workflow ) - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3900"' in workflow assert ( 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}s"' in workflow ) assert "OpenCode model pool exceeded the outer" in workflow - assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert re.search( r"Run OpenCode PR Review model pool[\s\S]{0,280}continue-on-error: true", workflow, @@ -1090,39 +1127,40 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): in workflow ) assert ( - 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' + 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-4.1 ' "openai/gpt-5.6-luna " - "github-models/openai/gpt-4.1 " + "github-models/deepseek/deepseek-v3-0324 " "github-models/openai/gpt-5 " "github-models/openai/gpt-5-chat " "github-models/openai/o3 " "github-models/deepseek/deepseek-r1-0528 " 'github-models/deepseek/deepseek-r1"' ) in workflow - assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_MODEL_ATTEMPTS: "2"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "180"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow - assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "3900"' in workflow + assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert ( "OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt" in workflow ) - assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow - assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400"' in workflow - assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700"' in workflow - assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0"' in workflow + assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow + assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "1800"' in workflow + assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "3600"' in workflow + assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow - assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_GITHUB_DEEPSEEK_RUN_TIMEOUT_SECONDS: "600"' in workflow + assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow publish_step = workflow.split(" - name: Publish OpenCode review outcome", 1)[ 1 @@ -1179,7 +1217,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "while :" in model_pool_runner assert "should_skip_model_candidate" in model_pool_runner assert "cap_model_run_timeout" in model_pool_runner - assert "constrained request-body limit" in model_pool_runner + assert "constrained or non-responsive endpoint" in model_pool_runner assert "run_central_adversarial_harness" not in model_pool_runner assert "finish_pool_without_model" in model_pool_runner assert "central-current-head-adversarial-harness" not in model_pool_runner @@ -1213,9 +1251,9 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-5-nano"' not in workflow ) assert ( - 'OPENCODE_MODEL_CANDIDATES: "github-models/deepseek/deepseek-v3-0324 ' + 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-4.1 ' "openai/gpt-5.6-luna " - "github-models/openai/gpt-4.1 " + "github-models/deepseek/deepseek-v3-0324 " "github-models/openai/gpt-5 " "github-models/openai/gpt-5-chat " "github-models/openai/o3 " @@ -1581,7 +1619,7 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): assert "uv run --no-project" not in measure assert "uv run --no-build" not in measure assert "Trusted offline Python test toolchain" in measure - assert "python3 -m coverage run -m pytest tests" in measure + assert '"$2" -m coverage run -m pytest tests' in measure assert 'chmod 0444 "$implementation_changed_files"' in measure assert "npm ci --ignore-scripts" in coverage_job assert "pnpm install --frozen-lockfile --ignore-scripts" in coverage_job @@ -1617,6 +1655,18 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): ) < target_job.index( "Exchange OpenCode app token for target repository review reads" ) + materialize_step = target_job.split( + " - name: Materialize inert pull request blobs for OpenCode review data", 1 + )[1].split("\n - name:", 1)[0] + assert 'git init --bare "$OPENCODE_SOURCE_GIT_DIR"' in materialize_step + assert 'git --git-dir="$OPENCODE_SOURCE_GIT_DIR" fetch' in materialize_step + assert "python3 scripts/ci/materialize_pr_review_source.py" in materialize_step + assert 'git worktree add --detach "$OPENCODE_SOURCE_WORKDIR"' not in target_job + assert 'find "$OPENCODE_SOURCE_WORKDIR" -type l' in materialize_step + assert 'find "$OPENCODE_SOURCE_WORKDIR" -type f -perm /111' in materialize_step + assert materialize_step.index("materialize_pr_review_source.py") < materialize_step.index( + 'git -C "$OPENCODE_SOURCE_WORKDIR" rev-parse HEAD' + ) codegraph_step = target_job.split( " - name: Initialize CodeGraph index for OpenCode", 1 )[1].split("\n - name:", 1)[0] @@ -1752,9 +1802,9 @@ def test_opencode_review_publication_prefers_app_token_for_review_writes(): assert 'review_write_token="${OPENCODE_APP_TOKEN:-}"' in workflow assert 'post_pull_review_with_retry "fallback review"' not in workflow assert "OPENCODE_REVIEW_IDENTITY_UNAVAILABLE" in workflow - assert "OPENCODE_REVIEW_STALE_HEAD" in workflow + assert "OPENCODE_REVIEW_STALE_IDENTITY" in workflow assert "OPENCODE_OVERVIEW_STALE_HEAD" in workflow - assert "review_live_head_sha()" in workflow + assert "review_live_pr_identity()" in workflow assert "validate_published_review_head()" in workflow assert "dismiss_stale_published_review()" in workflow assert 'review_head_guard_token="${GH_TOKEN:-$review_write_token}"' in workflow diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 7602b2a1..1ed7281c 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -12,6 +12,8 @@ HEAD = "a" * 40 +BASE_REF = "main" +BASE_SHA = "b" * 40 SOURCE_LINES = ( b"name: Required OpenCode Review", b"on:", @@ -68,6 +70,8 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): "OPENCODE_ARTIFACT_MANIFEST_SHA256", hashlib.sha256(manifest.read_bytes()).hexdigest(), ) + opencode_review_normalize_output.current_changed_files.cache_clear() + opencode_review_normalize_output.trusted_execution_receipts.cache_clear() def valid_body(head: str = HEAD) -> str: @@ -92,12 +96,26 @@ def valid_body(head: str = HEAD) -> str: ], "residual_risk": "Hosted token permissions remain externally enforced.", } + control = { + "head_sha": head, + "run_id": "123", + "run_attempt": "2", + "result": "APPROVE", + "reason": "No blocking findings after adversarial validation.", + "summary": "The exact current-head evidence passed the strict review gate.", + "adversarial_validation": evidence, + "findings": [], + } return "\n".join( ( "## Pull request overview", "", gate.PRIMARY_APPROVAL_MARKER, "", + "", + "", "## Adversarial validation", "", "```json", @@ -105,6 +123,8 @@ def valid_body(head: str = HEAD) -> str: "```", "", "- Result: APPROVE", + f"- Base ref: `{BASE_REF}`", + f"- Base SHA: `{BASE_SHA}`", f"- Head SHA: `{head}`", "- Workflow run: 123", "- Workflow attempt: 2", @@ -129,7 +149,9 @@ def review(**overrides): def test_flatten_reviews_and_accept_real_model_approval(payload): reviews = gate.flatten_reviews(payload) log = io.StringIO() - assert gate.has_reusable_real_model_approval(reviews, HEAD, log=log) + assert gate.has_reusable_real_model_approval( + reviews, HEAD, BASE_REF, BASE_SHA, log=log + ) assert "accepted real-model review" in log.getvalue() @@ -145,6 +167,55 @@ def test_extract_adversarial_evidence_uses_last_parseable_block(): assert gate.extract_adversarial_evidence("none") is None +def test_reusable_approval_accepts_structurally_bound_location(): + """Published evidence need not duplicate structured path and line in prose.""" + evidence = gate.extract_adversarial_evidence(valid_body()) + assert evidence is not None + for probe, source_line in zip(evidence["probes"], SOURCE_LINES, strict=True): + probe["evidence"] = ( + "Focused regression command passed with exit code 0. " + "source-line-sha256=" + hashlib.sha256(source_line).hexdigest() + ) + body = f"## Adversarial validation\n```json\n{json.dumps(evidence)}\n```" + + assert gate.adversarial_rejection_reason(body) is None + + +def test_extract_control_payload_rejects_ambiguous_and_malformed_blocks(): + """Reusable approval bodies contain exactly one object-shaped control block.""" + assert gate.extract_control_payload("none")[1] == ( + "review body must contain exactly one OpenCode control block" + ) + one = "" + assert gate.extract_control_payload(f"{one}\n{one}")[1] == ( + "review body must contain exactly one OpenCode control block" + ) + assert "parseable JSON" in gate.extract_control_payload( + "" + )[1] + assert "JSON object" in gate.extract_control_payload( + "" + )[1] + + body = valid_body() + control_start = body.index("", control_start) + len("-->") + missing_control = review(body=body[:control_start] + body[control_end:]) + assert "exactly one" in gate.review_rejection_reason( + missing_control, HEAD, BASE_REF, BASE_SHA + ) + wrong_head = review( + body=valid_body().replace( + f'"head_sha": "{HEAD}"', + f'"head_sha": "{"c" * 40}"', + 1, + ) + ) + assert "control head" in gate.review_rejection_reason( + wrong_head, HEAD, BASE_REF, BASE_SHA + ) + + @pytest.mark.parametrize( ("mutate", "reason"), [ @@ -169,12 +240,77 @@ def test_extract_adversarial_evidence_uses_last_parseable_block(): ), "APPROVE result", ), + ( + lambda value: value.update( + body=value["body"] + "\n- Result: REQUEST_CHANGES" + ), + "unambiguous APPROVE result", + ), + ( + lambda value: value.update( + body=value["body"].replace( + '"result": "APPROVE"', + '"result": "REQUEST_CHANGES"', + ) + ), + "control result", + ), + ( + lambda value: value.update( + body=value["body"].replace('"run_id": "123"', '"run_id": "999"') + ), + "control workflow run", + ), + ( + lambda value: value.update( + body=value["body"].replace( + '"run_attempt": "2"', '"run_attempt": "9"' + ) + ), + "control workflow attempt", + ), ( lambda value: value.update( body=value["body"].replace(f"- Head SHA: `{HEAD}`", "") ), "current-head", ), + ( + lambda value: value.update( + body=value["body"].replace(f"- Base ref: `{BASE_REF}`", "") + ), + "base ref", + ), + ( + lambda value: value.update( + body=value["body"].replace( + f"- Base ref: `{BASE_REF}`", f"> - Base ref: `{BASE_REF}`" + ) + ), + "base ref", + ), + ( + lambda value: value.update( + body=value["body"].replace(f"- Base SHA: `{BASE_SHA}`", "") + ), + "base SHA", + ), + ( + lambda value: value.update( + body=value["body"].replace( + f"- Base ref: `{BASE_REF}`", "- Base ref: `release`" + ) + ), + "base ref", + ), + ( + lambda value: value.update( + body=value["body"].replace( + f"- Base SHA: `{BASE_SHA}`", f"- Base SHA: `{'c' * 40}`" + ) + ), + "base SHA", + ), ( lambda value: value.update( body=value["body"].replace("- Workflow run: 123", "") @@ -198,7 +334,17 @@ def test_extract_adversarial_evidence_uses_last_parseable_block(): def test_review_rejection_reason_rejects_non_model_evidence(mutate, reason): value = review() mutate(value) - assert reason in gate.review_rejection_reason(value, HEAD) + assert reason in gate.review_rejection_reason(value, HEAD, BASE_REF, BASE_SHA) + + +def test_review_rejection_reason_accepts_uppercase_sha_bullets(): + """Hex SHA identity is case-insensitive while the base ref remains exact.""" + value = review() + value["body"] = value["body"].replace( + f"- Head SHA: `{HEAD}`", f"- Head SHA: `{HEAD.upper()}`" + ).replace(f"- Base SHA: `{BASE_SHA}`", f"- Base SHA: `{BASE_SHA.upper()}`") + + assert gate.review_rejection_reason(value, HEAD, BASE_REF, BASE_SHA) is None @pytest.mark.parametrize( @@ -298,6 +444,8 @@ def test_has_reusable_real_model_approval_logs_rejected_candidates(): fallback, ], HEAD, + BASE_REF, + BASE_SHA, log=log, ) assert "rejected same-head review" in log.getvalue() @@ -310,11 +458,13 @@ def test_opencode_app_only_mode_rejects_github_actions_approval(): strict_log = io.StringIO() assert not gate.has_reusable_real_model_approval( - [actions_review], HEAD, log=default_log + [actions_review], HEAD, BASE_REF, BASE_SHA, log=default_log ) assert not gate.has_reusable_real_model_approval( [actions_review], HEAD, + BASE_REF, + BASE_SHA, log=strict_log, approval_authors=gate.OPENCODE_APP_APPROVAL_AUTHORS, ) @@ -328,6 +478,8 @@ def test_opencode_app_only_mode_accepts_app_approval(): assert gate.has_reusable_real_model_approval( [review()], HEAD, + BASE_REF, + BASE_SHA, log=log, approval_authors=gate.OPENCODE_APP_APPROVAL_AUTHORS, ) @@ -405,30 +557,48 @@ def test_adversarial_validation_rejects_forged_traversal_receipt(): def test_parse_args_and_main(monkeypatch, capsys): - args = gate.parse_args(["--head", HEAD]) + gate_args = ["--head", HEAD, "--base-ref", BASE_REF, "--base-sha", BASE_SHA] + args = gate.parse_args(gate_args) assert args.head == HEAD + assert args.base_ref == BASE_REF + assert args.base_sha == BASE_SHA assert not args.require_opencode_app - strict_args = gate.parse_args(["--head", HEAD, "--require-opencode-app"]) + strict_args = gate.parse_args([*gate_args, "--require-opencode-app"]) assert strict_args.require_opencode_app monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps([[review()]]))) - assert gate.main(["--head", HEAD]) == 0 + assert gate.main(gate_args) == 0 monkeypatch.setattr(sys, "stdin", io.StringIO("not-json")) - assert gate.main(["--head", HEAD]) == 2 + assert gate.main(gate_args) == 2 assert "could not parse reviews" in capsys.readouterr().err monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) - assert gate.main(["--head", "short"]) == 2 + assert ( + gate.main(["--head", "short", "--base-ref", BASE_REF, "--base-sha", BASE_SHA]) + == 2 + ) assert "40-character" in capsys.readouterr().err monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) - assert gate.main(["--head", HEAD]) == 1 + assert gate.main( + ["--head", HEAD, "--base-ref", "bad ref", "--base-sha", BASE_SHA] + ) == 2 + assert "valid base ref" in capsys.readouterr().err + + monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) + assert gate.main( + ["--head", HEAD, "--base-ref", BASE_REF, "--base-sha", "short"] + ) == 2 + assert "40-character base SHA" in capsys.readouterr().err + + monkeypatch.setattr(sys, "stdin", io.StringIO("[]")) + assert gate.main(gate_args) == 1 monkeypatch.setattr( sys, "stdin", io.StringIO(json.dumps([[review(user={"login": "github-actions[bot]"})]])), ) - assert gate.main(["--head", HEAD, "--require-opencode-app"]) == 1 + assert gate.main([*gate_args, "--require-opencode-app"]) == 1 diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..d7daaa89 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -653,8 +653,9 @@ def test_github_gpt5_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: assert result.returncode == 1 assert ( - "OpenCode github-models/openai/gpt-5 runtime cap selected 3s instead of 9s " - "because this installation has returned a constrained request-body limit for that endpoint." + "OpenCode github-models/openai/gpt-5 provider-specific queue cap selected " + "3s instead of 9s so a constrained or non-responsive endpoint cannot " + "monopolize the organization review queue." ) in result.stdout attempt_budget = re.search( r"OpenCode github-models/openai/gpt-5 attempt 1/1 using (\d+)s run timeout " @@ -667,6 +668,49 @@ def test_github_gpt5_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: assert run_timeout <= remaining_budget <= 30 +def test_github_deepseek_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: + """A silent GitHub DeepSeek endpoint cannot consume the whole review job.""" + result = run_failed_model( + tmp_path, + extra_env={ + "OPENCODE_GITHUB_DEEPSEEK_RUN_TIMEOUT_SECONDS": "2", + "OPENCODE_RUN_TIMEOUT_SECONDS": "9", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert ( + "OpenCode github-models/deepseek/deepseek-v3-0324 provider-specific " + "queue cap selected 2s instead of 9s so a constrained or non-responsive " + "endpoint cannot monopolize the organization review queue." + ) in result.stdout + assert re.search( + r"OpenCode github-models/deepseek/deepseek-v3-0324 attempt 1/1 " + r"using 2s run timeout with \d+s retry budget remaining\.", + result.stdout, + ) + + +def test_timeout_skips_second_attempt_for_same_model(tmp_path: Path) -> None: + """A silent endpoint falls through while response validation remains retryable.""" + result = run_failed_model( + tmp_path, + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "3", + "OPENCODE_MODEL_ATTEMPTS": "2", + "OPENCODE_RUN_TIMEOUT_SECONDS": "1", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "10", + }, + model_candidates="github-models/openai/gpt-4.1", + ) + + assert result.returncode == 1 + assert "attempt 1/2 timed out after 1s" in result.stdout + assert "skipping remaining attempts for this model" in result.stdout + assert "attempt 2/2" not in result.stdout + + def test_github_models_openai_prompt_references_evidence_without_inlining( tmp_path: Path, ) -> None: diff --git a/tests/test_opencode_review_normalize_output.py b/tests/test_opencode_review_normalize_output.py index 590fb3e5..a3217dd7 100644 --- a/tests/test_opencode_review_normalize_output.py +++ b/tests/test_opencode_review_normalize_output.py @@ -957,6 +957,25 @@ def test_adversarial_evidence_rejects_exact_changed_path_without_independent_pro ) +def test_structured_probe_accepts_observed_evidence_without_repeated_path(): + """Path, line, and digest remain bound even when prose omits the location.""" + value = adversarial_validation() + for index, probe in enumerate(value["probes"], start=7): + probe["evidence"] = ( + "Focused regression command passed with exit code 0. " + + source_line_receipt(f"line {index}") + ) + + assert ( + norm.adversarial_validation_error( + value, + result="APPROVE", + findings=[], + ) + == "" + ) + + @pytest.mark.parametrize( "unsafe_path", [ diff --git a/tests/test_opencode_security_boundaries.py b/tests/test_opencode_security_boundaries.py index a194f0e6..0f4222ab 100644 --- a/tests/test_opencode_security_boundaries.py +++ b/tests/test_opencode_security_boundaries.py @@ -161,6 +161,11 @@ def test_safe_pytest_argv_classifier_rejects_empty_argv() -> None: "pytest `id`", "pytest $(id)", "pytest 'unterminated", + "./pytest -q", + "../pytest -q", + "/tmp/pytest -q", + ".\\pytest -q", + "C:\\pytest -q", "", ], ) @@ -172,21 +177,298 @@ def test_safe_pytest_parser_rejects_shell_and_non_pytest_execution(command: str) def test_safe_pytest_executor_never_uses_a_shell(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """The realistic configured-command boundary executes validated argv with shell disabled.""" observed: dict[str, object] = {} - virtualenv_bin = tmp_path / ".venv" / "bin" - virtualenv_bin.mkdir(parents=True) + project_dir = tmp_path / "project" + project_dir.mkdir() + local_virtualenv_bin = project_dir / ".venv" / "bin" + local_virtualenv_bin.mkdir(parents=True) + (local_virtualenv_bin / "pytest").write_text("untrusted", encoding="utf-8") + trusted_executable = tmp_path / "runner-tools" / "pytest" + trusted_executable.parent.mkdir() + trusted_executable.write_text("#!/bin/sh\n", encoding="utf-8") + trusted_executable.chmod(0o755) def fake_run(argv, *, cwd, env, shell, check): observed.update(argv=argv, cwd=cwd, env=env, shell=shell, check=check) return subprocess.CompletedProcess(argv, 0) + monkeypatch.setenv("PATH", os.defpath) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_DIRS", + (trusted_executable.parent,), + ) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_OWNER_UID", + os.getuid(), + ) monkeypatch.setattr(safe_pytest.subprocess, "run", fake_run) - assert safe_pytest.execute_command(tmp_path, ["pytest", "-q", "tests"]) == 0 - assert observed["argv"] == ["pytest", "-q", "tests"] - assert observed["cwd"] == tmp_path + monkeypatch.setattr( + safe_pytest.shutil, "which", lambda executable, path: str(trusted_executable) + ) + assert safe_pytest.execute_command(project_dir, ["pytest", "-q", "tests"]) == 0 + assert observed["argv"] == [str(trusted_executable), "-q", "tests"] + assert observed["cwd"] == project_dir assert observed["shell"] is False assert observed["check"] is False assert observed["env"]["PYTHONPATH"] == "." - assert observed["env"]["PATH"].split(os.pathsep)[0] == str(virtualenv_bin) + assert observed["env"]["PATH"] == str(trusted_executable.parent) + + +@pytest.mark.parametrize("unsafe_path", [None, "", ".", f"/usr/bin{os.pathsep}", f"relative{os.pathsep}/usr/bin"]) +def test_safe_pytest_executor_rejects_missing_or_ambiguous_path( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + unsafe_path: str | None, +) -> None: + """An absent, empty, or relative PATH cannot select a project-local executable.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + monkeypatch.delenv("OPENCODE_PYTHON_ENV_BIN", raising=False) + if unsafe_path is None: + monkeypatch.delenv("PATH", raising=False) + else: + monkeypatch.setenv("PATH", unsafe_path) + + with pytest.raises(ValueError, match="trusted PATH is missing or unsafe"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + +def test_safe_pytest_executor_rejects_project_symlink_and_relative_resolution( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """PATH lookup cannot select a local venv shim, symlink, or relative executable.""" + project_dir = tmp_path / "project" + project_bin = project_dir / ".venv" / "bin" + project_bin.mkdir(parents=True) + trusted_target = tmp_path / "runner-tools" / "pytest" + trusted_target.parent.mkdir() + trusted_target.write_text("#!/bin/sh\n", encoding="utf-8") + trusted_target.chmod(0o755) + local_symlink = project_bin / "pytest" + local_symlink.symlink_to(trusted_target) + + monkeypatch.setenv("PATH", os.defpath) + monkeypatch.setattr( + safe_pytest.shutil, "which", lambda executable, path: str(local_symlink) + ) + with pytest.raises(ValueError, match="trust validation"): + safe_pytest.execute_command(project_dir, ["pytest", "-q"]) + + monkeypatch.setattr(safe_pytest.shutil, "which", lambda executable, path: "pytest") + with pytest.raises(ValueError, match="absolute path"): + safe_pytest.execute_command(project_dir, ["pytest", "-q"]) + + +def test_safe_pytest_executor_uses_only_validated_base_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """An explicitly supplied base environment is owner and permission checked.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + trusted_root = tmp_path / "base-python-envs" + trusted_bin = trusted_root / "lock-000" / "bin" + trusted_bin.mkdir(parents=True) + trusted_target = trusted_root / "runtime" / "python" + trusted_target.parent.mkdir() + trusted_target.write_text("#!/bin/sh\n", encoding="utf-8") + trusted_target.chmod(0o755) + trusted_executable = trusted_bin / "python" + trusted_executable.symlink_to(trusted_target) + observed: dict[str, object] = {} + + def fake_run(argv, *, cwd, env, shell, check): + observed.update(argv=argv, cwd=cwd, env=env, shell=shell, check=check) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(safe_pytest, "TRUSTED_PYTHON_ENV_ROOT", trusted_root) + monkeypatch.setattr( + safe_pytest, "TRUSTED_PYTHON_ENV_OWNER_UID", os.getuid() + ) + monkeypatch.setenv("OPENCODE_PYTHON_ENV_BIN", str(trusted_bin)) + monkeypatch.setattr(safe_pytest.subprocess, "run", fake_run) + + assert safe_pytest.execute_command(project_dir, ["python", "-m", "pytest"]) == 0 + assert observed["argv"] == [str(trusted_target), "-m", "pytest"] + assert observed["env"]["PATH"] == str(trusted_bin) + + +def test_safe_pytest_executor_ignores_user_path_entries( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Inherited PATH cannot select executables outside the fixed runtime allowlist.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + attacker_bin = tmp_path / "attacker-bin" + attacker_bin.mkdir() + (attacker_bin / "pytest").write_text("#!/bin/sh\n", encoding="utf-8") + trusted_bin = tmp_path / "trusted-runtime" + trusted_bin.mkdir() + trusted_target = trusted_bin / "pytest-real" + trusted_target.write_text("#!/bin/sh\n", encoding="utf-8") + trusted_target.chmod(0o755) + trusted_link = trusted_bin / "pytest" + trusted_link.symlink_to(trusted_target) + observed: dict[str, object] = {} + + def fake_which(executable: str, path: str) -> str: + observed.update(executable=executable, path=path) + return str(trusted_link) + + def fake_run(argv, *, cwd, env, shell, check): + observed.update(argv=argv, cwd=cwd, env=env, shell=shell, check=check) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.delenv("OPENCODE_PYTHON_ENV_BIN", raising=False) + monkeypatch.setenv("PATH", f"{attacker_bin}{os.pathsep}{trusted_bin}") + monkeypatch.setattr( + safe_pytest, "TRUSTED_INHERITED_EXECUTABLE_DIRS", (trusted_bin,) + ) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_OWNER_UID", + os.getuid(), + ) + monkeypatch.setattr(safe_pytest.shutil, "which", fake_which) + monkeypatch.setattr(safe_pytest.subprocess, "run", fake_run) + + assert safe_pytest.execute_command(project_dir, ["pytest", "-q"]) == 0 + assert observed["path"] == str(trusted_bin) + assert observed["argv"] == [str(trusted_target), "-q"] + assert observed["env"]["PATH"] == str(trusted_bin) + + +def test_safe_pytest_rejects_inherited_directory_from_untrusted_owner( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """An owner-writable user directory cannot enter the privileged PATH allowlist.""" + user_bin = tmp_path / "user-bin" + user_bin.mkdir() + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_DIRS", + (user_bin,), + ) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_OWNER_UID", + os.getuid() + 1, + ) + + assert safe_pytest._trusted_inherited_search_dirs() == [] + + +def test_safe_pytest_rejects_unavailable_inherited_search_dirs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Missing fixed directories cannot produce an empty privileged PATH allowlist.""" + monkeypatch.delenv("OPENCODE_PYTHON_ENV_BIN", raising=False) + monkeypatch.setenv("PATH", os.defpath) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_DIRS", + (tmp_path / "missing-runtime",), + ) + + assert safe_pytest._trusted_inherited_search_dirs() == [] + with pytest.raises(ValueError, match="allowlist is unavailable"): + safe_pytest.execute_command(tmp_path, ["pytest"]) + + +def test_safe_pytest_rejects_nested_inherited_executable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Inherited executables must live directly in an allowlisted directory.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + trusted_root = tmp_path / "trusted-runtime" + nested_dir = trusted_root / "nested" + nested_dir.mkdir(parents=True) + nested_executable = nested_dir / "pytest" + nested_executable.write_text("#!/bin/sh\n", encoding="utf-8") + nested_executable.chmod(0o755) + monkeypatch.delenv("OPENCODE_PYTHON_ENV_BIN", raising=False) + monkeypatch.setenv("PATH", os.defpath) + monkeypatch.setattr( + safe_pytest, "TRUSTED_INHERITED_EXECUTABLE_DIRS", (trusted_root,) + ) + monkeypatch.setattr( + safe_pytest, + "TRUSTED_INHERITED_EXECUTABLE_OWNER_UID", + os.getuid(), + ) + monkeypatch.setattr( + safe_pytest.shutil, + "which", + lambda executable, path: str(nested_executable), + ) + + with pytest.raises(ValueError, match="outside the trusted allowlist"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + +def test_safe_pytest_executor_reports_unavailable_and_invalid_executables( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Every failed trusted-path lookup leaves a specific fail-closed result.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + + monkeypatch.setenv("PATH", os.defpath) + monkeypatch.setattr(safe_pytest.shutil, "which", lambda executable, path: None) + with pytest.raises(ValueError, match="unavailable from the trusted PATH"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + missing = tmp_path / "runner-tools" / "pytest" + monkeypatch.setattr( + safe_pytest.shutil, "which", lambda executable, path: str(missing) + ) + with pytest.raises(ValueError, match="executable path is unavailable"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + +def test_safe_pytest_executor_rejects_invalid_trusted_environment_layout( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The explicit base environment must be a direct, owner-controlled bin path.""" + project_dir = tmp_path / "project" + project_dir.mkdir() + trusted_root = tmp_path / "base-python-envs" + trusted_root.mkdir() + monkeypatch.setattr(safe_pytest, "TRUSTED_PYTHON_ENV_ROOT", trusted_root) + monkeypatch.setattr( + safe_pytest, "TRUSTED_PYTHON_ENV_OWNER_UID", os.getuid() + ) + monkeypatch.setenv("OPENCODE_PYTHON_ENV_BIN", str(trusted_root)) + with pytest.raises(ValueError, match="environment path failed validation"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + trusted_bin = trusted_root / "lock-000" / "bin" + nested_bin = trusted_bin / "nested" + nested_bin.mkdir(parents=True) + nested_executable = nested_bin / "pytest" + nested_executable.write_text("#!/bin/sh\n", encoding="utf-8") + nested_executable.chmod(0o755) + monkeypatch.setenv("OPENCODE_PYTHON_ENV_BIN", str(trusted_bin)) + monkeypatch.setattr( + safe_pytest.shutil, + "which", + lambda executable, path: str(nested_executable), + ) + with pytest.raises(ValueError, match="environment executable failed validation"): + safe_pytest.execute_command(project_dir, ["pytest"]) + + +def test_safe_pytest_executor_rejects_untrusted_supplied_environment( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A caller cannot redirect configured pytest through a PR-controlled bin path.""" + untrusted_bin = tmp_path / "attacker-env" / "bin" + untrusted_bin.mkdir(parents=True) + monkeypatch.setenv("OPENCODE_PYTHON_ENV_BIN", str(untrusted_bin)) + + with pytest.raises(ValueError, match="trusted Python environment path"): + safe_pytest.execute_command(tmp_path, ["python", "-m", "pytest"]) def test_configured_pytest_discovery_drops_injected_workflow_command(tmp_path: Path) -> None: @@ -247,13 +529,20 @@ def test_safe_pytest_cli_paths_and_invalid_execution( assert exc.value.code == 0 +BASE_REF = "main" +BASE_SHA = "b" * 40 + + def approval_review(head_sha: str, **overrides: object) -> dict[str, object]: """Build one exact-current-head OpenCode approval review.""" review: dict[str, object] = { "state": "APPROVED", "commit_id": head_sha, "user": {"login": "opencode-agent[bot]"}, - "body": f"- Result: APPROVE\n- Head SHA: `{head_sha}`", + "body": ( + f"- Result: APPROVE\n- Base ref: `{BASE_REF}`\n" + f"- Base SHA: `{BASE_SHA}`\n- Head SHA: `{head_sha}`" + ), } review.update(overrides) return review @@ -266,7 +555,12 @@ def test_dispatch_status_requires_live_current_head_approval_and_coverage() -> N model_outcome="success", coverage_result="success", expected_head=head, - pull_request={"head": {"sha": head}}, + expected_base_ref=BASE_REF, + expected_base_sha=BASE_SHA, + pull_request={ + "head": {"sha": head}, + "base": {"ref": BASE_REF, "sha": BASE_SHA}, + }, reviews=[approval_review(head)], ) @@ -286,13 +580,74 @@ def test_dispatch_status_latest_current_head_decision_is_authoritative() -> None model_outcome="success", coverage_result="success", expected_head=head, - pull_request={"head": {"sha": head}}, + expected_base_ref=BASE_REF, + expected_base_sha=BASE_SHA, + pull_request={ + "head": {"sha": head}, + "base": {"ref": BASE_REF, "sha": BASE_SHA}, + }, reviews=reviews, ) assert decision["state"] == "failure" +def test_dispatch_status_ignores_incidental_identity_text() -> None: + """Quoted findings cannot override the anchored identity bullet lines.""" + head = "a" * 40 + review = approval_review(head) + review["body"] = ( + f"{review['body']}\n\n" + "Quoted finding text:\n" + "Base ref: `release`\n" + f"Base SHA: `{'c' * 40}`\n" + f"Head SHA: `{'d' * 40}`" + ) + + assert dispatch_status._has_current_approval( + [review], head, BASE_REF, BASE_SHA + ) + + +@pytest.mark.parametrize( + ("live_base_ref", "live_base_sha", "review_body"), + [ + ("release", BASE_SHA, None), + (BASE_REF, "c" * 40, None), + ( + BASE_REF, + BASE_SHA, + f"- Result: APPROVE\n- Base ref: `release`\n" + f"- Base SHA: `{BASE_SHA}`\n- Head SHA: `{'a' * 40}`", + ), + ], +) +def test_dispatch_status_rejects_unchanged_head_base_retarget( + live_base_ref: str, + live_base_sha: str, + review_body: str | None, +) -> None: + """A same-head approval cannot be replayed after the base identity changes.""" + head = "a" * 40 + review = approval_review(head) + if review_body is not None: + review["body"] = review_body + decision = dispatch_status.decide_status( + model_outcome="success", + coverage_result="success", + expected_head=head, + expected_base_ref=BASE_REF, + expected_base_sha=BASE_SHA, + pull_request={ + "head": {"sha": head}, + "base": {"ref": live_base_ref, "sha": live_base_sha}, + }, + reviews=[review], + ) + + assert decision["state"] == "failure" + + @pytest.mark.parametrize( ("model_outcome", "coverage_result", "live_head", "review_overrides"), [ @@ -318,7 +673,12 @@ def test_dispatch_status_fails_closed_without_validated_approval( model_outcome=model_outcome, coverage_result=coverage_result, expected_head=head, - pull_request={"head": {"sha": observed_head}}, + expected_base_ref=BASE_REF, + expected_base_sha=BASE_SHA, + pull_request={ + "head": {"sha": observed_head}, + "base": {"ref": BASE_REF, "sha": BASE_SHA}, + }, reviews=[approval_review(head, **review_overrides)], ) @@ -335,7 +695,15 @@ def test_dispatch_status_cli_and_evidence_shape_validation( head = "a" * 40 pr_file = tmp_path / "pr.json" reviews_file = tmp_path / "reviews.json" - pr_file.write_text(json.dumps({"head": {"sha": head}}), encoding="utf-8") + pr_file.write_text( + json.dumps( + { + "head": {"sha": head}, + "base": {"ref": BASE_REF, "sha": BASE_SHA}, + } + ), + encoding="utf-8", + ) reviews_file.write_text(json.dumps([approval_review(head)]), encoding="utf-8") args = [ "--model-outcome", @@ -344,6 +712,10 @@ def test_dispatch_status_cli_and_evidence_shape_validation( "success", "--expected-head", head, + "--expected-base-ref", + BASE_REF, + "--expected-base-sha", + BASE_SHA, "--pull-request-file", str(pr_file), "--reviews-file", diff --git a/tests/test_pr_review_fix_scheduler.py b/tests/test_pr_review_fix_scheduler.py index a0ea3fe6..cbda0275 100644 --- a/tests/test_pr_review_fix_scheduler.py +++ b/tests/test_pr_review_fix_scheduler.py @@ -28,13 +28,58 @@ def make_pr(**overrides): def test_recent_fix_marker_is_head_scoped(): - """Fix markers are scoped to the exact PR head.""" + """Only exact-head markers from the token actor and past time are trusted.""" head = "a" * 40 - comments = [{"body": f"{fix.FIX_MARKER} head_sha={head} epoch={int(time.time())} -->"}] + actor = "github-actions[bot]" + now = int(time.time()) + comments = [ + { + "body": f"{fix.FIX_MARKER} head_sha={head} epoch={now} -->", + "user": {"login": actor}, + } + ] + + assert fix.recent_fix_marker_exists(comments, head, 24 * 3600, actor) + assert not fix.recent_fix_marker_exists(comments, "b" * 40, 24 * 3600, actor) + assert not fix.recent_fix_marker_exists( + [{"body": f"{fix.FIX_MARKER} head_sha={head} epoch=oops -->"}], + head, + 24 * 3600, + actor, + ) + oversized_epoch_marker = { + "body": f"{fix.FIX_MARKER} head_sha={head} epoch={'9' * 5000} -->", + "user": {"login": actor}, + } + assert not fix.recent_fix_marker_exists( + [oversized_epoch_marker], head, 24 * 3600, actor + ) + assert fix.recent_fix_marker_exists( + [comments[0], oversized_epoch_marker], head, 24 * 3600, actor + ) + forged = [{**comments[0], "user": {"login": "contributor"}}] + assert not fix.recent_fix_marker_exists(forged, head, 24 * 3600, actor) + future = [ + { + **comments[0], + "body": f"{fix.FIX_MARKER} head_sha={head} epoch={now + 86_400} -->", + } + ] + assert not fix.recent_fix_marker_exists(future, head, 24 * 3600, actor) + assert not fix.recent_fix_marker_exists(comments, head, 24 * 3600, "") + - assert fix.recent_fix_marker_exists(comments, head, 24 * 3600) - assert not fix.recent_fix_marker_exists(comments, "b" * 40, 24 * 3600) - assert not fix.recent_fix_marker_exists([{"body": f"{fix.FIX_MARKER} head_sha={head} epoch=oops -->"}], head, 24 * 3600) +def test_current_token_actor_fails_closed(monkeypatch): + monkeypatch.setattr(fix, "run_json", lambda _args: {"login": "scheduler[bot]"}) + assert fix.current_token_actor() == "scheduler[bot]" + monkeypatch.setattr(fix, "run_json", lambda _args: []) + assert fix.current_token_actor() == "" + monkeypatch.setattr( + fix, + "run_json", + lambda _args: (_ for _ in ()).throw(json.JSONDecodeError("bad", "x", 0)), + ) + assert fix.current_token_actor() == "" def test_needs_autofix_uses_current_head_evidence(): @@ -135,6 +180,7 @@ def test_process_queue_dispatches_same_repo_current_head(monkeypatch, capsys): pr = make_pr() calls = [] + monkeypatch.setattr(fix, "current_token_actor", lambda: "scheduler[bot]") monkeypatch.setattr(fix, "fetch_open_prs", lambda repo, max_prs: [pr]) monkeypatch.setattr(fix, "needs_autofix", lambda pr: (True, ("current-head OpenCode requested changes",))) monkeypatch.setattr(fix, "issue_comments", lambda repo, number: []) @@ -163,6 +209,52 @@ def test_process_queue_dispatches_same_repo_current_head(monkeypatch, capsys): assert payload["autofix_dispatches"] == 1 +def test_process_queue_resolves_token_actor_once_for_multiple_prs(monkeypatch): + """One queue sweep must not spend one actor lookup per candidate PR.""" + prs = [make_pr(number=7), make_pr(number=8)] + actor_calls = [] + monkeypatch.setattr( + fix, + "current_token_actor", + lambda: actor_calls.append("lookup") or "scheduler[bot]", + ) + monkeypatch.setattr(fix, "fetch_open_prs", lambda repo, max_prs: prs) + monkeypatch.setattr(fix, "needs_autofix", lambda pr: (True, ("reason",))) + monkeypatch.setattr(fix, "issue_comments", lambda repo, number: []) + monkeypatch.setattr( + fix, + "inspect_pr", + lambda repo, pr, args, **kwargs: ("skip", (kwargs["trusted_actor"],)), + ) + args = fix.parse_args( + ["--repo", "owner/repo", "--base-branch", "main", "--dry-run"] + ) + + assert fix.process_queue(args) == 0 + assert actor_calls == ["lookup"] + + +def test_process_queue_blocks_candidates_when_token_actor_is_unknown(monkeypatch): + """Unknown mutation identity must not bypass author-bound marker dedupe.""" + pr = make_pr() + comment_calls = [] + monkeypatch.setattr(fix, "fetch_open_prs", lambda repo, max_prs: [pr]) + monkeypatch.setattr(fix, "needs_autofix", lambda pr: (True, ("reason",))) + monkeypatch.setattr(fix, "current_token_actor", lambda: "") + monkeypatch.setattr( + fix, + "issue_comments", + lambda repo, number: comment_calls.append((repo, number)) or [], + ) + args = fix.parse_args( + ["--repo", "owner/repo", "--base-branch", "main", "--dry-run"] + ) + + with pytest.raises(RuntimeError, match="mutation credential actor"): + fix.process_queue(args) + assert comment_calls == [] + + def test_autofix_context_filters_outdated_threads_and_renders_checks(): """The context helper filters stale threads and renders compact checks.""" assert context.repo_parts("owner/repo") == ("owner", "repo") @@ -402,6 +494,8 @@ def fake_run(argv, *, stdin=None): def _approved_dirty_pr(**overrides): """Return an approved PR that GitHub reports as conflicting.""" + head = "a" * 40 + base = "b" * 40 fields = { "mergeStateStatus": "DIRTY", "reviews": { @@ -409,8 +503,13 @@ def _approved_dirty_pr(**overrides): { "state": "APPROVED", "author": {"login": "opencode-agent"}, - "commit": {"oid": "a" * 40}, - "body": "Approved.", + "commit": {"oid": head}, + "body": ( + "Approved.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{base}`\n" + f"- Head SHA: `{head}`" + ), } ] }, @@ -431,6 +530,14 @@ def test_needs_conflict_resolution_requires_approved_and_conflicting(): assert fix.needs_conflict_resolution( _approved_dirty_pr(mergeStateStatus="CLEAN") ) == (False, ()) + # A base retarget or base advance invalidates the prior approval even when + # the head commit itself did not change. + assert fix.needs_conflict_resolution( + _approved_dirty_pr(baseRefName="release") + ) == (False, ()) + assert fix.needs_conflict_resolution( + _approved_dirty_pr(baseRefOid="c" * 40) + ) == (False, ()) def test_dispatch_autofix_passes_resolve_conflict_flag(capsys): @@ -479,6 +586,7 @@ def test_dispatch_autofix_rejects_selectable_workflow_and_invalid_repository(): def test_inspect_pr_dispatches_conflict_resolution(monkeypatch): """An approved conflicting PR dispatches autofix in resolve_conflict mode.""" captured = {} + monkeypatch.setattr(fix, "current_token_actor", lambda: "scheduler[bot]") monkeypatch.setattr(fix, "issue_comments", lambda repo, number: []) monkeypatch.setattr( fix, @@ -498,6 +606,7 @@ def test_inspect_pr_dispatches_conflict_resolution(monkeypatch): def test_process_queue_includes_conflict_resolution_candidates(monkeypatch, capsys): """The queue pre-filter fetches comments for approved conflicting PRs too.""" pr = _approved_dirty_pr() + monkeypatch.setattr(fix, "current_token_actor", lambda: "scheduler[bot]") monkeypatch.setattr(fix, "fetch_open_prs", lambda repo, max_prs: [pr]) monkeypatch.setattr(fix, "issue_comments", lambda repo, number: []) monkeypatch.setattr( @@ -527,7 +636,17 @@ def test_fix_inspect_skip_wait_and_error_paths(monkeypatch): ) monkeypatch.setattr(fix, "needs_autofix", lambda pr: (True, ("reason",))) - monkeypatch.setattr(fix, "issue_comments", lambda repo, number: [{"body": f"{fix.FIX_MARKER} head_sha={'a' * 40} epoch={int(time.time())} -->"}]) + monkeypatch.setattr(fix, "current_token_actor", lambda: "scheduler[bot]") + monkeypatch.setattr( + fix, + "issue_comments", + lambda repo, number: [ + { + "body": f"{fix.FIX_MARKER} head_sha={'a' * 40} epoch={int(time.time())} -->", + "user": {"login": "scheduler[bot]"}, + } + ], + ) assert fix.inspect_pr("owner/repo", make_pr(), args) == ("wait", ("recent autofix marker exists for this head",)) pr1 = make_pr(number=1) diff --git a/tests/test_pr_review_fix_scheduler_coverage.py b/tests/test_pr_review_fix_scheduler_coverage.py index 11c5d48f..7ceb8203 100644 --- a/tests/test_pr_review_fix_scheduler_coverage.py +++ b/tests/test_pr_review_fix_scheduler_coverage.py @@ -44,6 +44,7 @@ def make_pr(number=1, **kwargs): monkeypatch.setattr(fix, "fetch_open_prs", lambda repo, max_prs: [pr1, pr2]) monkeypatch.setattr(fix, "needs_autofix", lambda pr: (True, ("reason",))) + monkeypatch.setattr(fix, "current_token_actor", lambda: "github-actions[bot]") def raise_error(repo, number): raise RuntimeError("boom") diff --git a/tests/test_pr_review_merge_scheduler.py b/tests/test_pr_review_merge_scheduler.py index 1fe10da7..642c6d18 100644 --- a/tests/test_pr_review_merge_scheduler.py +++ b/tests/test_pr_review_merge_scheduler.py @@ -30,6 +30,9 @@ def fake_fine_grained_github_token(body): return "github" + TOKEN_SEPARATOR + "pat" + TOKEN_SEPARATOR + body +TEST_BASE_SHA = "c" * 40 + + def make_pr(**overrides): value = { "number": 1, @@ -40,7 +43,7 @@ def make_pr(**overrides): "restMergeableState": "", "reviewDecision": "REVIEW_REQUIRED", "baseRefName": "main", - "baseRefOid": "base", + "baseRefOid": TEST_BASE_SHA, "headRefName": "feature", "headRefOid": "head", "isCrossRepository": False, @@ -78,6 +81,12 @@ def opencode_review( "author": {"login": login}, "submittedAt": submitted_at, "commit": {"oid": commit}, + "body": ( + "OpenCode review evidence.\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{commit}`" + ), } @@ -1013,6 +1022,19 @@ def test_review_state_and_failed_checks(): assert not sched.has_current_head_approval( make_pr(headRefOid="", reviews={"nodes": [opencode_review("APPROVED", "head")]}) ) + current_approval = opencode_review("APPROVED", "head") + assert not sched.has_current_head_approval( + make_pr( + baseRefName="release", + reviews={"nodes": [current_approval]}, + ) + ) + assert not sched.has_current_head_approval( + make_pr( + baseRefOid="advanced-base", + reviews={"nodes": [current_approval]}, + ) + ) exact_head = "a" * 40 stale_body_head = "b" * 40 body_sha_mismatch = make_pr( @@ -1034,19 +1056,76 @@ def test_review_state_and_failed_checks(): "nodes": [ { **opencode_review("APPROVED", exact_head), - "body": f"## Gate evidence\n\n- Head SHA: `{exact_head.upper()}`", + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{exact_head.upper()}`" + ), } ] }, ) assert sched.has_current_head_approval(body_sha_match) + body_with_incidental_identity_text = make_pr( + headRefOid=exact_head, + reviews={ + "nodes": [ + { + **opencode_review("APPROVED", exact_head), + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{exact_head}`\n\n" + f"Quoted finding prose Head SHA: `{stale_body_head}`\n" + "Untrusted prose Head SHA: `not-a-sha`" + ), + } + ] + }, + ) + assert ( + sched.review_body_head_sha( + body_with_incidental_identity_text["reviews"]["nodes"][0] + ) + == exact_head + ) + assert sched.has_current_head_approval(body_with_incidental_identity_text) + commit_bound_review_ignores_invalid_body_token = make_pr( + reviews={ + "nodes": [ + { + **opencode_review("APPROVED", "head"), + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + "- Head SHA: `not-a-sha`" + ), + } + ] + } + ) + assert ( + sched.review_body_head_sha( + commit_bound_review_ignores_invalid_body_token["reviews"]["nodes"][0] + ) + is None + ) + assert sched.has_current_head_approval(commit_bound_review_ignores_invalid_body_token) body_sha_only_match = make_pr( headRefOid=exact_head, reviews={ "nodes": [ { **opencode_review("APPROVED", ""), - "body": f"## Gate evidence\n\n- Head SHA: `{exact_head}`", + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{exact_head}`" + ), } ] }, @@ -1145,7 +1224,7 @@ def test_review_state_and_failed_checks(): ] } ) - assert sched.has_current_head_approval(missing_review_time) + assert not sched.has_current_head_approval(missing_review_time) human_review_only = make_pr( reviews={"nodes": [opencode_review("APPROVED", "head", login="human")]} ) @@ -1196,7 +1275,12 @@ def test_review_state_and_failed_checks(): exact_head_approval = { **opencode_review("APPROVED", exact_head), "databaseId": 302, - "body": f"## Gate evidence\n\n- Head SHA: `{exact_head}`", + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{exact_head}`" + ), } stale_approval_history = make_pr( headRefOid=exact_head, @@ -1268,7 +1352,10 @@ def test_review_state_and_failed_checks(): } } ) - assert sched.failed_status_checks(manual_strix_supersedes_pr_target_failure) == ["lint"] + assert sched.failed_status_checks(manual_strix_supersedes_pr_target_failure) == [ + "strix", + "lint", + ] opencode_pr_target_failure_without_status = make_pr( statusCheckRollup={ "contexts": { @@ -1290,7 +1377,10 @@ def test_review_state_and_failed_checks(): } } ) - assert sched.failed_status_checks(manual_opencode_supersedes_pr_target_failure) == ["lint"] + assert sched.failed_status_checks(manual_opencode_supersedes_pr_target_failure) == [ + "opencode-review", + "lint", + ] def test_workflow_run_followup_defers_deterministic_fallback_retry(monkeypatch): @@ -1345,7 +1435,12 @@ def test_body_head_sha_approval_prevents_same_run_opencode_rerun(monkeypatch): "nodes": [ { **opencode_review("APPROVED", ""), - "body": f"## Gate evidence\n\n- Head SHA: `{head}`", + "body": ( + "## Gate evidence\n\n" + "- Base ref: `main`\n" + f"- Base SHA: `{TEST_BASE_SHA}`\n" + f"- Head SHA: `{head}`" + ), } ] }, @@ -4482,27 +4577,55 @@ def test_action_error_guidance_distinguishes_update_branch_from_merge(): assert "PR head likely changed after inspection" in stale_head_error assert "reads the new head before mutating" in stale_head_error + def test_parse_conflict_reason_success(): """Test parse_conflict_reason with valid complete conflict strings.""" - assert sched.parse_conflict_reason("merge conflict: DIRTY; base=main,head=feature-branch") == ("DIRTY", "main", "feature-branch") - assert sched.parse_conflict_reason("Some prior text. merge conflict: BEHIND; base=develop,head=feat/123") == ("BEHIND", "develop", "feat/123") - assert sched.parse_conflict_reason("merge conflict: DIRTY; foo=bar; base=master,head=bugfix; other=stuff") == ("DIRTY", "master", "bugfix") + assert sched.parse_conflict_reason( + "merge conflict: DIRTY; base=main,head=feature-branch" + ) == ("DIRTY", "main", "feature-branch") + assert sched.parse_conflict_reason( + "Some prior text. merge conflict: BEHIND; base=develop,head=feat/123" + ) == ("BEHIND", "develop", "feat/123") + assert sched.parse_conflict_reason( + "merge conflict: DIRTY; foo=bar; base=master,head=bugfix; other=stuff" + ) == ("DIRTY", "master", "bugfix") + def test_parse_conflict_reason_no_prefix(): """Test parse_conflict_reason returns None when prefix is missing.""" assert sched.parse_conflict_reason("no conflict here") is None assert sched.parse_conflict_reason("merge conflict: space issue") is None + def test_parse_conflict_reason_empty_state(): """Test parse_conflict_reason defaults state to UNKNOWN if missing or empty.""" - assert sched.parse_conflict_reason("merge conflict: ; base=main,head=feature") == ("UNKNOWN", "main", "feature") - assert sched.parse_conflict_reason("merge conflict: ") == ("UNKNOWN", "base", "head") + assert sched.parse_conflict_reason("merge conflict: ; base=main,head=feature") == ( + "UNKNOWN", + "main", + "feature", + ) + assert sched.parse_conflict_reason("merge conflict: ") == ( + "UNKNOWN", + "base", + "head", + ) + def test_parse_conflict_reason_missing_branches(): """Test parse_conflict_reason uses defaults when branch info is missing or malformed.""" - assert sched.parse_conflict_reason("merge conflict: DIRTY; some other segment") == ("DIRTY", "base", "head") - assert sched.parse_conflict_reason("merge conflict: DIRTY; base=,head=something") == ("DIRTY", "base", "something") - assert sched.parse_conflict_reason("merge conflict: DIRTY; base=main,head=") == ("DIRTY", "main", "head") + assert sched.parse_conflict_reason("merge conflict: DIRTY; some other segment") == ( + "DIRTY", + "base", + "head", + ) + assert sched.parse_conflict_reason( + "merge conflict: DIRTY; base=,head=something" + ) == ("DIRTY", "base", "something") + assert sched.parse_conflict_reason("merge conflict: DIRTY; base=main,head=") == ( + "DIRTY", + "main", + "head", + ) def test_run_masks_secrets(): diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 8e66277f..8bac05b1 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -207,6 +207,53 @@ def test_strix_install_normalizes_executable_permissions_before_hashing() -> Non ) +def test_strix_install_accepts_only_same_repo_exact_head_dependency_lock() -> None: + """Only the central repo's exact PR-head hash lock may replace the base lock.""" + workflow = workflow_text("strix.yml") + materialize_step = workflow_step( + workflow, "Materialize central Strix dependency lock from PR head" + ) + install_step = workflow_step(workflow, "Install Strix") + + assert "github.event_name == 'pull_request_target'" in materialize_step + assert materialize_step.count( + "github.event.pull_request.base.repo.full_name == 'ContextualWisdomLab/.github'" + ) == 1 + assert materialize_step.count( + "github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github'" + ) == 1 + assert "github.event.pull_request.user.login == 'dependabot[bot]'" in materialize_step + assert ( + "startsWith(github.event.pull_request.head.ref, 'dependabot/pip/')" + in materialize_step + ) + assert "PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}" in materialize_step + assert '[[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]' in materialize_step + assert '[[ "$PR_BASE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]' in materialize_step + assert ( + 'git -C "$TRUSTED_WORKSPACE" diff --name-only -z ' + '"$PR_BASE_SHA...$PR_HEAD_SHA"' in materialize_step + ) + assert '"${#changed_files[@]}" -ne 1' in materialize_step + assert '"${changed_files[0]}" != "requirements-strix-ci-hashes.txt"' in materialize_step + assert '"$lock_mode" != "100644"' in materialize_step + assert '"$lock_type" != "blob"' in materialize_step + assert ( + 'git -C "$TRUSTED_WORKSPACE" show ' + '"$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" > ' + '"$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt"' + in materialize_step + ) + assert ( + 'trusted_lock="$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt"' + in install_step + ) + assert '[ ! -f "$trusted_lock" ] || [ -L "$trusted_lock" ]' in install_step + assert 'resolved_trusted_lock="$(realpath "$trusted_lock")"' in install_step + assert '"$TRUSTED_STRIX_SOURCE"/*' in install_step + assert '--require-hashes -r "$resolved_trusted_lock"' in install_step + + def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() -> None: workflows = ( "close-empty-pr.yml", @@ -334,6 +381,37 @@ def test_noema_review_credentials_and_llm_configuration_fail_closed() -> None: assert "Noema app token is unavailable; review skipped." not in workflow +def test_noema_dispatch_is_authorized_and_bound_to_live_pr_before_token_mint() -> None: + """Untrusted dispatch payloads must never select a token repository or PR.""" + workflow = workflow_text("noema-review.yml") + validation = workflow.index( + "- name: Bind Noema inputs to the live organization pull request" + ) + credential = workflow.index("- name: Select fail-closed Noema reviewer credential") + mint = workflow.index("- name: Mint repository-scoped Noema GitHub App token") + evidence = workflow.index("- name: Materialize exact OpenCode approval evidence") + review = workflow.index("- name: Run Noema LLM review and submit verdict") + + assert validation < credential < mint < evidence < review + assert "NOEMA_REPOSITORY_DISPATCH_ACTOR" in workflow + assert "NOEMA_REPOSITORY_DISPATCH_TARGETS" in workflow + assert '"$DISPATCH_ACTOR" != "$ALLOWED_DISPATCH_ACTOR"' in workflow + assert '"$DISPATCH_SENDER" != "$ALLOWED_DISPATCH_ACTOR"' in workflow + assert 'gh api "repos/${TARGET_REPOSITORY}/pulls/${PR_NUMBER}"' in workflow + assert '"$base_repository" != "$TARGET_REPOSITORY"' in workflow + assert '"$head_repository" != "$TARGET_REPOSITORY"' in workflow + assert "payload does not match the live base/head identity" in workflow + assert "repositories: ${{ steps.live_pr.outputs.repository_name }}" in workflow + assert workflow.count( + "TARGET_REPOSITORY: ${{ steps.live_pr.outputs.target_repository }}" + ) >= 3 + assert "PR_NUMBER: ${{ steps.live_pr.outputs.pr_number }}" in workflow + assert "git check-ref-format" in workflow + assert "materialize_pr_review_source.py" in workflow + assert "OPENCODE_REQUIRE_ADVERSARIAL_VALIDATION: \"true\"" in workflow + assert "OPENCODE_ARTIFACT_MANIFEST_SHA256" in workflow + + def test_noema_workflow_run_without_pull_request_skips_before_token_exchange() -> None: workflow = workflow_text("noema-review.yml") @@ -381,7 +459,7 @@ def test_noema_review_mints_a_least_privilege_github_app_token() -> None: assert "client-id: ${{ vars.NOEMA_GITHUB_APP_CLIENT_ID }}" in workflow assert "private-key: ${{ secrets.NOEMA_GITHUB_APP_PRIVATE_KEY }}" in workflow assert "owner: ContextualWisdomLab" in workflow - assert "repositories: ${{ steps.noema_credential.outputs.repository }}" in workflow + assert "repositories: ${{ steps.live_pr.outputs.repository_name }}" in workflow for permission in ( "permission-actions: read", "permission-checks: read", @@ -428,6 +506,21 @@ def test_unassociated_review_workflow_runs_do_not_scan_the_whole_pr_queue() -> N assert "github.event.workflow_run.pull_requests[0].number" in workflow +def test_scheduler_repository_dispatch_is_actor_gated_and_default_branch_bound() -> None: + """Generic repository dispatch cannot select a privileged merge target branch.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + + assert workflow.count("github.actor == 'github-actions[bot]'") >= 2 + assert workflow.count("github.event.sender.login == 'github-actions[bot]'") >= 2 + assert ( + "DEFAULT_BRANCH: ${{ github.event_name == 'workflow_call' && " + "inputs.base_branch || github.event.repository.default_branch }}" + ) in workflow + assert ( + "DEFAULT_BRANCH: ${{ github.event.client_payload.base_branch ||" not in workflow + ) + + def test_org_queue_sweep_covers_target_repositories_on_a_heartbeat() -> None: """Guard the org-wide approved-PR fallback sweep contract. @@ -559,6 +652,24 @@ def test_org_queue_sweep_manual_cadence_inputs_reach_the_sweep_job() -> None: assert 'if [ "$ORG_SWEEP_UPDATE_BRANCHES" = "true" ]; then' in workflow +def test_org_queue_sweep_can_be_scoped_to_one_exact_repository() -> None: + """A targeted repair must not wake unrelated organization PR queues.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + + assert ( + "ORG_SWEEP_TARGET_REPOSITORY: " + "${{ github.event.client_payload.target_repository || '' }}" + ) in workflow + assert ( + '"$ORG_SWEEP_TARGET_REPOSITORY" =~ ' + "^ContextualWisdomLab/[A-Za-z0-9_.-]+$" + ) in workflow + assert 'jq -r --arg target "$ORG_SWEEP_TARGET_REPOSITORY"' in workflow + assert "select(.full_name == $target)" in workflow + assert 'if [ "${#sweep_targets[@]}" -ne 1 ]; then' in workflow + assert "Scoped organization sweep target was not found" in workflow + + def test_org_queue_sweep_active_run_aggregation_tolerates_error_payloads() -> None: """An inaccessible Actions page must not add a secondary jq null error.""" jq = shutil.which("jq") @@ -885,21 +996,23 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence -def test_strix_provider_outage_without_findings_is_neutralized() -> None: +def test_strix_provider_outage_without_report_fails_closed() -> None: workflow = workflow_text("strix.yml") - assert "RateLimitError|Too many requests" in workflow - assert "exceeded your current quota" in workflow - assert "billing details" in workflow - assert "LLM warm-up failed" in workflow - assert "zero_vulnerabilities_signal" not in workflow - assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow - assert "before producing a vulnerability report" in workflow - assert "genuine findings still fail the check" in workflow - assert ( - '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow - ) + assert "a required security check must fail" in workflow + assert 'strix_rc="${PIPESTATUS[0]}"' in workflow + assert 'exit "$strix_rc"' in workflow + assert "Treating as a neutral skip" not in workflow + assert "backend_unavailable_signal=" not in workflow + + +def test_strix_github_models_openai_id_preserves_publisher() -> None: + gate = (REPO_ROOT / "scripts/ci/strix_quick_gate.sh").read_text(encoding="utf-8") + + assert "github_models/openai/*)" in gate + assert "catalog ID openai/" in gate + assert "printf 'openai/%s\\n' \"${model#github_models/}\"" in gate def test_strix_cross_repo_dispatch_uses_target_token_for_pr_scoping() -> None: