From 6feeb25aaf159992a8a2347bb28270d793d0903a Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Wed, 19 Aug 2026 21:36:23 +0000 Subject: [PATCH 1/2] ci: run module scorecard check on community (fork) PRs --- .github/workflows/module-scorecard-check.yaml | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/module-scorecard-check.yaml b/.github/workflows/module-scorecard-check.yaml index 7b0dc23e1..09805703c 100644 --- a/.github/workflows/module-scorecard-check.yaml +++ b/.github/workflows/module-scorecard-check.yaml @@ -4,8 +4,19 @@ # regressions, celebrates improvements, and confirms unchanged scores. # # Never creates or updates discussions; the comparison is read-only against -# GitHub and only writes the PR comment. Skipped for fork PRs because it -# requires secrets. +# GitHub and only writes the PR comment. +# +# Runs on community (fork) PRs too. Because scoring needs secrets, the +# trigger is pull_request_target, which runs in the base-repo context with +# secrets available. To keep that safe against "pwn request" attacks: +# - The checkout below is the trusted base ref (pull_request_target's +# default), so the workflow and the .github/scorecard scripts always +# run from main, never from the PR. +# - PR content is fetched separately and materialized ONLY under +# registry/coder/modules/, where it is read as inert text for the +# scoring prompt. Nothing from the PR head is ever executed. +# Combined with the repository's Actions approval settings for outside +# contributors, this bounds fork-PR risk to the comment body itself. # # Required repository secrets: # SCORECARD_ANTHROPIC_API_KEY Anthropic API key used for scoring @@ -16,7 +27,7 @@ name: Module Scorecard Check on: - pull_request: + pull_request_target: paths: - "registry/coder/modules/**" @@ -32,15 +43,13 @@ concurrency: jobs: check: name: Compare module scores against main - if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 30 steps: - - name: Checkout + - name: Checkout trusted base ref + # Deliberately NOT the PR head: the scorecard scripts must come + # from main so a PR cannot alter the code that runs with secrets. uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - # The merge commit plus both parents, for the HEAD^1 diff below. - fetch-depth: 2 - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -49,18 +58,31 @@ jobs: - name: Determine changed modules id: changed - # The checkout is GitHub's test merge commit (PR head merged into - # current main), so HEAD^1 is the exact main tip this PR is applied + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + # Fetch GitHub's test merge commit (PR head merged into current + # main) so FETCH_HEAD^1 is the exact main tip this PR is applied # to and the diff is precisely what the PR changes. Diffing against # github.event.pull_request.base.sha is wrong: that SHA is the base # tip from when the PR was opened, so on stale PRs it picks up every # module merged to main since the PR branched and scores unrelated # modules (REG-74). run: | - MODULES=$(git diff --name-only HEAD^1 HEAD | { grep -oP '^registry/coder/modules/\K[^/]+' || true; } | sort -u | paste -sd, -) + git fetch --depth=2 origin "refs/pull/${PR_NUMBER}/merge" + MODULES=$(git diff --name-only FETCH_HEAD^1 FETCH_HEAD | { grep -oP '^registry/coder/modules/\K[^/]+' || true; } | sort -u | paste -sd, -) echo "modules=${MODULES}" >> "${GITHUB_OUTPUT}" echo "Changed modules: ${MODULES:-none}" + - name: Materialize PR module content (data only) + if: steps.changed.outputs.modules != '' + # Overlay ONLY the module directories from the PR merge commit onto + # the trusted checkout. The scoring script reads these files as + # plain text for the LLM prompt; it never executes them. Everything + # under .github/ stays at the base ref. + run: | + git rm -rq --ignore-unmatch registry/coder/modules + git checkout FETCH_HEAD -- registry/coder/modules + - name: Score changed modules if: steps.changed.outputs.modules != '' env: From 770ee287a25fc9a7adcbb8e8987567a494025077 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Thu, 20 Aug 2026 13:24:55 +0000 Subject: [PATCH 2/2] ci: gate fork-PR scorecard runs behind a maintainer /scorecard comment --- .github/workflows/module-scorecard-check.yaml | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/.github/workflows/module-scorecard-check.yaml b/.github/workflows/module-scorecard-check.yaml index 09805703c..2e4ce1dd4 100644 --- a/.github/workflows/module-scorecard-check.yaml +++ b/.github/workflows/module-scorecard-check.yaml @@ -6,17 +6,18 @@ # Never creates or updates discussions; the comparison is read-only against # GitHub and only writes the PR comment. # -# Runs on community (fork) PRs too. Because scoring needs secrets, the -# trigger is pull_request_target, which runs in the base-repo context with -# secrets available. To keep that safe against "pwn request" attacks: -# - The checkout below is the trusted base ref (pull_request_target's -# default), so the workflow and the .github/scorecard scripts always -# run from main, never from the PR. -# - PR content is fetched separately and materialized ONLY under -# registry/coder/modules/, where it is read as inert text for the -# scoring prompt. Nothing from the PR head is ever executed. -# Combined with the repository's Actions approval settings for outside -# contributors, this bounds fork-PR risk to the comment body itself. +# Two triggers: +# - pull_request: runs automatically for same-repo PRs. Fork PRs are +# skipped because secrets are unavailable to them. +# - issue_comment: a maintainer comments "/scorecard" on any PR +# (including fork PRs) to run the check on demand. Restricted to +# OWNER/MEMBER/COLLABORATOR, so every fork-PR scoring is an explicit +# maintainer decision. +# +# Both paths execute only trusted code: the checkout is the base +# repository, and PR content is fetched separately and materialized ONLY +# under registry/coder/modules/, where the scoring script reads it as +# inert text for the LLM prompt. Nothing from the PR head is executed. # # Required repository secrets: # SCORECARD_ANTHROPIC_API_KEY Anthropic API key used for scoring @@ -27,9 +28,11 @@ name: Module Scorecard Check on: - pull_request_target: + pull_request: paths: - "registry/coder/modules/**" + issue_comment: + types: [created] permissions: contents: read @@ -37,19 +40,30 @@ permissions: pull-requests: write concurrency: - group: module-scorecard-check-${{ github.event.pull_request.number }} + group: module-scorecard-check-${{ github.event.pull_request.number || github.event.issue.number }} cancel-in-progress: true jobs: check: name: Compare module scores against main + if: >- + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/scorecard') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) runs-on: ubuntu-latest timeout-minutes: 30 steps: - - name: Checkout trusted base ref - # Deliberately NOT the PR head: the scorecard scripts must come - # from main so a PR cannot alter the code that runs with secrets. + - name: Checkout trusted base + # For issue_comment this is the default branch; for pull_request it + # is the test merge commit of a same-repo PR. Either way the + # .github/scorecard scripts executed below come from the base + # repository, never from a fork. uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Bun uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -59,7 +73,7 @@ jobs: - name: Determine changed modules id: changed env: - PR_NUMBER: ${{ github.event.pull_request.number }} + PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} # Fetch GitHub's test merge commit (PR head merged into current # main) so FETCH_HEAD^1 is the exact main tip this PR is applied # to and the diff is precisely what the PR changes. Diffing against @@ -71,6 +85,7 @@ jobs: git fetch --depth=2 origin "refs/pull/${PR_NUMBER}/merge" MODULES=$(git diff --name-only FETCH_HEAD^1 FETCH_HEAD | { grep -oP '^registry/coder/modules/\K[^/]+' || true; } | sort -u | paste -sd, -) echo "modules=${MODULES}" >> "${GITHUB_OUTPUT}" + echo "pr=${PR_NUMBER}" >> "${GITHUB_OUTPUT}" echo "Changed modules: ${MODULES:-none}" - name: Materialize PR module content (data only) @@ -78,7 +93,7 @@ jobs: # Overlay ONLY the module directories from the PR merge commit onto # the trusted checkout. The scoring script reads these files as # plain text for the LLM prompt; it never executes them. Everything - # under .github/ stays at the base ref. + # under .github/ stays at the trusted base. run: | git rm -rq --ignore-unmatch registry/coder/modules git checkout FETCH_HEAD -- registry/coder/modules @@ -95,7 +110,7 @@ jobs: if: steps.changed.outputs.modules != '' env: GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} + PR_NUMBER: ${{ steps.changed.outputs.pr }} run: | # No report means every changed module was skipped (for example, # internal building blocks); leave no comment in that case.