Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude/skills/deep-code-review/INDEX.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ Generated by `scripts/skill_index.py` (do not hand-edit; rerun it after changing
| `references/skill-authoring-and-size.md` | the target repository ships or installs agent skills (/SKILL.md, .claude/skills/, .cursor/skills/,… | 991 |
| `references/small-diff-read-budget.md` | reviewing a small DIFF (the fast path in SKILL.md), before the first read or command. | 131 |
| `references/standards-index.md` | citing a standard URL or verification date from an installed copy of the skill. | 117 |
| `references/supply-chain-check.md` | gating a deploy on dependency advisories, hardening a repo's GitHub Actions, or adding code scannin… | 794 |
| `references/supply-chain-check.md` | gating a deploy on dependency advisories, hardening a repo's GitHub Actions, or adding code scannin… | 855 |
| `references/testing-ai-evals.md` | the target calls an LLM, retrieves for generation (RAG), or runs a tool-using agent. | 2564 |
| `references/testing-and-evals.md` | Read this to judge whether the target is *actually* verified — not whether a coverage number is hig… | 3667 |
| `references/testing-ml.md` | the target trains or serves a classical ML model, makes consequential model decisions about people,… | 2743 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ silent pass: it prints `SKIPPED <tool>: <reason>` and the run exits 3 (INCOMPLET
Pre-deploy: `--online --fail-on high`; block the release on any non-zero exit. osv-scanner exit 128 (no package
sources found) is reported as a failure, not confirmed as "nothing to scan".

osv rows carry a reach tag: an npm package no source file imports prints `(reach: dev-only|transitive)`; only dev-only skips the gate unless `SUPPLY_CHAIN_STRICT=1` (transitive may be reached via a direct dependency); unknown reach is gated.

## Sandbox degradation

Under the macOS agent sandbox, `supply_chain_check.sh` retries a crashed `osv-scanner` CLI (Go TLS x509 -26276) via the
Expand Down
52 changes: 35 additions & 17 deletions .claude/skills/deep-code-review/scripts/supply_chain_check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
# osv-scanner CLI crash (rc>1) retries via the OSV API with curl and prints "osv: api-fallback"; python3+curl needed.
# Exit: 0 every check ran clean; 1 finding at/above threshold or a tool failed; 2 usage;
# 3 INCOMPLETE: a check was skipped (tool or python3 missing, osv/scorecard not requested), nothing failed.
# Reach: an npm osv row no source file imports prints "(reach: dev-only|transitive)"; only dev-only skips the gate
# unless SUPPLY_CHAIN_STRICT=1 (transitive may be reached via a direct dependency); unknown reach is gated.
# --allow-skip turns a skip-only run into exit 0. A finding (1) wins over INCOMPLETE (3). Needs python3 for osv.
set -u
online=${SUPPLY_CHAIN_ONLINE:-0}; fail_on=high; sc_repo=; allow_skip=0; skipped=
Expand Down Expand Up @@ -74,7 +76,7 @@ pkgs = []
for (e, n, v), r in zip(pk, res):
ids = [x["id"] for x in r.get("vulns") or []]
for x in ids: vs.setdefault(x, None)
pkgs.append({"package": {"name": n, "version": v}, "groups": [], "vulnerabilities": ids})
pkgs.append({"package": {"name": n, "version": v, "ecosystem": e}, "groups": [], "vulnerabilities": ids})
for x in vs: vs[x] = api("https://api.osv.dev/v1/vulns/" + x)
for p in pkgs: p["vulnerabilities"] = [vs[x] for x in p["vulnerabilities"]]
print(json.dumps({"results": [{"packages": pkgs}]}))
Expand Down Expand Up @@ -103,22 +105,37 @@ else
elif [ "$osv_rc" -gt 1 ]; then echo "FAIL osv-scanner: exit $osv_rc"; rc=1
else
printf '%s' "$out" | python3 -I -c '
import json, os, sys
import json, os, re, sys
order = ["low", "moderate", "high", "critical"]
# Reachability downgrade: a package npm marks "dev": true (reachable only via devDependencies), or osv puts in a dev/build/test
# dependency group, is "dev-only": noted, and capped at moderate so it cannot fail --fail-on high/critical.
devs, prods = set(), set()
for d, ds, fs in os.walk(sys.argv[2]):
ds[:] = [x for x in ds if x not in (".git", "node_modules", ".venv", "venv", "vendor")]
# Reach tag per row: imported (npm name in a JS/TS import, require or dynamic import outside node_modules/.git, and outside
# a root-level .venv/venv/vendor), dev-only (npm "dev": true on every occurrence, or an osv dev/build/test group), else
# transitive (it may be reached through a direct dependency, so it still gates). Every row not imported prints "(reach: <tag>)";
# only dev-only skips the --fail-on gate, unless SUPPLY_CHAIN_STRICT=1. Unknown (unreadable file, lockfile without a "packages"
# map, non-npm ecosystem, no ecosystem field) is imported = gated. PyPI dist names differ from module names, so PyPI is
# downgraded only by a dev group. ponytail: regex scan misses computed require(x) and config-string use; symlinks not followed.
devs, prods, imp, scan_ok = set(), set(), set(), True
JS = re.compile(r"""(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)["\x27\x60]([^"\x27\x60]+)["\x27\x60]""")
for d, ds, fs in os.walk(sys.argv[2], onerror=lambda e: globals().update(scan_ok=False)):
ds[:] = [x for x in ds if x not in (".git", "node_modules") and not (d == sys.argv[2] and x in (".venv", "venv", "vendor"))]
for f in fs:
if f.endswith((".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte", ".astro", ".mdx")):
try: txt = open(os.path.join(d, f), errors="replace").read()
except OSError: scan_ok = False; continue
for m in JS.findall(txt):
if m[0] not in ".#/" and ":" not in m: imp.add("/".join(m.split("/")[:2 if m[0] == "@" else 1]))
if "package-lock.json" in fs:
try:
for k, v in (json.load(open(os.path.join(d, "package-lock.json"))).get("packages") or {}).items():
for k, v in json.load(open(os.path.join(d, "package-lock.json")))["packages"].items(): # v1 has no "packages": fail closed
if "node_modules/" in k: (devs if v.get("dev") else prods).add(k.rsplit("node_modules/", 1)[1] + "@" + str(v.get("version")))
except (OSError, ValueError, AttributeError): pass
except (OSError, ValueError, AttributeError, KeyError): scan_ok = False
def devonly(p):
k = p["package"]["name"] + "@" + str(p["package"]["version"])
if p["package"].get("ecosystem") == "npm" and k in devs and k not in prods: return True # every occurrence is dev
if p["package"].get("ecosystem") == "npm" and (k in devs or k in prods): return k not in prods # the lock is authoritative: dev only if every occurrence is dev
return bool({str(g).lower() for g in p.get("dependency_groups") or []} & {"dev", "build", "test"})
def reach(p):
n, e = p["package"]["name"], p["package"].get("ecosystem")
if not scan_ok or (n in imp and e == "npm"): return "imported"
return "dev-only" if devonly(p) else "transitive" if e == "npm" else "imported"
def sev(v, g):
s = str((v.get("database_specific") or {}).get("severity", "")).lower()
if s == "medium": s = "moderate"
Expand All @@ -130,22 +147,23 @@ def fixed(v):
return sorted({e["fixed"] for a in v.get("affected", []) for r in a.get("ranges", []) for e in r.get("events", []) if "fixed" in e})
try: data = json.load(sys.stdin)
except ValueError: print("FAIL osv-scanner: unparseable output"); sys.exit(1)
counts, rows = {}, []
strict = os.environ.get("SUPPLY_CHAIN_STRICT") == "1"
counts, gate, rows, down = {}, {}, [], 0
for res in data.get("results") or []:
for p in res.get("packages") or []:
maxg = {i: g.get("max_severity") for g in p.get("groups") or [] for i in g.get("ids", [])}
for v in p.get("vulnerabilities") or []:
s, note = sev(v, maxg.get(v.get("id"))), ""
if devonly(p):
note = " [dev-only" + ("; capped from %s" % s if s in ("high", "critical") else "") + "]"
if s in ("high", "critical"): s = "moderate"
s, r = sev(v, maxg.get(v.get("id"))), reach(p)
counts[s] = counts.get(s, 0) + 1
rows.append((s, p["package"]["name"], p["package"]["version"], v.get("id"), (",".join(fixed(v)) or "none") + note))
if r != "dev-only" or strict: gate[s] = gate.get(s, 0) + 1
else: down += 1
rows.append((s, p["package"]["name"], p["package"]["version"], v.get("id"), (",".join(fixed(v)) or "none") + ("" if r == "imported" else " (reach: %s)" % r)))
print("osv-scanner: " + (", ".join("%s=%d" % (s, counts[s]) for s in reversed(order + ["unknown"]) if s in counts) or "no advisories"))
for s, n, ver, i, f in sorted(rows, key=lambda r: -order.index(r[0]) if r[0] in order else 1):
print(" %-8s %s@%s %s fixed in: %s" % (s, n, ver, i, f))
if down: print(" %d dev-only: excluded from the --fail-on gate (SUPPLY_CHAIN_STRICT=1 gates them)" % down)
t = order.index(sys.argv[1])
sys.exit(1 if any(s == "unknown" or order.index(s) >= t for s in counts) else 0)
sys.exit(1 if any(s == "unknown" or order.index(s) >= t for s in gate) else 0)
' "$fail_on" "$dir" || rc=1
fi
fi
Expand Down
6 changes: 3 additions & 3 deletions SHA256SUMS
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ d98f3e4380da3496061ba63be7beebe75017ab4ee1734eda9a3e798e19007b0e .claude/skills
92d22cc5d0a60ba6bdb1f4a9b597823dec5b8412cec09a5be6cf93b3973f4226 .claude/skills/contribution/scripts/prefile_check.sh
18f20db537ab784460ad33cf218f7a01ac0a172fc38413f14c0e19e1ce6821cc .claude/skills/contribution/scripts/share_learning.py
9b6b9befc4342355101283d8c561e870e11a0197b3dd13306728254c4e6d4fcb .claude/skills/deep-code-review/INDEX-scripts.md
b38c24f26272582bf66d4a8062cec4bef05da0be6135fe1d6cb9db2a69822c76 .claude/skills/deep-code-review/INDEX.md
fa8ffafc8edd32a10b6b426c6ba6f6a06a592e9c9926391b4a43123bc11af95d .claude/skills/deep-code-review/INDEX.md
7934e28b62c1b4c8a817c90baabe36e2f6bf0479c195b3b144468d63d7ecab3a .claude/skills/deep-code-review/SKILL.md
798f91a62c68f4b7accf376835c503f3f167c38321db1fb8c6d0d93d5b1c0c86 .claude/skills/deep-code-review/VERSION
392371349dc5757dbabc27d7a1e7fc38742b92c18e42d3fc7e5fb82b50d2c230 .claude/skills/deep-code-review/evals/evals.json
Expand Down Expand Up @@ -330,7 +330,7 @@ d8075eb0c9482e05a96c8ded3f885f98b165a43cf19b52ba461fb90339b00cd7 .claude/skills
e22c43592533cdeaa5dd6d6967d2a5519124dc2e5401b595caa182e7e0e68fb5 .claude/skills/deep-code-review/references/skill-authoring-and-size.md
df698c4cbc9eb45aec1ffe62191801d713c9d6f923ed36914a703d3c9c3e71f1 .claude/skills/deep-code-review/references/small-diff-read-budget.md
a0ed7ad061b7e60230601ee9b38a00400841d6ddf3af386edbd4b34a981c9798 .claude/skills/deep-code-review/references/standards-index.md
996be31761037cf0ebef1d9383958f5ce32d8b0d19fcb355dfc580591d64ff08 .claude/skills/deep-code-review/references/supply-chain-check.md
3aa290fc569016e1e433ffc4583606e63e1585feccc5306226b6e747770491bd .claude/skills/deep-code-review/references/supply-chain-check.md
1d42030a463bd30ec43ed5a57af49551749d29ce3f3f4d0095ee59d3fafecd6f .claude/skills/deep-code-review/references/testing-ai-evals.md
65e11f80668f360d8516f3ddae7ea923df1c46f97edee08401024fb1c80bfd02 .claude/skills/deep-code-review/references/testing-and-evals.md
2463c659a2591017ee207a137da11ed263fee5e9b899c22873039e7c78fc40ce .claude/skills/deep-code-review/references/testing-ml.md
Expand Down Expand Up @@ -393,7 +393,7 @@ f1108e2700864195a239744b319b1d04953879589f8a2b83708bc45a080e3d86 .claude/skills
859c67d6a6a781c873f47d5662d403da2321da53e6994ca3d0ccdbf74afd39ac .claude/skills/deep-code-review/scripts/score_review.py
51c50eef1e4c5f417e56ee89b6804068052333ac8b8764c3b6ffcb2f3b8c185e .claude/skills/deep-code-review/scripts/security_verify.py
761ccb5730e297f41d865aeec9babd44b00b5cc029a8d4e7b2cb37b1122222be .claude/skills/deep-code-review/scripts/source_scan_tests.py
d09fad6e40f0355f5f82ddf87c4f501b084b4b406e0cd9f070310d7c41b2c1ad .claude/skills/deep-code-review/scripts/supply_chain_check.sh
222d0d1f1f47c9992616a90cc73324359244b833435c62da0c2aa2152f7f2d6e .claude/skills/deep-code-review/scripts/supply_chain_check.sh
01ce284157850e6f9b146023dbcf12a8ed014db578ee5f6f27ddf21a2a3a0b5e .claude/skills/deep-code-review/scripts/token_differ.py
317acae212ed73279df66eb7b419b8426733bb1be05345b0e1c8b2c6790ad8a1 .claude/skills/deep-code-review/scripts/validate_status_claims.py
12a01a5082b5c4466c40aab5f0114cd8442f271df1aa7ef8ddd186512681a7b8 .claude/skills/deep-code-review/templates/dcr-gates.sh
Expand Down
3 changes: 3 additions & 0 deletions changelog.d/wave-osv-reach-tags.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Added
- deep-code-review `supply_chain_check.sh`: each osv row for an npm package gets a reach tag. `imported` (a JS/TS/Vue/Svelte/Astro/MDX source file outside node_modules and .git, and outside a root-level .venv, venv or vendor, imports or requires it), else `dev-only` (npm `dev: true` on every occurrence of that name@version, or, when the lockfile does not list it, an osv dev/build/test group), else `transitive`. Rows that are not `imported` print `(reach: <tag>)` and stay in the severity counts. Only `dev-only` rows skip the `--fail-on` gate (unless `SUPPLY_CHAIN_STRICT=1`) and a summary line counts them; `transitive` rows still gate because a direct dependency may reach them. Unknown reach (unreadable file, a lockfile without a `packages` map such as lockfile v1, non-npm ecosystem, PyPI without a dev group) is treated as imported. This replaces the earlier dev-only cap at moderate. The osv API fallback now includes the ecosystem so it gets the same tags. Tests: `scripts/test_supply_chain_reach.py`.
size-budget-raise: .claude/skills/deep-code-review/references/supply-chain-check.md 3178→3421 osv reach tag and SUPPLY_CHAIN_STRICT documented
2 changes: 2 additions & 0 deletions scripts/ci-tests.d/supply-chain-reach.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# supply_chain_check.sh osv reach tags (imported, dev-only, transitive); non-imported rows skip the gate unless SUPPLY_CHAIN_STRICT=1.
python3 scripts/test_supply_chain_reach.py
2 changes: 1 addition & 1 deletion scripts/test-scanner-offline-db.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ bash "$S/review_checks.sh" --repo "$T/none" >/dev/null 2>&1 && bad "--repo bad d
# 6. supply_chain_check: dev-only cap. Same critical advisory on a dev-only package fails --fail-on high only when prod-reachable.
cd "$T/repo"
rc=0; o=$(REVIEW_OSV_DB_DIR="$T/db" bash "$S/supply_chain_check.sh" --online --allow-skip --fail-on high "$T/repo" 2>&1) || rc=$?
[ "$rc" = 0 ] && printf '%s' "$o" | grep -q 'moderate=1' && printf '%s' "$o" | grep -q 'devpkg@1.0.0 GHSA-d fixed in: none \[dev-only; capped from critical\]' && ok "dev-only critical capped to moderate, noted" || { bad "dev-only cap (rc=$rc)"; printf '%s\n' "$o"; }
[ "$rc" = 0 ] && printf '%s' "$o" | grep -q 'critical=1' && printf '%s' "$o" | grep -q 'devpkg@1.0.0 GHSA-d fixed in: none (reach: dev-only)' && ok "dev-only critical skips the gate, noted" || { bad "dev-only cap (rc=$rc)"; printf '%s\n' "$o"; }
printf '{"packages":{"node_modules/devpkg":{"version":"1.0.0"}}}' >package-lock.json
rc=0; o=$(REVIEW_OSV_DB_DIR="$T/db" bash "$S/supply_chain_check.sh" --online --allow-skip --fail-on high "$T/repo" 2>&1) || rc=$?
[ "$rc" = 1 ] && printf '%s' "$o" | grep -q 'critical=1' && ! printf '%s' "$o" | grep -q 'dev-only' && ok "prod-reachable critical is not capped" || { bad "prod critical (rc=$rc)"; printf '%s\n' "$o"; }
Expand Down
89 changes: 89 additions & 0 deletions scripts/test_supply_chain_reach.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""supply_chain_check.sh reach tags: fake osv-scanner + tiny fixture tree, no network.
Run: python3 scripts/test_supply_chain_reach.py"""
import json, os, pathlib, shutil, subprocess, tempfile, unittest

SC = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/deep-code-review/scripts/supply_chain_check.sh"
NPM = ["impkg", "@sc/imp", "devpkg", "devonly", "transpkg", "nmonly", "btick", "astropkg", "vendpkg", "rootvend", "prodgrp"]
ECO = {**{n: "npm" for n in NPM}, "Pyd": "PyPI"}


def osv(names):
return json.dumps({"results": [{"packages": [
{"package": {"name": n, "version": "1.0.0", "ecosystem": ECO[n]}, "groups": [],
**({"dependency_groups": ["dev"]} if n == "prodgrp" else {}), # osv says dev, the lock says prod: lock wins
"vulnerabilities": [{"id": "GHSA-" + n.strip("@").replace("/", "-"), "database_specific": {"severity": "CRITICAL"}}]}
for n in names]}]})


class Reach(unittest.TestCase):
def setUp(self):
t = self.t = pathlib.Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, t)
(t / "bin").mkdir()
(t / "bin/osv-scanner").write_text("#!/bin/sh\ncat '%s'\n" % (t / "osv.json"))
(t / "bin/osv-scanner").chmod(0o755)
r = t / "repo"
for d in ("src/vendor", "node_modules/x", "vendor"): (r / d).mkdir(parents=True)
(r / "src/app.js").write_text("import a from 'impkg/sub';\nconst b = await import(\"@sc/imp/x\");\nimport './local.js';\n")
(r / "src/t.js").write_text("require('devpkg')\n") # imported beats dev-only
(r / "src/b.js").write_text("const c = require(`btick`)\n") # backtick string
(r / "src/p.astro").write_text("---\nimport x from 'astropkg'\n---\n")
(r / "src/vendor/v.js").write_text("require('vendpkg')\n") # nested vendor is app code
(r / "vendor/r.js").write_text("require('rootvend')\n") # root vendor is pruned
(r / "node_modules/x/i.js").write_text("require('nmonly')\n") # node_modules never counts
self.lock(devpkg=True, devonly=True)

def lock(self, **dev):
pk = {"node_modules/" + n: {"version": "1.0.0", **({"dev": True} if dev.get(n) else {})} for n in NPM}
(self.t / "repo/package-lock.json").write_text(json.dumps({"packages": pk}))

def run_check(self, names, strict=False):
(self.t / "osv.json").write_text(osv(names))
env = {**os.environ, "PATH": "%s:%s" % (self.t / "bin", os.environ["PATH"])}
env.pop("SUPPLY_CHAIN_STRICT", None)
if strict: env["SUPPLY_CHAIN_STRICT"] = "1"
r = subprocess.run(["bash", str(SC), "--online", "--allow-skip", "--fail-on", "high", str(self.t / "repo")],
capture_output=True, text=True, env=env)
return r.returncode, r.stdout

@staticmethod
def row(out, name):
return next(ln for ln in out.splitlines() if (name + "@") in ln)

def test_tags(self):
rc, out = self.run_check(list(ECO))
self.assertEqual(rc, 1, out)
for n in ("impkg", "@sc/imp", "devpkg", "btick", "astropkg", "vendpkg", "Pyd"): # Pyd: PyPI without dev group -> fail closed
self.assertNotIn("(reach:", self.row(out, n), n)
self.assertIn("(reach: dev-only)", self.row(out, "devonly"))
for n in ("transpkg", "nmonly", "rootvend", "prodgrp"):
self.assertIn("(reach: transitive)", self.row(out, n), n)
self.assertIn("1 dev-only", out)

def test_only_dev_only_skips_gate_unless_strict(self):
rc, out = self.run_check(["devonly"])
self.assertEqual((rc, "critical=1" in out), (0, True), out)
rc, out = self.run_check(["devonly"], strict=True)
self.assertEqual(rc, 1, out)
self.assertNotIn("dev-only:", out)

def test_transitive_still_gates(self):
rc, out = self.run_check(["transpkg", "prodgrp"])
self.assertEqual(rc, 1, out)

def test_lockfile_v1_fails_closed(self):
(self.t / "repo/package-lock.json").write_text(json.dumps({"lockfileVersion": 1, "dependencies": {}}))
rc, out = self.run_check(["devonly"])
self.assertEqual(rc, 1, out)
self.assertNotIn("(reach:", out)

def test_bad_lockfile_fails_closed(self):
(self.t / "repo/package-lock.json").write_text("{not json")
rc, out = self.run_check(["devonly"])
self.assertEqual(rc, 1, out)
self.assertNotIn("(reach:", out)


if __name__ == "__main__":
unittest.main()
Loading