diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml
index a8d68c73..fe9d23eb 100644
--- a/.github/workflows/check-docs.yml
+++ b/.github/workflows/check-docs.yml
@@ -34,14 +34,30 @@ jobs:
Your task: check whether the project documentation is still accurate
and complete given the code changes introduced by this PR.
+ The documentation is structured, and each kind of change has a home:
+ - A verb's flags, output, JSON shape, or exit codes → its page in docs/cli/
+ (one page per verb, plus the overview's exit-code contract).
+ - An engine module's responsibility, key symbols, or invariants → its page
+ in docs/api/ (hand-written module tours) and, for cross-cutting shifts,
+ docs/architecture.md.
+ - User-visible behavior (scaffold contents, states, refusal messages,
+ environment model, SLURM, publication) → docs/user/ (getting-started
+ quotes real console output; troubleshooting quotes real refusals) and
+ README.md's quick start.
+ - Test structure, dev workflow, or conventions → docs/contributing/.
+
Steps to follow:
1. Run: git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }}
to get the list of changed files.
- 2. Read the changed source files (focus on .py, .ts, .js, .go, .rs files).
- 3. Read existing documentation files (README*, docs/**/*.md, CHANGELOG*, CONTRIBUTING*).
- 4. SKIP any files under skills/ or named SKILL.md — these are not user-facing docs.
- 5. Identify any documentation that is now inaccurate, incomplete, or missing
- given the code changes.
+ 2. Read the changed source files (focus on src/**/*.py and the workflows).
+ 3. Read the documentation pages the map above points at for those changes.
+ 4. SKIP CLAUDE.md and evals/ — agent instructions and the eval harness are
+ maintained separately, not user-facing docs.
+ 5. Identify documentation that is now inaccurate, incomplete, or missing.
+ Two failure modes matter most here: a quoted console output or refusal
+ message that no longer matches what the CLI prints, and a documented
+ flag, verb, state, or file that no longer exists (the docs must never
+ describe more than the code delivers — no foreshadowing).
Then:
- Post a comment on PR #${{ github.event.pull_request.number }} summarising
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 0973decb..949205c9 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -3,11 +3,14 @@ name: Eval
# Minimal agentic eval: scaffold a project from the eval seed on the
# runner, let Claude Code (with the astra plugin) build it using the
# branch's lc/astra, then check the results with astra validate and
-# lc status.
+# lc materialize --check.
on:
- pull_request:
workflow_dispatch:
+ pull_request:
+ # `ready_for_review` beside the defaults, so flipping a draft to
+ # ready triggers the run the draft guard below skips.
+ types: [opened, synchronize, reopened, ready_for_review]
# Only one eval per PR at a time — cancel in-progress runs
concurrency:
@@ -28,20 +31,35 @@ jobs:
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
+ with:
+ # Full history so hatch-vcs sees the tags and stamps a truthful
+ # dev version into the installed engine (and every manifest's
+ # lc_version) instead of a 0.1.dev fallback
+ fetch-depth: 0
- name: Set up uv
+ # No python-version input: it would export an ambient UV_PYTHON,
+ # which lc's install-settings scrub then rightly warns about on
+ # every single invocation the agent makes. The interpreter is
+ # pinned per tool install below instead.
uses: astral-sh/setup-uv@v6
with:
- python-version: "3.12"
enable-cache: true
- activate-environment: true
- - name: Install lightcone-cli (branch under test)
+ - name: Install lightcone-cli + astra (uv tools)
+ # The engine is the host's uv tool — install the branch under
+ # test the way a user installs a release: `uv tool install`,
+ # which links `lc` and (via the mirrored entry points) the
+ # git-annex executables into ~/.local/bin. The commit under test
+ # is the checkout itself, stamped into the version by hatch-vcs.
+ # astra's executable belongs to astra-tools, so it is its own
+ # tool install, pinned to the version the branch depends on. No
+ # venv is activated: the agent's shell sees the tools exactly as
+ # an end user's would.
run: |
- uv sync
- # Scientific deps the task's analysis scripts will need — the
- # agent can add more with `uv pip install`
- uv pip install numpy scipy matplotlib
+ uv tool install --python 3.12 "$GITHUB_WORKSPACE"
+ uv tool install --python 3.12 "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)"
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Install Claude Code + astra plugin
run: |
@@ -70,6 +88,12 @@ jobs:
rm -f universes/baseline.yaml
astra universe generate -n baseline \
-d 'Default configuration using standard practices'
+ # No scientific deps are pre-seeded — declaring what the
+ # recipes import (`uv add`) is part of what the eval measures
+ # lc materialize refuses a dirty tree — commit the seed so the
+ # agent starts from a clean one
+ git add -A
+ git commit -q -m 'Seed the eval task'
- name: Run Claude Code
id: agent
@@ -92,11 +116,14 @@ jobs:
working-directory: ${{ env.PROJECT }}
run: |
astra validate astra.yaml
- lc status --universe baseline
- # Fail unless every materializable output is ok
- lc status --json | jq -e \
- '[.universes[].outputs[] | select(.status != "alias")]
- | length > 0 and all(.status == "ok")'
+ lc status
+ # The gate: nothing failed, nothing left to make — and at least
+ # one output actually got materialized
+ lc materialize --check --json | jq -e \
+ '.up_to_date and ((.current | length) > 0)'
+ # The publication view: declaring a license and materializing
+ # once more is part of the task
+ test -f ro-crate-metadata.json
- name: Build report
if: always()
@@ -136,7 +163,7 @@ jobs:
echo 'lc status
'
echo ""
echo '```'
- (cd "$PROJECT" && lc status --universe baseline) || echo "unavailable"
+ (cd "$PROJECT" && lc status) || echo "unavailable"
echo '```'
echo ""
echo '