diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 9594e56e..5b24e23e 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -3,10 +3,12 @@ name: Eval
# Minimal agentic eval: scaffold a project from the eval seed on the
# runner, let Claude Code (with the astra plugin) build it using the
# branch's lc/astra, then check the results with astra validate and
-# lc status.
+# lc materialize --check.
on:
workflow_dispatch:
+ pull_request:
+ types: [labeled]
# Only one eval per PR at a time — cancel in-progress runs
concurrency:
@@ -19,7 +21,7 @@ permissions:
jobs:
eval:
- if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false
+ if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-eval'
runs-on: ubuntu-latest
# Step-level timeout on the agent step (below) is what actually
# bounds the run: a job-level timeout would cancel the always()
@@ -33,14 +35,21 @@ jobs:
with:
python-version: "3.12"
enable-cache: true
- activate-environment: true
- - name: Install lightcone-cli (branch under test)
+ - name: Install lightcone-cli + astra (uv tools)
+ # The engine is the host's uv tool — install the branch under
+ # test the way a user installs a release: `uv tool install`,
+ # which links `lc` and (via the mirrored entry points) the
+ # git-annex executables into ~/.local/bin. The commit under test
+ # is the checkout itself, stamped into the version by hatch-vcs.
+ # astra's executable belongs to astra-tools, so it is its own
+ # tool install, pinned to the version the branch depends on. No
+ # venv is activated: the agent's shell sees the tools exactly as
+ # an end user's would.
run: |
- uv sync
- # Scientific deps the task's analysis scripts will need — the
- # agent can add more with `uv pip install`
- uv pip install numpy scipy matplotlib
+ uv tool install "$GITHUB_WORKSPACE"
+ uv tool install "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)"
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Install Claude Code + astra plugin
run: |
@@ -69,6 +78,12 @@ jobs:
rm -f universes/baseline.yaml
astra universe generate -n baseline \
-d 'Default configuration using standard practices'
+ # No scientific deps are pre-seeded — declaring what the
+ # recipes import (`uv add`) is part of what the eval measures
+ # lc materialize refuses a dirty tree — commit the seed so the
+ # agent starts from a clean one
+ git add -A
+ git commit -q -m 'Seed the eval task'
- name: Run Claude Code
id: agent
@@ -91,11 +106,14 @@ jobs:
working-directory: ${{ env.PROJECT }}
run: |
astra validate astra.yaml
- lc status --universe baseline
- # Fail unless every materializable output is ok
- lc status --json | jq -e \
- '[.universes[].outputs[] | select(.status != "alias")]
- | length > 0 and all(.status == "ok")'
+ lc status
+ # The gate: nothing failed, nothing left to make — and at least
+ # one output actually got materialized
+ lc materialize --check --json | jq -e \
+ '.up_to_date and ((.current | length) > 0)'
+ # The publication view: declaring a license and materializing
+ # once more is part of the task
+ test -f ro-crate-metadata.json
- name: Build report
if: always()
@@ -135,7 +153,7 @@ jobs:
echo 'lc status
'
echo ""
echo '```'
- (cd "$PROJECT" && lc status --universe baseline) || echo "unavailable"
+ (cd "$PROJECT" && lc status) || echo "unavailable"
echo '```'
echo ""
echo ' '
@@ -198,11 +216,12 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: eval-project
- # include-hidden-files keeps the .lightcone-manifest.json
- # provenance files; the venv and snakemake scratch are excluded
+ # include-hidden-files keeps the manifests, the run records in
+ # .git, and the annexed results; the venv and lc's per-run
+ # machine state are excluded
path: |
${{ env.PROJECT }}
!${{ env.PROJECT }}/.venv/**
- !${{ env.PROJECT }}/.snakemake/**
+ !${{ env.PROJECT }}/.lightcone/**
include-hidden-files: true
if-no-files-found: warn
diff --git a/evals/prompt.md b/evals/prompt.md
index 15a7a9db..c0f948f4 100644
--- a/evals/prompt.md
+++ b/evals/prompt.md
@@ -8,19 +8,32 @@ This project is driven by two CLIs — use them rather than improvising:
`astra validate astra.yaml` checks it against the schema. If an `astra`
skill or plugin is available in your environment, load it before reading
or editing `astra.yaml` — it documents the full spec format.
-- `lc` (lightcone-cli) is the execution layer, a thin shim over Snakemake:
- - `lc run --universe baseline` materializes an output (and
- anything upstream of it) by running the recipe commands declared in
- `astra.yaml`. With no output ids it builds everything. It is
- idempotent: re-running only rebuilds what is stale or missing.
- - `lc status --universe baseline` reports each output as `ok`, `stale`,
- or `missing`; `lc status --json` is the machine-readable form.
+- `lc` (lightcone-cli) is the execution layer:
+ - `lc materialize` makes every output the spec declares, running each
+ recipe in dependency order and committing each result to git as it
+ lands, together with a provenance manifest. It refuses to start on
+ a dirty tree: commit your own edits first, with plain `git add` and
+ `git commit` — the project's git-annex filter handles large files
+ transparently, so never run a git-annex command yourself.
+ - `lc materialize ` (or `/`) narrows
+ a run to one output and whatever it depends on. Re-running is
+ idempotent: only what is stale gets remade — an output the spec now
+ defines differently, or one whose declared inputs changed.
+ - `lc status` reports each output as `current`, `stale`, or `behind`,
+ with the commit it was made at; `lc status --json` is the
+ machine-readable form. It always exits 0. The pass/fail gate is
+ `lc materialize --check`, which exits 1 while anything still needs
+ making.
+ - `lc run ` runs an ad-hoc command in the project
+ environment under the same isolation a recipe gets — useful for
+ probing why a recipe would fail.
- Outputs land in `results/baseline//`, each with a
- `.lightcone-manifest.json` provenance manifest written by the engine.
- Files placed in `results/` by hand have no manifest and fail
- verification — never write there yourself.
- - When `lc run` fails, read the error and the Snakemake log it points
- to, fix the script or spec, and re-run.
+ `.lightcone-manifest.json` manifest written and committed by the
+ engine. Never write into `results/` yourself: a hand-placed file
+ has no run record, and the engine detects the foreign write and
+ remakes the output.
+ - When a recipe fails, `lc materialize` reports which output failed
+ and why; fix the script or the spec, commit, and re-run.
## Recipe template grammar
@@ -49,37 +62,59 @@ is how the engine orders the build.
## Environment
-Recipes and your interactive shell run in two different environments —
-keep them straight:
-
-- **Recipe commands run by `lc run`** may execute inside a container
- built from the project's `Containerfile` + `requirements.txt`
- (whenever `astra.yaml` declares a `container:` and a runtime is
- available). Every package a recipe script imports must therefore be
- listed in `requirements.txt` — add it there *before* running, and the
- engine rebuilds the content-addressed image automatically. Host-side
- installs never reach the container.
-- **Your own shell commands** run on the host in an activated uv-managed
- virtual environment with numpy, scipy, and matplotlib pre-installed.
- For ad-hoc host tools use `uv pip install ` — plain `pip` is
- not available in this venv.
+Recipes run in the project's own locked environment (`pyproject.toml` +
+`uv.lock` + `.venv`), sandboxed: the project tree is read-only apart from
+each recipe's own output directory under `results/`, and only declared
+tools are executable.
+
+- The project is managed by uv and starts with **no dependencies**.
+ Every package a recipe script imports must be declared before
+ materializing: run `uv add [ ...]` in the project
+ root (e.g. `uv add numpy scipy`). That updates `pyproject.toml`,
+ re-locks `uv.lock`, and syncs `.venv` in one step — commit all of it
+ along with your scripts, like any other edit.
+- To remove a package use `uv remove `; to pin a version,
+ `uv add 'numpy>=2'`. Do **not** use plain `pip` or `uv pip install` —
+ an install that bypasses the lock reaches nothing a recipe sees.
+- A sandbox denial names the path or tool that was denied and the
+ remedy — follow the remedy rather than working around the sandbox.
## Build loop
`astra.yaml` is the single source of truth: inputs, outputs, recipes, and
-methodological decisions all live there — read it first. For each output
-that needs materializing:
+methodological decisions all live there — read it first. The seed spec is
+deliberately incomplete: recipe commands do not yet pass their inputs,
+decisions, or output directory, and outputs may be missing entries in
+their `inputs:` / `decisions:` contracts. Completing the spec is part of
+the task. For each output:
-1. Read the recipe's `command` to see what script and arguments it expects.
+1. Complete the recipe `command` so it references `{output}` and the
+ `{inputs.}` / `{decisions.}` the computation needs, and
+ declare everything it references in that output's `inputs:` /
+ `decisions:` lists.
2. Write the script at the path the command names, parameterizing every
decision via argparse — never hardcode option values.
-3. Run `lc run --universe baseline` to materialize it through
- the engine.
-4. Commit progress as you go.
+3. Commit your edits, then run `lc materialize` (or
+ `lc materialize `) to build through the engine.
+
+Build iteratively from upstream outputs to downstream. `lc status` shows
+where every output stands.
+
+## Publication
+
+Once every output is materialized, prepare the repository for
+publication:
+
+1. Declare a license in `pyproject.toml`, as an SPDX expression under
+ `[project]` — e.g. `license = "CC-BY-4.0"`. Declaring one is what
+ turns publication on: from then on `lc materialize` also maintains
+ `ro-crate-metadata.json` at the project root, an RO-Crate view of
+ the project and its provenance.
+2. Commit the edit, then run `lc materialize` once more — nothing is
+ remade, but the crate document is generated and committed.
-Build iteratively from upstream outputs to downstream. `lc status
---universe baseline` shows you what's `ok`, `stale`, or `missing` — you're
-done when every output shows `ok` and `astra validate astra.yaml` passes.
+You're done when `astra validate astra.yaml` and
+`lc materialize --check` pass and `ro-crate-metadata.json` exists.
Skip plan approval and interactive confirmations — this is an automated
eval run.
diff --git a/evals/tasks/snae/astra.yaml b/evals/tasks/snae/astra.yaml
index 3bdf0fb5..41b9d647 100644
--- a/evals/tasks/snae/astra.yaml
+++ b/evals/tasks/snae/astra.yaml
@@ -9,8 +9,6 @@ description: |
using maximum-likelihood (MAP) point estimation. This provides best-fit
cosmological parameters as a building block for a larger analysis.
-container: Containerfile
-
inputs:
- id: union21
type: data
diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md
index 1058b5f9..120f4eb0 100644
--- a/evals/trace-analysis-prompt.md
+++ b/evals/trace-analysis-prompt.md
@@ -8,7 +8,7 @@ Write a section titled exactly `### Confusion & pain points` containing
- failed commands or errored tool results, and their root cause
- moments where the agent misunderstood the spec format, the CLI
- surface, or the execution/container environment
+ surface, the sandbox, or the execution environment
- detours: reverse-engineering source code, probing the environment,
redoing work, or fixing things the harness should have provided
- workarounds the agent invented that hint at a product gap (these are