From 1f1a82df46d75c03f78a5bec810ccca744b5e8b5 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Mon, 17 Aug 2026 16:05:21 +0200 Subject: [PATCH 01/23] reset on clean basis --- .github/workflows/eval.yml | 1 - .gitmodules | 0 CLAUDE.md | 316 ++-- justfile | 88 -- pyproject.toml | 55 +- src/lightcone/cli/__init__.py | 3 + src/lightcone/cli/commands.py | 1393 +---------------- src/lightcone/engine/__init__.py | 21 +- src/lightcone/engine/cloudbuild.py | 420 ----- src/lightcone/engine/constants.py | 26 + src/lightcone/engine/container.py | 877 ----------- src/lightcone/engine/dask_cluster.py | 516 ------ src/lightcone/engine/manifest.py | 216 --- src/lightcone/engine/project.py | 390 +++++ src/lightcone/engine/runner.py | 110 -- src/lightcone/engine/scratch.py | 208 --- src/lightcone/engine/site_registry.py | 205 --- src/lightcone/engine/snakefile.py | 487 ------ src/lightcone/engine/status.py | 170 -- src/lightcone/engine/templates/__init__.py | 154 ++ .../engine/templates/files/gitignore.tmpl | 9 + .../engine/templates/files/index.md.tmpl | 29 + .../engine/templates/files/myst.yml.tmpl | 12 + .../templates/files/pyproject.toml.tmpl | 13 + .../templates/files/results-README.md.tmpl | 5 + src/lightcone/engine/tree.py | 375 ----- src/lightcone/engine/validation.py | 180 --- src/lightcone/engine/verify.py | 140 -- src/lightcone/engine/wrroc.py | 885 ----------- .../__init__.py | 29 - .../executor.py | 229 --- tests/conftest.py | 51 + tests/test_cli.py | 508 ++---- tests/test_cloudbuild.py | 275 ---- tests/test_container.py | 750 --------- tests/test_dask_cluster.py | 604 ------- tests/test_dask_plugin.py | 220 --- tests/test_eval_tasks.py | 31 - tests/test_manifest.py | 390 ----- tests/test_project.py | 397 +++++ tests/test_runner.py | 149 -- tests/test_scratch.py | 231 --- tests/test_site_registry.py | 155 -- tests/test_snakefile.py | 599 ------- tests/test_status.py | 203 --- tests/test_templates.py | 96 ++ tests/test_tree.py | 61 - tests/test_validation.py | 197 --- tests/test_verify.py | 296 ---- tests/test_wrroc.py | 785 ---------- 50 files changed, 1580 insertions(+), 11980 deletions(-) delete mode 100644 .gitmodules delete mode 100644 justfile delete mode 100644 src/lightcone/engine/cloudbuild.py create mode 100644 src/lightcone/engine/constants.py delete mode 100644 src/lightcone/engine/container.py delete mode 100644 src/lightcone/engine/dask_cluster.py delete mode 100644 src/lightcone/engine/manifest.py create mode 100644 src/lightcone/engine/project.py delete mode 100644 src/lightcone/engine/runner.py delete mode 100644 src/lightcone/engine/scratch.py delete mode 100644 src/lightcone/engine/site_registry.py delete mode 100644 src/lightcone/engine/snakefile.py delete mode 100644 src/lightcone/engine/status.py create mode 100644 src/lightcone/engine/templates/__init__.py create mode 100644 src/lightcone/engine/templates/files/gitignore.tmpl create mode 100644 src/lightcone/engine/templates/files/index.md.tmpl create mode 100644 src/lightcone/engine/templates/files/myst.yml.tmpl create mode 100644 src/lightcone/engine/templates/files/pyproject.toml.tmpl create mode 100644 src/lightcone/engine/templates/files/results-README.md.tmpl delete mode 100644 src/lightcone/engine/tree.py delete mode 100644 src/lightcone/engine/validation.py delete mode 100644 src/lightcone/engine/verify.py delete mode 100644 src/lightcone/engine/wrroc.py delete mode 100644 src/snakemake_executor_plugin_dask/__init__.py delete mode 100644 src/snakemake_executor_plugin_dask/executor.py delete mode 100644 tests/test_cloudbuild.py delete mode 100644 tests/test_container.py delete mode 100644 tests/test_dask_cluster.py delete mode 100644 tests/test_dask_plugin.py delete mode 100644 tests/test_eval_tasks.py delete mode 100644 tests/test_manifest.py create mode 100644 tests/test_project.py delete mode 100644 tests/test_runner.py delete mode 100644 tests/test_scratch.py delete mode 100644 tests/test_site_registry.py delete mode 100644 tests/test_snakefile.py delete mode 100644 tests/test_status.py create mode 100644 tests/test_templates.py delete mode 100644 tests/test_tree.py delete mode 100644 tests/test_validation.py delete mode 100644 tests/test_verify.py delete mode 100644 tests/test_wrroc.py diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 0973decb..9594e56e 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -6,7 +6,6 @@ name: Eval # lc status. on: - pull_request: workflow_dispatch: # Only one eval per PR at a time — cancel in-progress runs diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29b..00000000 diff --git a/CLAUDE.md b/CLAUDE.md index 12a3d49c..4229909c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,16 +2,96 @@ ## Project Overview -**lightcone-cli** is Lightcone Research's execution layer for ASTRA (Agentic Schema for Transparent Research Analysis). It ships the `lc` executable — an agent-agnostic CLI; it bundles no agent-specific skills, hooks, or plugins. +**lightcone-cli** is Lightcone Research's execution layer for ASTRA +(Agentic Schema for Transparent Research Analysis). It ships the `lc` +executable — an agent-agnostic CLI; it bundles no agent-specific skills, +hooks, or plugins. + +- **ASTRA** = pure specification: schema, validation, prior insights & + findings, evidence verification, helpers, minimal CLI +- **lightcone-cli** = execution layer: project scaffolding, execution, + environment identity, hermeticity enforcement + +lightcone-cli depends on ASTRA. The `astra` CLI handles spec operations; +the `lc` CLI handles execution. + +## ⚠️ This repository is a clean rebuild in progress + +The codebase is being **re-added layer by layer** on top of the normative +design spec: + +> **`../lightcone-cli/docs/design/execution-environment.md`** — *"the +> locked environment is the execution environment"*, v6.1. Normative. +> Read it before adding anything. It lives in the sibling checkout for +> now (branch `redesign_prototype`), alongside its decision records +> (rationale, substrate tradeoffs, hermeticity enforcement, the v6 +> review); it moves into this repo's `docs/design/` when the docs are +> rewritten at the end of the rebuild. + +The pre-rebuild codebase (Snakemake shim, authored Containerfiles, +`container:` in `astra.yaml`, vendored dask executor plugin, WRROC export) +was stripped deliberately. Functionality comes back **one layer at a +time**, each layer landing with its own tests and dependencies — never +speculatively. + +### Layers + +| # | Layer | State | +|---|-------|-------| +| 1 | **Project scaffolding** — `lc init` | ✅ **done** | +| 2 | Environment layer — `env_version`, lock scan, manifest schema | ⬜ | +| 3 | The `lc` entrypoint — launcher: discover → mode-detect → scrub `UV_*` → converge → delegate | ⬜ | +| 4 | Fabric — `lc materialize`, worker sequence, mid-run relock gate | ⬜ | +| 5 | Sandbox layer — Landlock / Seatbelt, exec-shim, denial UX | ⬜ | +| 6 | Container hatch — `[tool.lightcone.image]`, generated Containerfile, `lc build` | ⬜ | +| 7 | Venues — Perlmutter, hub/GKE, Cloud Build | ⬜ | +| 8 | `lc status`, `lc verify`, WRROC export | ⬜ | + +The spec's §11 (Migration) is the authoritative ordering; the table above +is the working map. **Layer boundaries are also dependency boundaries** — +a dependency enters `pyproject.toml` with the layer that needs it, not +before. + +### Rules while rebuilding + +- **No dead code.** If nothing in the current layer calls it, it doesn't + land yet. `lc --help` advertises only verbs that work. +- **`docs/` is frozen at its pre-rebuild state, by decision.** It still + describes the old Snakemake architecture and the full command set. + Don't patch it layer by layer — it gets rewritten in one pass once the + rebuild is complete. Same for `README.md` and `zensical.toml`. +- **Port with intent.** Prior implementations (this repo's git history, + and the `redesign_prototype` branch of the sibling `lightcone-cli` + checkout) are references, not sources of truth. The spec is. +- **Every layer ships tests.** See the per-layer test list in spec §11. + +## Architecture (target) + +A project is `pyproject.toml` + `uv.lock` + `.python-version` — **uv is +the only environment substrate**. Mode is *derived, not configured*: -- **ASTRA** = pure specification: schema, validation, prior insights & findings, evidence verification, helpers, minimal CLI -- **lightcone-cli** = execution layer: project scaffolding, **Snakemake-based execution**, container builds +``` +direct mode (default) containerized mode +───────────────────── ────────────────── +.venv in the project tree [tool.lightcone.image] declared +no image, ever ⇒ content-addressed image is the +recipes run under Landlock/ execution world: driver, workers, + Seatbelt recipes, probes all run in-image + from the baked /opt/venv +``` -lightcone-cli depends on ASTRA. The `astra` CLI handles spec operations; the `lc` CLI handles execution. +Identity: `env_version = sha256(uv.lock ‖ .python-version ‖ canonical +install-settings ‖ canonical [tool.lightcone.image] ‖ Containerfile.extra +hash)`, sitting inside `code_version`. Every output records what +enforcement it actually ran under (`hermeticity`). See spec §1–§3, §7. ### Namespace contract -`lightcone-cli` ships the `lightcone.*` namespace via PEP 420 implicit namespace packages. **`src/lightcone/` must not contain an `__init__.py`** — that would turn the namespace into a regular package and break coexistence with future sibling distributions (`lightcone-ui`, etc.). +`lightcone-cli` ships the `lightcone.*` namespace via PEP 420 implicit +namespace packages. **`src/lightcone/` must not contain an +`__init__.py`** — that would turn the namespace into a regular package +and break coexistence with future sibling distributions +(`lightcone-ui`, etc.). Any new `lightcone-*` package must: @@ -19,136 +99,144 @@ Any new `lightcone-*` package must: 2. Not create `src/lightcone/__init__.py`. 3. Ship only its own subpackage under `src/lightcone//`. -## Architecture - -The execution layer is a thin shim over Snakemake. The integrity layer (per-output content-addressed manifests) is the only thing we own substantively. - -``` -astra.yaml ── snakefile generator ──> .lightcone/Snakefile - │ - snakemake (CLI subprocess) - │ - ┌───────────────────────────────┼───────────────────────────────┐ - │ │ │ │ │ - DAG resolution staleness cluster submission container exec conda - (Snakemake) (mtime+code) (slurm plugin) (apptainer/docker) - │ - └─── per-rule run: block: shell() recipe + write_manifest() - │ - results///... - results///.lightcone-manifest.json -``` - -**What Snakemake owns** (we do not write code for any of this): DAG construction, topological execution, parallelism (`--cores`, `--jobs`), cluster submission (`snakemake-executor-plugin-slurm`), per-rule resources, profiles, dry-run, DAG visualization, staleness detection (`--rerun-triggers`), locking, log capture, retry, container runtime invocation. - -**What we own**: a Snakefile generator, the manifest layer (write/read/verify), a status walker, and a verify routine. - -## Repository Structure +## Repository Structure (current) ``` src/lightcone/ # namespace — NO __init__.py -├── cli/ # Click surface -│ ├── __init__.py # exposes main() -│ └── commands.py # init, run, status, verify, build -├── engine/ # execution substrate — Snakemake-based -│ ├── __init__.py -│ ├── manifest.py # write_manifest, sha256_dir, code_version — the integrity layer -│ ├── snakefile.py # generates .lightcone/Snakefile from astra.yaml -│ ├── container.py # Content-addressed container builds (Docker, podman-hpc, apptainer) -│ ├── cloudbuild.py # GCP Cloud Build backend (JupyterHub deployments; no local OCI runtime) -│ ├── status.py # Manifest-driven status walker (no Snakemake import) -│ ├── verify.py # Recompute hashes; validate provenance chain -│ ├── tree.py # Sub-analysis tree traversal (kept from before) -│ ├── validation.py # Post-materialization output shape checks -│ └── site_registry.py # Known HPC site defaults (Perlmutter, etc.) - -evals/ # Agentic eval: prompt.md + tasks// seed files; - # driven by .github/workflows/eval.yml (no Python harness) -tests/ # pytest — mirrors src/ structure -pyproject.toml # hatchling + hatch-vcs, ASTRA + Snakemake as deps +├── cli/ # the CLI only: flags, rendering, exit codes +│ ├── __init__.py # exposes main(); the launcher hooks in here (layer 3) +│ └── commands.py # lc init — the only verb so far +└── engine/ + ├── __init__.py # lc_version() + ├── constants.py # engine constants (interpreter pin, uv floor) + ├── project.py # what a project IS: discovery + convergence + └── templates/ # the scaffold's file templates + ├── __init__.py # loader + one renderer per scaffolded file + └── files/*.tmpl # the templates themselves, as real files + +evals/ # agentic eval seed: prompt.md + tasks// +tests/ # pytest — mirrors src/ ``` ## Development Commands ```bash -uv sync --group dev # installs pytest, ruff, mypy +uv sync --group dev # installs pytest, ruff, mypy uv run pytest -uv run ruff check src/ tests/ +uv run ruff check src/ tests/ # --fix to apply uv run mypy src/ +uv build # wheel + sdist ``` -A `justfile` is available for common tasks — run `just` to see all recipes: - -```bash -just test # run pytest -just lint # ruff + mypy -just docs # build the documentation site -``` - -## Architecture & Data Flow - -``` -astra.yaml ── snakefile.generate() ──> .lightcone/Snakefile + .lightcone/snakefile-config.json - │ - snakemake -s ... -d ... - │ - per-rule run: - │ - shell(recipe) ────────────────► write_manifest() - (in container if container: set; (host-side) - Snakemake handles invocation) - │ - results///data.txt - results///.lightcone-manifest.json -``` - -- `snakefile.generate(project, universes=[...])` reads `astra.yaml`, writes `.lightcone/Snakefile`, and writes a sidecar JSON keyed by `(rule, universe)` containing the recipe text, container image, decisions, and precomputed `code_version`. -- The Snakefile body for each rule is a `run:` block: `shell(params.cfg["recipe"])` then `write_manifest(...)`. -- `code_version = sha256(recipe + container_image + decisions)`. Embedded in the rule's shell command literally so Snakemake's built-in `code` rerun-trigger detects drift. -- `data_version = sha256_dir(output_dir)`. Written into the manifest after the recipe completes; used by `lc verify` to detect tampering. Excludes the manifest file itself and `.snakemake_timestamp`. -- The manifest is a *declared output* of every rule. A missing manifest causes Snakemake to re-run the rule, blocking the agent-faked-file scenario. - -## Key Invariants - -**Spec & execution:** -- `astra.yaml` is the single source of truth — all inputs, outputs, recipes, decisions, containers -- Output paths are always `results///` for root and inline sub-analyses; `/results///` for path-rooted sub-analyses -- Container image hashes are deterministic: SHA256(Containerfile + dependency files) → `lc--` -- The Snakefile and snakefile-config.json are regenerated on every `lc run` — never edit them by hand - -**Integrity:** -- Every materialized output has `/.lightcone-manifest.json` recording code_version, data_version, container, recipe, decisions, input_versions, git_sha, lc_version, host -- `lc verify` recomputes data_version and walks the chain; failures surface as `tampered_data`, `broken_chain`, or `missing_manifest` -- `lc status` reads only manifests — works offline, no Snakemake or DB needed - -**CLI surface:** -- `lc init` — idempotently converge a project (astra.yaml, .gitignore, .lightcone/, results/, universes/, Containerfile, MyST report template); `--check` reports drift without writing, `--json` emits the report -- `lc run [outputs...]` — generate Snakefile, invoke snakemake -- `lc status` — manifest-driven status report -- `lc verify` — chain integrity check -- `lc build` — pre-build container images from Containerfiles - -Global config (`~/.lightcone/config.yaml`) is auto-created with defaults on first invocation. +These four are the whole loop, and they are exactly what CI runs +(`.github/workflows/{tests,lint}.yml`). There is deliberately no task +runner in between — the pre-rebuild `justfile` was 90 lines of wrappers +around them plus recipes for the frozen docs and the dormant eval. + +## Key Invariants (layer 1) + +**The CLI/engine split.** `cli/commands.py` owns flags, console rendering, +and exit codes — nothing else. Everything about *what a project is* lives +in `engine/project.py`: + +| Symbol | Role | +|---|---| +| `find_root(start)` | Discovery: nearest ancestor holding an `astra.yaml`, else `None` | +| `project_name(dir)` | PEP 503-ish name from the directory name | +| `converge(dir, *, write, git, sync)` | The whole scaffold operation | +| `ConvergenceReport` | `created` / `repaired` / `unchanged` / `warnings`, `.converged`, `.as_dict()` | +| `ProjectError` | The one engine exception; `_EngineErrorGroup` in the CLI turns it into a clean `ClickException` | + +The engine never imports click and never prints. `converge(write=False)` +is check mode — the *same* decision path with side effects switched off +(via `_Converger.item` / `.file`), which is what keeps `--check` honest +rather than a second implementation. + +**Templates are files, not string literals.** `engine/templates/files/*.tmpl` +are package data, loaded through `importlib.resources`; `templates/__init__.py` +exposes one function per scaffolded file. Placeholders are +`string.Template` (`${name}`) — **not** `str.format` — because several +templates legitimately contain braces (TOML tables, MyST `{astra}` roles). +Substitution is strict, so a missing key raises. + +**`.gitignore` converges entry-wise, not by marker.** +`templates.gitignore_entries()` is the template minus comments and blanks; +`missing_gitignore_entries(text)` is what a repair appends, in template +order. Idempotency is therefore structural — a pattern already in the file +is never re-added, whoever wrote it — and a pattern introduced by a later +lc release still reaches projects that already have a `.gitignore`, which +a "marker present ⇒ done" check would have skipped. `GITIGNORE_HEADER` is +cosmetic only; never make correctness depend on it. + +**What `lc init` converges** — idempotently, never overwriting a file the +user owns: + +| Path | Role | +|---|---| +| `astra.yaml`, `universes/baseline.yaml`, `src/` | ASTRA spec scaffold (from astra's boilerplate; the `container:` key is stripped — ASTRA carries no environment) | +| `pyproject.toml` | The uv project: **virtual** (no `[build-system]`), `lightcone-cli` pinned in its own dependencies — the engine lives inside the experiment's lock | +| `.python-version` | Exact interpreter patch (`engine.constants.DEFAULT_PYTHON`) | +| `uv.lock`, `.venv` | `uv lock`, then `uv sync --locked --exact --compile-bytecode` | +| `.gitignore` | One managed block of patterns; convergence ensures each is present | +| `results/` + `README.md` | Where outputs land; the README states the materialize-don't-hand-write contract | +| `myst.yml`, `index.md` | Template MyST report referencing `astra.yaml` *by path* | + +- **Convergence, not scaffolding.** Each item is created if missing, + offered to a conservative `repair(text) -> str | None` hook otherwise, + and left alone when the hook returns `None`. `--check` computes the + same report without writing (exit 1 when not converged); `--json` + emits `{converged, created, repaired, unchanged, warnings}`. +- **An authored `Containerfile` is refused**, before any write: images + are generated from the lock, so a hand-written one would be silently + ignored. The user's own file operation is the consent to migrate — no + flag substitutes for it (spec §8). +- **uv is required** and is invoked through a single seam + (`commands._run_uv`) that tests monkeypatch. Every uv invocation + carries an explicit `--project` — uv's own walk-up discovery is never + trusted (spec §4). +- **Project discovery** is one rule (`engine.project.find_root`): the + nearest ancestor, including the start directory, holding an + `astra.yaml`. + +### Recorded deviations from the spec + +- **No `AGENTS.md` scaffolding** (spec §2 calls for an agent notes + stanza). Dropped by decision: it documented four verbs, three of which + don't exist yet, and the scaffold shouldn't assert behavior the CLI + can't deliver. Revisit at layer 4, when the verbs it describes are + real. `tests/test_project.py::test_converge_writes_no_agent_notes` + pins the absence, so re-adding it is a deliberate act. +- Scaffolded file bodies stay descriptive of what works today — see the + trimmed `results/README.md` for the same reason. ## Extending the Codebase | To... | Read | Key patterns | |---|---|---| -| Add a CLI command | `src/lightcone/cli/commands.py` | `@main.command()`, project discovery via `_project_root()` | -| Change manifest semantics | `src/lightcone/engine/manifest.py` + `tests/test_manifest.py` | Bump `SCHEMA_VERSION`; add a test | -| Change Snakefile shape | `src/lightcone/engine/snakefile.py` + `tests/test_snakefile.py` | Includes a `snakemake -n` parse test | -| Add container features | `src/lightcone/engine/container.py` | `compute_image_tag()`, build/resolve functions | +| Add the next layer | the spec (§11 = the layer ordering) | Land code + tests + deps together; update the layer table above. Docs are deliberately deferred | +| Change what a scaffolded file contains | `src/lightcone/engine/templates/files/` | Edit the `.tmpl`; add new ones to `TEMPLATE_NAMES` + a renderer | +| Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | +| Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | +| Add an engine constant | `src/lightcone/engine/constants.py` | Constants ship inside the locked engine — never resolved at run time | ## Test Patterns -- `tests/test_manifest.py` — pure-function tests for the integrity layer -- `tests/test_snakefile.py` — generator tests; final test runs `snakemake -n` on the output -- `tests/test_status.py` / `tests/test_verify.py` — end-to-end against a tmp project -- `tests/test_cli.py` — Click `CliRunner().invoke(main, [...])` patterns +- `tests/conftest.py` — the `fake_uv` autouse fixture stubs the uv seam + (`engine.project._run_uv`), emulating `uv lock` writing `uv.lock` and + `uv sync` making `.venv`, plus an isolated `$HOME`. Tests are hermetic: + no network, no real resolution. +- `tests/test_project.py` — discovery and convergence semantics, called + directly. This is where scaffold behavior is tested. +- `tests/test_cli.py` — the CLI surface only: flags reaching the engine, + rendering, exit codes, error translation. Rich wraps output at terminal + width, so assert on short unwrappable fragments. +- `tests/test_templates.py` — template loading (guards the packaging of + package data), strict substitution, marker presence. ## Conventions - Ruff for linting (E, F, I, N, W, UP), line length 100, target Python 3.11 -- mypy strict mode with `namespace_packages = true`, `explicit_package_bases = true` -- Manifest filename is fixed: `.lightcone-manifest.json` (don't change without bumping `SCHEMA_VERSION`) -- Snakemake's `directory()` outputs require excluding `.snakemake_timestamp` from the hash +- mypy strict mode with `namespace_packages = true`, + `explicit_package_bases = true` +- Docstrings and comments carry *why*, and cite the spec section + (`spec §7`) when they encode a design decision diff --git a/justfile b/justfile deleted file mode 100644 index 1977e37c..00000000 --- a/justfile +++ /dev/null @@ -1,88 +0,0 @@ -# lightcone-cli — justfile -# Install just: https://github.com/casey/just -# Usage: just - -# Show available recipes -default: - @just --list - -# ── Development ──────────────────────────────────────────────────────────────── - -# Sync all dependency groups (dev + docs) -install: - uv sync --all-groups - -# Sync only the dev group -install-dev: - uv sync --group dev - -# Run the test suite -test *ARGS: - uv run pytest {{ ARGS }} - -# Run tests with coverage report -test-cov: - uv run pytest --cov=src/lightcone --cov-report=term-missing --cov-report=html - -# Lint with ruff + mypy -lint: - uv run ruff check src/ tests/ - uv run mypy src/ - -# Auto-fix lint issues -fix: - uv run ruff check --fix src/ tests/ - uv run ruff format src/ tests/ - -# Format code -fmt: - uv run ruff format src/ tests/ - -# Run all checks (lint + tests) -check: lint test - -# ── Documentation ────────────────────────────────────────────────────────────── - -# Sync the docs dependency group -docs-install: - uv sync --group docs - -# Build the documentation site (outputs to site/) -docs: docs-install - uv run zensical build - -# Build with strict mode (fail on warnings; --strict is accepted but not yet enforced by zensical) -docs-strict: docs-install - uv run zensical build --strict - -# Serve documentation with live reload at http://127.0.0.1:8000 -docs-serve: docs-install - uv run zensical serve - -# Serve on a custom port -docs-serve-port port="8080": docs-install - uv run zensical serve --dev-addr 0.0.0.0:{{ port }} - -# Remove the built site directory -docs-clean: - rm -rf site/ - -# ── Package ──────────────────────────────────────────────────────────────────── - -# Build the wheel and sdist -build: - uv build - -# Show the current version (from git tag via hatch-vcs) -version: - uv run hatch version - -# Clean build artifacts -clean: - rm -rf dist/ build/ site/ *.egg-info src/*.egg-info - -# ── Evals ────────────────────────────────────────────────────────────────────── - -# The eval is a plain GitHub Actions workflow: .github/workflows/eval.yml -# scaffolds a project from evals/tasks/, runs Claude Code with -# evals/prompt.md, and checks the results with lc status + astra validate. diff --git a/pyproject.toml b/pyproject.toml index 6e2d2532..bc3763ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "lightcone-cli" dynamic = ["version"] -description = "Lightcone command-line toolchain" +description = "Lightcone Research's agentic execution layer" readme = "README.md" license = { text = "BSD-3-Clause" } requires-python = ">=3.11" @@ -23,25 +23,16 @@ classifiers = [ "Topic :: Scientific/Engineering", ] +# Clean rebuild: dependencies are re-added with the layer that needs +# them. Layer 1 (`lc init` scaffolding) needs the ASTRA spec tooling, the +# CLI framework, and console rendering — nothing else. dependencies = [ - # Pinned: astra-tools 0.2.14 made `astra init` idempotent, changing - # the callback signature this release's `lc init` delegates to. - "astra-tools==0.2.11", + # Pinned: the spec scaffold delegates to astra's boilerplate helper, + # whose surface has changed across releases (0.2.14 exposes the + # public `create_boilerplate`). Bump deliberately. + "astra-tools==0.2.14", "click>=8.0", - "pyyaml>=6.0", "rich>=13.0", - "jinja2>=3.0", - "snakemake>=9.0", - "snakemake-interface-executor-plugins>=9.0", - "snakemake-interface-common>=1.14", - "dask>=2024.1", - "distributed>=2024.1", - # Dask Gateway client for JupyterHub/Kubernetes deployments. A - # normal dependency (not an extra) so `lc run` works out of the box - # on a hub and scaffolded project images inherit it via their - # lightcone-cli pin. - "dask-gateway>=2024.1", - "rocrate>=0.11", ] [dependency-groups] @@ -50,7 +41,6 @@ dev = [ "pytest-cov", "ruff", "mypy", - "types-PyYAML", ] docs = [ "zensical>=0.0.33", @@ -68,16 +58,10 @@ build-backend = "hatchling.build" source = "vcs" [tool.hatch.build.targets.wheel] -packages = [ - "src/lightcone", - "src/snakemake_executor_plugin_dask", -] +packages = ["src/lightcone"] [tool.hatch.build.targets.sdist] -include = [ - "src/lightcone", - "src/snakemake_executor_plugin_dask", -] +include = ["src/lightcone"] [tool.ruff] target-version = "py311" @@ -93,22 +77,6 @@ namespace_packages = true explicit_package_bases = true mypy_path = "src" -[[tool.mypy.overrides]] -module = ["dask.*", "distributed.*"] -ignore_missing_imports = true -follow_untyped_imports = true - -[[tool.mypy.overrides]] -# Untyped and without explicit re-exports; skip so `from dask_gateway -# import Gateway` doesn't trip strict no_implicit_reexport. -module = ["dask_gateway", "dask_gateway.*"] -ignore_missing_imports = true -follow_imports = "skip" - -[[tool.mypy.overrides]] -module = ["rocrate.*"] -ignore_missing_imports = true - [[tool.mypy.overrides]] module = ["astra.*"] ignore_missing_imports = true @@ -116,6 +84,3 @@ follow_untyped_imports = true [tool.pytest.ini_options] testpaths = ["tests"] -markers = [ - "slow: tests that spin up real subsystems (dask cluster, etc.) — opt-in via -m slow", -] diff --git a/src/lightcone/cli/__init__.py b/src/lightcone/cli/__init__.py index ae164cc2..599ba55f 100644 --- a/src/lightcone/cli/__init__.py +++ b/src/lightcone/cli/__init__.py @@ -11,6 +11,9 @@ def main() -> None: + # The launcher (discover → mode-detect → scrub UV_* → converge → + # delegate into the project-locked engine, spec §4) hooks in here when + # its layer lands. Until then `lc` dispatches directly. from lightcone.cli.commands import main as _main _main() diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 51523b02..0e91cc32 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -1,40 +1,29 @@ """Command-line interface for lightcone-cli — the ASTRA execution layer. -The redesigned CLI is a thin shim over Snakemake. Provenance integrity -(per-output content-addressed manifests) is implemented in -:mod:`lightcone.engine.manifest`; ``lc run`` generates a Snakefile from -``astra.yaml`` and shells out to ``snakemake``. +This is a clean rebuild against the normative design spec, re-added layer +by layer. -Commands: -- ``lc init`` — idempotently converge a project scaffold (spec, Containerfile, - gitignore, MyST report template, venv); ``--check``/``--json`` for agents. -- ``lc run`` — generate Snakefile and run snakemake. -- ``lc status`` — manifest-driven status walk (no Snakemake needed). -- ``lc verify`` — recompute hashes and validate the provenance chain. -- ``lc build`` — build containers from Containerfiles. +**Layer 1 — project scaffolding.** The only verb is ``lc init``. What a +project *is* and how one is converged lives in +:mod:`lightcone.engine.project`; the file templates live in +:mod:`lightcone.engine.templates`. This module owns only the CLI: flags, +console rendering, and exit codes. -The global config at ``~/.lightcone/config.yaml`` is auto-created with -defaults on first invocation if missing. +The remaining verbs — ``lc materialize``, ``lc run``, ``lc status``, +``lc verify``, ``lc build`` — arrive with their layers. """ from __future__ import annotations import json import logging -import os -import re -import shutil -import subprocess import sys -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext from pathlib import Path import click -import yaml from rich.console import Console -from lightcone.engine.container import ContainerBuildError +from lightcone.engine.project import ConvergenceReport, ProjectError, converge console = Console() logger = logging.getLogger(__name__) @@ -43,68 +32,25 @@ class _EngineErrorGroup(click.Group): """Render engine errors as clean CLI errors instead of tracebacks. - The engine raises :class:`ContainerBuildError` (and its subclass - ``CloudBuildError``) from many entry points — tag hashing, builds, - status walks. Translating once at the group boundary keeps every - command, present and future, from leaking a raw traceback; click - prints ``ClickException`` messages cleanly and exits 1. + The engine raises its own exception types from many entry points. + Translating once at the group boundary keeps every command, present + and future, from leaking a raw traceback; click prints + ``ClickException`` messages cleanly and exits 1. """ def invoke(self, ctx: click.Context) -> object: try: return super().invoke(ctx) - except ContainerBuildError as e: + except ProjectError as e: raise click.ClickException(str(e)) from e -def _config_path() -> Path: - return Path.home() / ".lightcone" / "config.yaml" - - -def _ensure_global_config() -> None: - """Create ``~/.lightcone/config.yaml`` with defaults if missing.""" - config = _config_path() - if config.exists(): - return - config.parent.mkdir(parents=True, exist_ok=True) - config.write_text( - yaml.safe_dump( - { - # Container runtime used by `lc build` and embedded in every - # recipe by `lc run`. ``auto`` picks the first of - # podman/docker/podman-hpc found on PATH (skipping docker if - # its daemon is unreachable); set explicitly to pin. ``none`` - # disables containerization entirely. - "container": {"runtime": "auto"}, - } - ) - ) - - @click.group(cls=_EngineErrorGroup) @click.version_option(package_name="lightcone-cli") @click.pass_context def main(ctx: click.Context) -> None: """lightcone-cli — execution layer for ASTRA projects.""" ctx.ensure_object(dict) - _ensure_global_config() - - -# ============================================================================= -# Project discovery -# ============================================================================= - - -def _project_root(start: Path | None = None) -> Path: - """Walk up from cwd until we find ``astra.yaml``. Errors if absent.""" - p = (start or Path.cwd()).resolve() - for parent in [p, *p.parents]: - if (parent / "astra.yaml").is_file(): - return parent - raise click.ClickException( - "No astra.yaml found in current directory or any parent. " - "Run `lc init` to create one." - ) # ============================================================================= @@ -121,7 +67,11 @@ def _project_root(start: Path | None = None) -> Path: @main.command() @click.argument("directory", type=click.Path(file_okay=False, path_type=Path), default=".") @click.option("--no-git", is_flag=True, help="Skip git init") -@click.option("--no-venv", is_flag=True, help="Skip Python venv creation") +@click.option( + "--no-sync", + is_flag=True, + help="Skip materializing the .venv (uv lock still runs).", +) @click.option( "--check", "check_only", @@ -137,1289 +87,80 @@ def _project_root(start: Path | None = None) -> Path: is_flag=True, help="Emit the convergence report as JSON on stdout.", ) -@click.option( - "--scratch", - "scratch_override", - default=None, - type=str, - help=( - "Scratch root for snakemake state, dask spill, and run locks. " - "Overrides the site default. Shell expressions like $SCRATCH are " - "expanded at run time (kept verbatim in the project config)." - ), -) def init( directory: Path, no_git: bool, - no_venv: bool, + no_sync: bool, check_only: bool, as_json: bool, - scratch_override: str | None, ) -> None: """Converge DIRECTORY into an ASTRA project (idempotent). Safe to re-run at any time: creates whatever is missing, repairs the - pieces lightcone manages, and never overwrites files you own. - Problems it can see but must not fix (e.g. an unsupported directory - COPY in your Containerfile) are reported as warnings. A directory - that already holds an ``astra.yaml`` is adopted, not rejected. + pieces lightcone manages, and never overwrites files you own. A + directory that already holds an ``astra.yaml`` is adopted, not + rejected. The spec scaffold (``astra.yaml``, ``universes/baseline.yaml``) - follows the ``astra init`` boilerplate; on top of it sit the - lightcone pieces: ``Containerfile`` + ``requirements.txt``, - ``.gitignore`` entries, ``.lightcone/`` project state, a template - MyST report (``myst.yml`` + ``index.md``), and an optional venv. + follows the ``astra init`` boilerplate; on top of it sit the uv + project (``pyproject.toml`` with lightcone-cli locked in, + ``.python-version``, ``uv.lock``, ``.venv``), ``.gitignore`` entries, + ``results/``, and a template MyST report (``myst.yml`` + + ``index.md``). """ - from lightcone.engine.site_registry import detect_current_site - directory = directory.resolve() write = not check_only - report: dict[str, list[str]] = { - "created": [], - "repaired": [], - "unchanged": [], - "warnings": [], - } - - def _converge(name: str, present: bool, apply: Callable[[], object]) -> None: - if present: - report["unchanged"].append(name) - else: - report["created"].append(name) - if write: - apply() - - def _converge_file( - name: str, - path: Path, - template: str, - repair: Callable[[str], str | None] | None = None, - ) -> None: - """Create *path* from *template* if missing; else offer it to *repair*. - - ``repair`` receives the current text and returns the fixed text, - or ``None`` when the file is already fine. Repairs must be - conservative by construction — user-authored content is never - touched. - """ - if not path.exists(): - report["created"].append(name) - if write: - path.write_text(template) - elif repair is not None and (fixed := repair(path.read_text())) is not None: - report["repaired"].append(name) - if write: - path.write_text(fixed) - else: - report["unchanged"].append(name) - - if write: - if not as_json: - console.print(f"[cyan]{_LIGHTCONE}[/cyan]") - directory.mkdir(parents=True, exist_ok=True) - - # Spec scaffold: astra.yaml + universes/baseline.yaml. astra's init - # *command* refuses non-empty directories and overwrites .gitignore - # — both wrong for convergence — so use the bare scaffold API and - # manage .gitignore ourselves below. - def _scaffold_spec() -> None: - try: - # Public API (LightconeResearch/astra-tools#99). The ignore - # is for astra-tools releases that predate it; mypy will - # flag it as unused once the dependency pin catches up. - from astra.cli import create_boilerplate # type: ignore[attr-defined] - except ImportError: # astra-tools ≤ 0.2.x without the public API - create_boilerplate = None - if create_boilerplate is not None: - create_boilerplate(directory) - else: - from astra.cli import _create_boilerplate_astra_yaml - - (directory / "universes").mkdir(exist_ok=True) - _create_boilerplate_astra_yaml(directory) - # The boilerplate recipes reference scripts under src/ (e.g. - # ``python src/main.py``); astra's own init creates the - # directory, so the scaffold must too. - (directory / "src").mkdir(exist_ok=True) - # Point the spec at our project-local Containerfile. The astra - # boilerplate ships a registry image so the scaffold is runnable - # as-is, but we want lightcone projects to build their own image - # so dependencies can evolve under content-addressed rebuilds. - # Rewrite the top-level ``container:`` line whatever image the - # boilerplate names, so astra bumping its default doesn't - # silently disable the rewrite. - astra_yaml_path = directory / "astra.yaml" - rewritten = re.sub( - r"(?m)^container:.*$", - "container: Containerfile", - astra_yaml_path.read_text(), - count=1, - ) - if "container: Containerfile" not in rewritten: - report["warnings"].append( - "astra.yaml: no top-level `container:` line found to point " - "at the Containerfile; set it manually." - ) - astra_yaml_path.write_text(rewritten) - - _converge("astra.yaml", (directory / "astra.yaml").exists(), _scaffold_spec) - - # One scaffold everywhere — the Containerfile is agnostic to the - # execution environment. requirements.txt holds only the analysis - # dependencies; the execution stack (lightcone-cli, which carries - # snakemake, dask, distributed, dask-gateway) is a separate - # Containerfile layer so the same image can wrap recipes locally or - # run as a Dask Gateway worker pod on a hub, while the project venv - # stays free of it — `lc` lives outside the venv. Anything - # pod-specific (uid, mounts) is deployment configuration, not image - # content. - cf_path = directory / "Containerfile" - _converge_file( - "Containerfile", - cf_path, - _CONTAINERFILE_TEMPLATE.format(lc_requirement=_lightcone_requirement()), - ) - # Advisory: a Containerfile with directory COPY sources belongs to - # the user, so init won't edit it — but lc build / lc run will - # reject it, so say so now rather than at build time. - if cf_path.is_file(): - from lightcone.engine.container import directory_copy_sources + if write and not as_json: + console.print(f"[cyan]{_LIGHTCONE}[/cyan]") - if bad := directory_copy_sources(cf_path, directory): - report["warnings"].append( - "Containerfile: COPY/ADD of a directory " - f"({', '.join(repr(s) for s in bad)}) is not supported and " - "lc build/run will fail. The image is an environment — " - "recipes run against the live project tree, so remove the " - "line(s)." - ) - _converge_file( - "requirements.txt", - directory / "requirements.txt", - _REQUIREMENTS, - ) - - # .gitignore: create with base + lightcone entries if absent; append - # the block once to a user-owned file (keyed on the "# lightcone-cli" - # marker). - _converge_file( - ".gitignore", - directory / ".gitignore", - _GITIGNORE_BASE + _GITIGNORE_APPEND, - repair=_repair_gitignore, - ) - - # .lightcone/ project state dir + lightcone.yaml. An explicit - # --scratch converges the stored scratch_root; without it an - # existing config is left alone. A file we can't parse is left - # untouched and reported — init must stay safe to re-run. - cfg_path = directory / ".lightcone" / "lightcone.yaml" - cfg_name = ".lightcone/lightcone.yaml" - if not cfg_path.exists(): - report["created"].append(cfg_name) - if write: - cfg_path.parent.mkdir(exist_ok=True) - project_cfg: dict[str, object] = {"target": "local"} - if scratch_override: - project_cfg["scratch_root"] = scratch_override - cfg_path.write_text(yaml.safe_dump(project_cfg)) - else: - try: - existing_cfg = yaml.safe_load(cfg_path.read_text()) - except yaml.YAMLError: - existing_cfg = None - if not isinstance(existing_cfg, dict): - report["unchanged"].append(cfg_name) - report["warnings"].append( - f"{cfg_name} is not a valid YAML mapping; left untouched" - + (" (--scratch not applied)" if scratch_override else "") - + "." - ) - elif scratch_override and existing_cfg.get("scratch_root") != scratch_override: - report["repaired"].append(cfg_name) - if write: - existing_cfg["scratch_root"] = scratch_override - cfg_path.write_text(yaml.safe_dump(existing_cfg)) - else: - report["unchanged"].append(cfg_name) - - # results/ ships with a README explaining the materialization - # contract — the placeholder directory alone is invisible in git - # (empty + ignored), so the README is what actually tells a human - # or agent opening the project where outputs land and that they - # must come from `lc run`, not be written by hand. - results_dir = directory / "results" - if results_dir.exists() and not results_dir.is_dir(): - report["unchanged"].extend(["results/", "results/README.md"]) - report["warnings"].append( - "results exists but is not a directory; outputs cannot " - "materialize until it is one." - ) - else: - _converge("results/", results_dir.is_dir(), results_dir.mkdir) - _converge_file("results/README.md", results_dir / "README.md", _RESULTS_README) - - # Template MyST report. MyST support is a recommended add-on on top of - # the spec, not part of it — which is why the report scaffold lives here - # and not in `astra init`. - _converge_file("myst.yml", directory / "myst.yml", _MYST_YML) - project_name = directory.name or "My Analysis" - _converge_file("index.md", directory / "index.md", f"# {project_name}\n" + _INDEX_MD_BODY) - - if not no_git: - _converge( - ".git", - (directory / ".git").exists(), - lambda: subprocess.run(["git", "init", "-q"], cwd=directory, check=False), - ) - - if not no_venv: - _converge( - ".venv", - (directory / ".venv").exists(), - lambda: _create_venv(directory, quiet=as_json), - ) - - converged = not report["created"] and not report["repaired"] + report = converge(directory, write=write, git=not no_git, sync=not no_sync) if as_json: - payload: dict[str, object] = {"converged": converged, **report} - click.echo(json.dumps(payload, indent=2)) + click.echo(json.dumps(report.as_dict(), indent=2)) elif check_only: - if converged: - console.print(f"[green]✓[/green] {directory} is converged — nothing to do") - else: - for item in report["created"]: - console.print(f" [yellow]would create[/yellow] {item}") - for item in report["repaired"]: - console.print(f" [yellow]would repair[/yellow] {item}") - for warning in report["warnings"]: - console.print(f" [yellow]![/yellow] {warning}") + _render_check(report, directory) else: - for item in report["created"]: - console.print(f"[green]✓[/green] created {item}") - for item in report["repaired"]: - console.print(f"[green]✓[/green] repaired {item}") - for warning in report["warnings"]: - console.print(f"[yellow]![/yellow] {warning}") - if converged: - console.print(f"\n[green]Project already converged at[/green] {directory}") - else: - console.print(f"\n[green]Project converged at[/green] {directory}") - - # Surface the resolved scratch root if a known site was detected — - # gives users early visibility into where lc run will keep its - # operational state (snakemake metadata, dask spill, cross-node - # locks). On NERSC this is critical: $HOME and CFS are mounted via - # DVS (no flock, slow small-file I/O), so lightcone keeps - # everything on $SCRATCH (Lustre). - site = detect_current_site() - if site: - scratch_expr = scratch_override or site.get("scratch_root") - if scratch_expr: - console.print(f"\n[dim]Detected site:[/dim] {site.display_name}") - console.print( - f"[dim]Scratch root for lc run:[/dim] [cyan]{scratch_expr}[/cyan] " - f"[dim](resolved at run time)[/dim]" - ) + _render_run(report, directory) - # Next steps only make sense for a freshly scaffolded spec. - if "astra.yaml" in report["created"]: - console.print("\nNext steps:") - console.print( - f" • Go to the newly created directory [cyan]cd {directory}[/cyan]" - ) - console.print( - " • Describe your analysis in [cyan]astra.yaml[/cyan], " - "then materialize it with [cyan]lc run[/cyan]" - ) - console.print( - " • Preview the report with [cyan]myst start[/cyan] " - "(requires the MyST CLI: [cyan]npm i -g mystmd[/cyan])" - ) - - if check_only and not converged: + if check_only and not report.converged: sys.exit(1) -def _create_venv(directory: Path, quiet: bool = False) -> None: - """Create ``.venv`` in ``directory`` with the analysis dependencies. - - Installs ``requirements.txt`` only — deliberately *not* - lightcone-cli. The venv exists to run the analysis code; ``lc`` - itself lives outside it (e.g. ``uv tool install lightcone-cli``), - and a second copy inside the venv would shadow it with whatever - version PyPI resolves. - """ - - def _status(msg: str) -> AbstractContextManager[object]: - return nullcontext() if quiet else console.status(msg) - - if shutil.which("uv"): - with _status("[dim]Creating virtual environment…[/dim]"): - subprocess.run( - ["uv", "venv", "--python", "3.12", ".venv"], - cwd=directory, - check=False, - capture_output=True, - ) - with _status("[dim]Installing project requirements…[/dim]"): - subprocess.run( - [ - "uv", - "pip", - "install", - "--python", - ".venv/bin/python", - "-r", - "requirements.txt", - ], - cwd=directory, - check=False, - capture_output=True, - ) +def _render_check(report: ConvergenceReport, directory: Path) -> None: + if report.converged: + console.print(f"[green]✓[/green] {directory} is converged — nothing to do") else: - with _status("[dim]Creating virtual environment…[/dim]"): - subprocess.run( - ["python", "-m", "venv", ".venv"], - cwd=directory, - check=False, - capture_output=True, - ) - with _status("[dim]Installing project requirements…[/dim]"): - subprocess.run( - [ - ".venv/bin/python", - "-m", - "pip", - "install", - "-q", - "-r", - "requirements.txt", - ], - cwd=directory, - check=False, - capture_output=True, - ) - - -_CONTAINERFILE_TEMPLATE = """\ -FROM python:3.12-slim - -WORKDIR /app - -# Execution stack — lets this image run rules on any backend, including -# as a Dask Gateway worker pod. Kept out of requirements.txt so the -# project venv stays free of it (`lc` lives outside the venv), and -# installed first so this heavy layer stays cached across -# requirements.txt edits. -RUN pip install --no-cache-dir {lc_requirement} - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# No COPY of the project source: recipes run against the live project -# tree (bind-mounted locally, shared filesystem on a hub), so the image -# is a pure environment — it only rebuilds when dependencies change, -# never on code edits. -""" - - -_REQUIREMENTS = """\ -numpy -pandas -""" - - -def _repair_gitignore(text: str) -> str | None: - """Append the managed block once to a user-owned .gitignore. - - Keyed on the block's ``# lightcone-cli`` marker so re-runs never - duplicate it. This is `lc init`'s only repair hook: adoption of a - project that already has its own .gitignore. - """ - if "# lightcone-cli" not in text: - return text + _GITIGNORE_APPEND - # Legacy managed-block upgrades, applied line-by-line so everything - # else in the file stays user territory: - # * a bare ``results/`` rule ignores the whole directory, and git - # cannot re-include results/README.md beneath an excluded - # directory; - # * trailing-slash ``.snakemake/`` entries never match the symlink - # that ``.snakemake`` becomes under a scratch root. - slash_fixes = { - ".snakemake/": ".snakemake", - ".snakemake.legacy/": ".snakemake.legacy", - } - out: list[str] = [] - changed = False - for line in text.splitlines(): - stripped = line.strip() - if stripped == "results/": - out.append("results/*") - if "!results/README.md" not in text: - out.append("!results/README.md") - changed = True - elif stripped in slash_fixes: - out.append(slash_fixes[stripped]) - changed = True - else: - out.append(line) - if changed: - return "\n".join(out) + ("\n" if text.endswith("\n") else "") - return None - - -def _lightcone_requirement() -> str: - """The lightcone-cli requirement pinned into the project image. - - The project image must be able to execute rules on any backend — - including as a Dask Gateway worker pod, where the dask worker and - the child snakemake run *inside* the image. lightcone-cli carries - that whole stack (snakemake, dask, distributed, dask-gateway) as - normal dependencies, so one requirement covers it. The pin mirrors - the version running ``lc init`` to keep driver and image in - lockstep; dev builds fall back to unpinned (their version isn't - published). - """ - from importlib.metadata import PackageNotFoundError, version - - try: - v = version("lightcone-cli") - except PackageNotFoundError: - v = "" - return f"lightcone-cli=={v}" if v and "dev" not in v else "lightcone-cli" - - -# Written when the project has no .gitignore of its own; mirrors the base -# entries `astra init` would have written. -_GITIGNORE_BASE = """\ -# ASTRA Analysis -__pycache__/ -*.py[cod] -.venv/ -.ipynb_checkpoints/ -.DS_Store -""" - - -# Managed block appended to any pre-existing .gitignore; the -# "# lightcone-cli" marker keeps the append idempotent. The -# ``.snakemake`` entries have no trailing slash on purpose: when a -# scratch root is active the project's ``.snakemake`` is a *symlink* -# into scratch, and a trailing-slash pattern matches only real -# directories. -_GITIGNORE_APPEND = """ -# lightcone-cli -.lightcone/Snakefile -.lightcone/snakefile-config.json -.snakemake -.snakemake.legacy -results/* -!results/README.md - -# MyST build output -_build/ -""" - - -_RESULTS_README = """\ -# results/ - -Materialized outputs land here, one directory per universe and output: - - results/// - -Each output directory carries a `.lightcone-manifest.json` sidecar -recording exactly how it was produced: recipe, container image, -decisions, input hashes, and the output's content hash. - -- Produce or refresh outputs with `lc run` — never write files here by - hand. Hand-placed or edited files fail `lc verify` (the content hash - won't match, and a missing manifest forces a re-run). -- `lc status` shows what is materialized, stale, or missing. -- Everything in this directory except this README is git-ignored; - outputs are reproducible from `astra.yaml`, not versioned. -""" - - -# The template report references the boilerplate ``astra.yaml`` elements by -# path via the MySTRA plugin, so the ids used below must track the astra -# boilerplate (``example_method``, ``main_result``). -_MYST_YML = """\ -# MyST configuration for the analysis report (https://mystmd.org/). -# The MySTRA plugin resolves {astra}`...` references against astra.yaml. -# `latest` always tracks the newest MySTRA release; for a reproducible -# build, pin a tag instead, e.g. .../releases/download/v0.0.1/mystra.mjs -version: 1 -project: - plugins: - - https://github.com/LightconeResearch/MySTRA/releases/latest/download/mystra.mjs - toc: - - file: index.md -site: - template: book-theme -""" - - -_INDEX_MD_BODY = """ -> **TODO:** this report was scaffolded by `lc init`. It references the -> analysis elements declared in `astra.yaml` *by path* — figures, decisions, -> and numbers stay single-sourced in the analysis, so never hard-type a -> measured value here. Preview with `myst start` (requires the MyST CLI). - -## Introduction - -TODO: the research question, its context, and why it matters. - -## Methods - -TODO: describe the approach. Reference the decisions the analysis exposes -rather than restating them — for example, we adopt the -{astra}`decisions.example_method` for this analysis: - -:::{astra} decisions.example_method -::: - -## Results - -TODO: present the outputs. Once `lc run` has materialized results, pull -numbers in live, e.g.: - -% The analysis yields {astra:value}`outputs.main_result`. - -:::{astra} outputs -::: -""" - -# ============================================================================= -# lc run -# ============================================================================= - - -def _abort_on_perlmutter_login() -> None: - """Stop-gap: refuse ``lc run`` on a Perlmutter login node. - - NERSC sets ``NERSC_HOST=perlmutter`` on every node; SLURM sets - ``SLURM_JOB_ID`` only inside an allocation. Their conjunction (NERSC - host + no allocation) unambiguously marks a login node, where shared - CPU and the absence of compute resources make a real run a bad idea. - - Bypassed when ``DASK_SCHEDULER_ADDRESS`` is set, matching the branch - in ``cluster_for_run``: if the user is targeting an external - scheduler the login-node CPU does not matter. - - Remove once proper site-backend gating exists. - """ - if os.environ.get("LIGHTCONE_ALLOW_LOGIN_NODE"): - return - if os.environ.get("NERSC_HOST") != "perlmutter": - return - if "SLURM_JOB_ID" in os.environ: - return - if os.environ.get("DASK_SCHEDULER_ADDRESS"): - return - raise click.ClickException( - "Refusing to run on a Perlmutter login node — compute work must " - "run inside a SLURM allocation.\n" - " Start one with, e.g.:\n" - " salloc -N 1 -C gpu -q interactive -t 1:00:00 -A \n" - " then re-run `lc run` from inside." - ) - - -@main.command() -@click.argument("outputs", nargs=-1) -@click.option("--universe", "-u", default=None, help="Universe to materialize") -@click.option("--jobs", "-j", default=None, type=int, help="Parallel jobs") -@click.option( - "--rerun-triggers", - default="code,input,mtime,params", - help="Comma-separated rerun-triggers (default: code,input,mtime,params)", -) -@click.option("--force", "-f", is_flag=True, help="Force re-materialization") -@click.option("--verbose", "-v", is_flag=True, help="Show full executor output") -def run( - outputs: tuple[str, ...], - universe: str | None, - jobs: int | None, - rerun_triggers: str, - force: bool, - verbose: bool, -) -> None: - """Materialize outputs declared in astra.yaml. - - Always dispatches through a Dask cluster: a ``LocalCluster`` on a - workstation, srun-launched workers inside a SLURM allocation, a - run-scoped Dask Gateway cluster on a JupyterHub deployment, or an - existing scheduler if ``DASK_SCHEDULER_ADDRESS`` is set. - """ - _abort_on_perlmutter_login() - - from lightcone.engine.container import load_runtime - from lightcone.engine.dask_cluster import cluster_for_run, gateway_branch_active - from lightcone.engine.scratch import ( - RunLockBusyError, - acquire_run_lock, - ensure_snakemake_symlink, - prepare_run_dirs, - resolve_scratch_root, - ) - from lightcone.engine.snakefile import discover_universes, generate - - project = _project_root() - universes = [universe] if universe else discover_universes(project) - - # Resolve scratch and prepare per-run directories before anything - # else. Snakemake's ``.snakemake/`` is redirected via symlink so its - # workflow lock and metadata land on a filesystem that honours - # ``flock`` (Lustre on NERSC) rather than DVS-mounted home/CFS where - # locks are silent no-ops. Dask spill and our cross-node stdout lock - # live alongside it. - rundirs = prepare_run_dirs(project) - ensure_snakemake_symlink(project, rundirs.snakemake_state) - if verbose: - console.print(f"[dim]Scratch root:[/dim] {resolve_scratch_root(project)}") - - choice = load_runtime(project_path=project) - images = _ensure_images(project, runtime=choice.runtime) - snakefile_path, cfg_path = generate( - project, universes=universes, runtime=choice.runtime - ) - - # On the Gateway branch the cluster is created with one image — the - # worker pod is the container for every rule, so a spec declaring - # several distinct containers cannot be honoured per-rule. - worker_image: str | None = None - if gateway_branch_active(): - if len(images) > 1: - raise click.ClickException( - "This deployment runs recipes natively in worker pods, " - "which supports one container image per run; astra.yaml " - "declares several: " + ", ".join(images) + ". " - "Consolidate on a single Containerfile (or one shared " - "prebuilt image)." - ) - worker_image = images[0] if images else None - - # Provenance guard: when ``runtime: auto`` silently fell back to - # ``none`` and the spec declares any containers, the recipe will run - # on the host while the manifest's ``container_image`` field still - # records the declared image — i.e. a provenance lie. Warn loudly so - # the user installs a runtime, sets ``runtime: none`` explicitly, or - # removes the container declarations. - if choice.runtime == "none" and not choice.explicit: - cfg_data = json.loads(cfg_path.read_text()) - declared = sorted( - { - entry["container_image"] - for rule_entries in cfg_data.values() - for entry in rule_entries.values() - if entry.get("container_image") - } - ) - if declared: - console.print( - "[yellow]⚠ No container runtime found on PATH " - "(checked docker, podman, podman-hpc).[/yellow]\n" - " The following declared containers will be ignored:\n" - + "\n".join(f" [dim]•[/dim] {c}" for c in declared) - + "\n Recipes will run on the host without isolation, " - "but each manifest will still record\n" - " the declared [cyan]container_image[/cyan] — recorded " - "provenance will not match what executed.\n" - " Install [cyan]docker[/cyan], [cyan]podman[/cyan], or " - "[cyan]podman-hpc[/cyan], or set\n" - " [cyan]container: {runtime: none}[/cyan] in " - "[cyan]~/.lightcone/config.yaml[/cyan] to silence.\n" - ) - - targets: list[str] = [] - if outputs: - for o in outputs: - for u in universes: - targets.append(_target_for(project, o, u)) - # If no specific targets, pass nothing → snakemake runs `rule all`. - - n = str(jobs or os.cpu_count() or 1) - # Snakemake requires ``--cores`` to bound per-rule CPU; the dask - # plugin requires ``--jobs`` to bound parallel dispatch. We surface - # one knob and pass it as both. - cmd = _build_snakemake_cmd( - snakefile_path=snakefile_path, - project=project, - n=n, - rerun_triggers=rerun_triggers, - targets=targets, - force=force, - has_outputs=bool(outputs), - ) - - # Hold a project-level flock for the duration of the run. Acquiring - # it also clears any stale snakemake lock left by a previously - # crashed invocation — safe because we just proved we're alone on - # the project. Concurrent ``lc run`` on the same project bails - # cleanly rather than corrupting Snakemake state. - try: - run_lock_cm = acquire_run_lock(rundirs) - run_lock_cm.__enter__() - except RunLockBusyError as e: - raise click.ClickException(str(e)) - - with cluster_for_run( - verbose=verbose, - local_directory=str(rundirs.dask_local), - worker_image=worker_image, - max_workers=int(n), - ) as cluster_env: - env = {**os.environ, **cluster_env} - if verbose: - console.print(f"[dim]$ {' '.join(cmd)}[/dim]") - sys.exit( - _run_snakemake( - cmd, env=env, scratch_root=rundirs.root, verbose=verbose - ) - ) - - -def _run_snakemake( - cmd: list[str], - *, - env: dict[str, str], - scratch_root: Path, - verbose: bool, -) -> int: - """Run snakemake, forwarding the run's narrative output. - - The executor plugin prints each finished rule's block of - sentinel-prefixed lines (see :data:`lightcone.engine.runner.SENTINEL`) - on the snakemake process's stdout. We forward those lines — prefix - stripped — to the terminal and drop everything else snakemake emits - (DAG chatter, job stats), so a run reads as a clean narrative. - Verbose mode forwards the noise too. - - stderr is tailed into a bounded ring buffer so a workflow crash - leaves a real log behind without it being visible during a - successful run (verbose passes stderr straight through instead). - """ - from collections import deque - - from lightcone.engine.runner import SENTINEL - - proc = subprocess.Popen( - cmd, - env=env, - stdout=subprocess.PIPE, - stderr=None if verbose else subprocess.PIPE, - text=True, - bufsize=1, - ) - assert proc.stdout is not None - tail: deque[str] = deque(maxlen=400) - - def _pump_stderr() -> None: - assert proc.stderr is not None - for line in proc.stderr: - tail.append(line) - - import threading - - stderr_thread: threading.Thread | None = None - if not verbose: - stderr_thread = threading.Thread(target=_pump_stderr, daemon=True) - stderr_thread.start() - - for line in proc.stdout: - if line.startswith(SENTINEL): - sys.stdout.write(line[len(SENTINEL):]) - sys.stdout.flush() - elif verbose: - sys.stdout.write(line) - sys.stdout.flush() - - rc = proc.wait() - if stderr_thread is not None: - stderr_thread.join(timeout=5) - if rc != 0 and not verbose: - log = scratch_root / f"snakemake-stderr-{os.getpid()}.log" - try: - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text("".join(tail)) - console.print( - f"\n[red]✗ Workflow failed.[/red] " - f"Last snakemake stderr saved to [cyan]{log}[/cyan]." - ) - except OSError: - # Last-ditch: dump to stderr if scratch is unwritable. - sys.stderr.write("".join(tail)) - return rc - - -def _build_snakemake_cmd( - *, - snakefile_path: Path, - project: Path, - n: str, - rerun_triggers: str, - targets: list[str], - force: bool, - has_outputs: bool, -) -> list[str]: - """Build the snakemake argv list for ``lc run``. - - ``--rerun-triggers`` uses ``nargs=+`` in snakemake's argparse, so without - an explicit ``--`` separator it greedily consumes the first positional - target path as an extra trigger value, causing an "invalid choice" error. - - ``--shared-fs-usage`` lists everything *except* - ``software-deployment``. With it included (snakemake's default), - spawned job commands embed the *driver's* ``sys.executable`` — a - path that doesn't exist inside a Dask Gateway worker image. Without - it, workers invoke plain ``python`` from their own environment, - which is equally correct on the other backends: LocalCluster - threads and srun-launched SLURM workers inherit the driver's - activated environment (and SLURM setups already require it — see - the ``dask``-on-PATH check in the cluster module). One invocation - shape for every backend; everything else stays shared via the - common filesystem (persistence, inputs/outputs, sources). - """ - cmd: list[str] = [ - "snakemake", - "-s", - str(snakefile_path), - "-d", - str(project), - "--cores", - n, - "--jobs", - n, - "--executor", - "dask", - "--shared-fs-usage", - "persistence", - "input-output", - "sources", - "storage-local-copies", - "source-cache", - "--rerun-triggers", - *rerun_triggers.split(","), - ] - if force: - # ``--force`` scopes to explicit targets; ``rule all`` itself - # has no recipe, so force-all is the only useful sense when no - # targets were named. - cmd.append("--force" if has_outputs else "--forceall") - if targets: - cmd.append("--") - cmd.extend(targets) - return cmd - - -def _target_for(project: Path, output_id: str, universe: str) -> str: - """Translate an output id into a Snakemake target path (the manifest). - - Accepts either a bare ``output_id`` (root-level or unique sub-analysis - output) or a qualified ``analysis_id.output_id`` to disambiguate when - the same id appears in multiple sub-analyses. - """ - from astra.helpers import load_yaml, resolve_analysis_tree - - from lightcone.engine.manifest import MANIFEST_FILENAME - from lightcone.engine.tree import collect_tree_outputs, resolve_output_path - - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - matches = [] - for to in collect_tree_outputs(spec): - if to.output_def.get("recipe") is None: - continue - qualified = ( - f"{to.analysis_id}.{to.output_id}" if to.analysis_id else to.output_id - ) - if qualified == output_id or to.output_id == output_id: - matches.append((qualified, to)) - - if not matches: - raise click.ClickException( - f"Output '{output_id}' not found in astra.yaml or has no recipe." - ) - if len(matches) > 1: - opts = ", ".join(q for q, _ in matches) - raise click.ClickException( - f"Output '{output_id}' is ambiguous; qualify it as one of: {opts}" - ) - - _, to = matches[0] - target = ( - resolve_output_path(project, to, universe) / to.output_id / MANIFEST_FILENAME - ) - return str(target.relative_to(project)) - - -# ============================================================================= -# lc status -# ============================================================================= - - -@main.command() -@click.option("--universe", "-u", default=None) -@click.option( - "--json", - "as_json", - is_flag=True, - help="Emit machine-readable JSON instead of a styled table.", -) -def status(universe: str | None, as_json: bool) -> None: - """Report materialization status for every declared output.""" - from lightcone.engine.snakefile import discover_universes - from lightcone.engine.status import get_output_status - - project = _project_root() - universes = [universe] if universe else discover_universes(project) - - if as_json: - payload = { - "universes": [ - { - "universe_id": u, - "outputs": [ - { - "output_id": s.output_id, - "analysis_id": s.analysis_id, - "status": s.status, - "recipe_command": s.recipe_command, - } - for s in get_output_status(project, universe_id=u) - ], - } - for u in universes - ], - } - click.echo(json.dumps(payload, indent=2)) - return - - for u in universes: - console.print(f"\n[bold]Universe[/bold] [cyan]{u}[/cyan]") - for s in get_output_status(project, universe_id=u): - label = _status_label(s.status) - scope = f"[dim]{s.analysis_id}.[/dim]" if s.analysis_id else "" - console.print(f" {label} {scope}{s.output_id}") - - -_STATUS_STYLES = { - "ok": "[green]✓ ok[/green] ", - "stale": "[yellow]✸ stale[/yellow] ", - "missing": "[red]✗ miss[/red] ", - "alias": "[dim]→ alias[/dim] ", -} - - -def _status_label(s: str) -> str: - return _STATUS_STYLES.get(s, s) - - -# ============================================================================= -# lc verify -# ============================================================================= - - -@main.command() -@click.option("--universe", "-u", default=None) -def verify(universe: str | None) -> None: - """Validate the provenance chain by recomputing hashes.""" - from lightcone.engine.snakefile import discover_universes - from lightcone.engine.verify import verify_outputs - - project = _project_root() - universes = [universe] if universe else discover_universes(project) - - failed = 0 - for u in universes: - console.print(f"\n[bold]Universe[/bold] [cyan]{u}[/cyan]") - for r in verify_outputs(project, universe_id=u): - if r.passed: - console.print(f" [green]✓ ok[/green] {r.output_id}") - else: - failed += 1 - console.print( - f" [red]✗ {r.failure}[/red] {r.output_id} [dim]{r.detail}[/dim]" - ) - - if failed: - console.print(f"\n[bold red]{failed} integrity failure(s).[/bold red]") - sys.exit(1) - console.print("\n[green]All outputs verified.[/green]") - - -# ============================================================================= -# lc build -# ============================================================================= - - -@main.command() -@click.option("--force", is_flag=True, help="Rebuild all images even if cached") -@click.option( - "--runtime", - default=None, - help=( - "docker | podman | podman-hpc | kubernetes " - "(overrides ~/.lightcone/config.yaml)" - ), -) -def build(force: bool, runtime: str | None) -> None: - """Build container images declared in astra.yaml. - - Containerfile syntax is Dockerfile syntax — we use ``docker``, - ``podman``, or ``podman-hpc`` directly. Each Containerfile builds to - an OCI image tagged ``lc--`` in the runtime's local - image store. Pre-built registry images (``python:3.12-slim``, - ``ghcr.io/foo/bar:tag``) are skipped — the runtime pulls them at - ``lc run`` time. - - On a deployment without a local OCI runtime (the ``kubernetes`` - runtime on a lightcone JupyterHub), the same command builds through - the deployment's GCP Cloud Build service instead and pushes - ``/lc-:`` — same content-addressed - identity, zero configuration. - """ - from lightcone.engine.container import load_runtime - - project = _project_root() - resolved_runtime = runtime or load_runtime(project_path=project).runtime - - if resolved_runtime == "none": - console.print( - "[yellow]No container runtime available " - "(checked docker, podman, podman-hpc). " - "Install one to build images, or set [cyan]container.runtime[/cyan] " - "in [cyan]~/.lightcone/config.yaml[/cyan].[/yellow]" - ) - return - - _ensure_images(project, runtime=resolved_runtime, force=force) - console.print("[green]Done.[/green]") - - -def _ensure_images(project: Path, *, runtime: str, force: bool = False) -> list[str]: - """Build/pull every container image referenced in astra.yaml. - - Returns the distinct resolved images, in declaration order (local - tags or registry refs for Containerfile specs, prebuilt specs - as-is). No-op (and empty) when *runtime* is ``"none"``. - - Idempotent: skips images already present (local image store, or the - deployment registry on the ``kubernetes`` runtime — where builds go - through Cloud Build instead of a local OCI CLI). Used by ``lc build`` - (with ``--force`` exposed) and as a pre-flight by ``lc run`` so the - first invocation after editing a Containerfile doesn't fail mid-DAG - with a missing image. - """ - if runtime == "none": - return [] - - from astra.helpers import load_yaml, resolve_analysis_tree - - from lightcone.engine.container import ( - KUBERNETES, - build_image, - compute_image_tag, - image_exists_locally, - is_containerfile, - pull_image, - ) - from lightcone.engine.tree import collect_tree_outputs - - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - project_name = (spec.get("name") or project.name).lower().replace(" ", "-") - - images: list[str] = [] - seen: set[str] = set() - for to in collect_tree_outputs(spec): - recipe = to.output_def.get("recipe") or {} - spec_str = ( - recipe.get("container") - or to.analysis_spec.get("container") - or spec.get("container") - ) - if not spec_str or spec_str in seen: - continue - seen.add(spec_str) - if not is_containerfile(spec_str, project): - images.append(spec_str) - if runtime == KUBERNETES: - # Nothing to materialize: worker pods pull registry - # images straight from their source. - continue - # Pull so ``lc run`` can use ``--pull=never`` without - # depending on the runtime's registry resolution. - if image_exists_locally(spec_str, runtime=runtime) and not force: - continue - console.print(f"[cyan]Pulling[/cyan] {spec_str} [dim](via {runtime})[/dim]") - pull_image(spec_str, runtime=runtime) - continue - - if runtime == KUBERNETES: - images.append(_cloudbuild_image(project, spec_str, project_name, force)) - continue - - containerfile = project / spec_str - tag = compute_image_tag(project_name, containerfile, project) - images.append(tag) - if image_exists_locally(tag, runtime=runtime) and not force: - continue - console.print( - f"[cyan]Building[/cyan] {spec_str} → {tag} [dim](via {runtime})[/dim]" - ) - build_image(tag, containerfile, project, runtime=runtime) - return images - - -def _cloudbuild_image( - project: Path, spec_str: str, project_name: str, force: bool -) -> str: - """Ensure one Containerfile's image via GCP Cloud Build; return its ref.""" - from lightcone.engine.cloudbuild import ( - CloudBuildError, - cloudbuild_available, - ensure_image, - ) - - if not cloudbuild_available(): - raise click.ClickException( - "No image build backend on this host: the kubernetes runtime " - "has no local OCI CLI and this environment does not provide " - "the Cloud Build contract (LIGHTCONE_REGISTRY + " - "LIGHTCONE_BUILD_BUCKET). On a lightcone JupyterHub these are " - "injected into every user pod — ask the hub admin." - ) - - status = console.status(f"[cyan]Ensuring image for[/cyan] {spec_str} …") - status.start() - - def on_progress(phase: str, detail: str) -> None: - note = f" [dim]{detail}[/dim]" if detail else "" - status.update( - f"[cyan]Ensuring image for[/cyan] {spec_str}: {phase}{note}" - ) - - try: - ref = ensure_image( - project, - spec_str, - project_name=project_name, - force=force, - on_progress=on_progress, - ) - except CloudBuildError as e: - raise click.ClickException(str(e)) - finally: - status.stop() - console.print(f"[green]✓[/green] Worker image: [cyan]{ref}[/cyan]") - return ref - - -# ============================================================================= -# lc export -# ============================================================================= - - -@main.group() -def export() -> None: - """Export project artifacts in interoperable formats.""" - - -@export.command("wrroc") -@click.option( - "--output", - "-o", - type=click.Path(path_type=Path), - default=Path("./wrroc"), - help="Bundle directory (or .zip path with --zip).", - show_default=True, -) -@click.option( - "--universe", - "-u", - multiple=True, - help="Restrict to specific universes (default: all).", -) -@click.option( - "--author", - default=None, - help='Author override, e.g. "Name ". Default: git config.', -) -@click.option( - "--license", - "license_url", - default=None, - help="License URL or SPDX identifier. Default: CC-BY-4.0.", -) -@click.option( - "--zip/--no-zip", - "zip_bundle", - default=False, - help="Package the bundle as a .zip after building.", -) -@click.option( - "--metadata-only", - is_flag=True, - help="Skip data files; bundle manifests + astra.yaml + universes only.", -) -def export_wrroc_cmd( - output: Path, - universe: tuple[str, ...], - author: str | None, - license_url: str | None, - zip_bundle: bool, - metadata_only: bool, -) -> None: - """Export a Workflow Run RO-Crate (WRROC) bundle. - - The bundle is suitable for upload to WorkflowHub, Zenodo (with the - RO-Crate plugin), or any RO-Crate-aware archive. The lightcone - manifest format on disk is unchanged — this is a publication view - generated on demand. - - Examples: - - lc export wrroc # ./wrroc/ directory - lc export wrroc -o my-run.zip --zip # zip bundle - lc export wrroc --metadata-only # provenance, no data - lc export wrroc -u baseline -u alt # specific universes - """ - from lightcone.engine.wrroc import export_wrroc - - project = _project_root() - - try: - result = export_wrroc( - project_path=project, - output_path=output, - universes=list(universe) or None, - author=author, - license=license_url, - zip_bundle=zip_bundle, - include_data=not metadata_only, - ) - except FileExistsError as e: - raise click.ClickException(str(e)) + for item in report.created: + console.print(f" [yellow]would create[/yellow] {item}") + for item in report.repaired: + console.print(f" [yellow]would repair[/yellow] {item}") + for warning in report.warnings: + console.print(f" [yellow]![/yellow] {warning}") + + +def _render_run(report: ConvergenceReport, directory: Path) -> None: + for item in report.created: + console.print(f"[green]✓[/green] created {item}") + for item in report.repaired: + console.print(f"[green]✓[/green] repaired {item}") + for warning in report.warnings: + console.print(f"[yellow]![/yellow] {warning}") + if report.converged: + console.print(f"\n[green]Project already converged at[/green] {directory}") + else: + console.print(f"\n[green]Project converged at[/green] {directory}") - flavor = "zip bundle" if result.is_zip else "directory" - console.print( - f"[green]✓[/green] Wrote WRROC {flavor}: [cyan]{result.bundle_path}[/cyan]" - ) - if result.runs_included == 0: + # Next steps only make sense for a freshly scaffolded spec. + if "astra.yaml" in report.created: + console.print("\nNext steps:") + console.print(f" • Go to the newly created directory [cyan]cd {directory}[/cyan]") console.print( - "[yellow]Warning:[/yellow] no materialized outputs were found — " - "the bundle contains only the workflow definition.\n" - " This usually means recipes haven't been run yet (try [cyan]lc run[/cyan]) " - "or the [cyan].lightcone-manifest.json[/cyan] sidecars are missing.\n" - " Workflow-only bundles will not pass strict Provenance Run Crate " - "validation; that profile requires at least one materialized run." + " • Add analysis dependencies with [cyan]uv add[/cyan] " + "(e.g. [cyan]uv add numpy astropy[/cyan])" ) - else: - u_list = ", ".join(result.universes_included) + console.print(" • Describe your analysis in [cyan]astra.yaml[/cyan]") console.print( - f" Captured [bold]{result.runs_included}[/bold] runs across " - f"universes: [cyan]{u_list}[/cyan]" + " • Preview the report with [cyan]myst start[/cyan] " + "(requires the MyST CLI: [cyan]npm i -g mystmd[/cyan])" ) diff --git a/src/lightcone/engine/__init__.py b/src/lightcone/engine/__init__.py index 27db6d3a..0725493b 100644 --- a/src/lightcone/engine/__init__.py +++ b/src/lightcone/engine/__init__.py @@ -1,6 +1,19 @@ -"""Lightcone execution engine. +"""The lightcone execution engine. -Snakemake-backed orchestrator for materializing astra.yaml outputs. -Provenance is recorded in per-output content-addressed manifests -(``.lightcone-manifest.json``) co-located with each output. +Rebuild in progress: the engine is being re-added layer by layer on top +of the normative design spec (``docs/design/execution-environment.md``). +Layer 1 is project scaffolding only — :mod:`lightcone.engine.project` +(discovery) and :mod:`lightcone.engine.constants`. """ + +from __future__ import annotations + + +def lc_version() -> str: + """The running lightcone-cli version ("unknown" for broken installs).""" + try: + from importlib.metadata import version + + return version("lightcone-cli") + except Exception: + return "unknown" diff --git a/src/lightcone/engine/cloudbuild.py b/src/lightcone/engine/cloudbuild.py deleted file mode 100644 index 02fdd0ee..00000000 --- a/src/lightcone/engine/cloudbuild.py +++ /dev/null @@ -1,420 +0,0 @@ -"""Remote image builds through GCP Cloud Build. - -The build backend for deployments where no OCI runtime exists on the -host — a JupyterHub user pod on GKE. ``lc build`` tars the project's -**staged build context** (the exact file set the content-addressed tag -hashes), uploads it to a deployment-provided GCS bucket, and submits a -Cloud Build job that pushes the image to the deployment's Artifact -Registry. Auth is the pod's Workload Identity, spoken to the GCE -metadata server — no stored credentials, no SDK dependency, no git -remote required. - -Image identity is the same content-addressed scheme as everywhere else -(:func:`lightcone.engine.container.image_identity`); the pushed ref is -``$LIGHTCONE_REGISTRY/lc-:`` and "is the image up to -date" is a single registry HEAD on that ref — unchanged files never -rebuild, never even upload. - -Deployment contract (env vars injected into user pods — see the -hub-deploy ``lightcone`` hub config): - -- :data:`~lightcone.engine.container.REGISTRY_ENV` — Artifact Registry - prefix (``-docker.pkg.dev//``); also names the - GCP project builds run in. -- :data:`BUCKET_ENV` — GCS bucket for build sources and logs. Its - presence (with the registry) is what selects this backend. -- :data:`SERVICE_ACCOUNT_ENV` (optional) — dedicated build service - account; the deployment grants it registry-writer rights only. - -The pod's identity needs ``cloudbuild.builds.editor``, -``iam.serviceAccountUser`` on the build SA, object create/view on the -bucket, and ``artifactregistry.reader`` for the freshness probe. -""" - -from __future__ import annotations - -import io -import json -import os -import tarfile -import tempfile -import time -import urllib.error -import urllib.parse -import urllib.request -from collections.abc import Callable -from pathlib import Path - -from lightcone.engine.container import ( - ContainerBuildError, - _populate_build_context, - deployment_registry, - image_identity, - registry_image_ref, -) - -#: GCS bucket for build sources/logs. Presence selects this backend. -BUCKET_ENV = "LIGHTCONE_BUILD_BUCKET" - -#: Optional dedicated Cloud Build service account (bare email or full -#: ``projects/…/serviceAccounts/…`` resource name). -SERVICE_ACCOUNT_ENV = "LIGHTCONE_BUILD_SERVICE_ACCOUNT" - -#: Hard ceiling on one build, seconds (also sent as the Cloud Build -#: timeout). Project images are slim; single-digit minutes is typical. -_BUILD_DEADLINE_S = 1800 - -_POLL_INTERVAL_S = 5.0 - -_METADATA_TOKEN_URL = ( - "http://metadata.google.internal/computeMetadata/v1/" - "instance/service-accounts/default/token" -) - -#: Progress callback ``(phase, detail)``; phases are ``cached``, -#: ``staging``, then Cloud Build statuses lowercased (queued/working/…). -ProgressFn = Callable[[str, str], None] - - -class CloudBuildError(ContainerBuildError): - """An image could not be produced through Cloud Build. - - Subclasses :class:`ContainerBuildError` so one handler covers every - way an image can fail to materialize, local or cloud. - """ - - -def cloudbuild_available() -> bool: - """Is this environment configured for Cloud Build image builds?""" - return bool(os.environ.get(BUCKET_ENV)) and deployment_registry() is not None - - -# --------------------------------------------------------------------------- -# Auth + HTTP plumbing -# --------------------------------------------------------------------------- - - -def _metadata_access_token() -> str | None: - """OAuth2 access token from the GCE metadata server, or ``None``. - - On GKE with Workload Identity this returns a token for the - Kubernetes service account's bound GCP identity. Off-GCP the - metadata host doesn't resolve and we return ``None`` quickly. - """ - req = urllib.request.Request( - _METADATA_TOKEN_URL, headers={"Metadata-Flavor": "Google"} - ) - try: - with urllib.request.urlopen(req, timeout=5) as resp: - payload = json.loads(resp.read().decode("utf-8")) - except (urllib.error.URLError, OSError, ValueError): - return None - token = payload.get("access_token") - return token if isinstance(token, str) and token else None - - -def _token() -> str: - token = _metadata_access_token() - if token is None: - raise CloudBuildError( - "No GCP credentials available from the metadata server. The " - "Cloud Build backend needs Workload Identity (or another " - "metadata-served identity) with cloudbuild.builds.editor." - ) - return token - - -def _request( - method: str, - url: str, - token: str, - *, - body: bytes | None = None, - content_type: str = "application/json", -) -> tuple[int, bytes]: - req = urllib.request.Request( - url, - data=body, - method=method, - headers={ - "Authorization": f"Bearer {token}", - **({"Content-Type": content_type} if body is not None else {}), - }, - ) - try: - with urllib.request.urlopen(req, timeout=120) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as exc: - return exc.code, exc.read() - except (urllib.error.URLError, OSError) as exc: - raise CloudBuildError(f"Could not reach {url.split('?')[0]} ({exc}).") from exc - - -def _json_or_error(status: int, payload: bytes, what: str) -> dict[str, object]: - if not 200 <= status < 300: - detail = payload.decode("utf-8", errors="replace")[:500] - raise CloudBuildError(f"{what} failed: HTTP {status}\n{detail}") - try: - parsed = json.loads(payload.decode("utf-8") or "{}") - except ValueError as exc: - raise CloudBuildError(f"{what} returned unparseable JSON.") from exc - return parsed if isinstance(parsed, dict) else {} - - -# --------------------------------------------------------------------------- -# Registry freshness probe -# --------------------------------------------------------------------------- - - -def registry_image_exists(ref: str) -> bool | None: - """Does *ref* exist in its registry? ``None`` when unknowable. - - Speaks the Docker Registry v2 API with the metadata-server token - (Artifact Registry accepts OAuth2 access tokens as Bearer). Returns - ``None`` — not ``False`` — when there are no credentials or the - registry can't be reached, so callers can distinguish "absent, - build it" from "can't tell". - """ - host, _, path = ref.partition("/") - repo, _, tag = path.rpartition(":") - if not (host and repo and tag): - return None - token = _metadata_access_token() - if token is None: - return None - url = f"https://{host}/v2/{repo}/manifests/{urllib.parse.quote(tag, safe='')}" - req = urllib.request.Request( - url, - method="HEAD", - headers={ - "Authorization": f"Bearer {token}", - "Accept": ( - "application/vnd.oci.image.index.v1+json, " - "application/vnd.oci.image.manifest.v1+json, " - "application/vnd.docker.distribution.manifest.v2+json, " - "application/vnd.docker.distribution.manifest.list.v2+json" - ), - }, - ) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return bool(200 <= resp.status < 300) - except urllib.error.HTTPError as exc: - return None if exc.code in (401, 403) else False - except (urllib.error.URLError, OSError): - return None - - -# --------------------------------------------------------------------------- -# Source staging + upload -# --------------------------------------------------------------------------- - - -def _staged_context_tarball(project: Path, containerfile: Path) -> bytes: - """gzip tarball of the staged build context (the hashed file set).""" - with tempfile.TemporaryDirectory(prefix="lc-cloudbuild-") as tmp: - staged = Path(tmp) - _populate_build_context(staged, containerfile, project) - buf = io.BytesIO() - with tarfile.open(fileobj=buf, mode="w:gz") as tar: - for entry in sorted(staged.rglob("*")): - tar.add(entry, arcname=str(entry.relative_to(staged))) - return buf.getvalue() - - -def _upload_source(bucket: str, object_name: str, data: bytes, token: str) -> None: - url = ( - "https://storage.googleapis.com/upload/storage/v1/b/" - f"{urllib.parse.quote(bucket, safe='')}/o?uploadType=media&name=" - f"{urllib.parse.quote(object_name, safe='')}" - ) - status, payload = _request( - "POST", url, token, body=data, content_type="application/gzip" - ) - _json_or_error(status, payload, "Source upload to the build bucket") - - -def _fetch_log_tail(bucket: str, build_id: str, token: str, lines: int = 30) -> str: - object_name = urllib.parse.quote(f"logs/log-{build_id}.txt", safe="") - url = ( - "https://storage.googleapis.com/storage/v1/b/" - f"{urllib.parse.quote(bucket, safe='')}/o/{object_name}?alt=media" - ) - try: - status, payload = _request("GET", url, token) - except CloudBuildError: - return "" - if not 200 <= status < 300: - return "" - text = payload.decode("utf-8", errors="replace") - return "\n".join(text.splitlines()[-lines:]) - - -# --------------------------------------------------------------------------- -# Build submission + polling -# --------------------------------------------------------------------------- - - -def _gcp_project(registry: str) -> str: - """GCP project id out of an Artifact Registry prefix. - - ``-docker.pkg.dev//`` → ````. - """ - parts = registry.split("/") - if len(parts) < 2 or not parts[0].endswith("-docker.pkg.dev"): - raise CloudBuildError( - f"{registry!r} is not an Artifact Registry prefix " - "(expected -docker.pkg.dev//); the " - "Cloud Build backend only targets Artifact Registry." - ) - return parts[1] - - -def _submit_build( - *, - gcp_project: str, - bucket: str, - source_object: str, - containerfile_name: str, - image_ref: str, - token: str, -) -> str: - """Create the build; return its id.""" - build: dict[str, object] = { - "source": {"storageSource": {"bucket": bucket, "object": source_object}}, - "steps": [ - { - "name": "gcr.io/cloud-builders/docker", - "args": ["build", "-t", image_ref, "-f", containerfile_name, "."], - } - ], - "images": [image_ref], - "timeout": f"{_BUILD_DEADLINE_S}s", - # Logs into our own bucket: GCS_ONLY is required when running as - # a custom service account, and it is where the failure tail - # comes from. - "logsBucket": f"gs://{bucket}/logs", - "options": {"logging": "GCS_ONLY"}, - } - build_sa = (os.environ.get(SERVICE_ACCOUNT_ENV) or "").strip() - if build_sa: - if "/" not in build_sa: - build_sa = f"projects/{gcp_project}/serviceAccounts/{build_sa}" - build["serviceAccount"] = build_sa - - url = f"https://cloudbuild.googleapis.com/v1/projects/{gcp_project}/builds" - status, payload = _request("POST", url, token, body=json.dumps(build).encode()) - op = _json_or_error(status, payload, "Cloud Build submission") - meta = op.get("metadata") - build_info = meta.get("build") if isinstance(meta, dict) else None - build_id = build_info.get("id") if isinstance(build_info, dict) else None - if not isinstance(build_id, str) or not build_id: - raise CloudBuildError( - f"Cloud Build submission returned no build id (response keys: {sorted(op)})." - ) - return build_id - - -def _wait_for_build( - gcp_project: str, - build_id: str, - token: str, - on_progress: ProgressFn | None, -) -> str: - """Poll until a terminal status; return it.""" - url = ( - f"https://cloudbuild.googleapis.com/v1/projects/{gcp_project}" - f"/builds/{build_id}" - ) - deadline = time.monotonic() + _BUILD_DEADLINE_S + 120 - last_status = "" - while time.monotonic() < deadline: - status_code, payload = _request("GET", url, token) - build = _json_or_error(status_code, payload, "Cloud Build status poll") - status = str(build.get("status") or "") - if status != last_status: - last_status = status - if on_progress: - on_progress(status.lower(), "") - if status in ( - "SUCCESS", - "FAILURE", - "INTERNAL_ERROR", - "TIMEOUT", - "CANCELLED", - "EXPIRED", - ): - return status - time.sleep(_POLL_INTERVAL_S) - raise CloudBuildError(f"Timed out waiting for Cloud Build {build_id} to finish.") - - -# --------------------------------------------------------------------------- -# High-level entry point -# --------------------------------------------------------------------------- - - -def ensure_image( - project: Path, - containerfile_spec: str, - *, - project_name: str, - force: bool = False, - on_progress: ProgressFn | None = None, -) -> str: - """Make sure the project's image is in the registry; return its ref. - - Content-addressed and git-free: the tag hashes the staged build - context, so an unchanged environment is a single registry HEAD (no - build, no upload), and any change builds from the working tree as - it is right now. *force* skips the freshness probe and rebuilds. - """ - containerfile = project / containerfile_spec - if not containerfile.is_file(): - raise CloudBuildError( - f"Declared container {containerfile_spec!r} not found in {project}." - ) - registry = deployment_registry() - bucket = (os.environ.get(BUCKET_ENV) or "").strip().removeprefix("gs://").rstrip("/") - if registry is None or not bucket: - raise CloudBuildError( - "This environment is not configured for Cloud Build: both " - f"LIGHTCONE_REGISTRY and {BUCKET_ENV} must be set (they are " - "injected by the deployment)." - ) - ref = registry_image_ref(project_name, containerfile, project, registry=registry) - - if not force and registry_image_exists(ref) is True: - if on_progress: - on_progress("cached", f"{ref} already in the registry") - return ref - - token = _token() - gcp_project = _gcp_project(registry) - - if on_progress: - on_progress("staging", "uploading build context") - # Content-addressed object name: identical contexts collide into - # the same object, which is exactly right. - _, digest = image_identity(project_name, containerfile, project) - source_object = f"sources/lc-{project_name}-{digest}.tar.gz" - _upload_source( - bucket, source_object, _staged_context_tarball(project, containerfile), token - ) - - build_id = _submit_build( - gcp_project=gcp_project, - bucket=bucket, - source_object=source_object, - containerfile_name=containerfile.name, - image_ref=ref, - token=token, - ) - status = _wait_for_build(gcp_project, build_id, token, on_progress) - if status != "SUCCESS": - tail = _fetch_log_tail(bucket, build_id, token) - raise CloudBuildError( - f"Cloud Build {build_id} ended with status {status}." - + (f" Last build output:\n{tail}" if tail else "") - ) - return ref diff --git a/src/lightcone/engine/constants.py b/src/lightcone/engine/constants.py new file mode 100644 index 00000000..4a960b86 --- /dev/null +++ b/src/lightcone/engine/constants.py @@ -0,0 +1,26 @@ +"""Engine constants. + +Constants ship *inside* the locked engine, never resolved at run time: +a new value reaches a project only through an engine release plus a +relock, which is what keeps environment identity a pure function of the +repo plus the engine (spec §3). + +The image layer's own pinned digests (default base, uv distribution) +arrive with the container hatch; layer 1 needs only the interpreter pin. +""" + +from __future__ import annotations + +#: Exact interpreter patch scaffolded into ``.python-version`` by +#: ``lc init``. Projects may pin a different one — environment identity +#: follows the file, not this constant. +DEFAULT_PYTHON = "3.12.12" + +#: The floor written into the scaffolded ``requires-python``, derived +#: from :data:`DEFAULT_PYTHON` so the two can never drift. +DEFAULT_PYTHON_FLOOR = ".".join(DEFAULT_PYTHON.split(".")[:2]) + +#: Minimum uv scaffolded into ``[tool.uv] required-version``. 0.12 is +#: the release the spec's empirical evidence pass was run against +#: (spec §13). +MIN_UV_VERSION = "0.12" diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py deleted file mode 100644 index 890d6be5..00000000 --- a/src/lightcone/engine/container.py +++ /dev/null @@ -1,877 +0,0 @@ -"""Container runtime layer. - -We commit to **Dockerfile syntax** for ``Containerfile`` and **own** the -container invocation end-to-end — Snakemake's built-in ``container:`` -directive and ``--sdm apptainer`` pipeline are deliberately not used. A -single config knob picks the OCI runtime; building and running both go -through it. - -Two surfaces: - -* :func:`compute_image_tag` and :func:`build_image` cover the **build** - phase — ``lc build`` invokes them to produce ``lc--`` - in the runtime's local image store. - -* :func:`wrap_recipe` covers the **run** phase — the Snakefile generator - calls it to convert a raw recipe into a shell command that executes - inside the configured container runtime. - -Supported runtimes: - * ``docker`` / ``podman`` — local desktop or build host - * ``podman-hpc`` — NERSC-style login nodes; ``build`` migrates the - image so compute-node apptainer can read it. ``run`` still uses - ``podman-hpc`` directly. - * ``kubernetes`` — the execution environment (a Dask Gateway worker - pod) already *is* the container: ``lc run`` starts the cluster - with the project's image, so ``wrap_recipe`` is a passthrough and - images resolve to registry refs (``/lc-:``) - instead of local-store tags. Building goes through a remote - builder (:mod:`lightcone.engine.cloudbuild`), never a local OCI - CLI. - * ``none`` — no container; recipe runs on the host. Useful for - development and for projects that don't need isolation. -""" -from __future__ import annotations - -import hashlib -import json -import logging -import os -import re -import shlex -import shutil -import subprocess -import tempfile -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from pathlib import Path - -import yaml - -from lightcone.engine.site_registry import detect_current_site - -logger = logging.getLogger(__name__) - -#: Supported runtimes, in fallback detection order. The site registry can -#: move a site's declared ``container_runtime`` to the front (see -#: :func:`detect_runtime`). Order rationale: podman-hpc first because -#: anyone who installed the HPC wrapper did so on purpose and plain -#: podman would build images compute nodes can't read; then podman -#: (rootless, no daemon); docker last, gated behind a ``docker info`` -#: probe so a down daemon doesn't silently win over a healthy podman. -RUNTIMES: tuple[str, ...] = ("podman-hpc", "podman", "docker") - -#: The non-OCI-CLI runtime: recipes run directly inside a worker pod -#: that was started from the project's image. Never auto-detected from -#: PATH — it is selected by site detection (a Dask Gateway deployment) -#: or pinned explicitly in ``~/.lightcone/config.yaml``. -KUBERNETES = "kubernetes" - -#: Registry prefix images are pushed to / pulled from on a deployment -#: with a remote builder (e.g. ``europe-west1-docker.pkg.dev// -#: `` on a lightcone JupyterHub). Injected into user pods by the -#: deployment; its presence is half of the Cloud Build contract (see -#: :mod:`lightcone.engine.cloudbuild`). -REGISTRY_ENV = "LIGHTCONE_REGISTRY" - -#: Files whose contents contribute to the image tag hash. -DEPENDENCY_FILES = ( - "requirements.txt", - "requirements-dev.txt", - "requirements-test.txt", - "pyproject.toml", - "setup.py", - "setup.cfg", - "poetry.lock", - "Pipfile.lock", - "uv.lock", - "conda-lock.yml", - "environment.yml", - "environment.yaml", -) - -#: Matches a Dockerfile-style flag like ``--from=builder`` or ``--chown=u:g``. -_FLAG_RE = re.compile(r"^--[A-Za-z][A-Za-z0-9-]*(=\S+)?$") - - -class ContainerBuildError(Exception): - """Raised when a container image build fails.""" - - -@dataclass -class ContainerBuildResult: - """Result of building a container image.""" - - tag: str - already_existed: bool - exit_code: int = 0 - stdout: str = "" - stderr: str = "" - - -@dataclass -class ContainerStatus: - """Status information for a container spec.""" - - type: str # "none", "prebuilt", "build" - image: str | None = None - exists: bool | None = None - containerfile: str | None = None - - -@dataclass(frozen=True) -class RuntimeChoice: - """Result of resolving the container runtime to use. - - ``runtime`` is the resolved value (``docker | podman | podman-hpc | none``). - ``explicit`` is ``True`` when the user pinned this value in - ``~/.lightcone/config.yaml`` — i.e. they typed ``runtime: docker``, - ``runtime: podman``, … or ``runtime: none``. ``False`` means - ``runtime: auto`` (or no config), and the runtime is whatever - detection produced — including ``none`` as a silent fallback. - - Callers use ``explicit`` to decide whether silently running without - isolation is acceptable. When the user explicitly opted out, no - surprise. When auto fell back to ``none`` against the spec's - declared containers, the manifest's ``container_image`` field would - misrepresent what actually executed — that is a provenance hazard - and the caller should warn or refuse to proceed. - """ - - runtime: str - explicit: bool - - -# --------------------------------------------------------------------------- -# Runtime detection / config -# --------------------------------------------------------------------------- - - -def detect_runtime() -> str | None: - """Return the first usable runtime in :func:`_detection_order`, or ``None``. - - "Usable" means the binary is on PATH and (for docker) its daemon - answers ``docker info``. Site-declared preferences (e.g. Perlmutter - → podman-hpc) are *hints* — missing-from-PATH falls through to the - next candidate. Errors on missing-but-explicit user config are - :func:`load_runtime`'s job. - - A site that declares ``container_runtime: kubernetes`` (a Dask - Gateway deployment) short-circuits the PATH probing entirely — - there is no binary to find; the pod itself is the container. - """ - if _site_preferred_runtime() == KUBERNETES: - return KUBERNETES - for runtime in _detection_order(): - if shutil.which(runtime) is None: - continue - if runtime == "docker" and not _docker_daemon_up(): - continue - return runtime - return None - - -def _detection_order() -> tuple[str, ...]: - """RUNTIMES with the host site's preferred runtime moved to the front.""" - preferred = _site_preferred_runtime() - if preferred is None: - return RUNTIMES - return (preferred, *(r for r in RUNTIMES if r != preferred)) - - -def _site_preferred_runtime() -> str | None: - """Return the host site's declared ``container_runtime``, else ``None``. - - Returns ``None`` when no site matches, no preference is declared, or - the declared value is not a known runtime — never raises. - """ - preferred = detect_current_site().get("container_runtime") - return preferred if preferred in (*RUNTIMES, KUBERNETES) else None - - -def _docker_daemon_up() -> bool: - try: - result = subprocess.run( - ["docker", "info"], - capture_output=True, - timeout=5, - check=False, - ) - except (subprocess.TimeoutExpired, FileNotFoundError): - return False - return result.returncode == 0 - - -def _global_config_path() -> Path: - return Path.home() / ".lightcone" / "config.yaml" - - -def load_runtime(*, project_path: Path | None = None) -> RuntimeChoice: - """Resolve the container runtime to use. - - Reads ``container.runtime`` from ``~/.lightcone/config.yaml`` (the - project_path is accepted for future per-project overrides but is not - consulted today). Values: - - * ``auto`` (default) — first available runtime in :data:`RUNTIMES`, - else falls back to ``"none"`` with ``explicit=False``. On a site - that declares ``container_runtime: kubernetes``, auto resolves to - :data:`KUBERNETES` without any PATH probing. - * ``docker | podman | podman-hpc`` — explicit; binary must exist. - * ``kubernetes`` — explicit; no binary involved. - * ``none`` — explicit opt-out; recipes run on the host. - - Raises :class:`ContainerBuildError` if an explicit runtime is - configured but its binary is missing on PATH, or if the configured - value is unrecognised. - """ - cfg_path = _global_config_path() - requested = "auto" - if cfg_path.is_file(): - try: - data = yaml.safe_load(cfg_path.read_text()) or {} - requested = (data.get("container") or {}).get("runtime") or "auto" - except yaml.YAMLError: - logger.warning("Could not parse %s; using runtime: auto", cfg_path) - requested = "auto" - - if requested == "auto": - return RuntimeChoice(runtime=detect_runtime() or "none", explicit=False) - if requested in ("none", KUBERNETES): - return RuntimeChoice(runtime=requested, explicit=True) - if requested not in RUNTIMES: - raise ContainerBuildError( - f"Unknown container.runtime {requested!r} in {cfg_path}. " - f"Expected one of: auto, none, {KUBERNETES}, {', '.join(RUNTIMES)}." - ) - if shutil.which(requested) is None: - raise ContainerBuildError( - f"Configured container.runtime {requested!r} is not on PATH. " - f"Install {requested} or set container.runtime to a different value " - f"in {cfg_path}." - ) - return RuntimeChoice(runtime=requested, explicit=True) - - -# --------------------------------------------------------------------------- -# Image tag computation -# --------------------------------------------------------------------------- - - -def find_dependency_files(project_path: Path) -> list[Path]: - """Return sorted list of dependency files found in *project_path*.""" - found = [project_path / name for name in DEPENDENCY_FILES] - return sorted(p for p in found if p.is_file()) - - -def _hash_file_into(path: Path, h: hashlib._Hash) -> None: - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(64 * 1024), b""): - h.update(chunk) - - -def _hash_named_file(path: Path, label: str, h: hashlib._Hash) -> None: - """Mix *path*'s identity and contents into *h* with explicit framing. - - Path-prefix + null separators stop boundary-shifting collisions — - e.g. moving a line from ``requirements.txt`` to ``requirements-dev.txt`` - no longer yields the same digest as keeping it in place. - """ - h.update(label.encode("utf-8")) - h.update(b"\0") - h.update(path.name.encode("utf-8")) - h.update(b"\0") - _hash_file_into(path, h) - h.update(b"\0") - - -def hash_file_contents(files: list[Path]) -> str: - """Return a SHA-256 hex digest over the framed contents of *files*. - - The digest mixes each file's basename and a label byte in addition to - its contents, so reordering or relabelling produces different digests. - """ - h = hashlib.sha256() - for f in files: - _hash_named_file(f, "f", h) - return h.hexdigest() - - -def _iter_build_context_entries( - containerfile: Path, project_path: Path -) -> Iterator[tuple[str, Path]]: - """Yield ``(kind, path)`` for everything that contributes to a build. - - ``kind`` is one of ``"containerfile"``, ``"dep"``, ``"copy_file"``. - Sharing this iteration between :func:`compute_image_tag` and - :func:`_populate_build_context` guarantees by construction that the - hashed set and the staged set cover identical files — so the tag - can never invalidate against a stage that's missing inputs (or vice - versa). - - Sources behind ``--from=`` and URL/git ``ADD`` arguments are - skipped — they're not part of the host context. Directory sources - (including ``COPY . .``) are rejected: the image is an environment, - not a code snapshot — recipes run against the live project tree - (bind-mounted locally, shared filesystem on a hub), so baking - source directories in would only force pointless rebuilds and go - stale between them. - """ - text = containerfile.read_text(errors="replace") - copy_files: list[Path] = [] - bad: list[str] = [] - for src_str in _parse_copy_sources(text): - is_dir_source = False - for resolved in _expand_copy_source(src_str, project_path): - if resolved.is_dir(): - is_dir_source = True - elif resolved.is_file(): - copy_files.append(resolved) - if is_dir_source: - bad.append(src_str) - if bad: - raise ContainerBuildError( - f"{containerfile.name}: COPY/ADD of a directory " - f"({', '.join(repr(s) for s in bad)}) is not supported. The " - "image is a pure environment — recipes run against the live " - "project tree (bind-mounted locally, shared filesystem on a " - "hub), so project source never needs to be baked in. Remove " - "the line (e.g. `COPY . .`), or COPY individual files if the " - "build itself needs them." - ) - yield "containerfile", containerfile - for dep in find_dependency_files(project_path): - yield "dep", dep - for resolved in copy_files: - yield "copy_file", resolved - - -def directory_copy_sources(containerfile: Path, project_path: Path) -> list[str]: - """``COPY``/``ADD`` sources in *containerfile* that resolve to directories. - - Directory sources are unsupported (the image is an environment, not - a code snapshot); this is the shared detector behind the build-time - rejection in :func:`_iter_build_context_entries` and the advisory - warning in ``lc init``. - """ - text = containerfile.read_text(errors="replace") - bad: list[str] = [] - for src_str in _parse_copy_sources(text): - if any( - resolved.is_dir() - for resolved in _expand_copy_source(src_str, project_path) - ): - bad.append(src_str) - return bad - - -def image_identity( - project_name: str, - containerfile: Path, - project_path: Path, -) -> tuple[str, str]: - """Compute a content-addressed image identity ``(safe_name, digest)``. - - The digest (12-char sha256) covers the Containerfile, every - dependency file in :data:`DEPENDENCY_FILES`, and the contents of - every ``COPY``/``ADD`` source file referenced from the - Containerfile (directory sources are rejected — the image is an - environment, not a code snapshot). - - The identity is spelled two ways downstream — ``lc--`` - in a local image store (:func:`compute_image_tag`), ``…/lc-: - `` in a registry (:func:`registry_image_ref`) — but it is - one identity: the same digest everywhere, on every backend. - """ - h = hashlib.sha256() - for kind, path in _iter_build_context_entries(containerfile, project_path): - if kind == "containerfile": - _hash_named_file(path, "containerfile", h) - elif kind == "dep": - _hash_named_file(path, "dep", h) - else: # copy_file - rel = _safe_relpath(path, project_path) - h.update(b"copy\0") - h.update(rel.encode("utf-8")) - h.update(b"\0file\0") - _hash_file_into(path, h) - h.update(b"\0") - - return project_name.lower().replace(" ", "-"), h.hexdigest()[:12] - - -def compute_image_tag( - project_name: str, - containerfile: Path, - project_path: Path, -) -> str: - """Content-addressed local-store tag: ``lc--``. - - See :func:`image_identity` for what the digest covers. - """ - safe_name, digest = image_identity(project_name, containerfile, project_path) - return f"lc-{safe_name}-{digest}" - - -def registry_image_ref( - project_name: str, - containerfile: Path, - project_path: Path, - *, - registry: str, -) -> str: - """Content-addressed registry ref: ``/lc-:``. - - Same identity as :func:`compute_image_tag`, spelled for a registry: - the digest moves into the tag position so one repository per project - accumulates its image history. - """ - safe_name, digest = image_identity(project_name, containerfile, project_path) - return f"{registry.rstrip('/')}/lc-{safe_name}:{digest}" - - -def deployment_registry() -> str | None: - """The deployment-injected registry prefix, or ``None`` off-deployment.""" - registry = (os.environ.get(REGISTRY_ENV) or "").strip() - return registry.rstrip("/") or None - - -def runtime_registry(runtime: str) -> str | None: - """Registry prefix image identities resolve against under *runtime*. - - The single source of truth for "which spelling of the image identity - does this runtime use": the deployment registry on - :data:`KUBERNETES` (worker pods pull from a registry), ``None`` — - local-store tags — everywhere else. Shared by the Snakefile - generator and the status walker so their ``code_version``s can - never disagree about the image identity. - """ - return deployment_registry() if runtime == KUBERNETES else None - - -def _safe_relpath(path: Path, root: Path) -> str: - try: - return path.resolve().relative_to(root.resolve()).as_posix() - except ValueError: - return path.name - - -def _parse_copy_sources(containerfile_text: str) -> list[str]: - """Return raw source strings from ``COPY``/``ADD`` lines. - - Skips ``--from=`` copies (those reference another build stage, - not the host context) and URL/git arguments (network resources, not - part of the local context we can hash). Glob patterns and relative - paths are returned verbatim — :func:`_expand_copy_source` resolves - them against the project tree. - - Handles backslash line continuations and the JSON exec form - (``COPY ["src", "dest"]``). Heredoc COPY (``COPY < list[Path]: - """Resolve a ``COPY``/``ADD`` source to actual paths under *project_path*. - - Returns ``[project_path]`` for ``.`` (whole context). Globs are - expanded against *project_path*. Paths that escape the project root - are dropped — we don't hash arbitrary host filesystem. - """ - src = src.lstrip("/") - if not src or src == ".": - return [project_path] - if any(c in src for c in "*?["): - return sorted(project_path.glob(src)) - candidate = (project_path / src).resolve() - try: - candidate.relative_to(project_path.resolve()) - except ValueError: - return [] - if candidate.exists(): - return [candidate] - return [] - - -def is_containerfile(spec: str, project_path: Path) -> bool: - """Return ``True`` if *spec* refers to an existing file (Containerfile).""" - return (project_path / spec).is_file() - - -# --------------------------------------------------------------------------- -# Build -# --------------------------------------------------------------------------- - - -def image_exists_locally(tag: str, *, runtime: str) -> bool: - """Check whether *tag* exists in the runtime's local image store.""" - if runtime == "podman-hpc": - return image_exists_podman_hpc(tag) - try: - result = subprocess.run( - [runtime, "image", "inspect", tag], - capture_output=True, - check=False, - ) - return result.returncode == 0 - except FileNotFoundError: - return False - - -def image_exists_podman_hpc(tag: str) -> bool: - try: - result = subprocess.run( - ["podman-hpc", "image", "exists", tag], - capture_output=True, - check=False, - ) - return result.returncode == 0 - except FileNotFoundError: - return False - - -def _populate_build_context( - staged: Path, containerfile: Path, source_context: Path -) -> None: - """Mirror the Containerfile + its referenced sources into *staged*. - - Why this exists: NERSC's home and CFS filesystems are mounted via - Cray DVS, which doesn't implement ``llistxattr`` (returns ``EPROTO``). - Buildah's copier — used by ``podman``, ``podman-hpc``, and any other - buildah-backed runtime — calls ``llistxattr`` unconditionally on every - ``COPY`` source and crashes when the project lives on DVS. Staging - the build context into ``$TMPDIR`` (tmpfs on Linux) sidesteps the - issue entirely without forcing the user to relocate their project. - - The set of staged files is :func:`_iter_build_context_entries` — the - same iteration :func:`compute_image_tag` hashes — so the tag can't - invalidate against a stage that's missing files. - """ - src_root = source_context.resolve() - for kind, path in _iter_build_context_entries(containerfile, source_context): - if kind in ("containerfile", "dep"): - shutil.copy2(path, staged / path.name) - continue - try: # copy_file - rel = path.resolve().relative_to(src_root) - except ValueError: - continue - dest = staged / rel - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(path, dest) - - -def build_image( - tag: str, - containerfile: Path, - context: Path, - *, - runtime: str, - build_args: dict[str, str] | None = None, -) -> ContainerBuildResult: - """Build a container image with the given *runtime*. - - The build context is staged into a fresh tempdir before invocation - (see :func:`_populate_build_context`). For ``podman-hpc``, the image - is automatically migrated after build so compute nodes can access it. - - Raises :class:`ContainerBuildError` on failure. - """ - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported build runtime {runtime!r}; expected one of {RUNTIMES}." - ) - - with tempfile.TemporaryDirectory(prefix="lc-build-") as staged_str: - staged = Path(staged_str) - _populate_build_context(staged, containerfile, context) - staged_cf = staged / containerfile.name - cmd: list[str] = [runtime, "build", "-t", tag, "-f", str(staged_cf)] - for key, value in (build_args or {}).items(): - cmd += ["--build-arg", f"{key}={value}"] - cmd.append(str(staged)) - - try: - proc = subprocess.run(cmd, capture_output=True, text=True, check=False) - except FileNotFoundError: - raise ContainerBuildError( - f"{runtime} is not installed or not on PATH. " - f"Install {runtime} to build container images." - ) - - if proc.returncode != 0: - raise ContainerBuildError( - f"{runtime} build failed (exit code {proc.returncode}):\n{proc.stderr}" - ) - - if runtime == "podman-hpc": - _podman_hpc_migrate(tag) - - return ContainerBuildResult( - tag=tag, - already_existed=False, - exit_code=proc.returncode, - stdout=proc.stdout, - stderr=proc.stderr, - ) - - -def pull_image(image: str, *, runtime: str) -> None: - """Pull *image* into the runtime's local image store. - - Used by ``lc build`` so that pre-built registry images (e.g. - ``python:3.12-slim``) are present before ``lc run`` invokes the - runtime with ``--pull=never``. - - Raises :class:`ContainerBuildError` on failure or if *runtime* isn't - on PATH. - """ - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported runtime {runtime!r}; expected one of {RUNTIMES}." - ) - try: - proc = subprocess.run( - [runtime, "pull", image], - capture_output=True, - text=True, - check=False, - ) - except FileNotFoundError: - raise ContainerBuildError( - f"{runtime} is not installed or not on PATH." - ) - if proc.returncode != 0: - raise ContainerBuildError( - f"{runtime} pull {image} failed (exit code {proc.returncode}):\n" - f"{proc.stderr}" - ) - if runtime == "podman-hpc": - _podman_hpc_migrate(image) - - -def _podman_hpc_migrate(tag: str) -> None: - """Run ``podman-hpc migrate `` to make image available on compute nodes.""" - try: - proc = subprocess.run( - ["podman-hpc", "migrate", tag], - capture_output=True, - text=True, - check=False, - ) - except FileNotFoundError: - raise ContainerBuildError("podman-hpc not found — cannot migrate image.") - if proc.returncode != 0: - raise ContainerBuildError( - f"podman-hpc migrate failed (exit code {proc.returncode}):\n{proc.stderr}" - ) - logger.info("podman-hpc migrate %s succeeded.", tag) - - -# --------------------------------------------------------------------------- -# Run-time recipe wrap -# --------------------------------------------------------------------------- - - -def resolve_image_for_run( - spec: str | None, - *, - project_path: Path, - project_name: str, - registry: str | None = None, -) -> str | None: - """Translate an astra.yaml ``container:`` value into the image tag - that the runtime will execute. - - * ``None`` / empty → ``None`` (no container). - * Path to a Containerfile in the project → the content-addressed - identity ``lc build`` produces: a local-store tag - (``lc--``), or with *registry* set (a deployment with - a remote builder) the registry ref (``/lc-:``). - * Anything else (registry image, e.g. ``python:3.12-slim``, or a - pre-namespaced ``ghcr.io/foo/bar:tag``) → returned as-is for the - runtime to pull. - """ - if not spec: - return None - if is_containerfile(spec, project_path): - containerfile = project_path / spec - if registry is not None: - return registry_image_ref( - project_name, containerfile, project_path, registry=registry - ) - return compute_image_tag(project_name, containerfile, project_path) - return spec - - -def make_image_tag_resolver( - project_path: Path, - project_name: str, - *, - registry: str | None = None, -) -> Callable[[str | None], str | None]: - """Memoizing wrapper around :func:`resolve_image_for_run`. - - Multiple outputs typically share the same Containerfile, and resolving - a Containerfile path re-hashes it plus all dependency files - (lockfiles can be MB each). The returned closure caches by spec - string for the lifetime of the caller's loop. - """ - cache: dict[str | None, str | None] = {} - - def resolve(spec: str | None) -> str | None: - if spec in cache: - return cache[spec] - tag = resolve_image_for_run( - spec, - project_path=project_path, - project_name=project_name, - registry=registry, - ) - cache[spec] = tag - return tag - - return resolve - - -def wrap_recipe( - recipe: str, - *, - image: str | None, - runtime: str, -) -> str: - """Wrap *recipe* so it executes inside *image* under *runtime*. - - Returns a shell-command string suitable for Snakemake's ``shell()``. - Snakemake's ``{output[0]}`` / ``{input.X}`` / ``{wildcards.universe}`` - placeholders inside *recipe* are preserved — they substitute through - Python's ``str.format`` at execution time, after wrapping. - - No-op cases: - * *image* is ``None`` → recipe returned unchanged - * *runtime* is ``"none"`` → recipe returned unchanged - * *runtime* is :data:`KUBERNETES` → recipe returned unchanged: - the Dask worker pod executing it was started from *image*, so - wrapping would be containerizing twice. The image still flows - into ``code_version`` and the manifest — provenance records - the pod's image, which is what actually ran the recipe. - - The recipe is shell-quoted with :func:`shlex.quote` and passed as the - argument to ``bash -c`` inside the container, which keeps single - quotes, dollar signs, and other shell metacharacters intact across - the host bash → runtime CLI → container bash boundaries. - """ - if image is None or runtime in ("none", KUBERNETES): - return recipe - if runtime not in RUNTIMES: - raise ContainerBuildError( - f"Unsupported run runtime {runtime!r}; expected one of {RUNTIMES} or 'none'." - ) - inner = shlex.quote(recipe) - # ``--pull=never`` is critical for podman, which by default does - # short-name resolution against ``unqualified-search-registries`` - # in registries.conf — that fails for ``lc--`` tags - # produced by ``lc build`` even though the image sits in local - # storage. Telling the runtime not to fetch sidesteps the issue and - # is the same semantics on docker and podman-hpc. Registry images - # (``python:3.12-slim``, ``ghcr.io/...``) must be pulled in advance - # by ``lc build``. - # - # Bind-mount and chdir to $PWD so recipes that write to relative - # paths land in the project tree. Snakemake invokes us with - # cwd=project, so $PWD is the project root. - return ( - f'{runtime} run --rm --pull=never ' - f'-v "$PWD":"$PWD" -w "$PWD" ' - f'{image} bash -c {inner}' - ) - - -# --------------------------------------------------------------------------- -# Status -# --------------------------------------------------------------------------- - - -def get_container_status( - spec: str | None, - project_path: Path, - project_name: str, - *, - runtime: str, -) -> ContainerStatus: - """Return status information for a container spec without building.""" - if spec is None: - return ContainerStatus(type="none") - - if not is_containerfile(spec, project_path): - return ContainerStatus(type="prebuilt", image=spec) - - containerfile = project_path / spec - if runtime == KUBERNETES and (registry := deployment_registry()) is not None: - from lightcone.engine.cloudbuild import registry_image_exists - - ref = registry_image_ref( - project_name, containerfile, project_path, registry=registry - ) - return ContainerStatus( - type="build", - image=ref, - exists=registry_image_exists(ref), - containerfile=spec, - ) - tag = compute_image_tag(project_name, containerfile, project_path) - exists = ( - image_exists_locally(tag, runtime=runtime) - if runtime not in ("none", KUBERNETES) - else None - ) - return ContainerStatus( - type="build", - image=tag, - exists=exists, - containerfile=spec, - ) diff --git a/src/lightcone/engine/dask_cluster.py b/src/lightcone/engine/dask_cluster.py deleted file mode 100644 index cf6a69f0..00000000 --- a/src/lightcone/engine/dask_cluster.py +++ /dev/null @@ -1,516 +0,0 @@ -# mypy: disable-error-code="no-untyped-call" -"""Cluster lifecycle for ``lc run``. - -One context manager, four branches: - -- ``DASK_SCHEDULER_ADDRESS`` is already set → yield it as-is. We don't own - the cluster, so we don't tear it down. -- ``DASK_GATEWAY__ADDRESS`` is set (a JupyterHub/Dask Gateway - deployment) → **create** a run-scoped Gateway cluster with the - project's image and shut it down when the run finishes. Create/cull - per run is what makes image updates seamless: a Gateway cluster's - image is fixed at creation, so picking up a freshly built project - image *requires* a fresh cluster. Gateway scheduler addresses use a - custom ``gateway://`` comm scheme a bare ``distributed.Client`` - cannot dial, so this branch hands the executor the *cluster name* - (via :data:`GATEWAY_CLUSTER_ENV`) and the executor rejoins through - the authenticated Gateway API. -- ``SLURM_JOB_ID`` is set → start an in-process scheduler via - ``LocalCluster(n_workers=0)``, then ``srun`` one ``dask worker`` per node - across the allocation. Workers advertise the node's full resources; - per-rule ``threads`` / ``mem_mb`` / ``gpus`` map to per-task constraints. -- None of the above → ``LocalCluster()`` sized to the local machine. - -Outside the Gateway branch the scheduler is always in-process (driven -by ``lc run`` itself) so its lifetime equals the run's lifetime — no -service to manage, no orphaned schedulers if the driver crashes. On the -Gateway branch the Gateway server owns scheduling, and the same -lifetime contract is enforced there: the cluster is shut down on exit, -with the deployment's idle timeout as the backstop if lc dies uncleanly. -""" - -from __future__ import annotations - -import getpass -import logging -import os -import shutil -import socket -import subprocess -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass - -# Resource keys advertised by workers and requested per-task. These strings -# form a contract between the worker bootstrap (here) and the executor plugin -# (snakemake_executor_plugin_dask.executor). Dask matches by string equality. -RESOURCE_CPUS = "cpus" -RESOURCE_MEMORY = "memory" -RESOURCE_GPUS = "gpus" - -#: Parent→child rendezvous for the Gateway branch: ``cluster_for_run`` -#: sets this to the name of the cluster it created so the executor -#: plugin (running in the child snakemake process) can rejoin it via -#: ``Gateway().connect(name)``. Internal contract, not a user knob. -GATEWAY_CLUSTER_ENV = "LIGHTCONE_GATEWAY_CLUSTER" - -#: Bounds how long the Gateway branch waits for the first worker of a -#: cluster it created (seconds; default 600 — a first-time image pull -#: on a fresh node is minutes, not seconds). Without this bound an -#: unpullable image leaves the run sitting at zero workers forever. -GATEWAY_WORKER_TIMEOUT_ENV = "LIGHTCONE_GATEWAY_WORKER_TIMEOUT" - - -@dataclass -class _NodeShape: - """Per-node resources advertised by the dask worker.""" - - cpus: int - mem_bytes: int - gpus: int - - -def _detect_node_shape() -> _NodeShape: - """Read node capacity from SLURM env vars (with sensible fallbacks).""" - cpus = int(os.environ.get("SLURM_CPUS_ON_NODE") or os.cpu_count() or 1) - - mem_mb = os.environ.get("SLURM_MEM_PER_NODE") - if mem_mb: - mem_bytes = int(mem_mb) * 1_000_000 - else: - try: - import psutil # type: ignore[import-untyped] - - mem_bytes = psutil.virtual_memory().total - except ImportError: - mem_bytes = 0 # advisory: workers won't enforce memory caps - - gpus = int(os.environ.get("SLURM_GPUS_ON_NODE") or 0) - return _NodeShape(cpus=cpus, mem_bytes=mem_bytes, gpus=gpus) - - -def _resource_dict(shape: _NodeShape) -> dict[str, float]: - """Resource keys advertised by a worker for this node shape. - - Single source of truth for which keys workers expose — both the - in-process LocalCluster and the srun-launched ``dask worker``s - advertise the same set so the executor's per-task requests resolve - on either path. - """ - res: dict[str, float] = {RESOURCE_CPUS: float(shape.cpus)} - if shape.mem_bytes: - res[RESOURCE_MEMORY] = float(shape.mem_bytes) - if shape.gpus: - res[RESOURCE_GPUS] = float(shape.gpus) - return res - - -def _resources_arg(shape: _NodeShape) -> str: - """Format `--resources` for `dask worker`.""" - return " ".join(f"{k}={int(v)}" for k, v in _resource_dict(shape).items()) - - -def gateway_branch_active() -> bool: - """Would :func:`cluster_for_run` take the Gateway branch right now? - - Exposed so ``lc run`` can shape the snakemake invocation (e.g. NFS - latency tolerance) before entering the cluster context. Pure - function of the environment, in the same priority order as the - branches in :func:`cluster_for_run`. - """ - if os.environ.get("DASK_SCHEDULER_ADDRESS"): - return False - return bool(os.environ.get("DASK_GATEWAY__ADDRESS")) - - -@contextmanager -def cluster_for_run( - *, - verbose: bool = False, - local_directory: str | None = None, - worker_image: str | None = None, - max_workers: int | None = None, -) -> Iterator[dict[str, str]]: - """Yield the env overlay the child snakemake needs to reach the cluster. - - The parent (``lc run``) and the executor plugin live in different - processes, so connection info travels via environment variables. - Address-based branches yield ``{"DASK_SCHEDULER_ADDRESS": addr}``; - the Gateway branch yields ``{GATEWAY_CLUSTER_ENV: name}`` because - Gateway clusters are rejoined by name through the authenticated - Gateway API rather than dialled by address. - - *local_directory*, when given, is where dask workers stage their - spilled task data and internal state files. ``lc run`` resolves it - to a path under :mod:`lightcone.engine.scratch` so on NERSC the - spill lands on Lustre instead of DVS-mounted home/CFS (where small- - file I/O is slow and can pressure the gateway nodes). - - *worker_image* is the registry ref the project's declared container - resolves to; the Gateway branch creates its cluster with exactly - this image (``None`` → the deployment's default). Ignored by the - other branches — they realize containers by wrapping recipes, not - via pod images. - - *max_workers* bounds the adaptive scaling of a Gateway cluster - (``lc run`` passes its job bound — there is never a reason to hold - more workers than dispatchable rules). Ignored everywhere else. - """ - if addr := os.environ.get("DASK_SCHEDULER_ADDRESS"): - if verbose: - print(f"→ Using existing Dask scheduler at {addr}") - yield {"DASK_SCHEDULER_ADDRESS": addr} - return - - if os.environ.get("DASK_GATEWAY__ADDRESS"): - with _gateway_cluster( - verbose=verbose, worker_image=worker_image, max_workers=max_workers - ) as name: - yield {GATEWAY_CLUSTER_ENV: name} - return - - if "SLURM_JOB_ID" in os.environ: - with _slurm_backed_cluster( - verbose=verbose, local_directory=local_directory - ) as addr: - yield {"DASK_SCHEDULER_ADDRESS": addr} - return - - with _local_cluster( - verbose=verbose, local_directory=local_directory - ) as addr: - yield {"DASK_SCHEDULER_ADDRESS": addr} - - -@contextmanager -def _gateway_cluster( - *, - verbose: bool, - worker_image: str | None, - max_workers: int | None, -) -> Iterator[str]: - """Create a run-scoped Dask Gateway cluster; yield its name. - - The Gateway client is configured entirely by ambient dask config — - on a lightcone JupyterHub deployment the ``DASK_GATEWAY__*`` env - vars carry the API address, the JupyterHub auth mode, and the proxy - address, so ``Gateway()`` needs no arguments here. - """ - from dask_gateway import Gateway - - gateway = Gateway() - # The server-declared cluster options (merged with ambient config - # defaults) tell us what this deployment exposes and what the - # effective worker shape/image will be. - try: - declared = dict(gateway.cluster_options()) - except Exception: - declared = {} - options: dict[str, object] = {} - if worker_image: - # ``image`` is a server-side cluster option declared by the - # deployment's options handler; a deployment that doesn't - # expose it rejects the request — surfaced below with guidance. - options["image"] = worker_image - if "environment" in declared: - # Self-provision everything our executor needs from a worker - # pod through the *standard* ``environment`` option, so the - # deployment's options handler can stay stock — no - # lightcone-specific injection required server-side. - base_env = dict(declared.get("environment") or {}) - options["environment"] = { - **base_env, - **_worker_environment(declared, worker_image), - } - try: - cluster = gateway.new_cluster(shutdown_on_close=True, **options) - except Exception as exc: - detail = ( - f" (requested image={worker_image!r} — if the deployment does " - "not expose an `image` cluster option, ask the hub admin to " - "add it to the gateway's cluster-options handler)" - if worker_image - else "" - ) - raise RuntimeError( - f"Could not create a Dask Gateway cluster ({exc}){detail}." - ) from exc - - bound = max(1, max_workers or 1) - if verbose: - image_note = f" with image {worker_image}" if worker_image else "" - print( - f"→ Created Dask Gateway cluster {cluster.name}{image_note}; " - f"scaling adaptively up to {bound} worker(s) " - f"(dashboard: {cluster.dashboard_link})" - ) - try: - cluster.adapt(minimum=1, maximum=bound) - client = cluster.get_client() - try: - _wait_first_worker(client, image=worker_image) - _assert_worker_resources(client) - finally: - client.close() - yield str(cluster.name) - finally: - # We created it, we cull it. shutdown() stops the cluster - # server-side; if lc dies before reaching this, shutdown_on_close - # and the deployment's idle timeout are the backstops. - try: - cluster.shutdown() - except Exception: - cluster.close() - if verbose: - print(f"→ Shut down Dask Gateway cluster {cluster.name}") - - -def _worker_environment( - declared: dict[str, object], worker_image: str | None -) -> dict[str, str]: - """Env vars lc provisions into scheduler/worker pods. - - Passed through the deployment's standard ``environment`` cluster - option — everything worker pods need that only lc (or the driver's - own environment) knows, keeping the deployment's options handler - free of lightcone-specific injection: - - * ``HOME``/``USER``/``LOGNAME`` — always set. Project images are - environment-agnostic (no passwd entry for the pod uid), so - ``getpass.getuser()`` (called by snakemake at startup) crashes - in a worker unless the env vars are present — and the notebook - pod may not export ``USER``/``LOGNAME`` itself (its own passwd - entry covers it), so the name is *derived* on the driver via - ``getpass`` rather than merely forwarded. Home paths are - identical on both sides by construction (same NFS mount). - * ``DASK_DISTRIBUTED__WORKER__RESOURCES__*`` — the scheduling - resource contract, mirrored from the deployment's declared - ``worker_cores``/``worker_memory`` option values. Dask matches - resource keys by exact presence; without these every rule hangs. - * ``LIGHTCONE_WORKER_IMAGE`` — the image the cluster runs (ours, - or the deployment default), recorded by the manifest layer as - execution ground truth. - """ - env: dict[str, str] = {} - if home := os.environ.get("HOME"): - env["HOME"] = home - try: - # Checks USER/LOGNAME/... env vars first, then the driver's - # passwd — one of the two works in any sane notebook pod. - user = getpass.getuser() - except (KeyError, OSError): - # Driver can't determine a name either; any stable non-empty - # value keeps snakemake alive, and jovyan is the Jupyter - # convention for uid 1000. - user = "jovyan" - env["USER"] = user - env["LOGNAME"] = user - - cores = declared.get("worker_cores") - if isinstance(cores, (int, float)) and cores > 0: - env["DASK_DISTRIBUTED__WORKER__RESOURCES__CPUS"] = str(int(cores)) - memory = declared.get("worker_memory") - if isinstance(memory, (int, float)) and memory > 0: - # Deployments conventionally declare worker_memory in GB - # (float); anything implausibly large for GB is already bytes. - mem_bytes = int(memory * 1e9) if memory < 1e6 else int(memory) - env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] = str(mem_bytes) - env["DASK_DISTRIBUTED__WORKER__RESOURCES__GPUS"] = str( - int(declared.get("worker_gpus") or 0) # type: ignore[call-overload] - ) - - image = worker_image or declared.get("image") - if isinstance(image, str) and image: - env["LIGHTCONE_WORKER_IMAGE"] = image - return env - - -def _wait_first_worker(client: object, *, image: str | None) -> None: - """Block until the created cluster has one live worker. - - An unpullable image or an unschedulable pool otherwise leaves the - run sitting at zero workers with no error at all — the classic - silent-hang failure mode. - """ - try: - timeout = int(os.environ.get(GATEWAY_WORKER_TIMEOUT_ENV) or 600) - except ValueError: - timeout = 600 - try: - client.wait_for_workers(n_workers=1, timeout=timeout) # type: ignore[attr-defined] - except Exception as exc: - image_hint = f"the worker image ({image or 'deployment default'}) cannot be pulled" - raise RuntimeError( - f"No Dask Gateway worker became ready within {timeout}s " - f"({exc}). Likely causes: {image_hint}, or the node pool " - "cannot schedule a worker (capacity/quota). Check the " - "JupyterLab Dask panel for the cluster's state, or raise " - f"{GATEWAY_WORKER_TIMEOUT_ENV}." - ) from exc - - -def _assert_worker_resources(client: object) -> None: - """Fail fast when Gateway workers don't advertise the resource contract. - - Dask schedules a task only on workers advertising *every* requested - resource key. The executor requests ``cpus`` for every rule and - ``memory`` for any rule with ``mem_mb``, so a deployment that forgot - to inject ``DASK_DISTRIBUTED__WORKER__RESOURCES__*`` into worker - pods makes every rule hang with no error — refuse loudly instead. - ``gpus`` is deliberately not required: a CPU-only deployment - legitimately omits it. - """ - workers = client.scheduler_info().get("workers", {}) # type: ignore[attr-defined] - if not workers: - return - if any( - RESOURCE_CPUS in res and RESOURCE_MEMORY in res - for w in workers.values() - if (res := w.get("resources") or {}) is not None - ): - return - raise RuntimeError( - "Dask Gateway workers do not advertise the lightcone resource " - f"contract ({RESOURCE_CPUS}+{RESOURCE_MEMORY}, plus " - f"{RESOURCE_GPUS} on GPU pools); per-rule resource requests " - "would never schedule. lc provisions these via the gateway's " - "`environment` cluster option — this deployment likely doesn't " - "expose that option (or strips it); ask the hub admin to expose " - "the standard image/worker_cores/worker_memory/environment " - "options." - ) - - -@contextmanager -def _local_cluster( - *, verbose: bool, local_directory: str | None -) -> Iterator[str]: - from dask.distributed import LocalCluster - - shape = _detect_node_shape() - # Workers must advertise every key the executor may request — Dask - # matches by exact key presence — or rules with ``mem_mb`` / - # ``gpus_per_task`` would never schedule on a workstation. - cluster = LocalCluster( - n_workers=1, - threads_per_worker=shape.cpus, - resources=_resource_dict(shape), - dashboard_address=":0", - local_directory=local_directory, - silence_logs=logging.INFO if verbose else logging.WARNING, - ) - if verbose: - print( - f"→ Local Dask cluster ({shape.cpus} threads); " - f"scheduler at {cluster.scheduler_address}" - ) - try: - yield cluster.scheduler_address - finally: - cluster.close() - - -@contextmanager -def _slurm_backed_cluster( - *, verbose: bool, local_directory: str | None -) -> Iterator[str]: - from dask.distributed import LocalCluster - - if shutil.which("dask") is None: - raise RuntimeError( - "`dask` CLI is not on PATH inside the SLURM allocation. " - "Install lightcone-cli (and its `distributed` dep) into the " - "environment activated by your sbatch/salloc." - ) - - shape = _detect_node_shape() - nnodes = int(os.environ.get("SLURM_NNODES") or 1) - - # Default LocalCluster binds the scheduler to 127.0.0.1, which workers - # on remote nodes cannot reach. Bind to the driver's hostname so srun- - # launched workers across the allocation can connect. SLURMD_NODENAME - # is the SLURM-canonical name; gethostname() is a sane fallback. - scheduler_host = os.environ.get("SLURMD_NODENAME") or socket.gethostname() - cluster = LocalCluster( - n_workers=0, - host=scheduler_host, - dashboard_address=":0", - local_directory=local_directory, - silence_logs=logging.INFO if verbose else logging.WARNING, - ) - addr = cluster.scheduler_address - - if verbose: - print( - f"→ SLURM allocation detected ({nnodes} node(s), " - f"{shape.cpus} cpu/node, {shape.gpus} gpu/node); " - f"launching workers via srun. Scheduler: {addr}" - ) - - worker_cmd = [ - "srun", - f"--ntasks={nnodes}", - "--ntasks-per-node=1", - "dask", - "worker", - addr, - "--nthreads", - str(shape.cpus), - "--nworkers", - "1", - "--resources", - _resources_arg(shape), - "--no-dashboard", - # Each srun task is a single run-scoped worker; an auto-restart - # nanny adds no value (srun won't relaunch the task either) and - # logs "Worker process died unexpectedly" when retire_workers - # asks the worker to exit on shutdown. - "--no-nanny", - ] - if local_directory: - worker_cmd.extend(["--local-directory", local_directory]) - # Hide the worker's INFO-level connection chatter (Nanny start, - # scheduler registration, etc.) — useful only when debugging the - # cluster itself. WARNING+ still surface real issues. The newer - # `dask worker` CLI dropped `--silence-logs`, so we drive it via - # Dask's config env var instead; srun inherits env by default. - worker_env = dict(os.environ) - if not verbose: - worker_env.setdefault("DASK_LOGGING__DISTRIBUTED", "warning") - workers = subprocess.Popen(worker_cmd, env=worker_env) - - try: - from dask.distributed import Client - - client = Client(addr) - try: - client.wait_for_workers(n_workers=nnodes, timeout=120) - if verbose: - print(f"→ {nnodes} dask worker(s) registered.") - finally: - client.close() - yield addr - finally: - # Graceful shutdown: ask the scheduler to retire workers so each - # `dask worker` process exits on its own. srun then sees its task - # exit with code 0 and terminates silently. SIGTERM-ing srun - # directly (the prior path) prints "srun: forcing job - # termination" / "task 0: Killed" to stderr on every clean run. - try: - client = Client(addr, timeout="10s") - try: - client.retire_workers(close_workers=True, remove=True) - finally: - client.close() - except Exception: - pass - try: - workers.wait(timeout=20) - except subprocess.TimeoutExpired: - workers.terminate() - try: - workers.wait(timeout=10) - except subprocess.TimeoutExpired: - workers.kill() - workers.wait() - cluster.close() diff --git a/src/lightcone/engine/manifest.py b/src/lightcone/engine/manifest.py deleted file mode 100644 index b22fda1b..00000000 --- a/src/lightcone/engine/manifest.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Per-output content-addressed manifests. - -The integrity layer of lightcone-cli. Every materialized output gets a -sidecar JSON manifest at ``/.lightcone-manifest.json`` that -records: - -- ``code_version``: sha256(recipe + container image + decisions). Stored - in each rule's per-universe ``params.cfg`` so Snakemake's ``params`` - rerun-trigger detects drift automatically. (The ``code`` trigger only - sees the rule body source, which is universe-parameterized and never - changes — that is why ``lc run`` defaults to including ``params``.) -- ``data_version``: sha256 of the output directory's contents. Lets - ``lc verify`` prove the bytes on disk are what the manifest claims. -- ``input_versions``: each declared input's ``data_version`` (if it's a - materialized output) or ``(mtime, size)`` fingerprint (if it's an - external file). This is the chain. - -Manifests are written by :func:`write_manifest`, called from each rule's -``run:`` block on the host immediately after the recipe shell exits. The -``os.replace`` rename is the atomic commit point: either the rule produced -both data and manifest, or it failed and Snakemake will rerun it. -""" -from __future__ import annotations - -import hashlib -import json -import os -import socket -import time -from pathlib import Path -from typing import Any - -MANIFEST_FILENAME = ".lightcone-manifest.json" -SCHEMA_VERSION = 1 - -#: Filenames inside an output directory that the data_version hash MUST -#: ignore: the manifest itself (chicken-and-egg) and Snakemake's -#: ``directory()`` mtime marker (touched AFTER the rule body completes). -_HASH_EXCLUDE = frozenset({MANIFEST_FILENAME, ".snakemake_timestamp"}) - -__all__ = [ - "MANIFEST_FILENAME", - "SCHEMA_VERSION", - "code_version", - "fingerprint_external", - "read_manifest", - "sha256_dir", - "write_manifest", -] - - -def _hash_file(path: Path, h: hashlib._Hash) -> None: - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(64 * 1024), b""): - h.update(chunk) - - -def sha256_dir(path: Path) -> str: - """Deterministic content hash of a directory. - - Walks ``path`` recursively, hashes each file along with its relative - path (so renames change the hash), and excludes the manifest plus - Snakemake's directory-output timestamp marker. - """ - path = Path(path) - if not path.exists(): - raise FileNotFoundError(path) - h = hashlib.sha256() - files: list[Path] = [ - p for p in path.rglob("*") if p.is_file() and p.name not in _HASH_EXCLUDE - ] - for p in sorted(files, key=lambda x: x.relative_to(path).as_posix()): - rel = p.relative_to(path).as_posix().encode("utf-8") - h.update(b"path:") - h.update(rel) - h.update(b"\0") - h.update(b"data:") - _hash_file(p, h) - h.update(b"\0") - return f"sha256:{h.hexdigest()}" - - -def _sha256_bytes(data: bytes) -> str: - return f"sha256:{hashlib.sha256(data).hexdigest()}" - - -def _sha256_file(path: Path) -> str: - h = hashlib.sha256() - _hash_file(path, h) - return f"sha256:{h.hexdigest()}" - - -def fingerprint_external(path: Path, *, strict: bool = False) -> str: - """Fingerprint an external input. - - For files: ``(mtime, size)`` by default; sha256 when ``strict=True``. - For directories: always sha256. - For missing paths: returns the literal string ``"missing"``. - """ - if not path.exists(): - return "missing" - if path.is_dir(): - return sha256_dir(path) - if strict: - return _sha256_file(path) - st = path.stat() - return f"mtime-size:{st.st_mtime_ns}-{st.st_size}" - - -def code_version( - *, - recipe: str, - container_image: str | None, - decisions: dict[str, Any], -) -> str: - """Compute a deterministic code version for an output. - - Hashes the recipe text, container image identifier, and canonicalized - decisions. Anything that changes the materialization semantics flows - through this hash; the *runtime* used to invoke the container - (docker/podman/podman-hpc) is intentionally excluded — the same image - produces the same data regardless of which OCI tool launched it. - """ - payload = { - "recipe": recipe, - "container_image": container_image, - "decisions": decisions, - } - return _sha256_bytes( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") - ) - - -def read_manifest(output_dir: Path) -> dict[str, Any] | None: - """Read the manifest at ``/.lightcone-manifest.json``. - - Returns ``None`` if the manifest is missing or unparseable. ``OSError`` - (permission denied, I/O failure) is intentionally **not** caught — - those are real problems that should not silently look like a missing - manifest in ``lc verify`` / ``lc status`` output. - """ - p = Path(output_dir) / MANIFEST_FILENAME - if not p.exists(): - return None - try: - return json.loads(p.read_text()) # type: ignore[no-any-return] - except json.JSONDecodeError: - return None - - -def write_manifest( - *, - output_dir: Path, - inputs: dict[str, Path], - cfg: dict[str, Any], -) -> Path: - """Atomically write the manifest for an already-materialized output. - - Called from each rule's ``run:`` block after the recipe shell exits. - Hashes the output dir, resolves input versions (chaining to upstream - manifests when present, falling back to external fingerprints), and - commits the manifest via ``os.replace``. - - Args: - output_dir: Directory containing the materialized output files. - inputs: Mapping of declared input id → filesystem path. Each is - either a directory containing a sibling manifest (upstream - output) or an external file/dir. - cfg: Per-rule configuration. Required keys: ``output_id``, - ``universe_id``, ``recipe``, ``container_image``, ``decisions``, - ``code_version``, ``git_sha``, ``lc_version``. - """ - output_dir = Path(output_dir) - - input_versions: dict[str, str] = {} - for inp_id, inp_path in inputs.items(): - inp_path = Path(inp_path) - upstream = read_manifest(inp_path) - if upstream is not None: - input_versions[inp_id] = upstream["data_version"] - else: - input_versions[inp_id] = fingerprint_external(inp_path) - - manifest = { - "schema_version": SCHEMA_VERSION, - "output_id": cfg["output_id"], - "universe_id": cfg["universe_id"], - "code_version": cfg["code_version"], - "data_version": sha256_dir(output_dir), - "container_image": cfg.get("container_image"), - "recipe": cfg["recipe"], - "decisions": cfg.get("decisions", {}), - "input_versions": input_versions, - "git_sha": cfg.get("git_sha"), - # URL of the git origin remote at the time of materialization. - # Optional/additive — older manifests without this field still - # parse. Surfaced by ``lc export wrroc`` as a CodeRepository entity. - "git_remote": cfg.get("git_remote"), - "lc_version": cfg.get("lc_version"), - "finished_at": time.time(), - "host": socket.gethostname(), - "slurm_job_id": os.environ.get("SLURM_JOB_ID"), - # On a Dask Gateway deployment `lc run` provisions this into - # every scheduler/worker pod (via the `environment` cluster - # option) with the image the cluster was started with. - # ``container_image`` above is what the spec *declared*; this is - # the pod-reported ground truth of what actually executed. - # Optional/additive — None everywhere else. - "worker_image": os.environ.get("LIGHTCONE_WORKER_IMAGE"), - } - - final_path = output_dir / MANIFEST_FILENAME - tmp_path = final_path.with_suffix(final_path.suffix + ".tmp") - tmp_path.write_text(json.dumps(manifest, sort_keys=True, indent=2)) - os.replace(tmp_path, final_path) - return final_path diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py new file mode 100644 index 00000000..0bd62bc3 --- /dev/null +++ b/src/lightcone/engine/project.py @@ -0,0 +1,390 @@ +"""What a project *is*, and how to converge one. + +Two responsibilities, both about the project as an object rather than +about the CLI: + +- **discovery** — :func:`find_root`. One rule, shared by every verb (and, + once it lands, the launcher): the project root is the nearest ancestor, + including the start directory, that contains an ``astra.yaml``. uv's own + walk-up discovery is never trusted — every uv invocation downstream + carries an explicit ``--project `` (spec §4). +- **convergence** — :func:`converge`. Bring a directory to the state a + lightcone project is defined to have: an ASTRA spec sitting on a uv + project. Idempotent, and never destructive to files the user owns. + +Nothing here knows about Click or the console: convergence returns a +:class:`ConvergenceReport` and raises :class:`ProjectError`; rendering and +exit codes are the CLI's business. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + +from lightcone.engine import templates +from lightcone.engine.constants import DEFAULT_PYTHON + +SPEC_FILENAME = "astra.yaml" + + +class ProjectError(Exception): + """A project cannot be discovered, read, or converged.""" + + +# ============================================================================= +# Discovery +# ============================================================================= + + +def find_root(start: Path | None = None) -> Path | None: + """Walk up from *start* (default: cwd) to the project root, or ``None``.""" + p = (start or Path.cwd()).resolve() + for parent in [p, *p.parents]: + if (parent / SPEC_FILENAME).is_file(): + return parent + return None + + +def project_name(directory: Path) -> str: + """A PEP 503-ish project name derived from the directory name.""" + name = re.sub(r"[^A-Za-z0-9._-]+", "-", directory.name).strip("-._").lower() + return name or "analysis" + + +# ============================================================================= +# The convergence report +# ============================================================================= + + +@dataclass +class ConvergenceReport: + """What a convergence did, or — in check mode — would have done. + + ``warnings`` carries problems convergence can *see* but must not fix; + they never affect whether the project counts as converged. + """ + + created: list[str] = field(default_factory=list) + repaired: list[str] = field(default_factory=list) + unchanged: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def converged(self) -> bool: + return not self.created and not self.repaired + + def as_dict(self) -> dict[str, object]: + return { + "converged": self.converged, + "created": self.created, + "repaired": self.repaired, + "unchanged": self.unchanged, + "warnings": self.warnings, + } + + +class _Converger: + """Decide-then-maybe-write, so check mode reuses the real decisions. + + Every item routes through :meth:`item` or :meth:`file`, which record + the outcome first and only then apply it when ``write`` is set. That + is what keeps ``lc init --check`` honest: it is the same code path with + side effects switched off, not a second implementation. + """ + + def __init__(self, *, write: bool) -> None: + self.write = write + self.report = ConvergenceReport() + + def item(self, name: str, present: bool, apply: Callable[[], object]) -> None: + """Converge something whose *presence* is the whole question.""" + if present: + self.report.unchanged.append(name) + else: + self.report.created.append(name) + if self.write: + apply() + + def file( + self, + name: str, + path: Path, + template: str, + repair: Callable[[str], str | None] | None = None, + ) -> None: + """Create *path* from *template* if missing; else offer it to *repair*. + + ``repair`` receives the current text and returns the fixed text, or + ``None`` when the file is already fine. Repairs must be + conservative by construction — see :func:`_repair_gitignore`. + """ + if not path.exists(): + self.report.created.append(name) + if self.write: + path.write_text(template) + elif repair is not None and (fixed := repair(path.read_text())) is not None: + self.report.repaired.append(name) + if self.write: + path.write_text(fixed) + else: + self.report.unchanged.append(name) + + def warn(self, message: str) -> None: + self.report.warnings.append(message) + + +# ============================================================================= +# Convergence +# ============================================================================= + + +def converge( + directory: Path, + *, + write: bool = True, + git: bool = True, + sync: bool = True, +) -> ConvergenceReport: + """Converge *directory* into an ASTRA project. Idempotent. + + Creates whatever is missing, repairs the pieces lightcone manages, and + never overwrites a file the user owns. A directory that already holds + an ``astra.yaml`` is adopted, not rejected. + + With ``write=False`` nothing touches the filesystem — not even the + directory itself — and the returned report describes what a real run + would have done. + + Raises :class:`ProjectError` for the two states convergence must not + paper over: an authored ``Containerfile``, and a missing or failing + uv. + """ + directory = directory.resolve() + + _refuse_authored_containerfile(directory) + _require_uv() + + c = _Converger(write=write) + + if write: + directory.mkdir(parents=True, exist_ok=True) + + _converge_spec(c, directory) + _converge_uv_project(c, directory) + _converge_managed_files(c, directory) + _converge_results(c, directory) + _converge_report_template(c, directory) + + if git: + c.item( + ".git", + (directory / ".git").exists(), + lambda: subprocess.run(["git", "init", "-q"], cwd=directory, check=False), + ) + + # Lock and sync last: they are the expensive steps, and they are + # meaningless until pyproject.toml and .python-version exist. + c.item("uv.lock", (directory / "uv.lock").exists(), lambda: _uv_lock(directory)) + if sync: + c.item(".venv", (directory / ".venv").exists(), lambda: _uv_sync(directory)) + + return c.report + + +def _refuse_authored_containerfile(directory: Path) -> None: + """Refuse before any write. + + Images are generated from the lock, so a hand-authored Containerfile + would be silently ignored — a half-state the user has to resolve. The + user's own file operation is the consent to migrate; no flag can + substitute for it (spec §8). + """ + if (directory / "Containerfile").is_file(): + raise ProjectError( + f"{directory}/Containerfile: images are generated from the " + "lock — delete or rename it, then re-run `lc init`. Declare " + "system dependencies in [tool.lightcone.image] instead." + ) + + +def _require_uv() -> None: + if shutil.which("uv") is None: + raise ProjectError( + "uv is required (the environment substrate). Install it: " + "https://docs.astral.sh/uv/getting-started/installation/" + ) + + +def _converge_spec(c: _Converger, directory: Path) -> None: + """The ASTRA spec scaffold: astra.yaml + universes/ + src/.""" + c.item("astra.yaml", (directory / SPEC_FILENAME).exists(), lambda: _scaffold_spec(directory)) + + +def _scaffold_spec(directory: Path) -> None: + """Write astra's boilerplate, then make it environment-free. + + astra's ``init`` *command* refuses non-empty directories and + overwrites ``.gitignore`` — both wrong for convergence — so this uses + the bare scaffold API and manages ``.gitignore`` separately. The + public ``create_boilerplate`` (LightconeResearch/astra-tools#99) is + preferred, with the private helper as the fallback on releases that + predate it; resolved by ``getattr`` rather than an import guard so the + code type-checks under either. + """ + from astra import cli as astra_cli + + create_boilerplate = getattr(astra_cli, "create_boilerplate", None) + if create_boilerplate is not None: + create_boilerplate(directory) + else: + (directory / "universes").mkdir(exist_ok=True) + astra_cli._create_boilerplate_astra_yaml(directory) + + # The boilerplate recipes reference scripts under src/ (e.g. + # ``python src/main.py``); astra's own init creates the directory, so + # the scaffold must too. + (directory / "src").mkdir(exist_ok=True) + + # ASTRA carries analysis structure only — the environment lives in + # pyproject.toml + uv.lock. Strip the ``container:`` line the + # boilerplate ships. + spec_path = directory / SPEC_FILENAME + stripped = re.sub(r"(?m)^container:.*\n?", "", spec_path.read_text(), count=1) + spec_path.write_text(stripped) + + +def _converge_uv_project(c: _Converger, directory: Path) -> None: + """pyproject.toml + .python-version — the environment definition. + + An existing ``pyproject.toml`` is the user's: read, possibly warned + about, never edited. + """ + pyproject_path = directory / "pyproject.toml" + if not pyproject_path.exists(): + c.report.created.append("pyproject.toml") + if c.write: + pyproject_path.write_text(templates.pyproject(name=project_name(directory))) + else: + c.report.unchanged.append("pyproject.toml") + if "lightcone-cli" not in pyproject_path.read_text(): + c.warn( + "pyproject.toml does not depend on lightcone-cli — the " + "engine should live inside the experiment's lock: " + "`uv add lightcone-cli`." + ) + + c.file(".python-version", directory / ".python-version", f"{DEFAULT_PYTHON}\n") + + +def _converge_managed_files(c: _Converger, directory: Path) -> None: + c.file( + ".gitignore", + directory / ".gitignore", + templates.gitignore(), + repair=_repair_gitignore, + ) + + +def _converge_results(c: _Converger, directory: Path) -> None: + results_dir = directory / "results" + if results_dir.exists() and not results_dir.is_dir(): + c.report.unchanged.extend(["results/", "results/README.md"]) + c.warn( + "results exists but is not a directory; outputs cannot " + "materialize until it is one." + ) + return + c.item("results/", results_dir.is_dir(), results_dir.mkdir) + c.file("results/README.md", results_dir / "README.md", templates.results_readme()) + + +def _converge_report_template(c: _Converger, directory: Path) -> None: + """The MyST report. Recommended add-on on top of the spec, not part of + it — which is why it is scaffolded here and not by ``astra init``.""" + c.file("myst.yml", directory / "myst.yml", templates.myst_yml()) + c.file( + "index.md", + directory / "index.md", + templates.index_md(title=directory.name or "My Analysis"), + ) + + +# ============================================================================= +# Repairs — append behind a marker, never rewrite +# ============================================================================= + + +def _repair_gitignore(text: str) -> str | None: + """Ensure every pattern lightcone manages is present, appending only + the ones that are missing. + + Entry-wise rather than marker-based: idempotency is then structural — + a pattern already in the file is never added again, whoever put it + there — and a pattern introduced by a later lc release still reaches + projects that already have a ``.gitignore``. + + The append preserves template order, which is what keeps + ``!results/README.md`` after the ``results/*`` it negates. (A file + holding the negation *without* ``results/*`` would end up with them + inverted; that state can only be hand-written, and re-ordering + someone's ignores to fix it would be the more surprising behavior.) + """ + missing = templates.missing_gitignore_entries(text) + if not missing: + return None + + block = "\n".join(missing) + "\n" + # The header is cosmetic, so add it only when it isn't already there; + # a second repair later appends bare patterns under the first one. + if templates.GITIGNORE_HEADER not in text: + block = templates.GITIGNORE_HEADER + "\n" + block + if not text.strip(): + return block + return text.rstrip("\n") + "\n\n" + block + + +# ============================================================================= +# The uv seam +# ============================================================================= + + +def _run_uv(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + """One seam for every uv invocation (tests monkeypatch it).""" + return subprocess.run(["uv", *args], cwd=cwd, capture_output=True, text=True, check=False) + + +def _uv(args: list[str], *, directory: Path, what: str) -> None: + """Run uv, surfacing a failure as :class:`ProjectError`. + + Every invocation carries an explicit ``--project``: uv's native + walk-up discovery is never trusted (spec §4). A failure surfaces here + with uv's own stderr, because a silently broken lock would fail every + later verb more confusingly. + """ + proc = _run_uv([*args, "--project", str(directory)], cwd=directory) + if proc.returncode != 0: + raise ProjectError(f"`uv {what}` failed:\n{proc.stderr.strip()}") + + +def _uv_lock(directory: Path) -> None: + _uv(["lock"], directory=directory, what="lock") + + +def _uv_sync(directory: Path) -> None: + """Converge the environment, with the flags the spec fixes. + + ``--locked`` so a drifted lock is an error rather than a silent + relock; ``--exact`` because a plain sync is additive; and + ``--compile-bytecode`` because the environment is read-only at + execution time (spec §4, §6). + """ + _uv( + ["sync", "--locked", "--exact", "--compile-bytecode"], + directory=directory, + what="sync", + ) diff --git a/src/lightcone/engine/runner.py b/src/lightcone/engine/runner.py deleted file mode 100644 index 800f302a..00000000 --- a/src/lightcone/engine/runner.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Per-rule execution helper invoked from the generated Snakefile. - -Each rule's ``run:`` block boils down to one call to :func:`run_rule`. -The helper: - -* runs the rule's pre-rendered shell command (template substitution and - container wrapping happen at Snakefile-generation time — see - :func:`lightcone.engine.snakefile.render_recipe`) with stdout and - stderr captured, -* emits a ``▶ rule [universe]`` header, the recipe's output, and a - ``✓ rule [universe] `` (or ``✗ … exit=N``) trailer, - each line framed with a sentinel prefix the executor extracts, -* writes the per-output manifest on success, -* runs the validation hook on the materialized output, -* raises :class:`subprocess.CalledProcessError` on non-zero exit so - Snakemake records the job as failed and halts the DAG. - -The sentinel prefix (:data:`SENTINEL`) is what the dask executor's -``_run_shell`` looks for when it filters worker subprocess output — -anything else (snakemake bootstrap, dask logs, stray prints) is dropped -on the floor. This is the entire mechanism by which lc run shows clean, -narrative output without ever filtering against a moving target of -upstream log strings. -""" -from __future__ import annotations - -import subprocess -import sys -import time -from pathlib import Path -from typing import Any - -#: Lines from the runner are prefixed with this so ``_run_shell`` in the -#: dask executor can distinguish them from snakemake/dask noise. Chosen -#: to be vanishingly unlikely in real recipe output (printable ASCII, -#: column-0 anchored, distinctive). Kept short to minimise capture cost. -SENTINEL = "__LCSTREAM__::" - - -def _emit(line: str = "") -> None: - """Write one sentinel-prefixed line to stdout and flush. - - The flush matters: we run inside a child snakemake subprocess whose - stdout is captured by the worker's ``_run_shell``; without flushing - the recipe output would arrive after ``rule_end`` if Python decides - to block-buffer. - """ - sys.stdout.write(f"{SENTINEL}{line}\n") - sys.stdout.flush() - - -def run_rule( - *, - rule_key: str, - universe: str, - output_dir: Path, - inputs: dict[str, Path], - cfg: dict[str, Any], -) -> None: - """Execute one rule's pre-rendered shell command and write its manifest. - - Called from the generated Snakefile's ``run:`` block. Recipe stdout - and stderr are interleaved by capture order (stdout first, then - stderr) — Snakemake's own output capture has the same property and - most recipes are well-behaved enough that this is fine. - - On non-zero exit, the manifest is **not** written. Snakemake will - treat the rule as failed; ``lc verify`` won't see a stale manifest - pointing at incomplete data. - """ - from lightcone.engine.manifest import write_manifest - from lightcone.engine.validation import validate_output - - t0 = time.monotonic() - _emit(f"\033[2m▶\033[0m {rule_key} \033[2m[{universe}]\033[0m") - - proc = subprocess.run( - cfg["shell_command"], - shell=True, - capture_output=True, - text=True, - check=False, - ) - - for line in proc.stdout.splitlines(): - _emit(f" {line}") - for line in proc.stderr.splitlines(): - _emit(f" {line}") - - dt = time.monotonic() - t0 - if proc.returncode != 0: - _emit( - f"\033[31m✗\033[0m {rule_key} \033[2m[{universe}]\033[0m " - f"exit={proc.returncode} {dt:.1f}s" - ) - raise subprocess.CalledProcessError(proc.returncode, cfg["shell_command"]) - - write_manifest(output_dir=output_dir, inputs=inputs, cfg=cfg) - - for warning in validate_output( - output_dir, cfg.get("output_type"), cfg["output_id"] - ): - _emit(f" \033[33m⚠\033[0m {warning}") - - _emit( - f"\033[32m✓\033[0m {rule_key} \033[2m[{universe}]\033[0m {dt:.1f}s" - ) - - -__all__ = ["SENTINEL", "run_rule"] diff --git a/src/lightcone/engine/scratch.py b/src/lightcone/engine/scratch.py deleted file mode 100644 index 0b3d0cf0..00000000 --- a/src/lightcone/engine/scratch.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Resolve and prepare lightcone's scratch root. - -A single concept: where lightcone keeps its operational state — snakemake -metadata, dask worker spill, the run-exclusion lock. Resolved at the -start of every ``lc run``. Resolution precedence (first hit wins): - -1. ``LIGHTCONE_SCRATCH`` env var (escape hatch / CI override). -2. ``scratch_root`` in ``/.lightcone/lightcone.yaml`` (per-project pin). -3. ``scratch_root`` from the detected site in - :mod:`lightcone.engine.site_registry`. Stored as a shell expression - (e.g. ``$SCRATCH``) and expanded with :func:`os.path.expandvars`. -4. :func:`tempfile.gettempdir` fallback (single-node only). - -The resolved path is then used as the parent of ``.lightcone/`` — -multiple projects can share one scratch root without colliding because -snakemake state is keyed by a hash of the project's absolute path. - -Why this matters on NERSC: ``$HOME`` and ``/global/cfs`` are mounted on -compute nodes via DVS, which `does not support file locking -`_. Snakemake's workflow -lock, our run-exclusion lock, and any future coordination primitive -silently fail there. ``$SCRATCH`` is Lustre, which works correctly. -""" -from __future__ import annotations - -import contextlib -import fcntl -import hashlib -import os -import shutil -import tempfile -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path - -import yaml - -from lightcone.engine.site_registry import detect_current_site - -LIGHTCONE_SCRATCH_ENV = "LIGHTCONE_SCRATCH" - - -@dataclass(frozen=True) -class RunDirs: - """Per-run scratch directories. All paths are guaranteed to exist.""" - - root: Path # ``/.lightcone`` - snakemake_state: Path # ``/.lightcone/snakemake//.snakemake`` - dask_local: Path # ``/.lightcone/dask/`` - # Project-level sentinel for the run-exclusion flock. Held for the - # duration of one ``lc run`` to prevent concurrent invocations on - # the same project from interleaving Snakemake state updates. - run_lock_path: Path # ``/.lightcone/locks/.run-lock`` - - -def resolve_scratch_root(project_path: Path) -> Path: - """Resolve the scratch root directory for *project_path*. - - See module docstring for the precedence chain. Always returns a - ``Path``; never raises. The returned path is not created — callers - that need sub-directories should use :func:`prepare_run_dirs`. - """ - if env := os.environ.get(LIGHTCONE_SCRATCH_ENV): - return Path(os.path.expandvars(env)).expanduser() - - project_cfg = project_path / ".lightcone" / "lightcone.yaml" - if project_cfg.is_file(): - try: - data = yaml.safe_load(project_cfg.read_text()) or {} - except yaml.YAMLError: - data = {} - if val := data.get("scratch_root"): - return Path(os.path.expandvars(str(val))).expanduser() - - if val := detect_current_site().get("scratch_root"): - expanded = os.path.expandvars(str(val)) - # ``$VAR`` left intact means the env wasn't set — don't write - # to a literal path called ``$SCRATCH``. Fall through. - if not expanded.startswith("$") and "$" not in expanded: - return Path(expanded).expanduser() - - return Path(tempfile.gettempdir()) - - -def project_hash(project_path: Path) -> str: - """Stable short hash keyed on the absolute project path. - - Used to namespace snakemake state under the scratch root: two - different projects sharing one ``$SCRATCH`` get separate - ``.snakemake/`` dirs; the same project moved to a different machine - gets a fresh state (since absolute path differs). - """ - return hashlib.sha256(str(project_path.resolve()).encode("utf-8")).hexdigest()[:12] - - -def prepare_run_dirs(project_path: Path, *, run_id: str | None = None) -> RunDirs: - """Create and return per-run scratch sub-directories. - - *run_id* defaults to the current PID — unique per ``lc run`` - invocation, easily mappable to a process for debugging. Lock and - dask-local dirs are run-scoped (cleaned per invocation); snakemake - state is project-scoped (persistent across invocations). - """ - scratch = resolve_scratch_root(project_path) - root = scratch / ".lightcone" - rid = run_id or str(os.getpid()) - pkey = project_hash(project_path) - snakemake_state = root / "snakemake" / pkey / ".snakemake" - dask_local = root / "dask" / rid - run_lock_path = root / "locks" / f"{pkey}.run-lock" - for d in (root, snakemake_state.parent, dask_local, run_lock_path.parent): - d.mkdir(parents=True, exist_ok=True) - # Touch the lockfile so ``flock`` never races on ``O_CREAT``. Empty - # file is fine — flock is independent of contents. - run_lock_path.touch(exist_ok=True) - return RunDirs( - root=root, - snakemake_state=snakemake_state, - dask_local=dask_local, - run_lock_path=run_lock_path, - ) - - -def ensure_snakemake_symlink(project_path: Path, snakemake_state: Path) -> None: - """Repoint ``/.snakemake`` to *snakemake_state*. - - Snakemake stores its workflow lock and per-job metadata under - ``.snakemake/`` in the working directory. On NERSC-like sites where - the project lives on a DVS-mounted filesystem, that directory's - ``flock``s are silent no-ops and small-file I/O is slow. Redirecting - via symlink lets snakemake find its state at the canonical path - while the bytes actually live on Lustre. - - If a real (non-symlink) ``.snakemake/`` already exists from a prior - direct ``snakemake`` invocation, we move it aside with a - ``.snakemake.legacy`` suffix rather than deleting — losing somebody - else's job metadata silently is a worse failure than leaving a - backup. - """ - link = project_path / ".snakemake" - snakemake_state.mkdir(parents=True, exist_ok=True) - if link.is_symlink(): - try: - if link.resolve() == snakemake_state.resolve(): - return - except OSError: - pass - link.unlink() - elif link.exists(): - backup = link.with_name(".snakemake.legacy") - # If a backup already exists, keep the existing one — that's - # likely from an even older run; don't mask it. - if not backup.exists(): - link.rename(backup) - else: - import shutil - - shutil.rmtree(link) - link.symlink_to(snakemake_state, target_is_directory=True) - - -class RunLockBusyError(RuntimeError): - """Raised when another ``lc run`` already holds the project's run-lock.""" - - -@contextlib.contextmanager -def acquire_run_lock(rundirs: RunDirs) -> Iterator[None]: - """Hold an exclusive flock on the project's run-lock for the duration. - - The lock is at ``/.lightcone/locks/.run-lock`` - so each project gets its own. Acquired non-blocking — concurrent - ``lc run`` invocations on the same project hit :class:`RunLockBusyError` - rather than queueing silently. - - The kernel releases the lock automatically when the holding process - exits (clean shutdown, crash, or SIGKILL), so a previous run that - died ungracefully cannot leave us deadlocked. Any ``.snakemake/`` - workflow lock that survived a prior crash gets cleared inside this - context — safe to do because we hold the project-wide lock. - """ - fd = os.open(rundirs.run_lock_path, os.O_RDWR | os.O_CREAT, 0o644) - try: - try: - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError as e: - raise RunLockBusyError( - f"Another ``lc run`` holds the lock at " - f"{rundirs.run_lock_path}. Wait for it to finish, or " - f"if you're certain it's gone, delete the lockfile." - ) from e - # We're alone. Clear any leftover snakemake lock from a prior - # crashed run — snakemake stores zero-byte sentinel files in - # ``.snakemake/locks/`` that aren't tied to a process and would - # otherwise refuse the next workflow start. - snake_locks = rundirs.snakemake_state / "locks" - if snake_locks.is_dir(): - for entry in snake_locks.iterdir(): - with contextlib.suppress(OSError): - if entry.is_dir(): - shutil.rmtree(entry) - else: - entry.unlink() - try: - yield - finally: - fcntl.flock(fd, fcntl.LOCK_UN) - finally: - os.close(fd) diff --git a/src/lightcone/engine/site_registry.py b/src/lightcone/engine/site_registry.py deleted file mode 100644 index 9fd282dd..00000000 --- a/src/lightcone/engine/site_registry.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Known site defaults. - -When ``lc init`` runs on a known site, the matching entry below provides -the scratch root surfaced to the user and any deny rules used to keep -edits off shared filesystems. - -To add a new site, append an entry to :data:`SITE_DEFAULTS`. - -The high-level entry point for the rest of the codebase is -:func:`detect_current_site`, which returns a :class:`HostSite` bundling -the matched site key with its declared defaults — keeping the -``socket.gethostname() + detect_site + get_site_defaults`` chain in one -place. -""" -from __future__ import annotations - -import os -import socket -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Any - -#: Per-site defaults. ``suggested_options`` follows the same shape as the -#: target file's ``options`` section: an orthogonal map of axis → -#: ``{default, choices}`` (where ``choices`` is ``{value: guidance}``). -#: ``cache_key_overrides`` captures non-conventional sacctmgr naming (e.g. -#: Perlmutter's ``regular_1`` for the CPU ``regular`` queue). -SITE_DEFAULTS: dict[str, dict[str, Any]] = { - "perlmutter": { - "hostname_patterns": ["perlmutter", "saul"], - "display_name": "NERSC Perlmutter", - "backend": "slurm", - "connection": { - "hostname": "perlmutter.nersc.gov", - }, - "container_runtime": "podman-hpc", - # Where lightcone keeps its operational state (snakemake metadata, - # dask spill, cross-node stdout locks). NERSC's $HOME and CFS are - # mounted on compute via DVS, which silently swallows ``flock`` and - # is slow for small-file I/O — Lustre ($SCRATCH) is the only sane - # choice. Stored as a shell expression so it expands to each user's - # private scratch path at run time. - "scratch_root": "$SCRATCH", - "suggested_options": { - "qos": { - "default": "debug", - "choices": { - "debug": "quick iteration, testing", - "regular": "production runs, large jobs", - "preempt": "cheap batch, restartable after 2h", - "shared": "fractional node (1–2 GPUs)", - }, - }, - "constraint": { - "default": "gpu", - "choices": { - "gpu": "A100 40 GB — 1,536 nodes, 4 GPUs/node", - "cpu": "CPU only — 3,072 nodes, 128 cores/node", - "gpu&hbm80g": "A100 80 GB — 256 nodes", - }, - }, - "time_limit": { - "default": "30m", - "guidance": "debug caps at 30 min; regular allows up to 48 h", - }, - }, - # Perlmutter's sacctmgr names prefix GPU QoS with `gpu_` and - # suffix the CPU regular queue as `regular_1`. The first is - # handled by the default `{constraint}_{qos}` convention; the - # second needs an explicit override. - "cache_key_overrides": { - "regular/cpu": "regular_1", - }, - "scratch_paths": [ - "//pscratch/**", - "//global/cscratch1/**", - "//global/cfs/cdirs/**", - ], - }, - # A JupyterHub deployment with Dask Gateway (e.g. lightcone-hub on - # GKE). Unlike HPC sites, hostnames here are meaningless pod names — - # detection is by the env vars the deployment injects into every - # user pod. ``container_runtime: kubernetes`` routes recipe - # execution through worker pods running the project image. - # - # ``scratch_root`` must be declared even though "local" gets by - # without one: with no site scratch, resolution falls back to the - # tempdir — fine on a single machine, but a pod's ``/tmp`` is - # pod-local. The ``.snakemake`` state the driver symlinks into - # scratch has to live on the NFS home every worker pod mounts, or - # each worker would write its job metadata into its own ``/tmp`` - # (invisible to the driver) and every subsequent run would consider - # all outputs stale. ``$HOME`` *is* the shared filesystem here. - "jupyterhub": { - "hostname_patterns": [], - "env_markers": ["DASK_GATEWAY__ADDRESS"], - "display_name": "JupyterHub (Dask Gateway)", - "backend": "kubernetes", - "connection": {}, - "container_runtime": "kubernetes", - "scratch_root": "$HOME", - }, - "local": { - "hostname_patterns": [], - "display_name": "Local", - "backend": "local", - "connection": {}, - }, -} - - -def detect_site(hostname_or_name: str) -> str | None: - """Detect a known site from a hostname or site name.""" - normalized = hostname_or_name.lower() - for site_key, site in SITE_DEFAULTS.items(): - if site.get("backend") == "local": - continue - if site_key in normalized: - return site_key - for pattern in site.get("hostname_patterns", []): - if pattern in normalized: - return site_key - return None - - -def detect_site_from_env() -> str | None: - """Detect a site whose declared ``env_markers`` are all present. - - Deployment-style sites (JupyterHub pods) have arbitrary hostnames; - what identifies them is the environment the deployment injects. - """ - for site_key, site in SITE_DEFAULTS.items(): - markers = site.get("env_markers") or [] - if markers and all(os.environ.get(m) for m in markers): - return site_key - return None - - -def get_site_defaults(site_key: str) -> dict[str, Any] | None: - """Return defaults for a known site, or ``None``.""" - return SITE_DEFAULTS.get(site_key) - - -def list_known_sites() -> list[tuple[str, str]]: - """Return ``(site_key, display_name)`` for all known sites.""" - return [ - (key, site.get("display_name", key)) - for key, site in SITE_DEFAULTS.items() - ] - - -def get_site_scratch_deny_rules(site_key: str) -> list[str]: - """Return Edit deny rules for a site's scratch/shared filesystems.""" - site = SITE_DEFAULTS.get(site_key) - if not site: - return [] - scratch_paths = site.get("scratch_paths", []) - return [f"Edit({path})" for path in scratch_paths] - - -@dataclass(frozen=True) -class HostSite: - """The site (if any) the local host belongs to. - - Returned by :func:`detect_current_site`. Use ``if site:`` to test - whether a known site was matched; use :meth:`get` (or - :attr:`defaults`) to read declared fields. - - Adding a new "site asks for X" feature should not require a fourth - copy of the ``detect_site(socket.gethostname()) → get_site_defaults`` - boilerplate — extend this class (or its consumers) instead. - """ - - key: str | None - defaults: Mapping[str, Any] = field(default_factory=dict) - - def __bool__(self) -> bool: - return self.key is not None - - @property - def display_name(self) -> str: - return self.defaults.get("display_name") or self.key or "unknown" - - def get(self, name: str, default: Any = None) -> Any: - """Look up a field declared in the site's defaults.""" - return self.defaults.get(name, default) - - -_UNKNOWN_HOST_SITE = HostSite(key=None, defaults={}) - - -def detect_current_site() -> HostSite: - """Return the :class:`HostSite` for the local host. - - Single source of truth for "which site are we on?" — everything else - in the codebase should call this rather than re-deriving it from - :func:`socket.gethostname` and :func:`detect_site`. Environment - markers win over hostname patterns (a pod's hostname is noise; the - injected env is the signal). Returns a falsy :class:`HostSite` - (``key is None``) when nothing matches. - """ - key = detect_site_from_env() or detect_site(socket.gethostname()) - if key is None: - return _UNKNOWN_HOST_SITE - return HostSite(key=key, defaults=get_site_defaults(key) or {}) diff --git a/src/lightcone/engine/snakefile.py b/src/lightcone/engine/snakefile.py deleted file mode 100644 index 7bc77010..00000000 --- a/src/lightcone/engine/snakefile.py +++ /dev/null @@ -1,487 +0,0 @@ -"""Generate ``.lightcone/Snakefile`` from ``astra.yaml``. - -The Snakefile is a thin shell over the astra spec: one rule per output -with a recipe, parameterized by ``{universe}``. Each rule's body is a -``run:`` block that calls :func:`lightcone.engine.runner.run_rule` with -the per-(rule, universe) cfg blob — which already contains the rendered -and wrapped ``shell_command``. All template substitution and container -wrapping happen here, at generation time, where every value is concrete -for a given universe; the runner stays a thin executor. - -ASTRA v0.0.7 moved ``inputs`` and ``decisions`` declarations from -``Recipe`` up to ``Output``. The recipe body is a *template* using a -small placeholder grammar — see :func:`render_recipe`. We don't use -Snakemake's ``container:`` directive or ``--sdm apptainer``: the -generator wraps with the configured runtime end-to-end (see -:mod:`lightcone.engine.container`). - -The ``os.replace`` rename inside ``write_manifest`` (called by -``run_rule``) is the atomic commit point — either the rule produced -both data and manifest, or it failed and Snakemake reruns the rule. -""" -from __future__ import annotations - -import json -import string -import subprocess -from pathlib import Path -from typing import Any - -from astra.helpers import load_yaml, resolve_analysis_tree - -from lightcone.engine.container import ( - make_image_tag_resolver, - runtime_registry, - wrap_recipe, -) -from lightcone.engine.manifest import code_version -from lightcone.engine.tree import ( - TreeOutput, - collect_tree_outputs, - find_upstream_output, - resolve_container_spec, - resolve_external_input, - resolve_universe_decisions, -) - -LIGHTCONE_DIR = ".lightcone" -SNAKEFILE_NAME = "Snakefile" -CONFIG_NAME = "snakefile-config.json" - -_FORMATTER = string.Formatter() - - -def render_recipe( - template: str, - *, - inputs: dict[str, str], - decisions: dict[str, str], - output: str, -) -> str: - """Substitute v0.0.7 recipe template placeholders. - - Recognized placeholders: - - * ``{output}`` — directory the artifact is written to. - * ``{inputs}`` — space-separated values of every entry in - ``inputs`` (in declaration order). - * ``{inputs.}`` — the named upstream input's resolved value - (a sibling output's directory path or an - analysis-level Input's source string). - * ``{decisions.}`` — the active option ID for the named - decision in this universe. - - ``{{`` and ``}}`` collapse to literal ``{`` / ``}``. Unknown - placeholders, undeclared references, and format-spec or conversion - flags raise :class:`KeyError` / :class:`ValueError`. ``astra - validate`` should already have caught these — the strict behaviour - here is defense-in-depth. - """ - pieces: list[str] = [] - for literal, field, spec, conv in _FORMATTER.parse(template): - pieces.append(literal) - if field is None: - continue - if spec or conv: - raise ValueError( - f"Recipe placeholder '{{{field}}}' must not use a format " - "spec or conversion" - ) - if field == "output": - pieces.append(output) - continue - if field == "inputs": - pieces.append(" ".join(inputs.values())) - continue - head, dot, tail = field.partition(".") - if not dot: - raise ValueError( - f"Unknown recipe placeholder '{{{field}}}' " - "(use {inputs}, {inputs.}, {decisions.}, or {output})" - ) - if head == "inputs": - if tail not in inputs: - raise KeyError( - f"Recipe placeholder '{{inputs.{tail}}}' references an " - "input not declared on this Output" - ) - pieces.append(inputs[tail]) - elif head == "decisions": - if tail not in decisions: - raise KeyError( - f"Recipe placeholder '{{decisions.{tail}}}' references " - "a decision not declared on this Output" - ) - pieces.append(str(decisions[tail])) - else: - raise ValueError( - f"Unknown recipe placeholder namespace '{head}' in '{{{field}}}'" - ) - return "".join(pieces) - - -def _git_sha(project_path: Path) -> str | None: - try: - out = subprocess.run( - ["git", "-C", str(project_path), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=False, - ) - if out.returncode == 0: - return out.stdout.strip() - except FileNotFoundError: - pass - return None - - -def _git_remote(project_path: Path) -> str | None: - """URL of the ``origin`` git remote, if the project is a git clone. - - Captured into the manifest alongside ``git_sha`` so a published - bundle can identify *which repository* the commit belongs to. - SSH URLs (``git@host:owner/repo.git``) are normalised to ``https`` - form so consumers (e.g. WorkflowHub) can render them as clickable - links. - """ - try: - out = subprocess.run( - ["git", "-C", str(project_path), - "config", "--get", "remote.origin.url"], - capture_output=True, - text=True, - check=False, - ) - if out.returncode != 0: - return None - url = out.stdout.strip() - if not url: - return None - if url.startswith("git@"): - host_path = url.removeprefix("git@").replace(":", "/", 1) - url = f"https://{host_path}" - if url.endswith(".git"): - url = url.removesuffix(".git") - return url - except FileNotFoundError: - return None - - -def _lc_version() -> str: - try: - from importlib.metadata import version - - return version("lightcone-cli") - except Exception: - return "unknown" - - -def _output_dir_pattern(tree_out: TreeOutput) -> str: - """Wildcard path to this output's directory. - - Root + inline sub-analyses: ``results/{universe}/`` - Path-rooted sub-analyses: ``/results/{universe}/`` - """ - if tree_out.analysis_path: - base = tree_out.analysis_path.lstrip("./") - return f"{base}/results/{{universe}}/{tree_out.output_id}" - return f"results/{{universe}}/{tree_out.output_id}" - - -def _rule_key(tree_out: TreeOutput) -> str: - """Unique key into the cfg JSON. Avoids collisions when two - sub-analyses share an output_id.""" - if tree_out.analysis_id is None: - return tree_out.output_id - return f"{tree_out.analysis_id}.{tree_out.output_id}" - - -def _rule_name(tree_out: TreeOutput) -> str: - """Snakemake rule name. Mirrors the cfg key but with - Snakemake-friendly identifier characters (``.`` → ``__``).""" - return _rule_key(tree_out).replace(".", "__") - - -def _safe_input_key(raw_id: str) -> str: - """Snakemake's ``input:`` keys must be valid Python identifiers, - so ``sub.real`` becomes ``sub__real``. Raw IDs (with their dots - intact) are still used as keys in the manifest's ``input_versions`` - map and inside the runner — verify and write_manifest see the same - spelling the user wrote in ``Output.inputs``. - """ - return raw_id.replace(".", "__") - - -def _scoped_decisions_for_output( - tree_out: TreeOutput, - universe_decisions: dict[str, Any], -) -> dict[str, Any]: - """Pick the active option ID for each decision the Output declares. - - v0.0.7: ``Output.decisions`` lists the IDs of decisions that - parameterize this output. The runner only needs (and the recipe - template can only reference) those — anything else is out of scope. - """ - declared = tree_out.output_def.get("decisions") or [] - if not declared: - return {} - scoped: dict[str, Any] = {} - prefix = f"{tree_out.analysis_id}." if tree_out.analysis_id else "" - for dec_id in declared: - if prefix and (qualified := f"{prefix}{dec_id}") in universe_decisions: - scoped[dec_id] = universe_decisions[qualified] - elif dec_id in universe_decisions: - scoped[dec_id] = universe_decisions[dec_id] - return scoped - - -def _universe_decisions( - universe_id: str, - project_path: Path, - spec: dict[str, Any], -) -> dict[str, Any]: - universe_yaml = project_path / "universes" / f"{universe_id}.yaml" - if not universe_yaml.exists(): - return {} - try: - return resolve_universe_decisions(project_path, spec, universe_id) - except (FileNotFoundError, KeyError): - return {} - - -def _render_snakefile( - rules: list[dict[str, Any]], - universes: list[str], -) -> str: - """Render the Snakefile string from rule descriptors. - - Each rule descriptor has ``name`` (Snakemake-safe), ``key`` (cfg - lookup), ``output_dir`` (wildcard pattern), and ``inputs`` (list of - ``(raw_id, safe_key, path_pattern)`` triples — both sibling outputs - and analysis-level Inputs, in declaration order). External Input - patterns are static source strings; sibling-output patterns carry a - ``{universe}`` wildcard. - - The rule body is a thin call into :func:`runner.run_rule`. The - ``inputs`` dict it builds is keyed by the **raw** input IDs (with - their dots intact) so that ``write_manifest``'s ``input_versions`` - matches what ``verify`` walks for chain integrity. - """ - universes_repr = repr(universes) - rule_all_inputs = [] - for r in rules: - rule_all_inputs.append( - f' expand("{r["output_dir"]}/.lightcone-manifest.json", ' - f"universe=UNIVERSES)," - ) - rule_all_block = "\n".join(rule_all_inputs) or " []" - - lines: list[str] = [] - lines.append('"""Auto-generated from astra.yaml — do not edit by hand."""') - lines.append("import json") - lines.append("from pathlib import Path") - lines.append("from lightcone.engine.runner import run_rule") - lines.append("") - lines.append("PROJECT = Path(workflow.basedir).parent") - lines.append( - 'CFG = json.loads((PROJECT / ".lightcone" / "snakefile-config.json").read_text())' - ) - lines.append(f"UNIVERSES = {universes_repr}") - lines.append("") - lines.append("rule all:") - lines.append(" input:") - lines.append(rule_all_block) - lines.append("") - - for r in rules: - lines.append(f'rule {r["name"]}:') - if r["inputs"]: - lines.append(" input:") - for _raw, safe, pattern in r["inputs"]: - lines.append(f' {safe}="{pattern}",') - lines.append(" output:") - lines.append(f' data=directory("{r["output_dir"]}"),') - lines.append(f' manifest="{r["output_dir"]}/.lightcone-manifest.json",') - lines.append(" params:") - lines.append(f' cfg=lambda wc: CFG["{r["key"]}"][wc.universe],') - lines.append(" run:") - # Manifest input_versions are keyed by raw declared input IDs - # (e.g. "sub.real"), not the safe Snakemake-input-directive key - # ("sub__real"). Verify walks the raw IDs from Output.inputs; - # this dict literal is what bridges the two. - inp_pairs = ", ".join( - f'"{raw}": Path(input.{safe})' for raw, safe, _pattern in r["inputs"] - ) - lines.append(" run_rule(") - lines.append(f' rule_key="{r["key"]}",') - lines.append(" universe=wildcards.universe,") - lines.append(" output_dir=Path(output.data),") - lines.append(f" inputs={{{inp_pairs}}},") - lines.append(" cfg=dict(params.cfg),") - lines.append(" )") - lines.append("") - - return "\n".join(lines) + "\n" - - -def generate( - project_path: Path, - *, - universes: list[str], - runtime: str = "none", -) -> tuple[Path, Path]: - """Write ``.lightcone/Snakefile`` and ``.lightcone/snakefile-config.json``. - - Args: - project_path: Project root containing ``astra.yaml``. - universes: Universe ids to expand rules over. - runtime: Container runtime to wrap recipes with. One of - ``docker | podman | podman-hpc | kubernetes | none``. - ``none`` runs recipes on the host without isolation; - ``kubernetes`` leaves recipes unwrapped (the worker pod runs - the project image) and resolves Containerfile specs to - registry refs. Resolution is done here once, not per-rule, - so all rules use a consistent runtime. See - :func:`lightcone.engine.container.load_runtime`. - - Returns ``(snakefile_path, config_path)``. - """ - project_path = Path(project_path).resolve() - spec = resolve_analysis_tree(load_yaml(project_path / "astra.yaml"), project_path) - project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") - - tree_outputs = collect_tree_outputs(spec) - - rules: list[dict[str, Any]] = [] - cfg: dict[str, dict[str, dict[str, Any]]] = {} - - git_sha = _git_sha(project_path) - git_remote = _git_remote(project_path) - lc_version = _lc_version() - resolve_image = make_image_tag_resolver( - project_path, project_name, registry=runtime_registry(runtime) - ) - - for to in tree_outputs: - recipe = to.output_def.get("recipe") - if recipe is None: - continue # alias output (re-export via ``from:``) - - rule_key = _rule_key(to) - rule_name = _rule_name(to) - out_dir_pattern = _output_dir_pattern(to) - - # v0.0.7: declared upstream inputs live on the Output, not the - # Recipe. Each ID resolves to either a sibling output (a - # universe-templated path) or an analysis-level Input (a static - # source string). Both flow through the same Snakemake ``input:`` - # slot so write_manifest fingerprints them and Snakemake gets to - # enforce existence and detect mtime drift uniformly. - declared_inputs = to.output_def.get("inputs") or [] - recipe_command = recipe.get("command", "") - - rule_inputs: list[tuple[str, str, str]] = [] # (raw_id, safe_key, pattern) - for inp_id in declared_inputs: - up = find_upstream_output(to, inp_id, tree_outputs) - if up is not None: - pattern = _output_dir_pattern(up) - else: - ext = resolve_external_input(to, inp_id, spec) - if ext is None: - continue # unresolvable; ``astra validate`` flags it. - pattern = ext - rule_inputs.append((inp_id, _safe_input_key(inp_id), pattern)) - - container_image = resolve_container_spec(to, spec) - image_tag = resolve_image(container_image) - - rules.append( - { - "name": rule_name, - "key": rule_key, - "output_dir": out_dir_pattern, - "inputs": rule_inputs, - } - ) - - cfg.setdefault(rule_key, {}) - for u in universes: - universe_decisions = _universe_decisions(u, project_path, spec) - scoped_decisions = _scoped_decisions_for_output(to, universe_decisions) - - # Build the resolved input map in declaration order so - # ``{inputs}`` joins paths in the same order the user wrote - # them. The ``{universe}`` substitution is a no-op for static - # external paths. - resolved_inputs: dict[str, str] = { - raw: pat.replace("{universe}", u) for raw, _, pat in rule_inputs - } - - output_dir = out_dir_pattern.replace("{universe}", u) - rendered = render_recipe( - recipe_command, - inputs=resolved_inputs, - decisions=scoped_decisions, - output=output_dir, - ) - wrapped = wrap_recipe(rendered, image=image_tag, runtime=runtime) - # ``image_tag`` (not the raw spec string) so a Containerfile - # edit propagates through ``code_version`` to ``lc status``. - cv = code_version( - recipe=recipe_command, - container_image=image_tag, - decisions=scoped_decisions, - ) - # Prefix the executed command with a no-op ``:`` builtin - # carrying the code_version. This makes the wrapped command - # differ when the recipe / container / decisions drift, so - # (a) Snakemake's ``shellcmd`` trigger sees the change and - # (b) any shell trace carries a breadcrumb. The trigger - # that actually fires today is ``params`` (cfg is - # per-universe and contains ``shell_command``) — see - # ``lc run --rerun-triggers``. - shell_command = f": lc_code_version={cv};\n{wrapped}" - cfg[rule_key][u] = { - "output_id": to.output_id, - "output_type": to.output_def.get("type"), - "universe_id": u, - # Raw template, preserved so the manifest's ``recipe`` - # field records what the user authored. - "recipe": recipe_command, - "shell_command": shell_command, - "container_image": container_image, - "decisions": scoped_decisions, - "code_version": cv, - "git_sha": git_sha, - "git_remote": git_remote, - "lc_version": lc_version, - } - - lightcone_dir = project_path / LIGHTCONE_DIR - lightcone_dir.mkdir(parents=True, exist_ok=True) - snakefile_path = lightcone_dir / SNAKEFILE_NAME - config_path = lightcone_dir / CONFIG_NAME - - snakefile_path.write_text(_render_snakefile(rules, universes)) - config_path.write_text(json.dumps(cfg, indent=2, sort_keys=True)) - - return snakefile_path, config_path - - -def discover_universes(project_path: Path) -> list[str]: - """Discover universe ids from ``universes/*.yaml``. If none exist, - returns ``["default"]``. - """ - universes_dir = project_path / "universes" - if not universes_dir.exists(): - return ["default"] - ids = sorted(p.stem for p in universes_dir.glob("*.yaml")) - return ids or ["default"] - - -__all__ = [ - "LIGHTCONE_DIR", - "discover_universes", - "generate", - "render_recipe", -] diff --git a/src/lightcone/engine/status.py b/src/lightcone/engine/status.py deleted file mode 100644 index 8175420a..00000000 --- a/src/lightcone/engine/status.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Manifest-driven status walker. - -For each output declared in a project's ``astra.yaml``, determines whether -it is materialized, stale, missing, or an alias — by reading the per-output -manifest written at ``/.lightcone-manifest.json``. - -This module never imports Snakemake. ``lc status`` works on a fresh clone -with no ``.snakemake/`` directory and on frozen archives. -""" -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Literal - -from astra.helpers import load_yaml, resolve_analysis_tree - -from lightcone.engine.container import ( - load_runtime, - make_image_tag_resolver, - runtime_registry, -) -from lightcone.engine.manifest import code_version, read_manifest -from lightcone.engine.tree import ( - TreeOutput, - collect_tree_outputs, - resolve_container_spec, - resolve_output_path, - resolve_universe_decisions, -) - -StatusLiteral = Literal["ok", "stale", "missing", "alias"] - - -@dataclass -class OutputStatus: - output_id: str - universe_id: str - analysis_id: str | None - output_dir: Path - status: StatusLiteral - manifest: dict[str, Any] | None - recipe_command: str | None - - -def _decisions_for( - tree_output: TreeOutput, - universe_decisions: dict[str, Any], -) -> dict[str, Any]: - """Return the decisions visible to a given output for code_version - computation. - - v0.0.7: ``Output.decisions`` is the explicit provenance contract — - the set of decisions whose option choices can change this output. - The Snakefile generator hashes only those into ``code_version``; - we mirror that scoping here so ``lc status`` stays in sync. - Outputs that do not declare decisions hash an empty dict. - """ - declared = tree_output.output_def.get("decisions") or [] - if not declared: - return {} - scoped: dict[str, Any] = {} - prefix = f"{tree_output.analysis_id}." if tree_output.analysis_id else "" - for dec_id in declared: - if prefix and (qualified := f"{prefix}{dec_id}") in universe_decisions: - scoped[dec_id] = universe_decisions[qualified] - elif dec_id in universe_decisions: - scoped[dec_id] = universe_decisions[dec_id] - return scoped - - -def _load_universe_decisions( - project_path: Path, - spec: dict[str, Any], - universe_id: str, -) -> dict[str, Any]: - """Load merged universe decisions if the file exists; empty dict otherwise. - - Universe files are optional during interactive work, so we tolerate - their absence rather than erroring. - """ - universe_yaml = project_path / "universes" / f"{universe_id}.yaml" - if not universe_yaml.exists(): - return {} - try: - return resolve_universe_decisions(project_path, spec, universe_id) - except (FileNotFoundError, KeyError): - return {} - - -def get_output_status( - project_path: Path, - *, - universe_id: str, -) -> Iterator[OutputStatus]: - """Yield an :class:`OutputStatus` for every declared output in the project.""" - spec_path = project_path / "astra.yaml" - spec = resolve_analysis_tree(load_yaml(spec_path), project_path) - universe_decisions = _load_universe_decisions(project_path, spec, universe_id) - project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") - # Resolve image identities exactly as `lc run` would right now — - # on a kubernetes deployment that's the registry ref, not the local - # tag, or every freshly materialized output would read as stale. - registry = runtime_registry(load_runtime(project_path=project_path).runtime) - resolve_image = make_image_tag_resolver(project_path, project_name, registry=registry) - - for tree_out in collect_tree_outputs(spec): - out_dir = resolve_output_path(project_path, tree_out, universe_id) / tree_out.output_id - - # Aliases — outputs without their own recipe — are materialized as - # a side effect of their upstream. They have no independent status. - recipe = tree_out.output_def.get("recipe") - if recipe is None: - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="alias", - manifest=None, - recipe_command=None, - ) - continue - - recipe_command = recipe.get("command", "") - - manifest = read_manifest(out_dir) - if manifest is None: - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="missing", - manifest=None, - recipe_command=recipe_command, - ) - continue - - # Mirror the snakefile generator's image-tag resolution so the - # recomputed code_version matches what was written into the - # manifest at run time. - image_tag = resolve_image(resolve_container_spec(tree_out, spec)) - current_cv = code_version( - recipe=recipe_command, - container_image=image_tag, - decisions=_decisions_for(tree_out, universe_decisions), - ) - if manifest.get("code_version") != current_cv: - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="stale", - manifest=manifest, - recipe_command=recipe_command, - ) - continue - - yield OutputStatus( - output_id=tree_out.output_id, - universe_id=universe_id, - analysis_id=tree_out.analysis_id, - output_dir=out_dir, - status="ok", - manifest=manifest, - recipe_command=recipe_command, - ) diff --git a/src/lightcone/engine/templates/__init__.py b/src/lightcone/engine/templates/__init__.py new file mode 100644 index 00000000..448e014f --- /dev/null +++ b/src/lightcone/engine/templates/__init__.py @@ -0,0 +1,154 @@ +"""Default file templates for a scaffolded project. + +The templates live as real files under ``templates/files/`` rather than as +string literals in the code that writes them: they *are* files, and +keeping them as files means they can be read, diffed, and highlighted as +whatever they will become. ``lc init`` is the only consumer — editing a +template here changes what new projects look like. + +Placeholders use ``string.Template`` (``${name}``) rather than +``str.format``, because several templates legitimately contain braces +(TOML tables, MyST roles like ``{astra}``) that ``format`` would try to +interpret. Substitution is strict: a missing key raises rather than +silently emitting a placeholder. +""" + +from __future__ import annotations + +from importlib import resources +from string import Template + +from lightcone.engine.constants import ( + DEFAULT_PYTHON_FLOOR, + MIN_UV_VERSION, +) + +#: Comment introducing lightcone's entries in a ``.gitignore`` it did not +#: author. Cosmetic only — what makes a repair idempotent is the entry set +#: (:func:`gitignore_entries`), not this line. +GITIGNORE_HEADER = "# lightcone-cli" + +#: Every template shipped, by file name. The loader checks membership so a +#: typo fails loudly at the call site instead of as a packaging mystery. +TEMPLATE_NAMES = frozenset( + { + "pyproject.toml.tmpl", + "gitignore.tmpl", + "results-README.md.tmpl", + "myst.yml.tmpl", + "index.md.tmpl", + } +) + + +def read(name: str) -> str: + """Return the raw text of template *name*.""" + if name not in TEMPLATE_NAMES: + raise KeyError(f"unknown template: {name!r}") + return (resources.files(__name__) / "files" / name).read_text(encoding="utf-8") + + +def _render(name: str, /, **values: str) -> str: + return Template(read(name)).substitute(**values) + + +# ============================================================================= +# The uv project +# ============================================================================= + + +def pyproject(*, name: str) -> str: + """The scaffolded ``pyproject.toml`` for a project called *name*. + + Deliberately **virtual** — no ``[build-system]``. Containerized mode + builds the image ``--no-install-project`` (project code never enters + an image), so a packaged project's ``import my_analysis`` would fail + inside its own container (spec §1). + """ + return _render( + "pyproject.toml.tmpl", + name=name, + python_floor=DEFAULT_PYTHON_FLOOR, + min_uv=MIN_UV_VERSION, + lc_requirement=lightcone_requirement(), + ) + + +def lightcone_requirement() -> str: + """The ``lightcone-cli`` requirement pinned into the scaffold. + + The engine lives *inside the experiment's lock* — pinned to the + version that ran ``lc init``, so driver and project stay in lockstep + and the engine that produced a result stays recoverable. Dev builds + fall back to unpinned: their versions aren't published, so pinning one + would make the project's lock unsolvable. + """ + from importlib.metadata import PackageNotFoundError, version + + try: + v = version("lightcone-cli") + except PackageNotFoundError: + v = "" + return f"lightcone-cli=={v}" if v and "dev" not in v else "lightcone-cli" + + +# ============================================================================= +# Managed files +# ============================================================================= + + +def gitignore() -> str: + """The whole ``.gitignore``, for a project that has none.""" + return read("gitignore.tmpl") + + +def gitignore_entries() -> tuple[str, ...]: + """The patterns lightcone manages, in template order. + + The template minus its comments and blank lines. Convergence works + against *this set* rather than against a marker, so a project ends up + with the right ignores however its ``.gitignore`` got there — and + entries added in a later lc release reach projects that already have + one, instead of being skipped because a marker was present. + """ + return tuple(_patterns(read("gitignore.tmpl"))) + + +def _patterns(text: str) -> list[str]: + """The meaningful (non-comment, non-blank) lines of a gitignore.""" + return [ + stripped + for line in text.splitlines() + if (stripped := line.strip()) and not stripped.startswith("#") + ] + + +def missing_gitignore_entries(text: str) -> list[str]: + """Which managed patterns *text* does not already carry, in order.""" + present = set(_patterns(text)) + return [e for e in gitignore_entries() if e not in present] + + +def results_readme() -> str: + """``results/README.md`` — where outputs land. + + ``results/`` is git-ignored and starts empty, so without the README + the directory is invisible in a clone and nothing explains the + ``results///`` layout. + """ + return read("results-README.md.tmpl") + + +# ============================================================================= +# The MyST report +# ============================================================================= + + +def myst_yml() -> str: + return read("myst.yml.tmpl") + + +def index_md(*, title: str) -> str: + """The template report. References ``astra.yaml`` elements *by path*, + so numbers and figures stay single-sourced in the analysis.""" + return _render("index.md.tmpl", title=title) diff --git a/src/lightcone/engine/templates/files/gitignore.tmpl b/src/lightcone/engine/templates/files/gitignore.tmpl new file mode 100644 index 00000000..30c8ba2c --- /dev/null +++ b/src/lightcone/engine/templates/files/gitignore.tmpl @@ -0,0 +1,9 @@ +# lightcone-cli +__pycache__/ +*.py[cod] +.ipynb_checkpoints/ +.DS_Store +.venv/ +_build/ +results/* +!results/README.md diff --git a/src/lightcone/engine/templates/files/index.md.tmpl b/src/lightcone/engine/templates/files/index.md.tmpl new file mode 100644 index 00000000..832f92f7 --- /dev/null +++ b/src/lightcone/engine/templates/files/index.md.tmpl @@ -0,0 +1,29 @@ +# ${title} + +> **TODO:** this report was scaffolded by `lc init`. It references the +> analysis elements declared in `astra.yaml` *by path* — figures, decisions, +> and numbers stay single-sourced in the analysis, so never hard-type a +> measured value here. Preview with `myst start` (requires the MyST CLI). + +## Introduction + +TODO: the research question, its context, and why it matters. + +## Methods + +TODO: describe the approach. Reference the decisions the analysis exposes +rather than restating them — for example, we adopt the +{astra}`decisions.example_method` for this analysis: + +:::{astra} decisions.example_method +::: + +## Results + +TODO: present the outputs. Once `lc materialize` has produced results, +pull numbers in live, e.g.: + +% The analysis yields {astra:value}`outputs.main_result`. + +:::{astra} outputs +::: diff --git a/src/lightcone/engine/templates/files/myst.yml.tmpl b/src/lightcone/engine/templates/files/myst.yml.tmpl new file mode 100644 index 00000000..f2bb617a --- /dev/null +++ b/src/lightcone/engine/templates/files/myst.yml.tmpl @@ -0,0 +1,12 @@ +# MyST configuration for the analysis report (https://mystmd.org/). +# The MySTRA plugin resolves {astra}`...` references against astra.yaml. +# `latest` always tracks the newest MySTRA release; for a reproducible +# build, pin a tag instead, e.g. .../releases/download/v0.0.1/mystra.mjs +version: 1 +project: + plugins: + - https://github.com/LightconeResearch/MySTRA/releases/latest/download/mystra.mjs + toc: + - file: index.md +site: + template: book-theme diff --git a/src/lightcone/engine/templates/files/pyproject.toml.tmpl b/src/lightcone/engine/templates/files/pyproject.toml.tmpl new file mode 100644 index 00000000..48bfac78 --- /dev/null +++ b/src/lightcone/engine/templates/files/pyproject.toml.tmpl @@ -0,0 +1,13 @@ +[project] +name = "${name}" +version = "0.0.1" +requires-python = ">=${python_floor}" +# Analysis dependencies: add with `uv add ` — never install into +# another environment by hand. The engine (lightcone-cli) is a normal +# locked dependency: the experiment pins its own execution layer. +dependencies = [ + "${lc_requirement}", +] + +[tool.uv] +required-version = ">=${min_uv}" diff --git a/src/lightcone/engine/templates/files/results-README.md.tmpl b/src/lightcone/engine/templates/files/results-README.md.tmpl new file mode 100644 index 00000000..736ed80c --- /dev/null +++ b/src/lightcone/engine/templates/files/results-README.md.tmpl @@ -0,0 +1,5 @@ +# results/ + +Materialized outputs land here, one directory per universe and output: + + results/// diff --git a/src/lightcone/engine/tree.py b/src/lightcone/engine/tree.py deleted file mode 100644 index f464f0b3..00000000 --- a/src/lightcone/engine/tree.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Analysis tree helpers — walk resolved sub-analysis trees. - -After ``resolve_analysis_tree()`` from astra.helpers expands ``path:`` -references, this module provides utilities to: - -- Collect all outputs across the tree (with their sub-analysis context). -- Resolve declared ``Output.inputs`` IDs to concrete upstream artifacts - (sibling outputs, parent inputs reached via ``from:`` aliases, or - external dataset paths). -- Resolve sub-analysis decisions to merged universe values, honouring - the v0.0.7 ``from: ../id`` (and ``../../id``) grammar. -- Pick the right ``container:`` declaration for an output (recipe → - sub-analysis → root). - -ASTRA v0.0.7 (`from:` aliasing) reshapes how Inputs/Outputs/Decisions -reference each other: - -* ``Input.from`` uses ``../id`` for an ancestor input, - ``../../id`` for a grandparent, - ``../sibling.out_id`` for a sibling sub's output. -* ``Output.from`` is a re-export and uses ``child.out_id`` - (own child sub) or deeper. -* ``Decision.from`` is upward only: ``../id``, ``../../id``, … - -Aliased nodes carry only ``id`` + ``from`` (+ optional ``when``); the -content is inherited from the target. -""" -from __future__ import annotations - -import logging -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from astra.helpers import get_inputs, get_outputs, load_yaml - -logger = logging.getLogger(__name__) - - -@dataclass -class TreeOutput: - """An output from the resolved analysis tree, with its sub-analysis context.""" - - output_id: str - output_def: dict[str, Any] - analysis_id: str | None # None for root-level outputs - analysis_path: str | None # relative path, e.g. "./analyses/hod_fitting" - analysis_spec: dict[str, Any] # the sub-analysis spec dict - - -def collect_tree_outputs(spec: dict[str, Any]) -> list[TreeOutput]: - """Walk the resolved tree and collect all outputs with context. - - Returns outputs from root level (analysis_id=None) and from each - sub-analysis. Outputs declared with ``from:`` (re-exports) are - included; consumers that care only about materializable outputs - should filter on ``recipe is not None``. - """ - results: list[TreeOutput] = [] - - # Root-level outputs - for out in get_outputs(spec): - results.append(TreeOutput( - output_id=out.get("id", ""), - output_def=out, - analysis_id=None, - analysis_path=None, - analysis_spec=spec, - )) - - # Sub-analysis outputs - for analysis_id, analysis_node in (spec.get("analyses") or {}).items(): - sub_path = analysis_node.get("path") - for out in get_outputs(analysis_node): - results.append(TreeOutput( - output_id=out.get("id", ""), - output_def=out, - analysis_id=analysis_id, - analysis_path=sub_path, - analysis_spec=analysis_node, - )) - - return results - - -def collect_tree_inputs(spec: dict[str, Any]) -> dict[str, dict[str, Any]]: - """Collect all inputs from root and sub-analyses. - - Returns {qualified_id: input_def} where qualified_id is: - - "input_id" for root inputs - - "analysis_id.input_id" for sub-analysis inputs - """ - result: dict[str, dict[str, Any]] = {} - - for inp in get_inputs(spec): - inp_id = inp.get("id", "") - if inp_id: - result[inp_id] = inp - - for analysis_id, analysis_node in (spec.get("analyses") or {}).items(): - for inp in get_inputs(analysis_node): - inp_id = inp.get("id", "") - if inp_id: - result[f"{analysis_id}.{inp_id}"] = inp - - return result - - -def _strip_up_prefix(ref: str) -> tuple[int, str]: - """Split a ``from:`` path into (up_levels, remainder). - - ``../foo`` -> (1, "foo") - ``../../foo`` -> (2, "foo") - ``foo`` -> (0, "foo") - """ - up = 0 - while ref.startswith("../"): - up += 1 - ref = ref[3:] - return up, ref - - -def resolve_universe_decisions( - project_path: Path, - spec: dict[str, Any], - universe_id: str, -) -> dict[str, Any]: - """Load and merge universe decisions from root and sub-analysis universes. - - Returns a flat dict of all decisions for execution: - - Root decisions from ``universes/.yaml`` - - Sub-analysis decisions from ``/universes/.yaml`` - - ``from:`` decisions in sub-analyses are resolved to ancestor values - - The returned dict uses qualified keys for sub-analysis decisions: - ``{analysis_id}.{decision_id}`` to avoid collisions. - - v0.0.7 ``from:`` grammar: ``../id`` walks one scope up, ``../../id`` - walks two. We currently model a 2-level tree (root + sub), so any - ``../`` count above 1 falls off the top and is logged as a warning. - """ - # Load root universe - root_universe_file = project_path / "universes" / f"{universe_id}.yaml" - root_decisions: dict[str, Any] = {} - sub_universe_refs: dict[str, str] = {} - - if root_universe_file.exists(): - root_data = load_yaml(root_universe_file) - root_decisions = root_data.get("decisions", {}) - # Parse sub-analysis universe references - for analysis_id, ref in (root_data.get("analyses") or {}).items(): - if isinstance(ref, dict) and ref.get("universe"): - sub_universe_refs[analysis_id] = ref["universe"] - - merged: dict[str, Any] = dict(root_decisions) - - # Load sub-analysis universes - for analysis_id, analysis_node in (spec.get("analyses") or {}).items(): - sub_path = analysis_node.get("path") - if not sub_path: - continue - - sub_universe_id = sub_universe_refs.get(analysis_id, universe_id) - sub_dir = (project_path / sub_path).resolve() - sub_universe_file = sub_dir / "universes" / f"{sub_universe_id}.yaml" - - if sub_universe_file.exists(): - sub_data = load_yaml(sub_universe_file) - sub_decisions = sub_data.get("decisions", {}) - else: - sub_decisions = {} - - # Resolve from: references in sub-analysis decisions - for decision_id, decision_def in (analysis_node.get("decisions") or {}).items(): - if isinstance(decision_def, dict) and decision_def.get("from"): - from_ref = decision_def["from"] - up, target = _strip_up_prefix(from_ref) - if up == 1 and target in root_decisions: - merged[f"{analysis_id}.{decision_id}"] = root_decisions[target] - else: - logger.warning( - "Decision '%s' in '%s' references '%s' which is not " - "resolvable in the universe (only ../ is " - "currently supported)", - decision_id, analysis_id, from_ref, - ) - elif decision_id in sub_decisions: - merged[f"{analysis_id}.{decision_id}"] = sub_decisions[decision_id] - - # Also add un-referenced local decisions from the sub-universe - for decision_id, value in sub_decisions.items(): - key = f"{analysis_id}.{decision_id}" - if key not in merged: - merged[key] = value - - return merged - - -def get_decisions_for_analysis( - merged_decisions: dict[str, Any], - analysis_id: str | None, -) -> dict[str, Any]: - """Extract the decisions relevant to a specific analysis from merged dict. - - For root (analysis_id=None): returns unqualified keys. - For sub-analysis: returns decisions with matching prefix, stripped to local names. - Also includes root-level decisions (for from: references). - """ - if analysis_id is None: - # Root analysis: return all unqualified keys - return {k: v for k, v in merged_decisions.items() if "." not in k} - - prefix = f"{analysis_id}." - result: dict[str, Any] = {} - - # Add qualified decisions with prefix stripped - for k, v in merged_decisions.items(): - if k.startswith(prefix): - local_key = k[len(prefix):] - result[local_key] = v - - return result - - -def resolve_output_path( - project_path: Path, - tree_output: TreeOutput, - universe_id: str, -) -> Path: - """Resolve the results directory for an output. - - Root outputs: ``results//`` - Sub-analysis outputs: ``/results//`` - """ - if tree_output.analysis_path: - return (project_path / tree_output.analysis_path).resolve() / "results" / universe_id - return project_path / "results" / universe_id - - -def resolve_container_spec( - tree_output: TreeOutput, - root_spec: dict[str, Any], -) -> str | None: - """Pick the container declaration in priority order: - recipe-level > sub-analysis-level > root-level. - Returns the raw spec string (Containerfile path or registry image - tag), or ``None`` when no container is declared at any level. - """ - recipe = tree_output.output_def.get("recipe") or {} - if "container" in recipe: - return recipe["container"] # type: ignore[no-any-return] - if tree_output.analysis_id is not None: - sub = tree_output.analysis_spec.get("container") - if sub is not None: - return sub # type: ignore[no-any-return] - return root_spec.get("container") - - -def find_upstream_output( - consumer: TreeOutput, - inp_id: str, - all_outputs: list[TreeOutput], -) -> TreeOutput | None: - """Resolve a declared ``Output.inputs`` id to the producing :class:`TreeOutput`. - - Per v0.0.7, ``Output.inputs`` references use plain artifact IDs and - resolve through the surrounding analysis scope: a sibling output, a - local input, or a local input that is itself a ``from:`` alias of - something further up. - - Returns ``None`` for inputs that resolve to external sources (no - upstream rule produces them) — :func:`resolve_external_input` - handles those. - """ - by_qualified: dict[str, TreeOutput] = {} - by_bare: dict[str, TreeOutput] = {} - for to in all_outputs: - if to.output_def.get("recipe") is None: - continue - if to.analysis_id is not None: - by_qualified[f"{to.analysis_id}.{to.output_id}"] = to - else: - by_qualified[to.output_id] = to - by_bare.setdefault(to.output_id, to) - - if "." in inp_id and inp_id in by_qualified: - return by_qualified[inp_id] - - if consumer.analysis_id is not None: - qualified = f"{consumer.analysis_id}.{inp_id}" - if qualified in by_qualified: - return by_qualified[qualified] - - if inp_id in by_qualified: - return by_qualified[inp_id] - - # Resolve through ``from:`` aliases on the consumer's analysis-level - # inputs. v0.0.7 grammar: ``../id`` (parent input), ``../../id`` - # (grandparent), ``../sibling.out_id`` (sibling sub-analysis output). - analysis_inputs = {i.get("id"): i for i in get_inputs(consumer.analysis_spec)} - inp_def = analysis_inputs.get(inp_id) - if inp_def and inp_def.get("from"): - _, target = _strip_up_prefix(inp_def["from"]) - if target in by_qualified: - return by_qualified[target] - if "." not in target and target in by_bare: - return by_bare[target] - - return None - - -def resolve_external_input( - consumer: TreeOutput, - inp_id: str, - root_spec: dict[str, Any], -) -> str | None: - """Resolve a declared ``Output.inputs`` id to an external source string. - - Used when the input is not produced by another rule (so - :func:`find_upstream_output` returned ``None``). Walks the - surrounding-scope ``Input`` declarations to find one matching - ``inp_id``; if it has a ``source:``, returns that. If it has a - ``from:`` alias, walks one hop further to the source. - - Returns ``None`` when the id is unresolvable. Recipes that reference - such an id via ``{inputs.}`` will surface a runtime ``KeyError`` - — that scenario is also caught by ``astra validate``. - """ - # Inputs visible to the consumer: the surrounding analysis's own - # inputs, plus root inputs when the consumer is at root. - analysis_inputs = {i.get("id"): i for i in get_inputs(consumer.analysis_spec)} - inp_def = analysis_inputs.get(inp_id) - - # If the consumer is in a sub-analysis and the bare id isn't a - # local Input there, also try the root inputs (a ``../id`` alias - # would have already redirected us, but plain id resolution per the - # spec walks the scope chain). - if inp_def is None and consumer.analysis_id is not None: - root_inputs = {i.get("id"): i for i in get_inputs(root_spec)} - inp_def = root_inputs.get(inp_id) - if inp_def is None: - return None - - # Direct source. - src = inp_def.get("source") - if isinstance(src, str) and src: - return src - - # ``from:`` alias to an ancestor input — walk one hop. - from_ref = inp_def.get("from") - if isinstance(from_ref, str) and from_ref: - _, target = _strip_up_prefix(from_ref) - # Only resolve to root inputs (parent of a sub). Sibling-output - # references would have been handled by find_upstream_output. - if "." not in target: - root_inputs = {i.get("id"): i for i in get_inputs(root_spec)} - target_def = root_inputs.get(target) - if target_def is not None: - target_src = target_def.get("source") - if isinstance(target_src, str) and target_src: - return target_src - return None - - -__all__ = [ - "TreeOutput", - "collect_tree_inputs", - "collect_tree_outputs", - "find_upstream_output", - "get_decisions_for_analysis", - "resolve_container_spec", - "resolve_external_input", - "resolve_output_path", - "resolve_universe_decisions", -] diff --git a/src/lightcone/engine/validation.py b/src/lightcone/engine/validation.py deleted file mode 100644 index ae74eb24..00000000 --- a/src/lightcone/engine/validation.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Post-materialization result file validation for ASTRA outputs.""" -from __future__ import annotations - -import csv -import json -import logging -import math -from pathlib import Path -from typing import Any - -logger = logging.getLogger(__name__) - - -def validate_output( - output_dir: Path, - output_type: str | None, - output_id: str, -) -> list[str]: - """Validate result files after a successful recipe run. - - Returns a list of warning strings. An empty list means no issues detected. - Never raises — all errors are surfaced as warning strings. - """ - if not output_dir.exists(): - return [f"Output directory missing after successful run: {output_dir}"] - if not output_dir.is_dir(): - return [f"Output '{output_id}': expected a directory at {output_dir}, found a file"] - - try: - files = list(output_dir.iterdir()) - except OSError: - return [] - - if not files: - return [f"Output directory is empty after successful run: {output_dir}"] - - if output_type == "metric": - return _validate_metric(output_dir, output_id) - if output_type == "table": - return _validate_table(output_dir, output_id) - if output_type == "figure": - return _validate_figure(output_dir, output_id) - return [] - - -def _validate_metric(output_dir: Path, output_id: str) -> list[str]: - json_files = list(output_dir.glob("*.json")) - if not json_files: - return [ - f"Output '{output_id}' (type: metric) produced no JSON files in {output_dir}" - ] - - warnings: list[str] = [] - for json_file in json_files: - try: - data: Any = json.loads(json_file.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - warnings.append( - f"Output '{output_id}': metric file '{json_file.name}' " - f"is not valid JSON: {exc}" - ) - continue - - if _all_scalars_null(data): - warnings.append( - f"Output '{output_id}': metric file '{json_file.name}' " - f"contains only null/NaN values" - ) - - return warnings - - -def _validate_table(output_dir: Path, output_id: str) -> list[str]: - csv_files = list(output_dir.glob("*.csv")) - if not csv_files: - return [ - f"Output '{output_id}' (type: table) produced no CSV files in {output_dir}" - ] - - warnings: list[str] = [] - for csv_file in csv_files: - try: - warnings.extend(_check_csv_nan(csv_file, output_id)) - except (OSError, csv.Error) as exc: - warnings.append( - f"Output '{output_id}': could not validate table '{csv_file.name}': {exc}" - ) - return warnings - - -def _check_csv_nan(csv_file: Path, output_id: str) -> list[str]: - with open(csv_file, newline="", encoding="utf-8", errors="replace") as fh: - reader = csv.DictReader(fh) - rows = list(reader) - - if not rows: - return [ - f"Output '{output_id}': table file '{csv_file.name}' has no data rows" - ] - - fieldnames = list(rows[0].keys()) - all_nan_cols: list[str] = [] - numeric_cols: list[str] = [] - - for col in fieldnames: - numeric_vals: list[float] = [] - for row in rows: - raw = row.get(col, "") - try: - numeric_vals.append(float(raw)) - except (ValueError, TypeError): - pass - if numeric_vals: - numeric_cols.append(col) - if all(math.isnan(v) for v in numeric_vals): - all_nan_cols.append(col) - - if not numeric_cols: - return [] - - if len(all_nan_cols) == len(numeric_cols): - return [ - f"Output '{output_id}': table file '{csv_file.name}' " - f"has all-NaN values in every numeric column" - ] - if all_nan_cols: - cols = ", ".join(f"'{c}'" for c in all_nan_cols) - return [ - f"Output '{output_id}': table file '{csv_file.name}' " - f"has all-NaN values in column(s): {cols}" - ] - return [] - - -def _validate_figure(output_dir: Path, output_id: str) -> list[str]: - figure_exts = {".png", ".jpg", ".jpeg", ".svg", ".pdf", ".eps"} - figure_files = [ - f for f in output_dir.iterdir() - if f.is_file() and f.suffix.lower() in figure_exts - ] - - if not figure_files: - return [ - f"Output '{output_id}' (type: figure) produced no image files " - f"(.png, .jpg, .svg, .pdf, .eps) in {output_dir}" - ] - - return [ - f"Output '{output_id}': figure file '{fig.name}' is empty (0 bytes)" - for fig in figure_files - if fig.stat().st_size == 0 - ] - - -def _all_scalars_null(data: Any) -> bool: - """Return True if every scalar in the JSON structure is null or NaN.""" - scalars = _collect_scalars(data) - return bool(scalars) and all(_is_null_scalar(s) for s in scalars) - - -def _collect_scalars(data: Any) -> list[Any]: - if isinstance(data, dict): - result: list[Any] = [] - for v in data.values(): - result.extend(_collect_scalars(v)) - return result - if isinstance(data, list): - result = [] - for item in data: - result.extend(_collect_scalars(item)) - return result - return [data] - - -def _is_null_scalar(v: Any) -> bool: - if v is None: - return True - if isinstance(v, float) and math.isnan(v): - return True - return False diff --git a/src/lightcone/engine/verify.py b/src/lightcone/engine/verify.py deleted file mode 100644 index 51065300..00000000 --- a/src/lightcone/engine/verify.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Integrity verification for materialized outputs. - -For each output with a manifest, recomputes the on-disk data_version and -walks the recorded input_versions back through the upstream chain. Three -failure modes: - -- ``missing_manifest``: a result directory exists but has no manifest, or - the manifest is unparseable. This is the agent-forged-file scenario. -- ``tampered_data``: the manifest is present and well-formed, but the - bytes on disk no longer hash to the recorded ``data_version``. -- ``broken_chain``: the recorded ``input_versions`` reference an upstream - output whose own ``data_version`` no longer matches. - -Like ``status``, this module never imports Snakemake. -""" -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path -from typing import Literal - -from astra.helpers import load_yaml, resolve_analysis_tree - -from lightcone.engine.manifest import read_manifest, sha256_dir -from lightcone.engine.tree import ( - collect_tree_outputs, - find_upstream_output, - resolve_output_path, -) - -FailureKind = Literal["missing_manifest", "tampered_data", "broken_chain"] - - -@dataclass -class VerifyResult: - output_id: str - universe_id: str - output_dir: Path - passed: bool - failure: FailureKind | None - detail: str | None = None - - -def verify_outputs( - project_path: Path, - *, - universe_id: str, -) -> Iterator[VerifyResult]: - """Yield a :class:`VerifyResult` for every output with a recipe.""" - spec = resolve_analysis_tree(load_yaml(project_path / "astra.yaml"), project_path) - all_outputs = collect_tree_outputs(spec) - - for tree_out in all_outputs: - # Aliases have no own materialization to verify. - if tree_out.output_def.get("recipe") is None: - continue - out_dir = resolve_output_path(project_path, tree_out, universe_id) / tree_out.output_id - - if not out_dir.exists(): - # Not a verification failure — there's just nothing materialized - # to verify. ``lc status`` is the right report for that. - continue - - manifest = read_manifest(out_dir) - if manifest is None: - yield VerifyResult( - output_id=tree_out.output_id, - universe_id=universe_id, - output_dir=out_dir, - passed=False, - failure="missing_manifest", - detail="No manifest found at output directory", - ) - continue - - actual_dv = sha256_dir(out_dir) - if actual_dv != manifest.get("data_version"): - yield VerifyResult( - output_id=tree_out.output_id, - universe_id=universe_id, - output_dir=out_dir, - passed=False, - failure="tampered_data", - detail=( - f"recorded {manifest.get('data_version')!r} != " - f"actual {actual_dv!r}" - ), - ) - continue - - # Walk the recorded chain. ``Output.inputs`` (v0.0.7) carries - # the upstream artifact IDs; recipe blocks no longer hold them. - chain_failure: str | None = None - declared_inputs = tree_out.output_def.get("inputs") or [] - for inp_id in declared_inputs: - recorded = manifest.get("input_versions", {}).get(inp_id) - if recorded is None: - # The manifest doesn't even know about an upstream the - # current spec declares — broken chain. - chain_failure = f"input '{inp_id}' missing from manifest" - break - # Find the upstream output's current materialized state. We - # accept either: (a) it's a sibling output we can locate via - # the tree, or (b) it's an external input (then recorded is - # an mtime-size or sha256 fingerprint, no chain to walk). - up = find_upstream_output(tree_out, inp_id, all_outputs) - if up is None: - continue # external; nothing to chain to - up_dir = resolve_output_path(project_path, up, universe_id) / up.output_id - up_manifest = read_manifest(up_dir) - if up_manifest is None: - chain_failure = f"upstream '{inp_id}' missing manifest" - break - if up_manifest.get("data_version") != recorded: - chain_failure = ( - f"upstream '{inp_id}' data_version drifted" - ) - break - - if chain_failure: - yield VerifyResult( - output_id=tree_out.output_id, - universe_id=universe_id, - output_dir=out_dir, - passed=False, - failure="broken_chain", - detail=chain_failure, - ) - continue - - yield VerifyResult( - output_id=tree_out.output_id, - universe_id=universe_id, - output_dir=out_dir, - passed=True, - failure=None, - ) - - diff --git a/src/lightcone/engine/wrroc.py b/src/lightcone/engine/wrroc.py deleted file mode 100644 index af9f72f1..00000000 --- a/src/lightcone/engine/wrroc.py +++ /dev/null @@ -1,885 +0,0 @@ -"""Workflow Run RO-Crate (WRROC) exporter. - -Walks a project's per-output ``.lightcone-manifest.json`` sidecars and -emits a `Workflow Run RO-Crate `_ -bundle suitable for upload to WorkflowHub, Zenodo, or any RO-Crate-aware -archive. - -The lightcone manifest layer remains the canonical internal format. WRROC -is the **publication** view, generated on demand. We target the deepest -of the three WRROC profiles — *Provenance Run Crate* — because lightcone -already captures the per-step data it requires. - -Entity mapping --------------- - -============================== ===================================== -lightcone concept WRROC entity -============================== ===================================== -``astra.yaml`` ``ComputationalWorkflow`` -each universe ``PropertyValue`` set on the workflow -each materialized output dir ``Dataset`` (data files inside) -each recipe execution ``CreateAction`` - ``object`` upstream Datasets / external Files - ``result`` the output Dataset - ``instrument`` the recipe ``SoftwareApplication`` - ``agent`` the human author (``Person``) -each container image ``SoftwareApplication`` -each decision value ``PropertyValue`` on the workflow -============================== ===================================== - -The exporter is one-shot: ``export_wrroc()`` produces a directory (or -``--zip`` archive). We do not maintain a live crate; the user invokes -this when they're ready to publish. -""" -from __future__ import annotations - -import logging -import os -import subprocess -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from astra.helpers import get_decisions, load_yaml, resolve_analysis_tree - -from lightcone.engine.manifest import MANIFEST_FILENAME, read_manifest -from lightcone.engine.tree import ( - TreeOutput, - collect_tree_outputs, - resolve_output_path, - resolve_universe_decisions, -) - -logger = logging.getLogger(__name__) - -#: WRROC profile we target. Pinned explicitly — bump when upgrading. -PROVENANCE_RUN_CRATE_PROFILE = ( - "https://w3id.org/ro/wfrun/provenance/0.5" -) -WORKFLOW_RUN_CRATE_PROFILE = ( - "https://w3id.org/ro/wfrun/workflow/0.5" -) -PROCESS_RUN_CRATE_PROFILE = ( - "https://w3id.org/ro/wfrun/process/0.5" -) - - -@dataclass -class ExportResult: - """Returned by :func:`export_wrroc` so callers can act on the outcome.""" - - bundle_path: Path - runs_included: int - universes_included: list[str] - is_zip: bool - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - - -#: Default license URL when none is supplied. CC-BY-4.0 is widely accepted -#: by Zenodo and WorkflowHub for research outputs and is permissive enough -#: not to surprise users who didn't think about licensing. Can be -#: overridden via the ``license`` argument or ``--license`` CLI flag. -DEFAULT_LICENSE = "https://creativecommons.org/licenses/by/4.0/" - - -def export_wrroc( - project_path: Path, - output_path: Path, - *, - universes: list[str] | None = None, - author: str | None = None, - license: str | None = None, - zip_bundle: bool = False, - include_data: bool = True, -) -> ExportResult: - """Walk manifests in *project_path* and emit a WRROC bundle. - - Parameters - ---------- - project_path: - Project root containing ``astra.yaml``. - output_path: - Where to write the bundle. If *zip_bundle* is True, this is the - ``.zip`` file; otherwise it is the bundle directory. - universes: - Restrict to a subset of universes. ``None`` includes all - universes that have at least one materialized output. - author: - Override the author. If ``None``, falls back to ``git config - user.name``/``user.email`` then to ``LIGHTCONE_AUTHOR`` env var. - license: - License URL or SPDX-style identifier for the bundle. Required by - the Workflow RO-Crate profile; defaults to :data:`DEFAULT_LICENSE` - (CC-BY-4.0) when ``None``. - zip_bundle: - Package as a single zip after building. The zip contains the - bundle directory at its root. - include_data: - When False, only the manifests, ``astra.yaml`` and universe files - are bundled — useful for archiving provenance without re-uploading - large data files. - """ - # Lazy import to keep the module importable without rocrate installed. - from rocrate.rocrate import ROCrate - - project_path = Path(project_path).resolve() - spec_path = project_path / "astra.yaml" - if not spec_path.is_file(): - raise FileNotFoundError( - f"No astra.yaml at {project_path}; cannot export." - ) - - spec = resolve_analysis_tree(load_yaml(spec_path), project_path) - project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") - - # Resolve which universes to include. If the caller didn't restrict, - # discover from the universes/ directory. - if universes is None: - universes = _discover_all_universes(project_path) - - crate = ROCrate() - crate.name = spec.get("name") or project_name - # RO-Crate REQUIRES a description on the root. Fall back to a - # generated one when astra.yaml doesn't define one. - crate.description = spec.get("description") or ( - f"WRROC bundle exported from {crate.name} on " - f"{_format_finished_at(_now())}." - ) - crate.creativeWorkStatus = "Published" - - # License — required by Workflow RO-Crate. Caller supplies it - # explicitly or we fall back to a permissive default. - license_url = license or spec.get("license") or DEFAULT_LICENSE - crate.root_dataset["license"] = {"@id": license_url} - - # Mark the root as a workflow run crate. - crate.root_dataset["conformsTo"] = [ - {"@id": PROCESS_RUN_CRATE_PROFILE}, - {"@id": WORKFLOW_RUN_CRATE_PROFILE}, - {"@id": PROVENANCE_RUN_CRATE_PROFILE}, - ] - # The validator expects each profile URL referenced by conformsTo to - # also exist as a CreativeWork entity in the @graph — declare them. - from rocrate.model import ContextEntity - for profile_url, profile_name in [ - (PROCESS_RUN_CRATE_PROFILE, "Process Run Crate"), - (WORKFLOW_RUN_CRATE_PROFILE, "Workflow Run Crate"), - (PROVENANCE_RUN_CRATE_PROFILE, "Provenance Run Crate"), - ]: - crate.add(ContextEntity(crate, profile_url, properties={ - "@type": "CreativeWork", - "name": f"{profile_name} 0.5", - "version": "0.5", - })) - # The metadata file descriptor itself must conformsTo a specific - # RO-Crate spec version. rocrate-py 0.15 emits a 1.2 @context but - # the validator looks for a conformsTo on the descriptor — set both - # 1.1 (for backward-compat validators) and 1.2 explicitly. - crate.metadata["conformsTo"] = [ - {"@id": "https://w3id.org/ro/crate/1.1"}, - {"@id": "https://w3id.org/ro/crate/1.2"}, - ] - - builder = WRROCBuilder( - crate=crate, - project_path=project_path, - spec=spec, - author_str=author or _detect_author(project_path), - include_data=include_data, - ) - - # Add the workflow definition (astra.yaml). - builder.add_workflow() - - # Walk each universe's outputs and emit Datasets + CreateActions. - runs_added = 0 - included_universes: list[str] = [] - tree_outputs = collect_tree_outputs(spec) - - for universe_id in universes: - universe_runs = builder.add_universe_runs(universe_id, tree_outputs) - if universe_runs > 0: - runs_added += universe_runs - included_universes.append(universe_id) - - if runs_added == 0: - logger.warning( - "No materialized outputs found for universes %s — bundle will " - "contain only the workflow definition.", - universes, - ) - - # Render the bundle. - output_path = Path(output_path).resolve() - - if zip_bundle: - if output_path.exists(): - if output_path.is_dir(): - raise FileExistsError( - f"{output_path} is an existing directory; cannot overwrite with a zip. " - "Pass a file path (e.g. bundle.zip) or remove the existing directory." - ) - output_path.unlink() - crate.write_zip(output_path) - result_path = output_path - else: - if output_path.exists() and any(output_path.iterdir()): - raise FileExistsError( - f"{output_path} is non-empty; refuse to clobber. " - "Pass a fresh path or remove the existing one." - ) - crate.write(output_path) - result_path = output_path - - return ExportResult( - bundle_path=result_path, - runs_included=runs_added, - universes_included=included_universes, - is_zip=zip_bundle, - ) - - -# --------------------------------------------------------------------------- -# Builder — accumulates entities into a single ROCrate instance -# --------------------------------------------------------------------------- - - -class WRROCBuilder: - """Accumulates lightcone state into a WRROC ``ROCrate`` instance. - - The builder owns the @id minting strategy and the de-duplication of - repeated entities (e.g. the same recipe ``SoftwareApplication`` is - used by multiple ``CreateAction`` runs). - """ - - def __init__( - self, - crate: Any, # rocrate.rocrate.ROCrate - project_path: Path, - spec: dict[str, Any], - author_str: str | None, - include_data: bool, - ) -> None: - self.crate = crate - self.project_path = project_path - self.spec = spec - self.include_data = include_data - - self._workflow_id: str | None = None - self._dataset_ids: dict[tuple[str, str], str] = {} # (universe, output) → @id - self._software_ids: dict[str, str] = {} # recipe text → @id - self._container_ids: dict[str, str] = {} # image tag → @id - self._person_id: str | None = None - self._code_repo_id: str | None = None # set lazily from manifest's git_remote - - if author_str: - self._person_id = self._add_person(author_str) - - # ----- Workflow + universes ----- - - def add_workflow(self) -> str: - """Add the astra.yaml as a ``ComputationalWorkflow`` entity.""" - from rocrate.model import ContextEntity - - spec_path = self.project_path / "astra.yaml" - wf_id = "astra.yaml" - # Also bundle the file itself (always — it's small and central). - # WRROC requires a known ComputerLanguage. astra.yaml is a spec - # over Snakemake (the actual executor), so we tag the workflow - # language as "snakemake" — the truthful description of what runs. - wf = self.crate.add_workflow( - spec_path, - wf_id, - main=True, - lang="snakemake", - ) - wf["name"] = self.spec.get("name") or "ASTRA analysis" - if "description" in self.spec: - wf["description"] = self.spec["description"] - - # Decisions: declare each decision (root + sub-analysis) as a - # FormalParameter of the workflow. Per-universe values get - # attached as PropertyValue on the CreateAction (see - # add_universe_runs). ASTRA's decisions are a dict keyed by id; - # use get_decisions() so sub-analysis decisions are merged in. - for decision_id, decision in get_decisions(self.spec).items(): - param_id = f"#param-{decision_id}" - # additionalType is REQUIRED by the WRROC FormalParameter - # shape. Infer from the default value (or first option) so - # we report the right schema.org primitive. - sample_val = _decision_sample_value(decision) - param = ContextEntity( - self.crate, - param_id, - properties={ - "@type": "FormalParameter", - "name": decision_id, - "description": ( - decision.get("rationale") - or decision.get("label") - or decision.get("description", "") - ), - "additionalType": _infer_additional_type(sample_val), - }, - ) - self.crate.add(param) - wf.append_to("input", param) - - # Bundle universe files for full reproducibility. - universes_dir = self.project_path / "universes" - if universes_dir.is_dir(): - for u_file in sorted(universes_dir.glob("*.yaml")): - rel = u_file.relative_to(self.project_path) - self.crate.add_file(u_file, str(rel)) - - self._workflow_id = wf_id - return wf_id - - def add_universe_runs( - self, - universe_id: str, - tree_outputs: list[TreeOutput], - ) -> int: - """Add Datasets + CreateActions for every materialized output in the - given universe. Returns the count of CreateActions added. - """ - decisions = _safe_load_universe_decisions( - self.project_path, self.spec, universe_id - ) - runs_added = 0 - - for tree_out in tree_outputs: - recipe = tree_out.output_def.get("recipe") - if recipe is None: # alias output, no own materialization - continue - - out_dir = ( - resolve_output_path(self.project_path, tree_out, universe_id) - / tree_out.output_id - ) - # Best-effort manifest read: skip outputs whose directory - # is unreadable (permission-denied scratch entries, broken - # symlinks, mid-rsync states). For `lc export`, partial - # bundles are more useful than a total abort. - try: - manifest = read_manifest(out_dir) - except OSError as exc: - logger.warning( - "Skipping %s/%s: cannot read manifest (%s)", - universe_id, tree_out.output_id, exc, - ) - continue - if manifest is None: - continue # not yet materialized in this universe - - dataset_id = self._add_output_dataset( - tree_out, universe_id, out_dir, manifest - ) - self._add_create_action( - tree_out=tree_out, - universe_id=universe_id, - dataset_id=dataset_id, - manifest=manifest, - decisions=decisions, - tree_outputs=tree_outputs, - ) - runs_added += 1 - - return runs_added - - # ----- Datasets / runs ----- - - def _add_output_dataset( - self, - tree_out: TreeOutput, - universe_id: str, - out_dir: Path, - manifest: dict[str, Any], - ) -> str: - """Add a ``Dataset`` for an output directory and return its @id.""" - rel_dir = out_dir.relative_to(self.project_path).as_posix() - dataset_id = rel_dir + "/" - - # If this dataset was already added (which can happen if the user - # passes overlapping universes), return the existing @id. - cache_key = (universe_id, tree_out.output_id) - if cache_key in self._dataset_ids: - return self._dataset_ids[cache_key] - - if self.include_data: - self.crate.add_dataset(out_dir, dataset_id) - else: - # Metadata-only mode: we still want the dataset in the graph - # for chain integrity, but we don't copy data files. Bundle - # only the manifest itself. - self.crate.add_dataset(None, dataset_id) - manifest_src = out_dir / MANIFEST_FILENAME - if manifest_src.exists(): - self.crate.add_file( - manifest_src, - f"{dataset_id}{MANIFEST_FILENAME}", - ) - - ds = self.crate.dereference(dataset_id) - ds["name"] = f"{tree_out.output_id} (universe={universe_id})" - # schema.org's `version` is the standard place for a content hash - # / version identifier on a Dataset. dataVersion (lightcone term) - # is not in the RO-Crate context so the validator rejects it. - if data_version := manifest.get("data_version"): - ds["version"] = data_version - - self._dataset_ids[cache_key] = dataset_id - return dataset_id - - def _add_create_action( - self, - *, - tree_out: TreeOutput, - universe_id: str, - dataset_id: str, - manifest: dict[str, Any], - decisions: dict[str, Any], - tree_outputs: list[TreeOutput], - ) -> str: - """Add a ``CreateAction`` linking inputs → instrument → output.""" - from rocrate.model import ContextEntity - - action_id = f"#run-{universe_id}-{_qualified_id(tree_out)}" - recipe_cmd = (tree_out.output_def.get("recipe") or {}).get("command", "") - - instrument_id = self._add_recipe_software( - recipe_cmd, - manifest.get("container_image"), - tool_name=(tree_out.output_def.get("recipe") or {}).get("tool_name"), - output_id=tree_out.output_id, - ) - - # If the manifest carries a git_remote URL, surface it as a - # CodeRepository entity once (de-duplicated across all actions). - self._link_code_repository(manifest.get("git_remote")) - - # Resolve `object` (inputs to the action). Each upstream input - # references the producing dataset's @id (Provenance chain). - # External inputs we represent as ContextEntity File-with-fingerprint. - objects: list[dict[str, str]] = [] - for inp_id, version_str in (manifest.get("input_versions") or {}).items(): - obj_ref = self._resolve_input_reference( - inp_id=inp_id, - version_str=version_str, - consumer=tree_out, - universe_id=universe_id, - tree_outputs=tree_outputs, - ) - if obj_ref is not None: - objects.append(obj_ref) - - properties: dict[str, Any] = { - "@type": "CreateAction", - "name": f"Run of {tree_out.output_id} (universe={universe_id})", - "instrument": {"@id": instrument_id}, - "object": objects, - "result": [{"@id": dataset_id}], - "endTime": _format_finished_at(manifest.get("finished_at")), - "actionStatus": {"@id": "http://schema.org/CompletedActionStatus"}, - } - if self._person_id: - properties["agent"] = {"@id": self._person_id} - - action = ContextEntity(self.crate, action_id, properties=properties) - self.crate.add(action) - - # Per-decision PropertyValue entities, attached to the action as - # parameter values rather than to the workflow (so multiple - # universes don't trample each other). - for d_id, d_value in decisions.items(): - pv_id = f"#pv-{universe_id}-{_qualified_id(tree_out)}-{d_id}" - pv = ContextEntity( - self.crate, - pv_id, - properties={ - "@type": "PropertyValue", - "name": d_id, - "value": _coerce_value(d_value), - }, - ) - self.crate.add(pv) - action.append_to("object", pv) - - # Workflow & manifest provenance metadata - action.append_to( - "object", - self._add_property_value( - f"#pv-{universe_id}-{_qualified_id(tree_out)}-code_version", - "code_version", - manifest.get("code_version", ""), - ), - ) - action.append_to( - "object", - self._add_property_value( - f"#pv-{universe_id}-{_qualified_id(tree_out)}-data_version", - "data_version", - manifest.get("data_version", ""), - ), - ) - - return action_id - - # ----- Resolved references ----- - - def _resolve_input_reference( - self, - *, - inp_id: str, - version_str: str, - consumer: TreeOutput, - universe_id: str, - tree_outputs: list[TreeOutput], - ) -> dict[str, str] | None: - """Return a ``{"@id": ...}`` reference for an action's input. - - For upstream-produced inputs, the @id points at the producing - dataset. For external inputs, we synthesize a File ContextEntity - with the fingerprint as its sha256 / mtime-size note. - """ - from lightcone.engine.tree import find_upstream_output - - upstream = find_upstream_output(consumer, inp_id, tree_outputs) - if upstream is not None: - # Reference an existing dataset @id (must already have been - # added — order-preserving collect_tree_outputs handles the - # common case; out-of-order DAGs still work because rocrate - # tolerates forward refs at write time). - cache_key = (universe_id, upstream.output_id) - if cache_key in self._dataset_ids: - return {"@id": self._dataset_ids[cache_key]} - # Forward reference: predict the @id deterministically. - out_dir = ( - resolve_output_path(self.project_path, upstream, universe_id) - / upstream.output_id - ) - return { - "@id": out_dir.relative_to(self.project_path).as_posix() + "/" - } - - # External input. Synthesize a stable @id from the input id + - # fingerprint so identical files de-duplicate across runs. - external_id = f"#ext-{inp_id}" - if not self.crate.dereference(external_id): - from rocrate.model import ContextEntity - - ext = ContextEntity( - self.crate, - external_id, - properties={ - "@type": "File", - "name": inp_id, - "description": f"External input fingerprint: {version_str}", - }, - ) - # Encode the fingerprint as a checksum/contentSize note. - if version_str.startswith("sha256:"): - ext["sha256"] = version_str.removeprefix("sha256:") - else: - ext["fingerprint"] = version_str - self.crate.add(ext) - return {"@id": external_id} - - def _link_code_repository(self, git_remote: str | None) -> None: - """Idempotently add a CodeRepository entity for the project repo. - - Called once per manifest read; later calls with the same URL are - no-ops. The repository entity is also linked from the workflow - via ``codeRepository`` so consumers can discover the source. - """ - if not git_remote: - return - if self._code_repo_id is not None: - return # already added - from rocrate.model import ContextEntity - - repo = ContextEntity(self.crate, git_remote, properties={ - "@type": ["CodeRepository", "SoftwareSourceCode"], - "name": git_remote.rsplit("/", 1)[-1] or git_remote, - "url": git_remote, - }) - self.crate.add(repo) - if self._workflow_id is not None: - wf = self.crate.dereference(self._workflow_id) - if wf is not None: - wf["codeRepository"] = {"@id": git_remote} - self._code_repo_id = git_remote - - def _add_recipe_software( - self, - recipe_cmd: str, - container_image: str | None, - *, - tool_name: str | None = None, - output_id: str | None = None, - ) -> str: - """De-duplicate recipes — return the @id of the SoftwareApplication. - - ``SoftwareApplication.name`` resolution order: - - 1. Explicit ``recipe.tool_name`` from astra.yaml (best — author-chosen). - 2. Heuristic from the command (e.g. ``scripts/analyze.py``). - 3. The output id (always available, always meaningful). - - The full command is always preserved as ``description``. - """ - from rocrate.model import ContextEntity - - key = recipe_cmd or "" - if key in self._software_ids: - return self._software_ids[key] - - sw_id = f"#recipe-{len(self._software_ids)}" - name = ( - tool_name - or _heuristic_tool_name(recipe_cmd) - or output_id - or "(empty recipe)" - ) - props: dict[str, Any] = { - "@type": "SoftwareApplication", - "name": name, - "description": recipe_cmd, - } - if container_image: - props["softwareRequirements"] = { - "@id": self._add_container_software(container_image) - } - sw = ContextEntity(self.crate, sw_id, properties=props) - self.crate.add(sw) - - # WRROC: ComputationalWorkflow MUST refer to its orchestrated - # tools via hasPart. Link each recipe back to the workflow. - if self._workflow_id is not None: - wf = self.crate.dereference(self._workflow_id) - if wf is not None: - wf.append_to("hasPart", sw) - - self._software_ids[key] = sw_id - return sw_id - - def _add_container_software(self, image_tag: str) -> str: - """De-duplicate container images.""" - from rocrate.model import ContextEntity - - if image_tag in self._container_ids: - return self._container_ids[image_tag] - cid = f"#container-{len(self._container_ids)}" - sw = ContextEntity( - self.crate, - cid, - properties={ - "@type": ["SoftwareApplication", "ContainerImage"], - "name": image_tag, - "softwareVersion": image_tag, - }, - ) - self.crate.add(sw) - self._container_ids[image_tag] = cid - return cid - - def _add_property_value(self, pv_id: str, name: str, value: Any) -> Any: - from rocrate.model import ContextEntity - - existing = self.crate.dereference(pv_id) - if existing: - return existing - pv = ContextEntity( - self.crate, - pv_id, - properties={ - "@type": "PropertyValue", - "name": name, - "value": _coerce_value(value), - }, - ) - self.crate.add(pv) - return pv - - def _add_person(self, author_str: str) -> str: - from rocrate.model import ContextEntity - - name, email = _parse_author(author_str) - person_id = f"#author-{(email or name).replace('@', '_at_')}" - if self.crate.dereference(person_id): - return person_id - props: dict[str, Any] = {"@type": "Person", "name": name} - if email: - props["email"] = email - person = ContextEntity(self.crate, person_id, properties=props) - self.crate.add(person) - return person_id - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _qualified_id(tree_out: TreeOutput) -> str: - """A filesystem-safe qualified id for use in @id minting.""" - if tree_out.analysis_id: - return f"{tree_out.analysis_id}.{tree_out.output_id}" - return tree_out.output_id - - -def _discover_all_universes(project_path: Path) -> list[str]: - """List every universe with at least one universe yaml present.""" - universes_dir = project_path / "universes" - if not universes_dir.exists(): - return ["baseline"] - ids = sorted(p.stem for p in universes_dir.glob("*.yaml")) - return ids or ["baseline"] - - -def _safe_load_universe_decisions( - project_path: Path, - spec: dict[str, Any], - universe_id: str, -) -> dict[str, Any]: - """Like resolve_universe_decisions but tolerant of missing/unreadable files.""" - universe_yaml = project_path / "universes" / f"{universe_id}.yaml" - try: - if not universe_yaml.exists(): - return {} - return resolve_universe_decisions(project_path, spec, universe_id) - except (FileNotFoundError, KeyError, OSError): - return {} - - -def _decision_sample_value(decision: dict[str, Any]) -> Any: - """Pick a representative value for a decision to infer its type. - - ASTRA's decisions schema uses a dict of options keyed by option id - (e.g. ``options: {bins_8: {label: '8 bins'}, ...}``). The default is - typically one of those keys. For type inference, prefer: - - 1. The `default` value if set (always a primitive). - 2. The first option key if options is a dict. - 3. The first option's `value` field if options is a list. - """ - if "default" in decision: - return decision["default"] - opts = decision.get("options") - if isinstance(opts, dict) and opts: - return next(iter(opts)) - if isinstance(opts, list) and opts: - first = opts[0] - if isinstance(first, dict): - return first.get("value") - return first - return None - - -def _heuristic_tool_name(recipe_cmd: str) -> str | None: - """Best-effort SoftwareApplication.name from a bash command. - - Looks for the first script-like token (``foo.py``, ``./bin/foo``, - ``foo.sh``) and returns just that. Returns None if no obvious tool - can be extracted, falling through to the output_id fallback. - """ - if not recipe_cmd: - return None - for token in recipe_cmd.split(): - # Skip env assignments, redirects, shell builtins - if "=" in token and not token.startswith("-"): - continue - if token in {"python", "python3", "bash", "sh", "uv", "run"}: - continue - # Must look like a path with an extension or a leading ./ - if "/" in token or token.startswith("./"): - return token - if "." in token and not token.startswith("-"): - ext = token.rsplit(".", 1)[-1] - if ext in {"py", "sh", "R", "jl", "rb", "pl"}: - return token - return None - - -def _infer_additional_type(value: Any) -> dict[str, str]: - """Map a sample decision value to a schema.org primitive type @id. - - WRROC's FormalParameter shape requires ``additionalType`` to indicate - the parameter's expected value type — Text/Integer/Float/Boolean. - """ - if isinstance(value, bool): - return {"@id": "http://schema.org/Boolean"} - if isinstance(value, int): - return {"@id": "http://schema.org/Integer"} - if isinstance(value, float): - return {"@id": "http://schema.org/Float"} - return {"@id": "http://schema.org/Text"} - - -def _coerce_value(value: Any) -> Any: - """schema.org PropertyValue.value should be a primitive (str/num/bool). - - Lightcone decisions can be arbitrary YAML — coerce non-primitives to - a JSON string so the PropertyValue stays valid. - """ - if isinstance(value, (str, int, float, bool)) or value is None: - return value - import json as _json - return _json.dumps(value, sort_keys=True) - - -def _format_finished_at(ts: float | None) -> str | None: - """Convert a unix timestamp to ISO 8601 for schema:endTime.""" - if ts is None: - return None - from datetime import UTC, datetime - return datetime.fromtimestamp(ts, tz=UTC).isoformat() - - -def _now() -> float: - import time - return time.time() - - -def _parse_author(s: str) -> tuple[str, str | None]: - """Parse ``"Name "`` or just ``"Name"`` into (name, email).""" - s = s.strip() - if "<" in s and s.endswith(">"): - name, _, rest = s.rpartition("<") - return name.strip(), rest.removesuffix(">").strip() or None - return s, None - - -def _detect_author(project_path: Path) -> str | None: - """Pull author from git config or environment, else return None.""" - if env := os.environ.get("LIGHTCONE_AUTHOR"): - return env - try: - name = subprocess.run( - ["git", "config", "user.name"], - cwd=project_path, capture_output=True, text=True, timeout=5, - ).stdout.strip() - email = subprocess.run( - ["git", "config", "user.email"], - cwd=project_path, capture_output=True, text=True, timeout=5, - ).stdout.strip() - if name and email: - return f"{name} <{email}>" - return name or None - except (subprocess.SubprocessError, FileNotFoundError, OSError): - return None - - -__all__ = [ - "ExportResult", - "PROCESS_RUN_CRATE_PROFILE", - "PROVENANCE_RUN_CRATE_PROFILE", - "WORKFLOW_RUN_CRATE_PROFILE", - "WRROCBuilder", - "export_wrroc", -] diff --git a/src/snakemake_executor_plugin_dask/__init__.py b/src/snakemake_executor_plugin_dask/__init__.py deleted file mode 100644 index 3cff3f63..00000000 --- a/src/snakemake_executor_plugin_dask/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Snakemake executor plugin: dispatches each rule's shell command to a -running ``dask.distributed`` cluster. - -The cluster rendezvous is read from the environment: either a plain -scheduler address in ``DASK_SCHEDULER_ADDRESS`` or a Dask Gateway -cluster name in ``LIGHTCONE_GATEWAY_CLUSTER`` (rejoined through the -Gateway API — ``gateway://`` schedulers cannot be dialled directly). -``lc run`` is responsible for setting one of them — typically by -constructing a ``LocalCluster()`` for the duration of the run, backed by -``srun``-launched workers inside a SLURM allocation, or by creating a -run-scoped Gateway cluster on a JupyterHub deployment. - -The plugin is intentionally minimal: each Snakemake job becomes a -``client.submit(_run_shell, cmd, resources={...})`` call. Workers run the -shell command as-is (recipes are already containerized at Snakefile -generation time, so the worker just shells out). -""" - -from snakemake_interface_executor_plugins.settings import ( # type: ignore[import-untyped] - CommonSettings, -) - -from .executor import DaskExecutor as Executor # noqa: F401 - -common_settings = CommonSettings( - job_deploy_sources=True, - non_local_exec=True, - implies_no_shared_fs=False, -) diff --git a/src/snakemake_executor_plugin_dask/executor.py b/src/snakemake_executor_plugin_dask/executor.py deleted file mode 100644 index 03d2a52a..00000000 --- a/src/snakemake_executor_plugin_dask/executor.py +++ /dev/null @@ -1,229 +0,0 @@ -# mypy: disable-error-code="no-untyped-call" -from __future__ import annotations - -import os -import shlex -import subprocess -import sys -from collections.abc import AsyncGenerator - -from snakemake_interface_common.exceptions import WorkflowError -from snakemake_interface_executor_plugins.executors.base import ( # type: ignore[import-untyped] - SubmittedJobInfo, -) -from snakemake_interface_executor_plugins.executors.remote import ( # type: ignore[import-untyped] - RemoteExecutor, -) -from snakemake_interface_executor_plugins.jobs import ( # type: ignore[import-untyped] - JobExecutorInterface, -) - -from lightcone.engine.dask_cluster import ( - GATEWAY_CLUSTER_ENV, - RESOURCE_CPUS, - RESOURCE_GPUS, - RESOURCE_MEMORY, -) -from lightcone.engine.runner import SENTINEL - -#: On a failure with no sentinel-framed output at all — the child -#: snakemake died before reaching the rule body (import error, missing -#: package in the worker image, broken snakefile) — forward this many -#: raw trailing lines so the failure is debuggable from the driver. -_RAW_TAIL_LINES = 60 - - -def _run_shell(cmd: str) -> tuple[int, str]: - """Worker-side: run the child snakemake command; return - ``(exit_code, output_block)``. - - The command is a child snakemake invocation that loads the generated - Snakefile and executes one rule's ``run:`` block. That block calls - :func:`lightcone.engine.runner.run_rule`, which streams structured - output prefixed with :data:`lightcone.engine.runner.SENTINEL`. - - The block travels back to the driver as part of the task result — - the only channel that works uniformly across LocalCluster threads, - srun-launched SLURM workers, and Dask Gateway worker pods (whose - stdout goes to pod logs, not the user's terminal). Sentinel-prefixed - lines are kept verbatim (prefix included: ``lc run`` filters on it); - everything else (snakemake bootstrap, dask noise, stray prints) is - dropped — unless the child failed without producing a single - sentinel line, in which case a bounded raw tail is forwarded so - bootstrap failures don't vanish into worker logs. - """ - p = subprocess.run( - cmd, shell=True, capture_output=True, text=True, check=False - ) - - lines = [ - line - for stream in (p.stdout, p.stderr) - for line in stream.splitlines() - if line.startswith(SENTINEL) - ] - if p.returncode != 0 and not lines: - raw = (p.stdout + p.stderr).splitlines()[-_RAW_TAIL_LINES:] - lines = [f"{SENTINEL} {line}" for line in raw] - - block = "\n".join(lines) + "\n" if lines else "" - return p.returncode, block - - -def _unpack_result(result: object) -> tuple[int, str]: - """Accept both the current ``(exit_code, block)`` result and the - bare ``int`` a worker running an older lightcone-cli release returns - (dask resolves ``_run_shell`` by module path on the worker, so - driver and worker versions can skew on image-based deployments).""" - if isinstance(result, tuple) and len(result) == 2: - return int(result[0]), str(result[1]) - return int(result), "" # type: ignore[call-overload] - - -def _build_resources(job: JobExecutorInterface) -> dict[str, float]: - """Translate Snakemake resources to Dask abstract resource units.""" - res: dict[str, float] = {} - cpus = job.resources.get("cpus_per_task") or job.threads - if cpus: - res[RESOURCE_CPUS] = float(cpus) - mem_mb = job.resources.get("mem_mb") - if mem_mb: - res[RESOURCE_MEMORY] = float(mem_mb) * 1e6 - gpus = job.resources.get("gpus_per_task") or job.resources.get("gpus") - if gpus: - res[RESOURCE_GPUS] = float(gpus) - return res - - -def _connect_client(): # type: ignore[no-untyped-def] - """Connect to the run's cluster. - - Two rendezvous modes, both set up by ``lc run``: - - - :data:`GATEWAY_CLUSTER_ENV` names a Dask Gateway cluster the - parent created. Gateway schedulers speak a ``gateway://`` comm - scheme with per-cluster TLS credentials held by the Gateway API — - a bare ``Client`` cannot dial them, so we rejoin through - ``Gateway().connect(name)``. - - Otherwise ``DASK_SCHEDULER_ADDRESS`` is a plain scheduler address. - - Returns ``(client, closer)`` where *closer* releases everything the - rendezvous opened. - """ - from dask.distributed import Client - - if name := os.environ.get(GATEWAY_CLUSTER_ENV): - from dask_gateway import Gateway - - # shutdown_on_close=False: the parent lc run owns the cluster - # lifecycle; the executor is a guest. - cluster = Gateway().connect(name, shutdown_on_close=False) - client = cluster.get_client() - - def closer() -> None: - client.close() - cluster.close() - - return client, closer - - addr = os.environ.get("DASK_SCHEDULER_ADDRESS") - if not addr: - raise WorkflowError( - "Neither DASK_SCHEDULER_ADDRESS nor " - f"{GATEWAY_CLUSTER_ENV} is set. `lc run` should set one " - "before invoking snakemake; if you're calling snakemake " - "directly, point it at a running dask scheduler." - ) - client = Client(addr) - return client, client.close - - -class DaskExecutor(RemoteExecutor): # type: ignore[misc] - def __init__(self, workflow, logger): # type: ignore[no-untyped-def] - super().__init__(workflow, logger) - try: - import dask.distributed # noqa: F401 - except ImportError as exc: - raise WorkflowError( - "dask.distributed is required for the dask executor " - "(`pip install distributed`)." - ) from exc - self._client, self._close_client = _connect_client() - - def get_job_exec_prefix(self, job: JobExecutorInterface) -> str: - # Spawned job commands carry no --directory: snakemake expects - # remote executors to cd into the workdir themselves (the - # official kubernetes executor does the same). Local and SLURM - # workers happen to inherit the driver's cwd, but a Dask - # Gateway worker pod starts in its image's WORKDIR (e.g. /app), - # where the child snakemake would resolve every relative path - # — and die on a read-only ``.snakemake``. - return f"cd {shlex.quote(self.workflow.workdir_init)}" - - def run_job(self, job: JobExecutorInterface) -> None: - cmd = self.format_job_exec(job) - self.logger.debug(cmd) - - resources = _build_resources(job) - future = self._client.submit( - _run_shell, - cmd, - resources=resources or None, - pure=False, - key=f"snakejob-{job.name}-{job.jobid}", - ) - - self.report_job_submission( - SubmittedJobInfo(job, external_jobid=future.key, aux={"future": future}) - ) - - async def check_active_jobs( - self, active_jobs: list[SubmittedJobInfo] - ) -> AsyncGenerator[SubmittedJobInfo, None]: - for j in active_jobs: - future = j.aux["future"] - if not future.done(): - yield j - continue - - exc = future.exception() - if exc is not None: - self.report_job_error( - j, msg=f"Dask task '{j.external_jobid}' raised: {exc!r}" - ) - continue - - exit_code, block = _unpack_result(future.result()) - if block: - # One atomic write per finished rule. We run inside the - # parent snakemake process, so this is naturally - # serialised — no cross-process locking needed. - sys.stdout.write(block) - sys.stdout.flush() - if exit_code != 0: - self.report_job_error( - j, msg=f"Dask task '{j.external_jobid}' exited {exit_code}." - ) - else: - self.report_job_success(j) - - def cancel_jobs(self, active_jobs: list[SubmittedJobInfo]) -> None: - # Snakemake calls cancel_jobs for partial cancellations as well as - # at terminal shutdown, so we MUST NOT close the client here — - # that would break any subsequent submissions in the same run. - # The client is closed in shutdown() exclusively. - for j in active_jobs: - future = j.aux.get("future") - if future is not None and not future.done(): - try: - future.cancel() - except Exception as exc: # noqa: BLE001 - self.logger.warning( - f"Failed to cancel dask task {j.external_jobid}: {exc}" - ) - - def shutdown(self) -> None: - try: - self._close_client() - finally: - super().shutdown() diff --git a/tests/conftest.py b/tests/conftest.py index baba0659..e7617194 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,54 @@ """Shared test fixtures for lightcone-cli tests.""" from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from click.testing import CliRunner + +#: Minimal lockfile body the faked ``uv lock`` writes. Its only job is to +#: exist and be non-empty — nothing in layer 1 parses a lock. +UV_LOCK_MIN = 'version = 1\nrequires-python = ">=3.12"\n' + + +@pytest.fixture +def runner() -> CliRunner: + return CliRunner() + + +@pytest.fixture(autouse=True) +def _isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Redirect ``~`` to a temp dir so tests can't touch the real home.""" + fake_home = tmp_path / "_home" + fake_home.mkdir() + monkeypatch.setattr(Path, "home", lambda: fake_home) + return fake_home + + +@pytest.fixture(autouse=True) +def fake_uv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """Fake the uv seam so convergence tests are hermetic — no network, no + real resolution. Emulates the observable effects: ``uv lock`` writes + ``uv.lock``, ``uv sync`` materializes ``.venv``. + + Returns the recorded argv lists, so a test can assert on *which* uv + commands ran with *which* flags. + """ + calls: list[list[str]] = [] + + def fake_run_uv(args: list[str], *, cwd: Path) -> MagicMock: + calls.append(list(args)) + project = Path(args[args.index("--project") + 1]) + if args[0] == "lock": + (project / "uv.lock").write_text(UV_LOCK_MIN) + elif args[0] == "sync": + (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) + return MagicMock(returncode=0, stdout="", stderr="") + + from lightcone.engine import project + + monkeypatch.setattr(project, "_run_uv", fake_run_uv) + monkeypatch.setattr(project.shutil, "which", lambda name, path=None: f"/usr/bin/{name}") + return calls diff --git a/tests/test_cli.py b/tests/test_cli.py index 4bec6685..618e1c01 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,475 +1,165 @@ -"""Tests for the redesigned lightcone CLI.""" +"""Tests for the CLI surface — flags, rendering, exit codes. + +Convergence *semantics* are tested against the engine in +`tests/test_project.py`; this file covers only what the CLI adds on top. +""" + from __future__ import annotations import json -import shutil -import subprocess from pathlib import Path -from unittest.mock import MagicMock import pytest from click.testing import CliRunner from lightcone.cli.commands import main - -@pytest.fixture -def runner() -> CliRunner: - return CliRunner() - - -@pytest.fixture(autouse=True) -def _isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Redirect ``~/.lightcone/`` to a temp dir so tests don't pollute the user's - real config. The global config is auto-created on first ``lc`` invocation.""" - fake_home = tmp_path / "_home" - fake_home.mkdir() - monkeypatch.setattr(Path, "home", lambda: fake_home) - return fake_home - - -# ---- top-level ------------------------------------------------------------ +# ---- top level ------------------------------------------------------------ -def test_help_lists_core_commands(runner: CliRunner) -> None: +def test_help_lists_the_implemented_verb(runner: CliRunner) -> None: result = runner.invoke(main, ["--help"]) assert result.exit_code == 0 - for cmd in ("init", "run", "status", "verify", "build"): - assert cmd in result.output + assert "init" in result.output -def test_help_does_not_advertise_removed_commands(runner: CliRunner) -> None: +def test_help_does_not_advertise_unbuilt_verbs(runner: CliRunner) -> None: + """Layer 1 ships `lc init` alone. The other verbs return with their + layers — advertising them before they work would be a lie the whole + rebuild is meant to avoid.""" result = runner.invoke(main, ["--help"]) - assert " dev " not in result.output - assert " cluster " not in result.output - assert " setup " not in result.output + for verb in ("materialize", "status", "verify", "build", "export"): + assert f" {verb}" not in result.output -def test_first_invocation_auto_creates_global_config( - runner: CliRunner, _isolated_home: Path, tmp_path: Path -) -> None: - config = _isolated_home / ".lightcone" / "config.yaml" - assert not config.exists() - # Any real subcommand triggers the group callback; ``init`` runs cleanly - # without a pre-existing project. - project = tmp_path / "proj" - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv"] - ) - assert result.exit_code == 0, result.output - assert config.exists() - assert "runtime: auto" in config.read_text() - - -# ---- lc init -------------------------------------------------------------- - - -def test_init_creates_project(runner: CliRunner, tmp_path: Path) -> None: - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - assert (project / "astra.yaml").exists() - assert (project / ".gitignore").exists() - assert (project / ".lightcone").is_dir() - assert (project / "results").is_dir() - assert (project / "universes").is_dir() - # The README is the durable hint that outputs materialize here via - # lc run — and the one file in results/ that stays tracked by git. - readme = (project / "results" / "README.md").read_text() - assert "lc run" in readme - gitignore = (project / ".gitignore").read_text() - assert "results/*" in gitignore - assert "!results/README.md" in gitignore - - -def test_init_creates_report_template(runner: CliRunner, tmp_path: Path) -> None: - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - - myst_yml = (project / "myst.yml").read_text() - assert "mystra.mjs" in myst_yml - assert "index.md" in myst_yml - - index_md = (project / "index.md").read_text() - assert index_md.startswith("# proj\n") - # References must track the astra init boilerplate element ids. - assert "{astra}`decisions.example_method`" in index_md - assert "{astra:value}`outputs.main_result`" in index_md - - assert "_build/" in (project / ".gitignore").read_text() - - -def test_init_adopts_existing_project(runner: CliRunner, tmp_path: Path) -> None: - """A directory that already holds an astra.yaml is converged, not rejected, - and user-owned files are never overwritten.""" +def test_engine_errors_render_cleanly(runner: CliRunner, tmp_path: Path) -> None: + """A ProjectError from anywhere in the engine surfaces as a one-line + CLI error (exit 1), not a traceback — the group boundary translates + it.""" project = tmp_path / "proj" project.mkdir() - (project / "astra.yaml").write_text("# user spec\n") - (project / ".gitignore").write_text("*.log\n") - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - # User files untouched (gitignore gains the managed block, keeps content). - assert (project / "astra.yaml").read_text() == "# user spec\n" - gitignore = (project / ".gitignore").read_text() - assert gitignore.startswith("*.log\n") - assert "# lightcone-cli" in gitignore - # Missing lightcone pieces were created. - assert (project / "Containerfile").exists() - assert (project / ".lightcone" / "lightcone.yaml").exists() - - -def test_init_is_idempotent(runner: CliRunner, tmp_path: Path) -> None: - """A second run reports everything unchanged and rewrites nothing.""" - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - before = {p: p.read_text() for p in project.rglob("*") if p.is_file()} - - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] - ) - assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert report["converged"] is True - assert report["created"] == [] - assert report["repaired"] == [] - assert {p: p.read_text() for p in project.rglob("*") if p.is_file()} == before - - # Gitignore block must not be duplicated across runs. - assert (project / ".gitignore").read_text().count("# lightcone-cli") == 1 - + (project / "Containerfile").write_text("FROM python:3.12-slim\n") -def test_init_check_reports_drift_without_writing( - runner: CliRunner, tmp_path: Path -) -> None: - project = tmp_path / "proj" - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--check", "--json"] - ) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 1 - report = json.loads(result.output) - assert report["converged"] is False - assert "astra.yaml" in report["created"] - assert not project.exists() # --check writes nothing, not even the dir + assert "delete or rename it" in result.output + assert "[tool.lightcone.image]" in result.output + assert "Traceback" not in result.output -def test_init_check_passes_on_converged_project( - runner: CliRunner, tmp_path: Path -) -> None: - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--check"] - ) - assert result.exit_code == 0, result.output +# ---- lc init: flags reach the engine -------------------------------------- -def test_init_warns_on_directory_copy(runner: CliRunner, tmp_path: Path) -> None: - """A user Containerfile with a directory COPY is never rewritten, but - the drift is surfaced through the warnings channel.""" +def test_init_creates_a_project(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - project.mkdir() - custom = "FROM python:3.12-slim\nRUN apt-get update\nCOPY src/ /app/src/\n" - (project / "Containerfile").write_text(custom) - (project / "src").mkdir() - (project / "src" / "a.py").write_text("a = 1\n") - - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] - ) + result = runner.invoke(main, ["init", str(project), "--no-git"]) assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert (project / "Containerfile").read_text() == custom - assert any("COPY/ADD of a directory" in w for w in report["warnings"]) + assert (project / "astra.yaml").exists() + assert (project / ".venv").exists() + assert not (project / ".git").exists() -def test_init_survives_malformed_lightcone_yaml( - runner: CliRunner, tmp_path: Path +def test_init_no_sync_skips_the_venv( + runner: CliRunner, tmp_path: Path, fake_uv: list[list[str]] ) -> None: - """'Safe to re-run at any time' includes a corrupted project config: - warn and leave it alone rather than crashing with a YAML traceback.""" - project = tmp_path / "proj" - (project / ".lightcone").mkdir(parents=True) - (project / ".lightcone" / "lightcone.yaml").write_text("target: [unclosed\n") - - result = runner.invoke( - main, - ["init", str(project), "--no-git", "--no-venv", "--scratch", "$SCRATCH", "--json"], - ) - assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert any("lightcone.yaml" in w for w in report["warnings"]) - assert (project / ".lightcone" / "lightcone.yaml").read_text() == "target: [unclosed\n" - - # A non-mapping file must not crash the --scratch merge either. - (project / ".lightcone" / "lightcone.yaml").write_text("local\n") - result = runner.invoke( - main, - ["init", str(project), "--no-git", "--no-venv", "--scratch", "$SCRATCH", "--json"], - ) - assert result.exit_code == 0, result.output - - -def test_init_points_spec_at_containerfile(runner: CliRunner, tmp_path: Path) -> None: - """The scaffolded spec must reference the project Containerfile — pins - the rewrite against drift in astra's boilerplate image name.""" project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - assert "container: Containerfile" in (project / "astra.yaml").read_text() + result = runner.invoke(main, ["init", str(project), "--no-git", "--no-sync"]) + assert result.exit_code == 0 + assert [c[0] for c in fake_uv] == ["lock"] + assert not (project / ".venv").exists() -def test_engine_errors_render_cleanly( - runner: CliRunner, monkeypatch: pytest.MonkeyPatch +def test_init_defaults_to_the_current_directory( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """ContainerBuildError from any command surfaces as a one-line CLI error - (exit 1), not a traceback — the group boundary translates it.""" - from lightcone.cli import commands - from lightcone.engine.container import ContainerBuildError - - def _boom(*args: object, **kwargs: object) -> Path: - raise ContainerBuildError("COPY of a directory is not supported") - - monkeypatch.setattr(commands, "_project_root", _boom) - result = runner.invoke(main, ["status"]) - assert result.exit_code == 1 - assert "not supported" in result.output - assert "Traceback" not in result.output + monkeypatch.chdir(tmp_path) + result = runner.invoke(main, ["init", "--no-git"]) + assert result.exit_code == 0 + assert (tmp_path / "astra.yaml").exists() -def test_cloudbuild_error_is_a_container_build_error() -> None: - """One boundary handler must cover both local and Cloud Build failures.""" - from lightcone.engine.cloudbuild import CloudBuildError - from lightcone.engine.container import ContainerBuildError +# ---- rendering ------------------------------------------------------------ - assert issubclass(CloudBuildError, ContainerBuildError) +def test_run_reports_what_it_created(runner: CliRunner, tmp_path: Path) -> None: + result = runner.invoke(main, ["init", str(tmp_path / "proj"), "--no-git"]) + assert "created astra.yaml" in result.output + assert "Project converged at" in result.output + # Next steps only make sense for a freshly scaffolded spec. + assert "Next steps" in result.output -def test_init_repairs_missing_piece(runner: CliRunner, tmp_path: Path) -> None: - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - (project / "Containerfile").unlink() - result = runner.invoke( - main, ["init", str(project), "--no-git", "--no-venv", "--json"] - ) - assert result.exit_code == 0, result.output - report = json.loads(result.output) - assert "Containerfile" in report["created"] - assert (project / "Containerfile").exists() - # The rest was left alone. - assert "astra.yaml" in report["unchanged"] - - -def test_init_venv_uses_uv_when_available( - runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_run_on_a_converged_project_says_so_and_skips_next_steps( + runner: CliRunner, tmp_path: Path ) -> None: - calls: list[list[str]] = [] + project = tmp_path / "proj" + assert runner.invoke(main, ["init", str(project), "--no-git"]).exit_code == 0 - def _fake_run(cmd: list[str], **kwargs: object) -> MagicMock: - calls.append(list(cmd)) - return MagicMock(returncode=0) + result = runner.invoke(main, ["init", str(project), "--no-git"]) + assert result.exit_code == 0 + assert "already converged" in result.output + assert "Next steps" not in result.output - monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/uv" if name == "uv" else None) - monkeypatch.setattr(subprocess, "run", _fake_run) +def test_run_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git"]) - assert result.exit_code == 0, result.output - - assert ["uv", "venv", "--python", "3.12", ".venv"] in calls - assert [ - "uv", "pip", "install", "--python", ".venv/bin/python", "-r", "requirements.txt", - ] in calls + project.mkdir() + (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') + result = runner.invoke(main, ["init", str(project), "--no-git"]) + assert result.exit_code == 0 + assert "does not depend on lightcone-cli" in result.output -def test_init_venv_falls_back_to_python_when_uv_missing( - runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - calls: list[list[str]] = [] - def _fake_run(cmd: list[str], **kwargs: object) -> MagicMock: - calls.append(list(cmd)) - return MagicMock(returncode=0) +# ---- --check / --json (the agent-facing surface) -------------------------- - monkeypatch.setattr(shutil, "which", lambda _: None) - monkeypatch.setattr(subprocess, "run", _fake_run) +def test_check_reports_drift_without_writing(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git"]) - assert result.exit_code == 0, result.output + result = runner.invoke(main, ["init", str(project), "--no-git", "--check"]) + assert result.exit_code == 1 + assert "would create" in result.output + assert not project.exists() - assert ["python", "-m", "venv", ".venv"] in calls - assert [ - ".venv/bin/python", "-m", "pip", "install", "-q", "-r", "requirements.txt", - ] in calls +def test_check_passes_on_a_converged_project(runner: CliRunner, tmp_path: Path) -> None: + project = tmp_path / "proj" + assert runner.invoke(main, ["init", str(project), "--no-git"]).exit_code == 0 -# ---- lc verify ------------------------------------------------------------ + result = runner.invoke(main, ["init", str(project), "--no-git", "--check"]) + assert result.exit_code == 0 + # Rich wraps on the terminal width, so assert on an unwrappable fragment. + assert "nothing to do" in result.output -def test_verify_clean_project_returns_zero( - runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """An empty project (no materialized outputs yet) is a clean state, not - a verification failure.""" +def test_json_report_is_machine_readable(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - project.mkdir() - (project / "astra.yaml").write_text( - "outputs:\n - id: foo\n recipe:\n command: echo\n" - ) - monkeypatch.chdir(project) - result = runner.invoke(main, ["verify"]) + result = runner.invoke(main, ["init", str(project), "--no-git", "--json"]) assert result.exit_code == 0 + payload = json.loads(result.output) + assert payload["converged"] is False + assert "astra.yaml" in payload["created"] + assert payload["warnings"] == [] -# ---- lc run command building ------------------------------------------------ - - -def test_run_cmd_inserts_separator_before_targets() -> None: - """Regression test for issue #87. - - snakemake's --rerun-triggers uses nargs=+ so it greedily consumes the - first positional target path as an extra trigger value, producing: - error: argument --rerun-triggers: invalid choice: - 'results/baseline/map_fit/.lightcone-manifest.json' - A '--' separator between the trigger values and target paths terminates - argparse flag processing and prevents this. - """ - from lightcone.cli.commands import _build_snakemake_cmd - - targets = ["results/baseline/map_fit/.lightcone-manifest.json"] - cmd = _build_snakemake_cmd( - snakefile_path=Path("/proj/.lightcone/Snakefile"), - project=Path("/proj"), - n="4", - rerun_triggers="code,input,mtime,params", - targets=targets, - force=False, - has_outputs=True, - ) - - assert "--" in cmd, "missing '--' separator; first target will be consumed as a trigger value" - sep_idx = cmd.index("--") - rt_idx = cmd.index("--rerun-triggers") - assert sep_idx > rt_idx, "'--' must appear after --rerun-triggers" - target_idx = cmd.index(targets[0]) - assert target_idx > sep_idx, "target path must appear after '--'" - - -def test_run_cmd_no_separator_when_no_targets() -> None: - """When no targets are supplied snakemake runs 'rule all'; '--' is unnecessary.""" - from lightcone.cli.commands import _build_snakemake_cmd - - cmd = _build_snakemake_cmd( - snakefile_path=Path("/proj/.lightcone/Snakefile"), - project=Path("/proj"), - n="4", - rerun_triggers="code,input,mtime,params", - targets=[], - force=False, - has_outputs=False, - ) - - assert "--" not in cmd - - -def test_run_cmd_multiple_triggers_all_before_separator() -> None: - """All four trigger tokens must precede the '--' separator.""" - from lightcone.cli.commands import _build_snakemake_cmd - - targets = ["results/baseline/out/.lightcone-manifest.json"] - cmd = _build_snakemake_cmd( - snakefile_path=Path("/proj/.lightcone/Snakefile"), - project=Path("/proj"), - n="1", - rerun_triggers="code,input,mtime,params", - targets=targets, - force=False, - has_outputs=True, - ) - - sep_idx = cmd.index("--") - for trigger in ("code", "input", "mtime", "params"): - assert trigger in cmd, f"trigger '{trigger}' missing from cmd" - assert cmd.index(trigger) < sep_idx, f"trigger '{trigger}' must come before '--'" - - -# ---- JupyterHub deployment paths ------------------------------------------ - - -def test_run_cmd_uniform_across_backends() -> None: - """One invocation shape for every backend: --shared-fs-usage drops - software-deployment so spawned jobs run plain `python` from the - worker's own environment (the worker image on a gateway, the - driver's activated env locally / on SLURM) instead of embedding the - driver's sys.executable. No gateway-specific flags exist.""" - from lightcone.cli.commands import _build_snakemake_cmd - - cmd = _build_snakemake_cmd( - snakefile_path=Path("/p/.lightcone/Snakefile"), - project=Path("/p"), - n="4", - rerun_triggers="mtime", - targets=["results/u/foo/.lightcone-manifest.json"], - force=False, - has_outputs=True, - ) - j = cmd.index("--shared-fs-usage") - values = cmd[j + 1 : cmd.index("--rerun-triggers")] - assert "software-deployment" not in values - assert "persistence" in values and "input-output" in values - assert "--latency-wait" not in cmd - # nargs=+ flags must never swallow the positional targets. - assert cmd.index("--") < cmd.index("results/u/foo/.lightcone-manifest.json") - - -def test_init_scaffold_is_environment_agnostic( - runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """The scaffold is identical on and off a hub: the Containerfile - carries no environment-specific content (pod identity is deployment - config, not image content), and the image gets the execution stack - — including dask-gateway — via a dedicated lightcone-cli layer.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - hub_project = tmp_path / "hub" - result = runner.invoke(main, ["init", str(hub_project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS") - local_project = tmp_path / "local" - result = runner.invoke(main, ["init", str(local_project), "--no-git", "--no-venv"]) - assert result.exit_code == 0, result.output - - containerfile = (hub_project / "Containerfile").read_text() - requirements = (hub_project / "requirements.txt").read_text() - assert containerfile == (local_project / "Containerfile").read_text() - assert requirements == (local_project / "requirements.txt").read_text() - assert "useradd" not in containerfile and "USER" not in containerfile - # The execution stack goes in the image, never the venv's - # requirements — `lc` lives outside the project venv. - assert "lightcone-cli" in containerfile - assert "lightcone-cli" not in requirements - + payload = json.loads(runner.invoke(main, ["init", str(project), "--no-git", "--json"]).output) + assert payload["converged"] is True + assert payload["created"] == [] and payload["repaired"] == [] -def test_lightcone_requirement_pins_running_version() -> None: - from importlib.metadata import version - from lightcone.cli.commands import _lightcone_requirement +def test_json_suppresses_the_banner(runner: CliRunner, tmp_path: Path) -> None: + """The report has to be parseable straight off stdout.""" + result = runner.invoke(main, ["init", str(tmp_path / "proj"), "--no-git", "--json"]) + json.loads(result.output) - req = _lightcone_requirement() - v = version("lightcone-cli") - if "dev" in v: - # Dev builds aren't published — unpinned fallback. - assert req == "lightcone-cli" - else: - assert req == f"lightcone-cli=={v}" - -def test_ensure_images_none_runtime_returns_empty(tmp_path: Path) -> None: - from lightcone.cli.commands import _ensure_images - - assert _ensure_images(tmp_path, runtime="none") == [] +def test_check_json_writes_nothing_and_exits_nonzero( + runner: CliRunner, tmp_path: Path +) -> None: + """`--check --json` together are the agent form: a drift report with no + side effects and an exit code to branch on.""" + project = tmp_path / "proj" + result = runner.invoke(main, ["init", str(project), "--no-git", "--check", "--json"]) + assert result.exit_code == 1 + assert json.loads(result.output)["converged"] is False + assert not project.exists() diff --git a/tests/test_cloudbuild.py b/tests/test_cloudbuild.py deleted file mode 100644 index 98461095..00000000 --- a/tests/test_cloudbuild.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Unit tests for the GCP Cloud Build backend. - -All GCP surfaces (metadata server, GCS, Cloud Build API, registry) are -mocked at the module's HTTP seams — ``_metadata_access_token`` and -``_request`` — so the tests exercise the real control flow: freshness -probe, staging, submission, polling, failure-tail reporting. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from lightcone.engine import cloudbuild -from lightcone.engine.cloudbuild import ( - BUCKET_ENV, - SERVICE_ACCOUNT_ENV, - CloudBuildError, - cloudbuild_available, - ensure_image, -) -from lightcone.engine.container import REGISTRY_ENV, registry_image_ref - -REGISTRY = "europe-west1-docker.pkg.dev/lightconehub/lightcone-images" - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: - for var in (REGISTRY_ENV, BUCKET_ENV, SERVICE_ACCOUNT_ENV): - monkeypatch.delenv(var, raising=False) - - -@pytest.fixture -def deployment(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(REGISTRY_ENV, REGISTRY) - monkeypatch.setenv(BUCKET_ENV, "lightconehub-lightcone-lc-build") - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - (tmp_path / "requirements.txt").write_text("numpy\n") - return tmp_path - - -# ---- backend selection ---------------------------------------------------- - - -def test_unavailable_without_env() -> None: - assert cloudbuild_available() is False - - -def test_available_with_full_contract(deployment: None) -> None: - assert cloudbuild_available() is True - - -def test_unavailable_with_bucket_only(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(BUCKET_ENV, "some-bucket") - assert cloudbuild_available() is False - - -# ---- registry ref / project parsing --------------------------------------- - - -def test_registry_ref_shape(project: Path) -> None: - ref = registry_image_ref("My Proj", project / "Containerfile", project, - registry=REGISTRY + "/") - repo, _, tag = ref.rpartition(":") - assert repo == f"{REGISTRY}/lc-my-proj" - assert len(tag) == 12 - - -def test_gcp_project_from_registry() -> None: - assert cloudbuild._gcp_project(REGISTRY) == "lightconehub" - - -def test_gcp_project_rejects_non_artifact_registry() -> None: - with pytest.raises(CloudBuildError, match="Artifact Registry"): - cloudbuild._gcp_project("ghcr.io/someorg") - - -# ---- ensure_image control flow -------------------------------------------- - - -def _fresh_probe(monkeypatch: pytest.MonkeyPatch, exists: bool | None) -> None: - monkeypatch.setattr(cloudbuild, "registry_image_exists", lambda ref: exists) - - -def test_ensure_image_cached_is_probe_only( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, True) - monkeypatch.setattr( - cloudbuild, "_request", lambda *a, **k: pytest.fail("no HTTP beyond probe") - ) - phases: list[str] = [] - ref = ensure_image( - project, "Containerfile", project_name="proj", - on_progress=lambda p, _d: phases.append(p), - ) - assert ref.startswith(f"{REGISTRY}/lc-proj:") - assert phases == ["cached"] - - -def test_ensure_image_builds_when_absent( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, False) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - monkeypatch.setenv(SERVICE_ACCOUNT_ENV, "builder@lightconehub.iam.gserviceaccount.com") - - calls: list[tuple[str, str]] = [] - poll_status = iter(["WORKING", "SUCCESS"]) - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - calls.append((method, url)) - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - payload = json.loads(body.decode()) - assert payload["images"][0].startswith(f"{REGISTRY}/lc-proj:") - assert payload["serviceAccount"] == ( - "projects/lightconehub/serviceAccounts/" - "builder@lightconehub.iam.gserviceaccount.com" - ) - assert payload["source"]["storageSource"]["bucket"] == ( - "lightconehub-lightcone-lc-build" - ) - return 200, json.dumps( - {"metadata": {"build": {"id": "build-123"}}} - ).encode() - if "/builds/build-123" in url: - return 200, json.dumps({"status": next(poll_status)}).encode() - raise AssertionError(f"unexpected request {method} {url}") - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - monkeypatch.setattr(cloudbuild.time, "sleep", lambda _s: None) - - phases: list[str] = [] - ref = ensure_image( - project, "Containerfile", project_name="proj", - on_progress=lambda p, _d: phases.append(p), - ) - assert ref.startswith(f"{REGISTRY}/lc-proj:") - assert phases[0] == "staging" - assert "working" in phases and "success" in phases - # Upload happened before submission. - assert "upload" in calls[0][1] - - -def test_ensure_image_failure_surfaces_log_tail( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, False) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - return 200, json.dumps( - {"metadata": {"build": {"id": "build-9"}}} - ).encode() - if "/builds/build-9" in url: - return 200, json.dumps({"status": "FAILURE"}).encode() - if "alt=media" in url: - return 200, b"step1 ok\nERROR: pip failed\n" - raise AssertionError(f"unexpected request {method} {url}") - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - monkeypatch.setattr(cloudbuild.time, "sleep", lambda _s: None) - - with pytest.raises(CloudBuildError, match="ERROR: pip failed"): - ensure_image(project, "Containerfile", project_name="proj") - - -def test_ensure_image_force_skips_probe( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - cloudbuild, - "registry_image_exists", - lambda ref: pytest.fail("force must skip the freshness probe"), - ) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - - def fake_request(method: str, url: str, token: str, *, body=None, content_type=""): - if "storage.googleapis.com/upload" in url: - return 200, b"{}" - if url.endswith("/builds") and method == "POST": - return 200, json.dumps( - {"metadata": {"build": {"id": "b"}}} - ).encode() - return 200, json.dumps({"status": "SUCCESS"}).encode() - - monkeypatch.setattr(cloudbuild, "_request", fake_request) - ensure_image(project, "Containerfile", project_name="proj", force=True) - - -def test_ensure_image_requires_credentials( - deployment: None, project: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _fresh_probe(monkeypatch, None) - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: None) - with pytest.raises(CloudBuildError, match="Workload Identity"): - ensure_image(project, "Containerfile", project_name="proj") - - -def test_ensure_image_missing_containerfile( - deployment: None, tmp_path: Path -) -> None: - with pytest.raises(CloudBuildError, match="not found"): - ensure_image(tmp_path, "Containerfile", project_name="proj") - - -def test_ensure_image_off_deployment(project: Path) -> None: - with pytest.raises(CloudBuildError, match="not configured for Cloud Build"): - ensure_image(project, "Containerfile", project_name="proj") - - -# ---- staged tarball -------------------------------------------------------- - - -def test_staged_tarball_matches_hashed_context(project: Path) -> None: - """The tarball must contain exactly the staged (= hashed) file set.""" - import io - import tarfile - - (project / "results").mkdir() - (project / "results" / "big.bin").write_text("x" * 10) - data = cloudbuild._staged_context_tarball(project, project / "Containerfile") - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: - names = set(tar.getnames()) - assert "Containerfile" in names - assert "requirements.txt" in names - assert not any("results" in n for n in names) - - -# ---- registry probe -------------------------------------------------------- - - -def test_registry_image_exists_parses_ref(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: "tok") - seen: dict[str, str] = {} - - class _Resp: - status = 200 - - def __enter__(self): - return self - - def __exit__(self, *a: object) -> None: - pass - - def fake_urlopen(req, timeout: int = 0): - seen["url"] = req.full_url - seen["method"] = req.get_method() - return _Resp() - - monkeypatch.setattr(cloudbuild.urllib.request, "urlopen", fake_urlopen) - assert cloudbuild.registry_image_exists(f"{REGISTRY}/lc-proj:abc123") is True - assert seen["method"] == "HEAD" - assert seen["url"] == ( - "https://europe-west1-docker.pkg.dev/v2/" - "lightconehub/lightcone-images/lc-proj/manifests/abc123" - ) - - -def test_registry_image_exists_none_without_credentials( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(cloudbuild, "_metadata_access_token", lambda: None) - assert cloudbuild.registry_image_exists(f"{REGISTRY}/lc-proj:abc") is None diff --git a/tests/test_container.py b/tests/test_container.py deleted file mode 100644 index 25d3b9ed..00000000 --- a/tests/test_container.py +++ /dev/null @@ -1,750 +0,0 @@ -"""Tests for the container runtime layer. - -Covers tag computation, build invocation, runtime detection/config, and -the recipe wrap that the Snakefile generator embeds into ``shell()``. -""" -from __future__ import annotations - -import shlex -from collections.abc import Iterator -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -import yaml - -from lightcone.engine.container import ( - RUNTIMES, - ContainerBuildError, - build_image, - compute_image_tag, - detect_runtime, - find_dependency_files, - get_container_status, - image_exists_locally, - image_exists_podman_hpc, - is_containerfile, - load_runtime, - pull_image, - resolve_image_for_run, - wrap_recipe, -) - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - """Minimal project with a Containerfile.""" - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - return tmp_path - - -@pytest.fixture -def project_with_deps(project: Path) -> Path: - (project / "requirements.txt").write_text("numpy\npandas\n") - (project / "pyproject.toml").write_text("[project]\nname = 'test'\n") - return project - - -# ---- find_dependency_files / compute_image_tag ---------------------------- - - -class TestFindDependencyFiles: - def test_finds_requirements_txt(self, project: Path) -> None: - (project / "requirements.txt").write_text("numpy\n") - found = find_dependency_files(project) - assert [f.name for f in found] == ["requirements.txt"] - - def test_finds_pyproject_toml(self, project: Path) -> None: - (project / "pyproject.toml").write_text("[project]\n") - found = find_dependency_files(project) - assert [f.name for f in found] == ["pyproject.toml"] - - def test_skips_missing_files(self, project: Path) -> None: - assert find_dependency_files(project) == [] - - def test_finds_multiple(self, project_with_deps: Path) -> None: - names = {f.name for f in find_dependency_files(project_with_deps)} - assert {"requirements.txt", "pyproject.toml"} <= names - - -class TestComputeImageTag: - def test_deterministic(self, project: Path) -> None: - cf = project / "Containerfile" - assert compute_image_tag("test", cf, project) == compute_image_tag("test", cf, project) - - def test_tag_format(self, project: Path) -> None: - tag = compute_image_tag("my-project", project / "Containerfile", project) - assert tag.startswith("lc-my-project-") - assert len(tag.removeprefix("lc-my-project-")) == 12 - - def test_changes_with_containerfile(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - cf.write_text("FROM ubuntu:22.04\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_changes_with_requirements(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - (project / "requirements.txt").write_text("numpy\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_sanitises_project_name(self, project: Path) -> None: - tag = compute_image_tag("My Project", project / "Containerfile", project) - assert tag.startswith("lc-my-project-") - - def test_changes_with_uv_lock(self, project: Path) -> None: - cf = project / "Containerfile" - tag1 = compute_image_tag("test", cf, project) - (project / "uv.lock").write_text("# v1\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_changes_with_copied_file(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY app.py /app/app.py\n") - (project / "app.py").write_text("print(1)\n") - tag1 = compute_image_tag("test", cf, project) - (project / "app.py").write_text("print(2)\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_directory_copy_source_is_rejected(self, project: Path) -> None: - """The image is an environment, not a code snapshot: directory - COPY sources (COPY src/, COPY . .) raise with guidance instead - of silently baking in a copy nothing executes.""" - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY src/ /app/src/\n") - (project / "src").mkdir() - (project / "src" / "a.py").write_text("a = 1\n") - with pytest.raises(ContainerBuildError, match="directory"): - compute_image_tag("test", cf, project) - - def test_copy_dot_is_rejected(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY . /app/\n") - with pytest.raises(ContainerBuildError, match="not supported"): - compute_image_tag("test", cf, project) - - def test_skips_from_stage_copy(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text( - "FROM python:3.12-slim AS builder\n" - "FROM python:3.12-slim\n" - "COPY --from=builder /tmp/x /app/x\n" - ) - # No real source on host, but parsing must not raise or expand. - tag = compute_image_tag("test", cf, project) - assert tag.startswith("lc-test-") - - def test_skips_url_add(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text( - "FROM python:3.12-slim\nADD https://example.com/x.tgz /app/x.tgz\n" - ) - tag = compute_image_tag("test", cf, project) - assert tag.startswith("lc-test-") - - def test_glob_copy_invalidates_on_match_change(self, project: Path) -> None: - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY *.py /app/\n") - (project / "main.py").write_text("x = 1\n") - tag1 = compute_image_tag("test", cf, project) - (project / "main.py").write_text("x = 2\n") - tag2 = compute_image_tag("test", cf, project) - assert tag1 != tag2 - - def test_swap_dep_file_names_not_collision(self, project: Path) -> None: - # Same total bytes, swapped between two dep files: must not collide - # (the old concat-without-delimiter scheme would have). - (project / "requirements.txt").write_text("numpy\n") - (project / "requirements-dev.txt").write_text("pandas\n") - tag1 = compute_image_tag("test", project / "Containerfile", project) - (project / "requirements.txt").write_text("pandas\n") - (project / "requirements-dev.txt").write_text("numpy\n") - tag2 = compute_image_tag("test", project / "Containerfile", project) - assert tag1 != tag2 - - -# ---- image_exists_locally / image_exists_podman_hpc ----------------------- - - -class TestImageExistsLocally: - @patch("lightcone.engine.container.subprocess.run") - def test_docker_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_locally("lc-foo", runtime="docker") is True - assert mock_run.call_args[0][0][0] == "docker" - - @patch("lightcone.engine.container.subprocess.run") - def test_podman_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_locally("lc-foo", runtime="podman") is True - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container.subprocess.run") - def test_not_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=1) - assert image_exists_locally("lc-foo", runtime="docker") is False - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_runtime_not_installed(self, mock_run: MagicMock) -> None: - assert image_exists_locally("lc-foo", runtime="docker") is False - - @patch("lightcone.engine.container.image_exists_podman_hpc", return_value=True) - def test_podman_hpc_delegates(self, mock_phpc: MagicMock) -> None: - assert image_exists_locally("lc-foo", runtime="podman-hpc") is True - mock_phpc.assert_called_once_with("lc-foo") - - -class TestImageExistsPodmanHpc: - @patch("lightcone.engine.container.subprocess.run") - def test_exists(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0) - assert image_exists_podman_hpc("img:v1") is True - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_not_installed(self, mock_run: MagicMock) -> None: - assert image_exists_podman_hpc("img:v1") is False - - -# ---- build_image ---------------------------------------------------------- - - -class TestBuildImage: - @patch("lightcone.engine.container.subprocess.run") - def test_docker_success(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="ok", stderr="") - result = build_image("lc-test", project / "Containerfile", project, runtime="docker") - assert result.tag == "lc-test" - cmd = mock_run.call_args[0][0] - assert cmd[0] == "docker" - assert cmd[1] == "build" - - @patch("lightcone.engine.container.subprocess.run") - def test_podman_success(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - result = build_image("lc-test", project / "Containerfile", project, runtime="podman") - assert result.tag == "lc-test" - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container._podman_hpc_migrate") - @patch("lightcone.engine.container.subprocess.run") - def test_podman_hpc_migrates_after_build( - self, mock_run: MagicMock, mock_migrate: MagicMock, project: Path - ) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - build_image("lc-test", project / "Containerfile", project, runtime="podman-hpc") - mock_migrate.assert_called_once_with("lc-test") - - @patch("lightcone.engine.container.subprocess.run") - def test_failure_raises(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError, match="docker build failed"): - build_image("lc-test", project / "Containerfile", project, runtime="docker") - - @patch("lightcone.engine.container.subprocess.run", side_effect=FileNotFoundError) - def test_runtime_missing_raises(self, mock_run: MagicMock, project: Path) -> None: - with pytest.raises(ContainerBuildError, match="podman is not installed"): - build_image("lc-test", project / "Containerfile", project, runtime="podman") - - def test_unsupported_runtime_raises(self, project: Path) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported build runtime"): - build_image( - "lc-test", project / "Containerfile", project, runtime="apptainer" - ) - - @patch("lightcone.engine.container.subprocess.run") - def test_build_args(self, mock_run: MagicMock, project: Path) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - build_image( - "lc-test", - project / "Containerfile", - project, - runtime="docker", - build_args={"PY_VERSION": "3.12"}, - ) - cmd = mock_run.call_args[0][0] - assert "--build-arg" in cmd - assert "PY_VERSION=3.12" in cmd - - def test_build_stages_context_off_source_tree(self, project: Path) -> None: - """Build context must be a tempdir, not the source project. - - On NERSC, projects living on DVS-mounted home/CFS hit - ``llistxattr EPROTO`` when buildah's copier walks COPY sources. - Staging into ``$TMPDIR`` (tmpfs) is what lets builds succeed there. - """ - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY app.py /app/app.py\n") - (project / "app.py").write_text("print('hi')\n") - - captured: dict = {} - - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd - ctx = Path(cmd[-1]) - captured["ctx"] = ctx - captured["files"] = sorted( - p.relative_to(ctx).as_posix() - for p in ctx.rglob("*") - if p.is_file() - ) - return MagicMock(returncode=0, stdout="", stderr="") - - with patch( - "lightcone.engine.container.subprocess.run", side_effect=fake_run - ): - build_image("lc-test", cf, project, runtime="podman") - - assert captured["ctx"].resolve() != project.resolve() - assert not captured["ctx"].exists() - assert "Containerfile" in captured["files"] - assert "app.py" in captured["files"] - - def test_build_rejects_copy_dot(self, project: Path) -> None: - """A ``COPY . .`` Containerfile fails the build with guidance - before any runtime is invoked.""" - cf = project / "Containerfile" - cf.write_text("FROM python:3.12-slim\nCOPY . /app/\n") - with pytest.raises(ContainerBuildError, match="environment"): - build_image("lc-test", cf, project, runtime="podman") - - @patch("lightcone.engine.container.subprocess.run") - def test_build_cleans_stage_on_failure( - self, mock_run: MagicMock, project: Path - ) -> None: - """Staged tempdir is removed even when the build fails.""" - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError): - build_image("lc-test", project / "Containerfile", project, runtime="docker") - ctx = Path(mock_run.call_args[0][0][-1]) - assert not ctx.exists() - - -# ---- pull_image ----------------------------------------------------------- - - -class TestPullImage: - @patch("lightcone.engine.container.subprocess.run") - def test_pull_success_docker(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="docker") - cmd = mock_run.call_args[0][0] - assert cmd == ["docker", "pull", "python:3.12-slim"] - - @patch("lightcone.engine.container.subprocess.run") - def test_pull_success_podman(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="podman") - assert mock_run.call_args[0][0][0] == "podman" - - @patch("lightcone.engine.container._podman_hpc_migrate") - @patch("lightcone.engine.container.subprocess.run") - def test_pull_podman_hpc_migrates( - self, mock_run: MagicMock, mock_migrate: MagicMock - ) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - pull_image("python:3.12-slim", runtime="podman-hpc") - mock_migrate.assert_called_once_with("python:3.12-slim") - - @patch("lightcone.engine.container.subprocess.run") - def test_pull_failure_raises(self, mock_run: MagicMock) -> None: - mock_run.return_value = MagicMock(returncode=1, stdout="", stderr="boom") - with pytest.raises(ContainerBuildError, match="docker pull"): - pull_image("python:3.12-slim", runtime="docker") - - def test_unsupported_runtime_raises(self) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported runtime"): - pull_image("img", runtime="apptainer") - - -# ---- detect_runtime / load_runtime --------------------------------------- - - -class TestDetectRuntime: - @pytest.fixture(autouse=True) - def _generic_hostname(self) -> Iterator[None]: - # Pin hostname to one that doesn't match any site so the default - # RUNTIMES order applies. Site-aware behaviour is exercised in - # TestSiteAwareDetection below. - with patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="generic-laptop", - ): - yield - - @patch("lightcone.engine.container.shutil.which") - def test_podman_hpc_preferred_when_present(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == "podman-hpc" - - @patch("lightcone.engine.container.shutil.which") - def test_podman_preferred_over_docker(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which") - def test_docker_only(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: "/usr/bin/docker" if name == "docker" else None - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=True - ): - assert detect_runtime() == "docker" - - @patch("lightcone.engine.container.shutil.which") - def test_docker_skipped_when_daemon_down(self, mock_which: MagicMock) -> None: - mock_which.side_effect = lambda name: "/usr/bin/docker" if name == "docker" else None - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=False - ): - assert detect_runtime() is None - - @patch("lightcone.engine.container.shutil.which") - def test_docker_daemon_down_falls_through_to_podman( - self, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - with patch( - "lightcone.engine.container._docker_daemon_up", return_value=False - ): - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which", return_value=None) - def test_none_available(self, mock_which: MagicMock) -> None: - assert detect_runtime() is None - - def test_no_apptainer(self) -> None: - # Apptainer/singularity must NOT be in the supported runtimes list — - # we own container invocation and only support OCI runtimes. - assert "apptainer" not in RUNTIMES - assert "singularity" not in RUNTIMES - - -class TestSiteAwareDetection: - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="login29.chn.perlmutter.nersc.gov", - ) - def test_perlmutter_picks_podman_hpc( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == "podman-hpc" - - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="login29.chn.perlmutter.nersc.gov", - ) - def test_falls_through_when_site_runtime_missing( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - # Site preference is a hint — explicit user config goes through - # load_runtime, which DOES error on missing binary. - mock_which.side_effect = lambda name: ( - None if name == "podman-hpc" else f"/usr/bin/{name}" - ) - assert detect_runtime() == "podman" - - @patch("lightcone.engine.container.shutil.which") - @patch( - "lightcone.engine.site_registry.socket.gethostname", - return_value="generic-laptop", - ) - def test_unknown_site_uses_default_order( - self, _hostname: MagicMock, mock_which: MagicMock - ) -> None: - mock_which.side_effect = lambda name: f"/usr/bin/{name}" - assert detect_runtime() == RUNTIMES[0] - - -class TestLoadRuntime: - def _write_config(self, tmp_path: Path, content: dict) -> None: - cfg_dir = tmp_path / ".lightcone" - cfg_dir.mkdir() - (cfg_dir / "config.yaml").write_text(yaml.safe_dump(content)) - - def test_no_config_uses_auto( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.detect_runtime", lambda: "docker" - ) - choice = load_runtime() - assert choice.runtime == "docker" - assert choice.explicit is False - - def test_auto_with_no_runtime_returns_none_implicitly( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - """auto + nothing on PATH → none, but explicit=False so the - caller can warn that this is a silent fallback.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.detect_runtime", lambda: None - ) - choice = load_runtime() - assert choice.runtime == "none" - assert choice.explicit is False - - def test_explicit_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """User opted out of containers — explicit=True, no warnings owed.""" - monkeypatch.setattr(Path, "home", lambda: tmp_path) - self._write_config(tmp_path, {"container": {"runtime": "none"}}) - choice = load_runtime() - assert choice.runtime == "none" - assert choice.explicit is True - - def test_explicit_runtime_present( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", - lambda name: f"/usr/bin/{name}" if name == "podman" else None, - ) - self._write_config(tmp_path, {"container": {"runtime": "podman"}}) - choice = load_runtime() - assert choice.runtime == "podman" - assert choice.explicit is True - - def test_explicit_runtime_missing_on_path_raises( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", lambda _: None - ) - self._write_config(tmp_path, {"container": {"runtime": "podman"}}) - with pytest.raises(ContainerBuildError, match="not on PATH"): - load_runtime() - - def test_unknown_runtime_raises( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - self._write_config(tmp_path, {"container": {"runtime": "apptainer"}}) - with pytest.raises(ContainerBuildError, match="Unknown container.runtime"): - load_runtime() - - -# ---- resolve_image_for_run ----------------------------------------------- - - -class TestResolveImageForRun: - def test_none_returns_none(self, project: Path) -> None: - assert resolve_image_for_run( - None, project_path=project, project_name="test" - ) is None - - def test_registry_image_passes_through(self, project: Path) -> None: - assert resolve_image_for_run( - "python:3.12-slim", project_path=project, project_name="test" - ) == "python:3.12-slim" - - def test_namespaced_registry_image_passes_through(self, project: Path) -> None: - assert resolve_image_for_run( - "ghcr.io/foo/bar:tag", project_path=project, project_name="test" - ) == "ghcr.io/foo/bar:tag" - - def test_containerfile_resolves_to_tag(self, project: Path) -> None: - result = resolve_image_for_run( - "Containerfile", project_path=project, project_name="test" - ) - assert result is not None - assert result.startswith("lc-test-") - - -# ---- wrap_recipe ---------------------------------------------------------- - - -class TestWrapRecipe: - def test_no_image_passthrough(self) -> None: - assert wrap_recipe("echo hi", image=None, runtime="podman") == "echo hi" - - def test_runtime_none_passthrough(self) -> None: - assert wrap_recipe( - "echo hi", image="python:3.12-slim", runtime="none" - ) == "echo hi" - - def test_podman_wrap_basic(self) -> None: - wrapped = wrap_recipe( - "echo hi", image="python:3.12-slim", runtime="podman" - ) - assert wrapped.startswith("podman run --rm --pull=never ") - assert "python:3.12-slim" in wrapped - # The recipe is shell-quoted to survive nested shells. - assert shlex.quote("echo hi") in wrapped - - def test_docker_wrap(self) -> None: - wrapped = wrap_recipe("echo hi", image="img:v1", runtime="docker") - assert wrapped.startswith("docker run --rm --pull=never ") - - def test_podman_hpc_wrap(self) -> None: - wrapped = wrap_recipe("echo hi", image="img:v1", runtime="podman-hpc") - assert wrapped.startswith("podman-hpc run --rm --pull=never ") - - def test_pull_never_short_name_safe(self) -> None: - """``--pull=never`` is what makes locally-built short-name images - like ``lc-foo-abc123`` work under podman, which would otherwise - try to resolve the name against unqualified-search-registries.""" - wrapped = wrap_recipe( - "echo", image="lc-foo-abc123", runtime="podman" - ) - assert "--pull=never" in wrapped - - def test_preserves_snakemake_placeholders(self) -> None: - """Snakemake's ``{output[0]}`` must survive the wrap so it can - substitute at exec time.""" - wrapped = wrap_recipe( - "echo > {output[0]}/x", image="img:v1", runtime="podman" - ) - assert "{output[0]}" in wrapped - - def test_preserves_recipe_with_single_quotes(self) -> None: - """Recipes may contain single quotes (e.g. ``python -c 'print(1)'``). - The shlex.quote escape must survive nested shell parsing.""" - recipe = """python -c 'print("hi")'""" - wrapped = wrap_recipe(recipe, image="img:v1", runtime="podman") - # Round-trip through shlex.split should yield the original recipe - # as the last argument (the bash -c argument). - tokens = shlex.split(wrapped) - assert tokens[-1] == recipe - - def test_unsupported_runtime_raises(self) -> None: - with pytest.raises(ContainerBuildError, match="Unsupported run runtime"): - wrap_recipe("echo", image="img:v1", runtime="apptainer") - - def test_bind_mounts_pwd(self) -> None: - """Recipes that write to relative paths need $PWD bind-mounted.""" - wrapped = wrap_recipe("echo", image="img:v1", runtime="podman") - assert '-v "$PWD":"$PWD"' in wrapped - assert '-w "$PWD"' in wrapped - - -# ---- get_container_status ------------------------------------------------- - - -class TestGetContainerStatus: - def test_none(self, project: Path) -> None: - s = get_container_status(None, project, "test", runtime="docker") - assert s.type == "none" - - def test_prebuilt(self, project: Path) -> None: - s = get_container_status("python:3.12", project, "test", runtime="docker") - assert s.type == "prebuilt" - assert s.image == "python:3.12" - - @patch("lightcone.engine.container.image_exists_locally", return_value=False) - def test_containerfile_not_built( - self, mock_exists: MagicMock, project: Path - ) -> None: - s = get_container_status("Containerfile", project, "test", runtime="docker") - assert s.type == "build" - assert s.exists is False - assert s.image is not None - - @patch("lightcone.engine.container.image_exists_locally", return_value=True) - def test_containerfile_built( - self, mock_exists: MagicMock, project: Path - ) -> None: - s = get_container_status("Containerfile", project, "test", runtime="docker") - assert s.type == "build" - assert s.exists is True - - def test_runtime_none_skips_existence_check(self, project: Path) -> None: - s = get_container_status("Containerfile", project, "test", runtime="none") - assert s.type == "build" - assert s.exists is None - - -# ---- is_containerfile ----------------------------------------------------- - - -class TestIsContainerfile: - def test_existing_file(self, project: Path) -> None: - assert is_containerfile("Containerfile", project) is True - - def test_missing_file(self, project: Path) -> None: - assert is_containerfile("python:3.12-slim", project) is False - - -# ---- kubernetes runtime --------------------------------------------------- - - -class TestKubernetesRuntime: - def test_wrap_recipe_is_passthrough(self) -> None: - """The worker pod already runs the image — wrapping would - containerize twice.""" - from lightcone.engine.container import KUBERNETES - - wrapped = wrap_recipe( - "python run.py", image="reg/lc-p:abc", runtime=KUBERNETES - ) - assert wrapped == "python run.py" - - def test_registry_ref_shares_identity_with_local_tag( - self, project: Path - ) -> None: - from lightcone.engine.container import registry_image_ref - - tag = compute_image_tag("proj", project / "Containerfile", project) - ref = registry_image_ref( - "proj", project / "Containerfile", project, registry="reg.io/ns/repo" - ) - digest = tag.rsplit("-", 1)[1] - assert ref == f"reg.io/ns/repo/lc-proj:{digest}" - - def test_resolve_image_uses_registry_when_given(self, project: Path) -> None: - ref = resolve_image_for_run( - "Containerfile", - project_path=project, - project_name="proj", - registry="reg.io/ns/repo", - ) - assert ref is not None and ref.startswith("reg.io/ns/repo/lc-proj:") - - def test_resolve_prebuilt_ignores_registry(self, project: Path) -> None: - assert ( - resolve_image_for_run( - "python:3.12-slim", - project_path=project, - project_name="proj", - registry="reg.io/ns/repo", - ) - == "python:3.12-slim" - ) - - def test_detect_runtime_site_kubernetes_skips_path_probe( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """A gateway deployment has no OCI binary to find — the site - preference short-circuits detection entirely.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dg") - monkeypatch.setattr( - "lightcone.engine.container.shutil.which", - lambda _: pytest.fail("no PATH probing on kubernetes sites"), - ) - assert detect_runtime() == "kubernetes" - - def test_load_runtime_explicit_kubernetes( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr(Path, "home", lambda: tmp_path) - cfg_dir = tmp_path / ".lightcone" - cfg_dir.mkdir() - (cfg_dir / "config.yaml").write_text( - yaml.safe_dump({"container": {"runtime": "kubernetes"}}) - ) - choice = load_runtime() - assert choice.runtime == "kubernetes" - assert choice.explicit is True diff --git a/tests/test_dask_cluster.py b/tests/test_dask_cluster.py deleted file mode 100644 index 099ee77b..00000000 --- a/tests/test_dask_cluster.py +++ /dev/null @@ -1,604 +0,0 @@ -"""Unit tests for the cluster bootstrap. - -We test the routing decision (which branch fires given env vars) and the -node-shape detection. The actual `LocalCluster` spin-up is exercised in a -single smoke test; the `srun`-backed path is mocked because real -multi-node testing requires SLURM. -""" - -from __future__ import annotations - -from contextlib import contextmanager -from unittest.mock import patch - -import pytest - -from lightcone.engine.dask_cluster import ( - RESOURCE_CPUS, - RESOURCE_GPUS, - RESOURCE_MEMORY, - _detect_node_shape, - _NodeShape, - _resources_arg, - cluster_for_run, -) - - -@pytest.fixture(autouse=True) -def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: - for var in ( - "DASK_SCHEDULER_ADDRESS", - "DASK_GATEWAY__ADDRESS", - "LIGHTCONE_GATEWAY_WORKER_TIMEOUT", - "SLURM_JOB_ID", - "SLURM_NNODES", - "SLURM_CPUS_ON_NODE", - "SLURM_MEM_PER_NODE", - "SLURM_GPUS_ON_NODE", - ): - monkeypatch.delenv(var, raising=False) - - -def test_detect_shape_falls_back_to_os(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("os.cpu_count", lambda: 8) - shape = _detect_node_shape() - assert shape.cpus == 8 - assert shape.gpus == 0 - - -def test_detect_shape_reads_slurm_env(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SLURM_CPUS_ON_NODE", "64") - monkeypatch.setenv("SLURM_MEM_PER_NODE", "256000") # 256 GB in MB - monkeypatch.setenv("SLURM_GPUS_ON_NODE", "4") - shape = _detect_node_shape() - assert shape.cpus == 64 - assert shape.mem_bytes == 256_000_000_000 - assert shape.gpus == 4 - - -def test_resources_arg_minimal() -> None: - arg = _resources_arg(_NodeShape(cpus=8, mem_bytes=0, gpus=0)) - assert arg == "cpus=8" - - -def test_resources_arg_full() -> None: - arg = _resources_arg(_NodeShape(cpus=64, mem_bytes=256_000_000_000, gpus=4)) - assert arg == "cpus=64 memory=256000000000 gpus=4" - - -def test_existing_scheduler_address_yields_unchanged( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://example:8786") - - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://example:8786"} - - -def test_no_env_uses_local_cluster() -> None: - """The local-cluster branch should actually start a (tiny) cluster.""" - sentinel: dict[str, str] = {} - - @contextmanager - def _fake_local(*, verbose: bool, local_directory: str | None = None): - sentinel["called"] = "local" - yield "tcp://stub:9999" - - with patch("lightcone.engine.dask_cluster._local_cluster", _fake_local): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://stub:9999"} - assert sentinel["called"] == "local" - - -def test_slurm_env_takes_slurm_path(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("SLURM_JOB_ID", "12345") - sentinel: dict[str, str] = {} - - @contextmanager - def _fake_slurm(*, verbose: bool, local_directory: str | None = None): - sentinel["called"] = "slurm" - yield "tcp://stub:9999" - - with patch("lightcone.engine.dask_cluster._slurm_backed_cluster", _fake_slurm): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://stub:9999"} - assert sentinel["called"] == "slurm" - - -def test_existing_scheduler_address_wins_over_slurm( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """If both are set, the explicit address takes precedence.""" - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - monkeypatch.setenv("SLURM_JOB_ID", "12345") - - @contextmanager - def _should_not_run(*, verbose: bool, local_directory: str | None = None): - raise AssertionError("slurm path should not have been taken") - yield # pragma: no cover - - with patch("lightcone.engine.dask_cluster._slurm_backed_cluster", _should_not_run): - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://existing:8786"} - - -def test_slurm_backed_cluster_binds_to_routable_host( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Multi-node SLURM allocations need the scheduler bound to a hostname - workers on other nodes can reach. The default LocalCluster host of - 127.0.0.1 fails silently with `wait_for_workers` timeouts. - """ - monkeypatch.setenv("SLURM_JOB_ID", "12345") - monkeypatch.setenv("SLURM_NNODES", "2") - monkeypatch.setenv("SLURMD_NODENAME", "nid001234") - monkeypatch.setattr( - "lightcone.engine.dask_cluster.shutil.which", lambda _: "/usr/bin/dask" - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://nid001234:8786" - - def close(self) -> None: - pass - - class _FakeClient: - def __init__(self, addr: str) -> None: - captured["client_addr"] = addr - - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - pass - - def close(self) -> None: - pass - - class _FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - captured["worker_cmd"] = cmd - captured["worker_kwargs"] = kwargs - - def terminate(self) -> None: - pass - - def wait(self, timeout: int | None = None) -> int: - return 0 - - def kill(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - monkeypatch.setattr("dask.distributed.Client", _FakeClient) - monkeypatch.setattr("subprocess.Popen", _FakePopen) - - from lightcone.engine.dask_cluster import _slurm_backed_cluster - - with _slurm_backed_cluster(verbose=False, local_directory=None) as addr: - assert addr == "tcp://nid001234:8786" - - assert captured.get("host") == "nid001234", ( - f"LocalCluster must be told to bind to the SLURM nodename so remote " - f"workers can connect; got host={captured.get('host')!r}" - ) - - -def test_slurm_backed_cluster_falls_back_to_gethostname( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Without SLURMD_NODENAME, fall back to socket.gethostname().""" - monkeypatch.setenv("SLURM_JOB_ID", "12345") - monkeypatch.setenv("SLURM_NNODES", "1") - monkeypatch.delenv("SLURMD_NODENAME", raising=False) - monkeypatch.setattr( - "lightcone.engine.dask_cluster.shutil.which", lambda _: "/usr/bin/dask" - ) - monkeypatch.setattr( - "lightcone.engine.dask_cluster.socket.gethostname", lambda: "host-fallback" - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://host-fallback:8786" - - def close(self) -> None: - pass - - class _FakeClient: - def __init__(self, addr: str) -> None: - pass - - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - pass - - def close(self) -> None: - pass - - class _FakePopen: - def __init__(self, cmd: list[str], **kwargs: object) -> None: - pass - - def terminate(self) -> None: - pass - - def wait(self, timeout: int | None = None) -> int: - return 0 - - def kill(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - monkeypatch.setattr("dask.distributed.Client", _FakeClient) - monkeypatch.setattr("subprocess.Popen", _FakePopen) - - from lightcone.engine.dask_cluster import _slurm_backed_cluster - - with _slurm_backed_cluster(verbose=False, local_directory=None): - pass - - assert captured.get("host") == "host-fallback" - - -def test_local_cluster_advertises_memory_and_gpus( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Dask only schedules a task on a worker that advertises every - requested resource key — so the local worker must expose mem and - gpus too, otherwise rules with ``mem_mb``/``gpus_per_task`` hang. - """ - monkeypatch.setattr( - "lightcone.engine.dask_cluster._detect_node_shape", - lambda: _NodeShape(cpus=4, mem_bytes=16_000_000_000, gpus=2), - ) - - captured: dict[str, object] = {} - - class _FakeCluster: - def __init__(self, **kwargs: object) -> None: - captured.update(kwargs) - self.scheduler_address = "tcp://stub:0" - - def close(self) -> None: - pass - - monkeypatch.setattr("dask.distributed.LocalCluster", _FakeCluster) - - from lightcone.engine.dask_cluster import _local_cluster - - with _local_cluster(verbose=False, local_directory=None): - pass - - resources = captured.get("resources") - assert isinstance(resources, dict) - assert set(resources.keys()) == {RESOURCE_CPUS, RESOURCE_MEMORY, RESOURCE_GPUS} - - -@pytest.mark.slow -def test_local_cluster_smoke() -> None: - """End-to-end: a real LocalCluster spins up, accepts a task, tears down.""" - from dask.distributed import Client - - from lightcone.engine.dask_cluster import _local_cluster - - with _local_cluster(verbose=False, local_directory=None) as addr: - client = Client(addr) - try: - assert client.submit(lambda x: x + 1, 41).result() == 42 - finally: - client.close() - -# --------------------------------------------------------------------------- -# Dask Gateway branch -# --------------------------------------------------------------------------- - - -def _install_fake_gateway( - monkeypatch: pytest.MonkeyPatch, - record: dict[str, object], - *, - worker_resources: dict[str, float] | None = None, - wait_raises: bool = False, - declared_options: dict[str, object] | None = None, -): - """Register a fake ``dask_gateway`` module and return it.""" - import sys - from types import SimpleNamespace - - resources = ( - worker_resources - if worker_resources is not None - else {"cpus": 2.0, "memory": 4e9} - ) - declared = ( - declared_options - if declared_options is not None - else { - "image": "notebook:latest", - "worker_cores": 2, - "worker_memory": 4.0, - "environment": {}, - } - ) - - class _FakeClient: - def wait_for_workers(self, n_workers: int, timeout: int) -> None: - record["waited"] = (n_workers, timeout) - if wait_raises: - raise TimeoutError("no workers") - - def scheduler_info(self) -> dict[str, object]: - return {"workers": {"w0": {"resources": resources}}} - - def close(self) -> None: - record["client_closed"] = True - - class _FakeCluster: - name = "hub.abc123" - dashboard_link = "http://dash" - - def adapt(self, minimum: int, maximum: int) -> None: - record["adapt"] = (minimum, maximum) - - def get_client(self) -> _FakeClient: - return _FakeClient() - - def shutdown(self) -> None: - record["shutdown"] = True - - def close(self) -> None: - record["closed"] = True - - class _FakeGateway: - def cluster_options(self) -> dict[str, object]: - return dict(declared) - - def new_cluster(self, shutdown_on_close: bool = True, **options: object): - record["shutdown_on_close"] = shutdown_on_close - record["options"] = options - return _FakeCluster() - - module = SimpleNamespace(Gateway=_FakeGateway) - monkeypatch.setitem(sys.modules, "dask_gateway", module) - return module - - -def test_gateway_env_takes_gateway_branch(monkeypatch: pytest.MonkeyPatch) -> None: - from lightcone.engine.dask_cluster import GATEWAY_CLUSTER_ENV - - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run(worker_image="reg/lc-p:abc", max_workers=4) as env: - assert env == {GATEWAY_CLUSTER_ENV: "hub.abc123"} - opts = record["options"] - assert opts["image"] == "reg/lc-p:abc" # type: ignore[index] - assert record["adapt"] == (1, 4) - assert "shutdown" not in record - - assert record["shutdown"] is True, "run-scoped cluster must be culled on exit" - - -def test_gateway_without_image_uses_deployment_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run() as _env: - pass - - assert "image" not in record["options"], "no image option → deployment default" # type: ignore[operator] - - -def test_explicit_scheduler_address_wins_over_gateway( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - - with cluster_for_run() as env: - assert env == {"DASK_SCHEDULER_ADDRESS": "tcp://existing:8786"} - - -def test_gateway_culled_when_body_raises(monkeypatch: pytest.MonkeyPatch) -> None: - """The cluster must be shut down even when the run fails.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with pytest.raises(RuntimeError, match="boom"): - with cluster_for_run(): - raise RuntimeError("boom") - - assert record["shutdown"] is True - - -def test_gateway_zero_workers_fails_loudly_and_culls( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv("LIGHTCONE_GATEWAY_WORKER_TIMEOUT", "7") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record, wait_raises=True) - - with pytest.raises(RuntimeError, match="within 7s"): - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass # pragma: no cover - - assert record["waited"] == (1, 7) - assert record["shutdown"] is True - - -def test_gateway_missing_resource_contract_refused( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record, worker_resources={}) - - with pytest.raises(RuntimeError, match="resource contract"): - with cluster_for_run(): - pass # pragma: no cover - - assert record["shutdown"] is True - - -def test_gateway_branch_active_matches_routing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.dask_cluster import gateway_branch_active - - assert gateway_branch_active() is False - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert gateway_branch_active() is True - monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://existing:8786") - assert gateway_branch_active() is False - - -def test_gateway_explicit_image_beats_ambient_default() -> None: - """The deployment injects DASK_GATEWAY__CLUSTER__OPTIONS__IMAGE - (= the notebook image) as the client's ambient default. lc run's - explicit ``image`` kwarg MUST override it — otherwise every cluster - would run the notebook image instead of the one `lc build` just - produced. Pinned against the real dask-gateway client merge logic. - """ - pytest.importorskip("dask_gateway") - import dask - from dask_gateway import Gateway - - captured: dict[str, object] = {} - - async def fake_request(self, method, url, json=None, **kwargs): # type: ignore[no-untyped-def] - captured["cluster_options"] = (json or {}).get("cluster_options") - - class _Resp: - async def json(self) -> dict[str, str]: - return {"name": "hub.fake"} - - return _Resp() - - with dask.config.set({"gateway.cluster.options": {"image": "notebook:latest"}}): - gateway = Gateway(address="http://gateway.invalid", auth="basic") - try: - with patch.object(Gateway, "_request", fake_request): - gateway.submit(image="reg/lc-proj:abc123") - assert captured["cluster_options"] == {"image": "reg/lc-proj:abc123"} - - gateway.submit() - assert captured["cluster_options"] == {"image": "notebook:latest"} - finally: - gateway.close() - - -def test_gateway_self_provisions_worker_environment( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """lc passes everything worker pods need through the STANDARD - `environment` cluster option — resource contract mirrored from the - deployment's declared worker shape, driver identity forwarded, - image ground truth — so the deployment's options handler needs no - lightcone-specific injection.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv("HOME", "/home/jovyan") - monkeypatch.setenv("USER", "jovyan") - monkeypatch.setenv("LOGNAME", "jovyan") - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, - record, - declared_options={ - "image": "notebook:latest", - "worker_cores": 2, - "worker_memory": 4.0, - "environment": {"EXTRA": "kept"}, - }, - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - env = record["options"]["environment"] # type: ignore[index] - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__CPUS"] == "2" - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == str(int(4.0 * 1e9)) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__GPUS"] == "0" - assert env["HOME"] == "/home/jovyan" - assert env["USER"] == "jovyan" - assert env["LOGNAME"] == "jovyan" - assert env["LIGHTCONE_WORKER_IMAGE"] == "reg/lc-p:abc" - assert env["EXTRA"] == "kept", "ambient environment defaults must survive" - - -def test_gateway_provisions_identity_without_user_env( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """USER/LOGNAME must be *derived*, not merely forwarded: notebook - pods often don't export them (their own passwd entry covers - getpass), while the environment-agnostic worker image has NO passwd - entry for the pod uid — so the child snakemake crashes at - ``getpass.getuser()`` unless lc always provisions the vars. - Regression: live run failed with ``getpwuid(): uid not found``.""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.delenv("USER", raising=False) - monkeypatch.delenv("LOGNAME", raising=False) - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, - record, - declared_options={"image": "notebook:latest", "environment": {}}, - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - import getpass - - env = record["options"]["environment"] # type: ignore[index] - assert env["USER"] == getpass.getuser() - assert env["LOGNAME"] == env["USER"] - - -def test_gateway_worker_image_env_falls_back_to_declared_default( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway(monkeypatch, record) - - with cluster_for_run(): # no project image → deployment default - pass - - env = record["options"]["environment"] # type: ignore[index] - assert env["LIGHTCONE_WORKER_IMAGE"] == "notebook:latest" - - -def test_gateway_no_environment_option_no_injection( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A deployment that doesn't expose `environment` gets no surprise - kwarg (the server would reject it).""" - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - record: dict[str, object] = {} - _install_fake_gateway( - monkeypatch, record, declared_options={"image": "notebook:latest"} - ) - - with cluster_for_run(worker_image="reg/lc-p:abc"): - pass - - assert "environment" not in record["options"] # type: ignore[operator] - - -def test_worker_environment_memory_bytes_heuristic() -> None: - from lightcone.engine.dask_cluster import _worker_environment - - env = _worker_environment({"worker_memory": 4294967296}, None) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == "4294967296" - env = _worker_environment({"worker_memory": 4.0}, None) - assert env["DASK_DISTRIBUTED__WORKER__RESOURCES__MEMORY"] == str(int(4e9)) diff --git a/tests/test_dask_plugin.py b/tests/test_dask_plugin.py deleted file mode 100644 index 8f77f8c8..00000000 --- a/tests/test_dask_plugin.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Unit tests for the dask Snakemake executor plugin. - -The Snakemake executor base classes are heavy and tied to a live Workflow -instance, so we don't instantiate the plugin's `Executor` directly here. -We test the pure helpers (`_run_shell`, `_build_resources`) and the -package-level discovery contract that Snakemake uses. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -from snakemake_executor_plugin_dask.executor import ( - _build_resources, - _run_shell, -) - - -def _job(threads: int = 1, **resources: float) -> SimpleNamespace: - return SimpleNamespace(threads=threads, resources=resources) - - -def test_run_shell_propagates_exit_code() -> None: - assert _run_shell("true")[0] == 0 - assert _run_shell("false")[0] != 0 - - -def test_run_shell_runs_under_shell() -> None: - """We rely on shell=True so recipes can use pipes and env expansion.""" - assert _run_shell("echo hi | grep hi >/dev/null")[0] == 0 - - -def test_build_resources_default_uses_threads() -> None: - res = _build_resources(_job(threads=4)) - assert res == {"cpus": 4.0} - - -def test_build_resources_cpus_per_task_overrides_threads() -> None: - res = _build_resources(_job(threads=4, cpus_per_task=8)) - assert res["cpus"] == 8.0 - - -def test_build_resources_mem_mb_to_bytes() -> None: - res = _build_resources(_job(threads=1, mem_mb=8000)) - assert res["memory"] == 8e9 - - -def test_build_resources_gpus_passthrough() -> None: - res = _build_resources(_job(threads=1, gpus=2)) - assert res["gpus"] == 2.0 - - -def test_build_resources_gpus_per_task_takes_precedence() -> None: - res = _build_resources(_job(threads=1, gpus=2, gpus_per_task=4)) - assert res["gpus"] == 4.0 - - -def test_build_resources_full_set() -> None: - res = _build_resources(_job(threads=8, mem_mb=32000, gpus=1)) - assert res == {"cpus": 8.0, "memory": 3.2e10, "gpus": 1.0} - - -def test_plugin_module_exposes_common_settings_and_executor() -> None: - """Snakemake imports the plugin module to read these on discovery.""" - import snakemake_executor_plugin_dask as mod - - assert mod.common_settings.non_local_exec is True - assert mod.Executor is not None - - -def test_cancel_jobs_does_not_close_client() -> None: - """Snakemake calls cancel_jobs for partial cancellations. The Dask - client must survive so subsequent submissions in the same run still - work — only ``shutdown()`` is allowed to close the client. - """ - from snakemake_executor_plugin_dask.executor import DaskExecutor - - closed = {"count": 0} - - class _FakeFuture: - def __init__(self) -> None: - self.cancelled = False - - def done(self) -> bool: - return False - - def cancel(self) -> None: - self.cancelled = True - - class _FakeClient: - def close(self) -> None: - closed["count"] += 1 - - class _FakeLogger: - def warning(self, _msg: str) -> None: - pass - - executor = DaskExecutor.__new__(DaskExecutor) - executor._client = _FakeClient() # type: ignore[attr-defined] - executor.logger = _FakeLogger() # type: ignore[attr-defined] - - future = _FakeFuture() - job = SimpleNamespace(external_jobid="x", aux={"future": future}) - executor.cancel_jobs([job]) # type: ignore[arg-type] - - assert future.cancelled is True - assert closed["count"] == 0, "cancel_jobs must not close the dask client" - - -def test_run_shell_returns_sentinel_block() -> None: - """Sentinel-prefixed lines come back (prefix intact) as the block; - everything else is dropped.""" - from lightcone.engine.runner import SENTINEL - - rc, block = _run_shell( - f"echo '{SENTINEL}hello'; echo noise; echo '{SENTINEL}world' >&2" - ) - assert rc == 0 - assert block == f"{SENTINEL}hello\n{SENTINEL}world\n" - - -def test_run_shell_failure_without_sentinel_forwards_raw_tail() -> None: - """A child snakemake that dies before the rule body (import error, - missing package in the worker image) must not vanish into worker - logs — its raw output comes back sentinel-framed.""" - from lightcone.engine.runner import SENTINEL - - rc, block = _run_shell("echo bootstrap-crash >&2; exit 3") - assert rc == 3 - assert block.startswith(SENTINEL) - assert "bootstrap-crash" in block - - -def test_run_shell_success_drops_noise() -> None: - rc, block = _run_shell("echo just-noise") - assert rc == 0 - assert block == "" - - -def test_unpack_result_accepts_legacy_int() -> None: - """Workers running an older lightcone-cli release return a bare int.""" - from snakemake_executor_plugin_dask.executor import _unpack_result - - assert _unpack_result(1) == (1, "") - assert _unpack_result((0, "block\n")) == (0, "block\n") - - -def test_connect_client_requires_rendezvous( - monkeypatch: object, -) -> None: - import pytest - from snakemake_interface_common.exceptions import WorkflowError - - from snakemake_executor_plugin_dask.executor import _connect_client - - monkeypatch.delenv("DASK_SCHEDULER_ADDRESS", raising=False) # type: ignore[attr-defined] - monkeypatch.delenv("LIGHTCONE_GATEWAY_CLUSTER", raising=False) # type: ignore[attr-defined] - with pytest.raises(WorkflowError, match="LIGHTCONE_GATEWAY_CLUSTER"): - _connect_client() - - -def test_connect_client_gateway_rendezvous_by_name( - monkeypatch: object, -) -> None: - """With LIGHTCONE_GATEWAY_CLUSTER set, the executor rejoins the run's - cluster through the Gateway API — never dials gateway:// directly.""" - import sys - from types import SimpleNamespace - - from snakemake_executor_plugin_dask.executor import _connect_client - - record: dict[str, object] = {} - - class _FakeClient: - def close(self) -> None: - record["client_closed"] = True - - class _FakeCluster: - def get_client(self) -> _FakeClient: - return _FakeClient() - - def close(self) -> None: - record["cluster_closed"] = True - - class _FakeGateway: - def connect(self, name: str, shutdown_on_close: bool = True): - record["connected"] = name - record["shutdown_on_close"] = shutdown_on_close - return _FakeCluster() - - monkeypatch.setitem( # type: ignore[attr-defined] - sys.modules, "dask_gateway", SimpleNamespace(Gateway=_FakeGateway) - ) - monkeypatch.setenv("LIGHTCONE_GATEWAY_CLUSTER", "hub.abc") # type: ignore[attr-defined] - monkeypatch.delenv("DASK_SCHEDULER_ADDRESS", raising=False) # type: ignore[attr-defined] - - client, closer = _connect_client() - assert record["connected"] == "hub.abc" - assert record["shutdown_on_close"] is False, ( - "the executor is a guest — closing it must not cull the run's cluster" - ) - closer() - assert record.get("client_closed") is True - assert record.get("cluster_closed") is True - - -def test_job_exec_prefix_cds_into_workdir() -> None: - """Gateway worker pods start in the image's WORKDIR, not the - project — every spawned job must cd into the workflow's workdir - first (spawned commands carry no --directory).""" - from types import SimpleNamespace - - from snakemake_executor_plugin_dask.executor import DaskExecutor - - executor = DaskExecutor.__new__(DaskExecutor) - executor.workflow = SimpleNamespace( # type: ignore[attr-defined] - workdir_init="/home/jovyan/my project" - ) - prefix = executor.get_job_exec_prefix(SimpleNamespace()) # type: ignore[arg-type] - assert prefix == "cd '/home/jovyan/my project'" diff --git a/tests/test_eval_tasks.py b/tests/test_eval_tasks.py deleted file mode 100644 index 63bc551e..00000000 --- a/tests/test_eval_tasks.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Validate every eval task's astra.yaml against the installed astra schema. - -The eval scaffold runs `lc init` and overlays the task's astra.yaml on -top, so the only spec the task itself ships must validate cleanly under -the same astra version the eval harness uses. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parent.parent -TASKS_DIR = REPO_ROOT / "evals" / "tasks" - - -def _task_specs() -> list[Path]: - return sorted(TASKS_DIR.glob("*/astra.yaml")) - - -@pytest.mark.parametrize( - "spec_path", - _task_specs(), - ids=lambda p: p.parent.name, -) -def test_task_astra_yaml_validates(spec_path: Path) -> None: - from astra.validation import validate_analysis_file - - errors = validate_analysis_file(spec_path) - assert not errors, "\n".join(str(e) for e in errors) diff --git a/tests/test_manifest.py b/tests/test_manifest.py deleted file mode 100644 index 73dce1c9..00000000 --- a/tests/test_manifest.py +++ /dev/null @@ -1,390 +0,0 @@ -"""Tests for the manifest module — the integrity layer.""" -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from lightcone.engine.manifest import ( - MANIFEST_FILENAME, - SCHEMA_VERSION, - code_version, - fingerprint_external, - read_manifest, - sha256_dir, - write_manifest, -) - - -def _write(path: Path, content: bytes | str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - if isinstance(content, str): - path.write_text(content) - else: - path.write_bytes(content) - - -# ---- sha256_dir ----------------------------------------------------------- - - -def test_sha256_dir_empty(tmp_path: Path) -> None: - d = tmp_path / "out" - d.mkdir() - h = sha256_dir(d) - assert h.startswith("sha256:") - assert len(h) == len("sha256:") + 64 - - -def test_sha256_dir_deterministic(tmp_path: Path) -> None: - d = tmp_path / "out" - _write(d / "a.txt", b"hello") - _write(d / "b/c.txt", b"world") - assert sha256_dir(d) == sha256_dir(d) - - -def test_sha256_dir_changes_on_content(tmp_path: Path) -> None: - d = tmp_path / "out" - _write(d / "a.txt", b"hello") - h1 = sha256_dir(d) - _write(d / "a.txt", b"hellooo") - h2 = sha256_dir(d) - assert h1 != h2 - - -def test_sha256_dir_changes_on_added_file(tmp_path: Path) -> None: - d = tmp_path / "out" - _write(d / "a.txt", b"hello") - h1 = sha256_dir(d) - _write(d / "b.txt", b"new") - h2 = sha256_dir(d) - assert h1 != h2 - - -def test_sha256_dir_independent_of_creation_order(tmp_path: Path) -> None: - d1 = tmp_path / "d1" - _write(d1 / "a.txt", b"A") - _write(d1 / "b.txt", b"B") - - d2 = tmp_path / "d2" - _write(d2 / "b.txt", b"B") - _write(d2 / "a.txt", b"A") - - assert sha256_dir(d1) == sha256_dir(d2) - - -def test_sha256_dir_excludes_manifest(tmp_path: Path) -> None: - """The manifest file itself MUST be excluded from data_version, - otherwise we'd create a chicken-and-egg problem.""" - d = tmp_path / "out" - _write(d / "data.csv", b"x,y\n1,2\n") - h_before = sha256_dir(d) - _write(d / MANIFEST_FILENAME, b'{"data_version": "sha256:foo"}') - h_after = sha256_dir(d) - assert h_before == h_after - - -def test_sha256_dir_excludes_snakemake_timestamp(tmp_path: Path) -> None: - """Snakemake touches ``.snakemake_timestamp`` AFTER the rule's run: - block completes (it's the marker for ``directory()`` outputs). Our - data_version must ignore it, otherwise verify always fails.""" - d = tmp_path / "out" - _write(d / "data.csv", b"x,y\n1,2\n") - h_before = sha256_dir(d) - _write(d / ".snakemake_timestamp", b"") - h_after = sha256_dir(d) - assert h_before == h_after - - -def test_sha256_dir_missing_dir_raises(tmp_path: Path) -> None: - with pytest.raises(FileNotFoundError): - sha256_dir(tmp_path / "nope") - - -# ---- fingerprint_external ------------------------------------------------- - - -def test_fingerprint_external_file_uses_mtime_size(tmp_path: Path) -> None: - f = tmp_path / "input.bin" - f.write_bytes(b"some bytes") - fp = fingerprint_external(f) - assert fp.startswith("mtime-size:") - - -def test_fingerprint_external_file_strict_uses_sha256(tmp_path: Path) -> None: - f = tmp_path / "input.bin" - f.write_bytes(b"some bytes") - fp = fingerprint_external(f, strict=True) - assert fp.startswith("sha256:") - - -def test_fingerprint_external_directory(tmp_path: Path) -> None: - d = tmp_path / "input_dir" - _write(d / "a.txt", b"data") - fp = fingerprint_external(d) - assert fp.startswith("sha256:") - - -def test_fingerprint_external_missing_returns_marker(tmp_path: Path) -> None: - fp = fingerprint_external(tmp_path / "nope.txt") - assert fp == "missing" - - -# ---- code_version --------------------------------------------------------- - - -def test_code_version_deterministic() -> None: - cv1 = code_version( - recipe="python script.py --x 1", - container_image="lc-foo-abc123", - decisions={"k": "a", "j": 1}, - ) - cv2 = code_version( - recipe="python script.py --x 1", - container_image="lc-foo-abc123", - decisions={"j": 1, "k": "a"}, - ) - assert cv1 == cv2 - assert cv1.startswith("sha256:") - - -def test_code_version_changes_on_recipe() -> None: - cv1 = code_version(recipe="a", container_image="c", decisions={}) - cv2 = code_version(recipe="b", container_image="c", decisions={}) - assert cv1 != cv2 - - -def test_code_version_changes_on_container() -> None: - cv1 = code_version(recipe="r", container_image="c1", decisions={}) - cv2 = code_version(recipe="r", container_image="c2", decisions={}) - assert cv1 != cv2 - - -def test_code_version_changes_on_decisions() -> None: - cv1 = code_version(recipe="r", container_image="c", decisions={"k": 1}) - cv2 = code_version(recipe="r", container_image="c", decisions={"k": 2}) - assert cv1 != cv2 - - -def test_code_version_handles_none_container() -> None: - cv = code_version(recipe="r", container_image=None, decisions={}) - assert cv.startswith("sha256:") - - -# ---- write_manifest ------------------------------------------------------- - - -def test_write_manifest_basic(tmp_path: Path) -> None: - out = tmp_path / "results" / "u1" / "foo" - _write(out / "data.csv", b"x,y\n1,2\n") - raw = tmp_path / "raw.txt" - raw.write_bytes(b"raw") - - manifest_path = write_manifest( - output_dir=out, - inputs={"raw_data": raw}, - cfg={ - "output_id": "foo", - "universe_id": "u1", - "recipe": "python script.py", - "container_image": "lc-foo-abc", - "decisions": {"k": 1}, - "code_version": "sha256:abc", - "git_sha": "deadbeef", - "git_remote": "https://github.com/dkn16/test-repo", - "lc_version": "0.4.1", - }, - ) - - assert manifest_path == out / MANIFEST_FILENAME - assert manifest_path.exists() - m = json.loads(manifest_path.read_text()) - - assert m["schema_version"] == SCHEMA_VERSION - assert m["output_id"] == "foo" - assert m["universe_id"] == "u1" - assert m["recipe"] == "python script.py" - assert m["container_image"] == "lc-foo-abc" - assert m["decisions"] == {"k": 1} - assert m["code_version"] == "sha256:abc" - assert m["git_sha"] == "deadbeef" - assert m["git_remote"] == "https://github.com/dkn16/test-repo" - assert m["lc_version"] == "0.4.1" - assert m["data_version"].startswith("sha256:") - assert "raw_data" in m["input_versions"] - assert m["input_versions"]["raw_data"].startswith("mtime-size:") - assert "finished_at" in m - assert "host" in m - - -def test_write_manifest_chains_upstream_data_version(tmp_path: Path) -> None: - """When an input is itself a materialized output (has a manifest), the - upstream's data_version flows into our manifest's input_versions. - """ - upstream = tmp_path / "results" / "u1" / "upstream" - _write(upstream / "out.csv", b"a,b\n") - write_manifest( - output_dir=upstream, - inputs={}, - cfg={ - "output_id": "upstream", - "universe_id": "u1", - "recipe": "echo", - "container_image": None, - "decisions": {}, - "code_version": "sha256:up", - "git_sha": "g", - "lc_version": "0.0", - }, - ) - upstream_manifest = json.loads((upstream / MANIFEST_FILENAME).read_text()) - upstream_dv = upstream_manifest["data_version"] - - downstream = tmp_path / "results" / "u1" / "downstream" - _write(downstream / "result.csv", b"r\n") - write_manifest( - output_dir=downstream, - inputs={"upstream": upstream}, - cfg={ - "output_id": "downstream", - "universe_id": "u1", - "recipe": "echo", - "container_image": None, - "decisions": {}, - "code_version": "sha256:dn", - "git_sha": "g", - "lc_version": "0.0", - }, - ) - dn_manifest = json.loads((downstream / MANIFEST_FILENAME).read_text()) - assert dn_manifest["input_versions"]["upstream"] == upstream_dv - - -def test_write_manifest_atomic(tmp_path: Path) -> None: - """No leftover .tmp file after a successful write.""" - out = tmp_path / "out" - _write(out / "x", b"1") - - cfg = { - "output_id": "o", - "universe_id": "u", - "recipe": "r", - "container_image": None, - "decisions": {}, - "code_version": "sha256:c", - "git_sha": "g", - "lc_version": "0.0", - } - write_manifest(output_dir=out, inputs={}, cfg=cfg) - assert (out / MANIFEST_FILENAME).exists() - assert not (out / (MANIFEST_FILENAME + ".tmp")).exists() - - -def test_write_manifest_data_version_matches_sha256_dir(tmp_path: Path) -> None: - """The data_version recorded by write_manifest is the same hash that - ``sha256_dir`` would compute — single source of truth for verify.""" - out = tmp_path / "out" - _write(out / "a.bin", b"abc") - _write(out / "nested" / "b.bin", b"def") - write_manifest( - output_dir=out, - inputs={}, - cfg={ - "output_id": "x", - "universe_id": "u", - "recipe": "r", - "container_image": None, - "decisions": {}, - "code_version": "sha256:c", - "git_sha": "g", - "lc_version": "0", - }, - ) - written_dv = json.loads((out / MANIFEST_FILENAME).read_text())["data_version"] - assert written_dv == sha256_dir(out) - - -# ---- read_manifest -------------------------------------------------------- - - -def test_read_manifest_present(tmp_path: Path) -> None: - out = tmp_path / "out" - _write(out / "x", b"d") - write_manifest( - output_dir=out, - inputs={}, - cfg={ - "output_id": "o", - "universe_id": "u", - "recipe": "r", - "container_image": None, - "decisions": {}, - "code_version": "sha256:c", - "git_sha": "g", - "lc_version": "0.0", - }, - ) - m = read_manifest(out) - assert m is not None - assert m["output_id"] == "o" - - -def test_read_manifest_missing_returns_none(tmp_path: Path) -> None: - out = tmp_path / "out" - out.mkdir() - assert read_manifest(out) is None - - -def test_read_manifest_corrupt_returns_none(tmp_path: Path) -> None: - out = tmp_path / "out" - out.mkdir() - (out / MANIFEST_FILENAME).write_text("not json") - assert read_manifest(out) is None - - -def test_read_manifest_propagates_oserror(tmp_path: Path) -> None: - """Permission errors must surface, not be silently confused with a - missing manifest by ``lc verify`` / ``lc status``.""" - import os - import sys - - if sys.platform == "win32" or os.geteuid() == 0: - pytest.skip("permission test requires non-root POSIX") - - out = tmp_path / "out" - out.mkdir() - manifest_path = out / MANIFEST_FILENAME - manifest_path.write_text("{}") - manifest_path.chmod(0o000) - try: - with pytest.raises(PermissionError): - read_manifest(out) - finally: - manifest_path.chmod(0o644) - - -def test_manifest_records_worker_image_from_env( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On a Gateway deployment the pod env carries the image it was - started with — the manifest records it as ground truth.""" - out = tmp_path / "out" - out.mkdir() - (out / "data.txt").write_text("x") - cfg = { - "output_id": "o", - "universe_id": "u", - "recipe": "echo", - "container_image": "Containerfile", - "decisions": {}, - "code_version": "sha256:x", - "git_sha": None, - "lc_version": "0", - } - monkeypatch.delenv("LIGHTCONE_WORKER_IMAGE", raising=False) - write_manifest(output_dir=out, inputs={}, cfg=cfg) - assert read_manifest(out)["worker_image"] is None - - monkeypatch.setenv("LIGHTCONE_WORKER_IMAGE", "reg/lc-p:abc") - write_manifest(output_dir=out, inputs={}, cfg=cfg) - assert read_manifest(out)["worker_image"] == "reg/lc-p:abc" diff --git a/tests/test_project.py b/tests/test_project.py new file mode 100644 index 00000000..e8334e77 --- /dev/null +++ b/tests/test_project.py @@ -0,0 +1,397 @@ +"""Tests for `lightcone.engine.project` — what a project is, and how one +converges.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from lightcone.engine.project import ( + SPEC_FILENAME, + ConvergenceReport, + ProjectError, + converge, + find_root, + project_name, +) + +#: Every path convergence is responsible for. +SCAFFOLD = ( + "astra.yaml", + "universes/baseline.yaml", + "src", + "pyproject.toml", + ".python-version", + ".gitignore", + "results/README.md", + "myst.yml", + "index.md", + "uv.lock", + ".venv", +) + + +# ---- discovery ------------------------------------------------------------ + + +def test_find_root_finds_the_spec_in_place(tmp_path: Path) -> None: + (tmp_path / SPEC_FILENAME).write_text("name: x\n") + assert find_root(tmp_path) == tmp_path.resolve() + + +def test_find_root_walks_up(tmp_path: Path) -> None: + (tmp_path / SPEC_FILENAME).write_text("name: x\n") + deep = tmp_path / "a" / "b" / "c" + deep.mkdir(parents=True) + assert find_root(deep) == tmp_path.resolve() + + +def test_find_root_returns_the_nearest_ancestor(tmp_path: Path) -> None: + """A sub-analysis nested inside a project is its own root.""" + (tmp_path / SPEC_FILENAME).write_text("name: outer\n") + inner = tmp_path / "sub" + inner.mkdir() + (inner / SPEC_FILENAME).write_text("name: inner\n") + assert find_root(inner) == inner.resolve() + + +def test_find_root_is_none_outside_a_project(tmp_path: Path) -> None: + assert find_root(tmp_path) is None + + +def test_find_root_ignores_a_directory_named_like_the_spec(tmp_path: Path) -> None: + (tmp_path / SPEC_FILENAME).mkdir() + assert find_root(tmp_path) is None + + +@pytest.mark.parametrize( + ("directory", "expected"), + [ + ("My Analysis", "my-analysis"), + ("SN_Cosmology", "sn_cosmology"), + ("2024.results", "2024.results"), + ("...", "analysis"), + ], +) +def test_project_name_normalizes_the_directory(directory: str, expected: str) -> None: + assert project_name(Path("/tmp") / directory) == expected + + +# ---- the report ----------------------------------------------------------- + + +def test_report_is_converged_only_when_nothing_changed() -> None: + assert ConvergenceReport().converged + assert ConvergenceReport(unchanged=["astra.yaml"]).converged + assert not ConvergenceReport(created=["astra.yaml"]).converged + assert not ConvergenceReport(repaired=[".gitignore"]).converged + + +def test_warnings_do_not_make_a_project_unconverged() -> None: + """Warnings are things convergence can see but must not fix — they + must not turn a converged project into a failing `--check`.""" + assert ConvergenceReport(unchanged=["astra.yaml"], warnings=["heads up"]).converged + + +# ---- the scaffold --------------------------------------------------------- + + +def test_converge_creates_the_whole_scaffold(tmp_path: Path) -> None: + project = tmp_path / "proj" + report = converge(project, git=False) + + assert not report.converged + for rel in SCAFFOLD: + assert (project / rel).exists(), f"missing {rel}" + + +def test_converge_scaffolds_a_uv_project(tmp_path: Path) -> None: + """The environment is the uv project: a *virtual* pyproject (no + [build-system]) carrying the engine in its own dependency list, plus + the exact interpreter pin (spec §2).""" + from lightcone.engine.constants import DEFAULT_PYTHON, DEFAULT_PYTHON_FLOOR + + project = tmp_path / "proj" + converge(project, git=False) + + pyproject = (project / "pyproject.toml").read_text() + assert 'name = "proj"' in pyproject + assert "lightcone-cli" in pyproject + assert f'requires-python = ">={DEFAULT_PYTHON_FLOOR}"' in pyproject + assert "[build-system]" not in pyproject + assert (project / ".python-version").read_text().strip() == DEFAULT_PYTHON + + +def test_converge_scaffolds_no_container_declaration(tmp_path: Path) -> None: + """Containerized mode is *derived* from a declaration the user makes, + never scaffolded — and ASTRA carries nothing about the environment, so + the boilerplate's `container:` key is stripped (spec §1, §2).""" + project = tmp_path / "proj" + converge(project, git=False) + + assert not (project / "Containerfile").exists() + assert "container:" not in (project / SPEC_FILENAME).read_text() + assert "[tool.lightcone.image]" not in (project / "pyproject.toml").read_text() + + +def test_converge_gitignores_results_but_keeps_its_readme(tmp_path: Path) -> None: + project = tmp_path / "proj" + converge(project, git=False) + + gitignore = (project / ".gitignore").read_text() + assert ".venv/" in gitignore + assert "results/*" in gitignore + assert "!results/README.md" in gitignore + + +def test_converge_titles_the_report_after_the_directory(tmp_path: Path) -> None: + project = tmp_path / "proj" + converge(project, git=False) + assert (project / "index.md").read_text().startswith("# proj\n") + + +# ---- idempotency and adoption -------------------------------------------- + + +def test_converge_is_idempotent(tmp_path: Path) -> None: + project = tmp_path / "proj" + converge(project, git=False) + before = {p: p.read_bytes() for p in project.rglob("*") if p.is_file()} + + report = converge(project, git=False) + assert report.converged + assert {p: p.read_bytes() for p in project.rglob("*") if p.is_file()} == before + + +def test_converge_adopts_an_existing_project(tmp_path: Path) -> None: + """A directory that already holds an astra.yaml is adopted, not + rejected — and the user's own file is never overwritten.""" + project = tmp_path / "proj" + project.mkdir() + (project / SPEC_FILENAME).write_text('name: "mine"\n') + + report = converge(project, git=False) + assert (project / SPEC_FILENAME).read_text() == 'name: "mine"\n' + assert "astra.yaml" in report.unchanged + assert (project / "pyproject.toml").exists() + + +def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: + project = tmp_path / "proj" + converge(project, git=False) + (project / "myst.yml").unlink() + + report = converge(project, git=False) + assert report.created == ["myst.yml"] + assert (project / "myst.yml").exists() + + +def test_gitignore_repair_preserves_user_content_and_adds_the_entries( + tmp_path: Path, +) -> None: + """A repair only ever appends: the user's own ignores survive intact.""" + from lightcone.engine import templates + + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text("mine.txt\nbuild/\n") + + report = converge(project, git=False) + assert ".gitignore" in report.repaired + + text = (project / ".gitignore").read_text() + assert text.startswith("mine.txt\nbuild/\n") + assert templates.missing_gitignore_entries(text) == [] + + +def test_gitignore_repair_appends_only_what_is_missing(tmp_path: Path) -> None: + """Entry-wise convergence, so a pattern the user already has is never + duplicated — whoever put it there.""" + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text("__pycache__/\n.venv/\n") + + converge(project, git=False) + text = (project / ".gitignore").read_text() + assert text.count("__pycache__/") == 1 + assert text.count(".venv/") == 1 + assert "results/*" in text + + +def test_gitignore_repair_is_idempotent(tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text("mine.txt\n") + + converge(project, git=False) + once = (project / ".gitignore").read_text() + report = converge(project, git=False) + + assert ".gitignore" in report.unchanged + assert (project / ".gitignore").read_text() == once + + +def test_gitignore_repair_reaches_a_file_that_has_the_header_already( + tmp_path: Path, +) -> None: + """The header is cosmetic: a project carrying it but missing an entry + — an older lc wrote it, or a hand edit removed a line — still + converges. This is the case a marker check would have skipped.""" + from lightcone.engine import templates + + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text(f"{templates.GITIGNORE_HEADER}\n.venv/\n") + + report = converge(project, git=False) + text = (project / ".gitignore").read_text() + + assert ".gitignore" in report.repaired + assert templates.missing_gitignore_entries(text) == [] + # Cosmetic, so still exactly one. + assert text.count(templates.GITIGNORE_HEADER) == 1 + + +def test_repair_of_an_empty_gitignore_adds_no_leading_blank_lines( + tmp_path: Path, +) -> None: + from lightcone.engine import templates + + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text("") + + converge(project, git=False) + assert (project / ".gitignore").read_text() == templates.gitignore() + + +def test_converge_writes_no_agent_notes(tmp_path: Path) -> None: + """`AGENTS.md` is deliberately not scaffolded — see the deviation note + in CLAUDE.md.""" + project = tmp_path / "proj" + converge(project, git=False) + assert not (project / "AGENTS.md").exists() + + +# ---- write=False (check mode) -------------------------------------------- + + +def test_check_mode_writes_nothing_at_all(tmp_path: Path) -> None: + """Not even the project directory — a drift report must be free of + side effects.""" + project = tmp_path / "proj" + report = converge(project, write=False, git=False) + + assert not report.converged + assert "astra.yaml" in report.created + assert not project.exists() + + +def test_check_mode_agrees_with_a_real_run(tmp_path: Path) -> None: + """Check mode is the same decisions with side effects off, so its + report must match what a real run reports.""" + project = tmp_path / "proj" + dry = converge(project, write=False, git=False) + wet = converge(project, git=False) + + assert dry.as_dict() == wet.as_dict() + assert converge(project, write=False, git=False).converged + + +# ---- warnings ------------------------------------------------------------- + + +def test_warns_when_pyproject_lacks_the_engine(tmp_path: Path) -> None: + """The engine belongs inside the experiment's lock; a pyproject we + didn't write is the user's, so warn rather than edit (spec §2).""" + project = tmp_path / "proj" + project.mkdir() + (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') + + report = converge(project, git=False) + assert any("does not depend on lightcone-cli" in w for w in report.warnings) + assert 'name = "mine"' in (project / "pyproject.toml").read_text() + + +def test_warns_when_results_is_not_a_directory(tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + (project / "results").write_text("not a directory\n") + + report = converge(project, git=False) + assert any("results exists but is not a directory" in w for w in report.warnings) + assert (project / "results").read_text() == "not a directory\n" + + +# ---- refusals ------------------------------------------------------------- + + +def test_refuses_an_authored_containerfile(tmp_path: Path) -> None: + """Images are generated from the lock; a hand-written Containerfile + would be silently ignored. Refuse — before any write (spec §8).""" + project = tmp_path / "proj" + project.mkdir() + (project / "Containerfile").write_text("FROM python:3.12-slim\n") + + with pytest.raises(ProjectError, match="delete or rename it"): + converge(project, git=False) + + assert not (project / SPEC_FILENAME).exists() + assert not (project / "pyproject.toml").exists() + + +def test_requires_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from lightcone.engine import project as project_mod + + monkeypatch.setattr(project_mod.shutil, "which", lambda name, path=None: None) + with pytest.raises(ProjectError, match="uv is required"): + converge(tmp_path / "proj", git=False) + + +# ---- the uv seam ---------------------------------------------------------- + + +def test_locks_then_syncs_exactly(tmp_path: Path, fake_uv: list[list[str]]) -> None: + """Converge once, with the flags the spec fixes: ``--locked --exact`` + (no drift, no additive sync) and ``--compile-bytecode`` (§4, §6).""" + project = tmp_path / "proj" + converge(project, git=False) + + assert [c[0] for c in fake_uv] == ["lock", "sync"] + sync = fake_uv[1] + for flag in ("--locked", "--exact", "--compile-bytecode"): + assert flag in sync + # uv's own walk-up discovery is never trusted (spec §4). + assert fake_uv[0][-2:] == ["--project", str(project.resolve())] + assert sync[-2:] == ["--project", str(project.resolve())] + + +def test_sync_false_still_locks(tmp_path: Path, fake_uv: list[list[str]]) -> None: + project = tmp_path / "proj" + converge(project, git=False, sync=False) + + assert [c[0] for c in fake_uv] == ["lock"] + assert (project / "uv.lock").exists() + assert not (project / ".venv").exists() + + +def test_check_mode_runs_no_uv(tmp_path: Path, fake_uv: list[list[str]]) -> None: + converge(tmp_path / "proj", write=False, git=False) + assert fake_uv == [] + + +def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A silently broken lock would fail every later verb more + confusingly — so the failure surfaces here, with uv's own stderr.""" + from unittest.mock import MagicMock + + from lightcone.engine import project as project_mod + + monkeypatch.setattr( + project_mod, + "_run_uv", + lambda args, *, cwd: MagicMock(returncode=1, stdout="", stderr="no solution found"), + ) + with pytest.raises(ProjectError, match="no solution found"): + converge(tmp_path / "proj", git=False) diff --git a/tests/test_runner.py b/tests/test_runner.py deleted file mode 100644 index c4e9ed0c..00000000 --- a/tests/test_runner.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Tests for the per-rule run_rule helper. - -The helper is invoked from the generated Snakefile's ``run:`` block; we -exercise it directly with a synthetic cfg, capturing stdout to assert on -the sentinel-prefixed framing the executor relies on. -""" -from __future__ import annotations - -import io -import re -import subprocess -from contextlib import redirect_stdout -from pathlib import Path - -from lightcone.engine.runner import SENTINEL, run_rule - -_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") - - -def _strip_ansi(s: str) -> str: - return _ANSI_RE.sub("", s) - - -def _capture(fn) -> tuple[str, BaseException | None]: - buf = io.StringIO() - err: BaseException | None = None - try: - with redirect_stdout(buf): - fn() - except BaseException as e: # noqa: BLE001 — we want CalledProcessError too - err = e - return buf.getvalue(), err - - -def _cfg(output_id: str = "foo", *, shell_command: str = "echo hi") -> dict: - """Minimal cfg matching what the Snakefile generator writes. - - ``manifest.write_manifest`` reads several keys; we provide the ones - it touches without standing up a real container/decision pipeline. - The runner reads ``shell_command`` directly — substitution and - container wrapping happen at generation time. - """ - return { - "output_id": output_id, - "output_type": "data", - "universe_id": "u1", - "recipe": "echo hi", - "shell_command": shell_command, - "container_image": None, - "decisions": {}, - "code_version": "abc", - "git_sha": None, - "lc_version": "test", - } - - -def test_emit_lines_carry_sentinel(tmp_path: Path) -> None: - out_dir = tmp_path / "out" - out_dir.mkdir() - output, err = _capture( - lambda: run_rule( - rule_key="foo", - universe="u1", - output_dir=out_dir, - inputs={}, - cfg=_cfg(shell_command="echo hello"), - ) - ) - assert err is None - # Every line we emit is sentinel-prefixed and column-0 anchored. - for line in output.splitlines(): - assert line.startswith(SENTINEL), line - # And the recipe's stdout reaches us framed. - body = _strip_ansi("\n".join(line[len(SENTINEL):] for line in output.splitlines())) - assert "▶ foo" in body - assert "hello" in body - assert "✓ foo" in body - - -def test_failed_recipe_raises_and_emits_cross(tmp_path: Path) -> None: - out_dir = tmp_path / "out" - out_dir.mkdir() - output, err = _capture( - lambda: run_rule( - rule_key="foo", - universe="u1", - output_dir=out_dir, - inputs={}, - cfg=_cfg(shell_command="false"), - ) - ) - assert isinstance(err, subprocess.CalledProcessError) - body = _strip_ansi("\n".join(line[len(SENTINEL):] for line in output.splitlines())) - assert "▶ foo" in body - assert "✗ foo" in body - assert "exit=1" in body - - -def test_no_manifest_on_failure(tmp_path: Path) -> None: - """A failing recipe must not leave a manifest behind — it would - poison ``lc verify``'s chain check by claiming completion of an - incomplete rule.""" - out_dir = tmp_path / "out" - out_dir.mkdir() - _, err = _capture( - lambda: run_rule( - rule_key="foo", - universe="u1", - output_dir=out_dir, - inputs={}, - cfg=_cfg(shell_command="false"), - ) - ) - assert err is not None - assert not (out_dir / ".lightcone-manifest.json").exists() - - -def test_manifest_written_on_success(tmp_path: Path) -> None: - out_dir = tmp_path / "out" - out_dir.mkdir() - _, err = _capture( - lambda: run_rule( - rule_key="foo", - universe="u1", - output_dir=out_dir, - inputs={}, - cfg=_cfg(shell_command=f"touch {out_dir}/data.txt"), - ) - ) - assert err is None - assert (out_dir / ".lightcone-manifest.json").is_file() - - -def test_recipe_stdout_and_stderr_both_forwarded(tmp_path: Path) -> None: - out_dir = tmp_path / "out" - out_dir.mkdir() - output, err = _capture( - lambda: run_rule( - rule_key="foo", - universe="u1", - output_dir=out_dir, - inputs={}, - cfg=_cfg(shell_command="echo on-stdout; echo on-stderr 1>&2"), - ) - ) - assert err is None - body = output - assert "on-stdout" in body - assert "on-stderr" in body diff --git a/tests/test_scratch.py b/tests/test_scratch.py deleted file mode 100644 index 3a884fb3..00000000 --- a/tests/test_scratch.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Tests for the scratch resolution layer. - -Covers the precedence chain (env > project > site > tempdir), the -per-run directory layout, and the snakemake symlink swap that ensures -snakemake's workflow lock and metadata land on a filesystem that -honours ``flock``. -""" -from __future__ import annotations - -import tempfile -from pathlib import Path - -import pytest -import yaml - -from lightcone.engine.scratch import ( - LIGHTCONE_SCRATCH_ENV, - RunLockBusyError, - acquire_run_lock, - ensure_snakemake_symlink, - prepare_run_dirs, - project_hash, - resolve_scratch_root, -) - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - p = tmp_path / "proj" - p.mkdir() - (p / "astra.yaml").write_text("outputs: []\n") - return p - - -@pytest.fixture(autouse=True) -def _no_known_site(monkeypatch: pytest.MonkeyPatch) -> None: - """Prevent the developer's actual hostname from leaking site detection - into resolution; tests pin the precedence chain explicitly.""" - import socket - monkeypatch.setattr(socket, "gethostname", lambda: "unknown-host-x") - - -# ---- resolve_scratch_root ------------------------------------------------- - - -def test_env_var_wins(monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path) -> None: - target = tmp_path / "env-scratch" - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, str(target)) - # Even with a project-level config set, env var takes precedence. - (project / ".lightcone").mkdir() - (project / ".lightcone" / "lightcone.yaml").write_text( - yaml.safe_dump({"scratch_root": str(tmp_path / "from-project")}) - ) - assert resolve_scratch_root(project) == target - - -def test_env_var_expands(monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path) -> None: - monkeypatch.setenv("MYSCRATCH", str(tmp_path / "x")) - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, "$MYSCRATCH/sub") - assert resolve_scratch_root(project) == tmp_path / "x" / "sub" - - -def test_project_config(monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path) -> None: - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - target = tmp_path / "proj-scratch" - (project / ".lightcone").mkdir() - (project / ".lightcone" / "lightcone.yaml").write_text( - yaml.safe_dump({"scratch_root": str(target)}) - ) - assert resolve_scratch_root(project) == target - - -def test_project_config_expands( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - monkeypatch.setenv("PROJSCRATCH", str(tmp_path / "expanded")) - (project / ".lightcone").mkdir() - (project / ".lightcone" / "lightcone.yaml").write_text( - yaml.safe_dump({"scratch_root": "$PROJSCRATCH"}) - ) - assert resolve_scratch_root(project) == tmp_path / "expanded" - - -def test_site_default_resolves_when_env_set( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - monkeypatch.setenv("SCRATCH", str(tmp_path / "lustre")) - import socket - monkeypatch.setattr(socket, "gethostname", lambda: "perlmutter-login01") - assert resolve_scratch_root(project) == tmp_path / "lustre" - - -def test_site_default_falls_through_when_env_missing( - monkeypatch: pytest.MonkeyPatch, project: Path -) -> None: - """If a known site's scratch_root is ``$SCRATCH`` and ``SCRATCH`` is - not set, the unexpanded ``$SCRATCH`` mustn't become a literal path — - we fall through to the tempdir fallback instead. - """ - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - monkeypatch.delenv("SCRATCH", raising=False) - import socket - monkeypatch.setattr(socket, "gethostname", lambda: "perlmutter-login01") - resolved = resolve_scratch_root(project) - assert "$" not in str(resolved) - assert resolved == Path(tempfile.gettempdir()) - - -def test_fallback_to_tempdir(monkeypatch: pytest.MonkeyPatch, project: Path) -> None: - monkeypatch.delenv(LIGHTCONE_SCRATCH_ENV, raising=False) - assert resolve_scratch_root(project) == Path(tempfile.gettempdir()) - - -# ---- prepare_run_dirs ----------------------------------------------------- - - -def test_prepare_run_dirs_creates_layout( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, str(tmp_path / "scratch")) - rd = prepare_run_dirs(project, run_id="42") - assert rd.root == tmp_path / "scratch" / ".lightcone" - assert rd.dask_local == rd.root / "dask" / "42" - assert rd.snakemake_state.parent.parent == rd.root / "snakemake" - # Every path that callers rely on must exist on return. - assert rd.root.is_dir() - assert rd.dask_local.is_dir() - assert rd.run_lock_path.is_file() - assert rd.snakemake_state.parent.is_dir() - - -def test_project_hash_is_path_keyed(tmp_path: Path) -> None: - a = tmp_path / "a" - b = tmp_path / "b" - a.mkdir() - b.mkdir() - assert project_hash(a) != project_hash(b) - assert project_hash(a) == project_hash(a) - - -def test_prepare_run_dirs_separates_projects( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """Two different projects sharing one scratch must not collide on - snakemake state — that would let one project's stale lock files - block another's runs.""" - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, str(tmp_path / "scratch")) - a = tmp_path / "proj-a" - a.mkdir() - b = tmp_path / "proj-b" - b.mkdir() - rd_a = prepare_run_dirs(a, run_id="x") - rd_b = prepare_run_dirs(b, run_id="x") - assert rd_a.snakemake_state != rd_b.snakemake_state - - -# ---- ensure_snakemake_symlink -------------------------------------------- - - -def test_symlink_created_when_absent(project: Path, tmp_path: Path) -> None: - target = tmp_path / "scratch" / "snakemake" - ensure_snakemake_symlink(project, target) - link = project / ".snakemake" - assert link.is_symlink() - assert link.resolve() == target.resolve() - - -def test_symlink_idempotent(project: Path, tmp_path: Path) -> None: - target = tmp_path / "scratch" / "snakemake" - ensure_snakemake_symlink(project, target) - ensure_snakemake_symlink(project, target) # second call is a no-op - assert (project / ".snakemake").is_symlink() - - -def test_symlink_repoints(project: Path, tmp_path: Path) -> None: - a = tmp_path / "scratch-a" / "snakemake" - b = tmp_path / "scratch-b" / "snakemake" - ensure_snakemake_symlink(project, a) - ensure_snakemake_symlink(project, b) - assert (project / ".snakemake").resolve() == b.resolve() - - -# ---- acquire_run_lock ----------------------------------------------------- - - -def test_run_lock_clears_stale_snakemake_locks( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - """A snakemake lock left by a prior crashed run must not block the - next ``lc run`` — once we hold our project-level flock, those zero- - byte sentinels are known-stale and safe to remove.""" - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, str(tmp_path / "scratch")) - rd = prepare_run_dirs(project, run_id="test") - snake_locks = rd.snakemake_state / "locks" - snake_locks.mkdir(parents=True) - (snake_locks / "0.input.lock").touch() - (snake_locks / "0.output.lock").touch() - with acquire_run_lock(rd): - assert not (snake_locks / "0.input.lock").exists() - assert not (snake_locks / "0.output.lock").exists() - - -def test_run_lock_rejects_concurrent_holder( - monkeypatch: pytest.MonkeyPatch, project: Path, tmp_path: Path -) -> None: - """Two concurrent ``lc run`` invocations on the same project must - not silently queue — the second one bails so the user sees the - collision and decides what to do.""" - monkeypatch.setenv(LIGHTCONE_SCRATCH_ENV, str(tmp_path / "scratch")) - rd = prepare_run_dirs(project, run_id="test") - with acquire_run_lock(rd): - with pytest.raises(RunLockBusyError): - with acquire_run_lock(rd): - pass # pragma: no cover - - -def test_legacy_real_dir_backed_up(project: Path, tmp_path: Path) -> None: - """A pre-existing real ``.snakemake/`` (left by a direct snakemake - invocation) must be moved aside, not deleted — losing real job - metadata silently is the worse failure mode.""" - legacy = project / ".snakemake" - legacy.mkdir() - (legacy / "marker").write_text("evidence") - target = tmp_path / "scratch" / "snakemake" - ensure_snakemake_symlink(project, target) - assert (project / ".snakemake").is_symlink() - backup = project / ".snakemake.legacy" - assert backup.is_dir() - assert (backup / "marker").read_text() == "evidence" diff --git a/tests/test_site_registry.py b/tests/test_site_registry.py deleted file mode 100644 index 939dbeeb..00000000 --- a/tests/test_site_registry.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Tests for the site registry — site detection and the HostSite wrapper.""" -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path - -import pytest - -from lightcone.engine import site_registry -from lightcone.engine.site_registry import ( - HostSite, - detect_current_site, - detect_site, -) - - -@pytest.fixture -def fake_hostname(monkeypatch: pytest.MonkeyPatch) -> Callable[[str], None]: - """Return a setter that pins ``socket.gethostname`` for the test.""" - - def _set(name: str) -> None: - monkeypatch.setattr(site_registry.socket, "gethostname", lambda: name) - - return _set - - -class TestDetectSite: - def test_matches_perlmutter_substring(self) -> None: - assert detect_site("login29.chn.perlmutter.nersc.gov") == "perlmutter" - - def test_matches_saul_pattern(self) -> None: - assert detect_site("saul01") == "perlmutter" - - def test_unknown_host(self) -> None: - assert detect_site("generic-laptop") is None - - def test_local_site_skipped(self) -> None: - # "local" has backend=local and is excluded from auto-detection. - assert detect_site("local") is None - - -class TestHostSite: - def test_matched_site_is_truthy(self) -> None: - site = HostSite(key="perlmutter", defaults={"display_name": "NERSC Perlmutter"}) - assert bool(site) is True - - def test_unmatched_site_is_falsy(self) -> None: - assert bool(HostSite(key=None)) is False - - def test_get_returns_field(self) -> None: - site = HostSite(key="perlmutter", defaults={"container_runtime": "podman-hpc"}) - assert site.get("container_runtime") == "podman-hpc" - - def test_get_missing_field_returns_default(self) -> None: - site = HostSite(key="perlmutter", defaults={}) - assert site.get("missing", "fallback") == "fallback" - assert site.get("missing") is None - - def test_display_name_from_defaults(self) -> None: - site = HostSite(key="perlmutter", defaults={"display_name": "NERSC Perlmutter"}) - assert site.display_name == "NERSC Perlmutter" - - def test_display_name_falls_back_to_key(self) -> None: - site = HostSite(key="perlmutter", defaults={}) - assert site.display_name == "perlmutter" - - def test_display_name_for_unknown_site(self) -> None: - assert HostSite(key=None).display_name == "unknown" - - -class TestDetectCurrentSite: - def test_known_host_returns_populated_site( - self, fake_hostname: Callable[[str], None] - ) -> None: - fake_hostname("login29.chn.perlmutter.nersc.gov") - site = detect_current_site() - assert site - assert site.key == "perlmutter" - assert site.get("container_runtime") == "podman-hpc" - assert site.display_name == "NERSC Perlmutter" - - def test_unknown_host_returns_empty_site( - self, fake_hostname: Callable[[str], None] - ) -> None: - fake_hostname("generic-laptop") - site = detect_current_site() - assert not site - assert site.key is None - assert site.get("container_runtime") is None - - def test_unknown_host_get_returns_default( - self, fake_hostname: Callable[[str], None] - ) -> None: - # Field access on an unmatched site shouldn't require an explicit - # truthiness guard at every call site — that's the whole point of - # returning an empty HostSite rather than None. - fake_hostname("generic-laptop") - assert detect_current_site().get("scratch_root", "/tmp") == "/tmp" - - -# ---- env-marker detection (JupyterHub deployments) ------------------------ - - -def test_detect_site_from_env_matches_jupyterhub( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.site_registry import detect_site_from_env - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS", raising=False) - assert detect_site_from_env() is None - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert detect_site_from_env() == "jupyterhub" - - -def test_env_markers_win_over_hostname(monkeypatch: pytest.MonkeyPatch) -> None: - """A pod's hostname is noise; the injected env is the signal.""" - from lightcone.engine.site_registry import detect_current_site - - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setattr( - "lightcone.engine.site_registry.socket.gethostname", - lambda: "login29.chn.perlmutter.nersc.gov", - ) - site = detect_current_site() - assert site.key == "jupyterhub" - assert site.get("container_runtime") == "kubernetes" - assert site.get("scratch_root") == "$HOME" - - -def test_no_markers_falls_back_to_hostname( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from lightcone.engine.site_registry import detect_current_site - - monkeypatch.delenv("DASK_GATEWAY__ADDRESS", raising=False) - monkeypatch.setattr( - "lightcone.engine.site_registry.socket.gethostname", - lambda: "login29.chn.perlmutter.nersc.gov", - ) - assert detect_current_site().key == "perlmutter" - - -def test_hub_scratch_resolves_to_home( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """No separate scratch space on the hub — home IS the shared volume.""" - import os - - from lightcone.engine.scratch import resolve_scratch_root - - project = tmp_path / "proj" - project.mkdir() - monkeypatch.delenv("LIGHTCONE_SCRATCH", raising=False) - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - assert resolve_scratch_root(project) == Path(os.environ["HOME"]) diff --git a/tests/test_snakefile.py b/tests/test_snakefile.py deleted file mode 100644 index aa64128c..00000000 --- a/tests/test_snakefile.py +++ /dev/null @@ -1,599 +0,0 @@ -"""Tests for engine/snakefile.py — the Snakefile generator.""" -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest -import yaml - -from lightcone.engine.snakefile import generate, render_recipe - - -def _spec(project_root: Path, spec: dict[str, Any]) -> None: - project_root.mkdir(parents=True, exist_ok=True) - (project_root / "astra.yaml").write_text(yaml.safe_dump(spec)) - - -def test_generate_simple_spec(tmp_path: Path) -> None: - _spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - { - "id": "bar", - "inputs": ["foo"], - "recipe": {"command": "echo bar"}, - }, - ] - }, - ) - snakefile, _ = generate(tmp_path, universes=["u1"]) - - assert (tmp_path / ".lightcone" / "Snakefile").exists() - assert (tmp_path / ".lightcone" / "snakefile-config.json").exists() - - snake_text = snakefile.read_text() - assert "rule foo:" in snake_text - assert "rule bar:" in snake_text - assert "rule all:" in snake_text - assert "results/{universe}/foo" in snake_text - - -def test_generate_universe_expansion(tmp_path: Path) -> None: - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) - _, cfg_path = generate(tmp_path, universes=["u1", "u2"]) - - cfg = json.loads(cfg_path.read_text()) - assert "foo" in cfg - assert set(cfg["foo"].keys()) == {"u1", "u2"} - - -def test_generate_skips_alias_outputs(tmp_path: Path) -> None: - """Outputs without a recipe (aliases) are NOT emitted as rules.""" - _spec( - tmp_path, - { - "outputs": [{"id": "alias", "from": "sub.real"}], - "analyses": { - "sub": { - "outputs": [{"id": "real", "recipe": {"command": "echo"}}], - } - }, - }, - ) - snakefile, _ = generate(tmp_path, universes=["u1"]) - text = snakefile.read_text() - assert "rule sub__real:" in text - assert "rule alias:" not in text - - -def test_generate_writes_code_version_per_universe(tmp_path: Path) -> None: - """code_version is part of the per-(rule, universe) cfg blob, so a - decision change in one universe doesn't poison another.""" - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) - _, cfg_path = generate(tmp_path, universes=["u1", "u2"]) - cfg = json.loads(cfg_path.read_text()) - assert "code_version" in cfg["foo"]["u1"] - assert "code_version" in cfg["foo"]["u2"] - - -def test_generate_includes_recipe_in_cfg(tmp_path: Path) -> None: - _spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "python script.py --arg 1"}}]}, - ) - _, cfg_path = generate(tmp_path, universes=["u1"]) - cfg = json.loads(cfg_path.read_text()) - # The raw recipe template (what the user wrote) is preserved so - # the manifest can record it. ``shell_command`` is the rendered + - # runtime-wrapped version, prefixed with a no-op carrying the - # code_version so drift is visible at the shell level. - assert cfg["foo"]["u1"]["recipe"] == "python script.py --arg 1" - sh = cfg["foo"]["u1"]["shell_command"] - assert "python script.py --arg 1" in sh - assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh - - -def test_generate_no_container_directive_emitted(tmp_path: Path) -> None: - """We own container invocation; the Snakemake ``container:`` directive - must never be emitted (we don't use --sdm apptainer).""" - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": {"command": "echo", "container": "python:3.12-slim"}, - } - ] - }, - ) - snakefile_path, _ = generate(tmp_path, universes=["u1"], runtime="podman") - text = snakefile_path.read_text() - assert "container:" not in text - - -def test_generate_wraps_recipe_with_runtime(tmp_path: Path) -> None: - """When a runtime is configured and the recipe has a container, the - wrapped shell command in cfg invokes the runtime with the image — - and the v0.0.7 ``{output}`` placeholder has been substituted to a - concrete per-universe path before the wrap.""" - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": { - "command": "echo hi > {output}/data.txt", - "container": "python:3.12-slim", - }, - } - ] - }, - ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="podman") - cfg = json.loads(cfg_path.read_text()) - sh = cfg["foo"]["u1"]["shell_command"] - assert "podman run --rm" in sh - assert "python:3.12-slim" in sh - # ``{output}`` is rendered at gen time to the concrete per-universe - # path; no placeholder survives the wrap. - assert "results/u1/foo/data.txt" in sh - assert "{output}" not in sh - # The code_version breadcrumb is prefixed onto the wrapped command. - assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh - - -def test_generate_no_wrap_for_runtime_none(tmp_path: Path) -> None: - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": {"command": "echo hi", "container": "python:3.12-slim"}, - } - ] - }, - ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="none") - cfg = json.loads(cfg_path.read_text()) - sh = cfg["foo"]["u1"]["shell_command"] - assert sh.endswith("echo hi") - assert f"lc_code_version={cfg['foo']['u1']['code_version']}" in sh - - -def test_generated_rules_delegate_to_run_rule(tmp_path: Path) -> None: - """Each rule body is a ``run:`` block that calls ``run_rule()``. - - The recipe execution, manifest write, and validation hook all live - inside :func:`lightcone.engine.runner.run_rule` — keeping the - generated Snakefile slim and behaviour in Python rather than in - shell strings. - """ - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo hi"}}]}) - snakefile, _ = generate(tmp_path, universes=["u1"]) - text = snakefile.read_text() - assert " run:" in text - assert " shell:" not in text - assert "run_rule(" in text - assert "from lightcone.engine.runner import run_rule" in text - # Direct shell()/write_manifest()/validate_output() calls in the - # generated body would mean we're double-executing or bypassing - # run_rule's lockable output frame. - assert " shell(" not in text - assert " write_manifest(" not in text - assert "_lc_finalize" not in text - - -def test_no_finalizer_script_written(tmp_path: Path) -> None: - """``_lc_finalize.py`` is gone — write_manifest runs on the host.""" - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) - generate(tmp_path, universes=["u1"]) - assert not (tmp_path / ".lightcone" / "_lc_finalize.py").exists() - - -def test_cfg_substitutes_inputs_per_universe(tmp_path: Path) -> None: - """``{inputs.}`` and ``{output}`` are substituted at gen time - to concrete per-universe paths; sibling output paths track the - universe wildcard so a ``u1`` rule can never reference ``u2`` data.""" - _spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo > {output}/data.txt"}}, - { - "id": "bar", - "inputs": ["foo"], - "recipe": {"command": "cat {inputs.foo}/data.txt > {output}/out.txt"}, - }, - ] - }, - ) - _, cfg_path = generate(tmp_path, universes=["u1", "u2"]) - cfg = json.loads(cfg_path.read_text()) - sh_u1 = cfg["bar"]["u1"]["shell_command"] - sh_u2 = cfg["bar"]["u2"]["shell_command"] - assert "results/u1/foo/data.txt" in sh_u1 - assert "results/u1/bar/out.txt" in sh_u1 - assert "results/u2/foo/data.txt" in sh_u2 - assert "results/u2/bar/out.txt" in sh_u2 - - -def test_recipe_edit_changes_params_for_rerun_trigger(tmp_path: Path) -> None: - """Editing a recipe must change ``params.cfg`` (which carries - ``code_version`` and ``shell_command``) so Snakemake's ``params`` - rerun-trigger fires. The rule body source itself does NOT change — - only ``params`` — which is why ``lc run`` defaults to including - ``params`` in ``--rerun-triggers``. - """ - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo v1"}}]}) - snakefile_v1, cfg_path_v1 = generate(tmp_path, universes=["u1"]) - body_v1 = snakefile_v1.read_text() - cfg_v1 = json.loads(cfg_path_v1.read_text())["foo"]["u1"] - - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo v2"}}]}) - snakefile_v2, cfg_path_v2 = generate(tmp_path, universes=["u1"]) - body_v2 = snakefile_v2.read_text() - cfg_v2 = json.loads(cfg_path_v2.read_text())["foo"]["u1"] - - assert body_v1 == body_v2, ( - "Rule body is universe-parameterized and must not change on a " - "recipe edit — that is the whole reason we rely on the params trigger." - ) - assert cfg_v1["code_version"] != cfg_v2["code_version"] - assert cfg_v1["shell_command"] != cfg_v2["shell_command"] - - -def test_containerfile_edit_changes_code_version(tmp_path: Path) -> None: - """Editing a Containerfile changes ``code_version`` so ``lc status`` - reports stale and the manifest records the image content faithfully. - """ - containerfile = tmp_path / "Containerfile" - containerfile.write_text("FROM python:3.12-slim\n") - _spec( - tmp_path, - { - "outputs": [ - { - "id": "foo", - "recipe": {"command": "echo", "container": "Containerfile"}, - } - ] - }, - ) - _, cfg_path_v1 = generate(tmp_path, universes=["u1"], runtime="podman") - cv_v1 = json.loads(cfg_path_v1.read_text())["foo"]["u1"]["code_version"] - - containerfile.write_text("FROM python:3.12-slim\nRUN pip install numpy\n") - _, cfg_path_v2 = generate(tmp_path, universes=["u1"], runtime="podman") - cv_v2 = json.loads(cfg_path_v2.read_text())["foo"]["u1"]["code_version"] - - assert cv_v1 != cv_v2, ( - "code_version must change when the Containerfile contents change " - "so that lc status correctly reports stale." - ) - - -def test_validation_runs_via_run_rule(tmp_path: Path) -> None: - """Validation now runs inside ``run_rule()`` rather than inline in - the generated Snakefile. We just check that the runner is imported - — the runner's own tests cover the validation-on-success path.""" - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo"}}]}) - snakefile, _ = generate(tmp_path, universes=["u1"]) - text = snakefile.read_text() - assert "from lightcone.engine.runner import run_rule" in text - - -def test_cfg_includes_output_type(tmp_path: Path) -> None: - """Validation needs the declared output type — pass it through cfg.""" - _spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "type": "metric", "recipe": {"command": "echo"}}, - ] - }, - ) - _, cfg_path = generate(tmp_path, universes=["u1"]) - cfg = json.loads(cfg_path.read_text()) - assert cfg["foo"]["u1"]["output_type"] == "metric" - - -def test_generated_snakefile_parses_with_snakemake(tmp_path: Path) -> None: - """End-to-end: the generated Snakefile must be valid Snakemake. - - Recipe uses the v0.0.7 ``{output}`` placeholder, which is rendered - to a concrete path by the generator — Snakemake never sees a - placeholder. Any leftover Snakemake-style ``{output[0]}`` would - have been a substitution failure inside ``render_recipe``. - """ - _spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo > {output}/data.txt"}}]}, - ) - generate(tmp_path, universes=["u1"]) - - import subprocess - proc = subprocess.run( - [ - "snakemake", - "-s", - str(tmp_path / ".lightcone" / "Snakefile"), - "-d", - str(tmp_path), - "-n", - "--cores", - "1", - ], - capture_output=True, - text=True, - ) - assert proc.returncode == 0, ( - f"snakemake -n failed:\nSTDOUT:\n{proc.stdout}\nSTDERR:\n{proc.stderr}" - ) - - -# ============================================================================ -# Unit tests for render_recipe — the v0.0.7 template substitution function -# ============================================================================ - - -def test_render_substitutes_output() -> None: - out = render_recipe( - 'python s.py --out {output}', - inputs={}, - decisions={}, - output='results/u1/foo', - ) - assert out == 'python s.py --out results/u1/foo' - - -def test_render_substitutes_named_input() -> None: - out = render_recipe( - 'cat {inputs.upstream}', - inputs={'upstream': 'results/u1/upstream'}, - decisions={}, - output='results/u1/foo', - ) - assert out == 'cat results/u1/upstream' - - -def test_render_substitutes_decisions() -> None: - out = render_recipe( - 'python s.py --scaling {decisions.scaling} --seed {decisions.seed}', - inputs={}, - decisions={'scaling': 'standard', 'seed': '42'}, - output='out', - ) - assert out == 'python s.py --scaling standard --seed 42' - - -def test_render_inputs_joined_in_declaration_order() -> None: - out = render_recipe( - 'merge {inputs} > {output}/merged', - inputs={'a': '/p/a', 'b': '/p/b', 'c': '/p/c'}, - decisions={}, - output='/p/out', - ) - assert out == 'merge /p/a /p/b /p/c > /p/out/merged' - - -def test_render_handles_brace_escapes() -> None: - out = render_recipe( - 'awk \'{{print $1}}\' {inputs.x}', - inputs={'x': '/p/x'}, - decisions={}, - output='out', - ) - assert out == "awk '{print $1}' /p/x" - - -def test_render_rejects_undeclared_input() -> None: - with pytest.raises(KeyError, match='not declared'): - render_recipe('cat {inputs.missing}', inputs={}, decisions={}, output='out') - - -def test_render_rejects_undeclared_decision() -> None: - with pytest.raises(KeyError, match='not declared'): - render_recipe( - 'python s.py --x {decisions.missing}', inputs={}, decisions={}, output='out' - ) - - -def test_render_rejects_unknown_namespace() -> None: - with pytest.raises(ValueError, match='Unknown'): - render_recipe('echo {wildcards.universe}', inputs={}, decisions={}, output='out') - - -def test_render_rejects_format_spec() -> None: - with pytest.raises(ValueError, match='format'): - render_recipe('echo {output:s}', inputs={}, decisions={}, output='out') - - -def test_generate_substitutes_decisions_into_shell_command(tmp_path: Path) -> None: - """Output.decisions resolves through the universe to actual option - IDs that get substituted into {decisions.} placeholders in the - rendered shell command.""" - _spec( - tmp_path, - { - 'outputs': [ - { - 'id': 'foo', - 'decisions': ['scaling'], - 'recipe': {'command': 'python s.py --scaling {decisions.scaling}'}, - } - ], - 'decisions': { - 'scaling': { - 'label': 'scaling', - 'default': 'standard', - 'options': {'standard': {'label': 'std'}, 'minmax': {'label': 'mm'}}, - } - }, - }, - ) - (tmp_path / 'universes').mkdir(exist_ok=True) - (tmp_path / 'universes' / 'u1.yaml').write_text('decisions:\n scaling: minmax\n') - _, cfg_path = generate(tmp_path, universes=['u1']) - cfg = json.loads(cfg_path.read_text()) - assert '--scaling minmax' in cfg['foo']['u1']['shell_command'] - assert cfg['foo']['u1']['decisions'] == {'scaling': 'minmax'} - - -def test_qualified_input_uses_raw_id_in_run_rule_call(tmp_path: Path) -> None: - """Sub-analysis output references like 'sub.real' reach run_rule's - inputs dict with their dots intact, so write_manifest's - input_versions matches what verify walks. The Snakemake input slot - itself uses the safe key 'sub__real' (must be a Python identifier).""" - _spec( - tmp_path, - { - 'outputs': [ - { - 'id': 'downstream', - 'inputs': ['sub.real'], - 'recipe': {'command': 'cat {inputs.sub.real}'}, - } - ], - 'analyses': { - 'sub': { - 'outputs': [{'id': 'real', 'recipe': {'command': 'echo r'}}], - } - }, - }, - ) - snakefile, _ = generate(tmp_path, universes=['u1']) - text = snakefile.read_text() - # Snakemake input directive uses the safe key. - assert 'sub__real=' in text - # The run_rule(inputs=...) dict literal uses the raw id. - assert '"sub.real": Path(input.sub__real)' in text - - -def test_external_input_flows_to_manifest(tmp_path: Path) -> None: - """An analysis-level Input with ``source:`` must reach the rule's - Snakemake input slot AND ``run_rule(inputs=...)`` so write_manifest - fingerprints it — otherwise ``lc verify`` reports broken_chain - because the spec declares an input that the manifest never recorded. - Regression test for issue #90. - """ - (tmp_path / "data").mkdir() - (tmp_path / "data" / "table.txt").write_text("z mu\n0.1 38.0\n") - _spec( - tmp_path, - { - "inputs": [ - {"id": "union21_table", "type": "data", "source": "data/table.txt"}, - ], - "outputs": [ - { - "id": "map_fit", - "inputs": ["union21_table"], - "recipe": {"command": "cp {inputs.union21_table} {output}/copy.txt"}, - } - ], - }, - ) - snakefile, _ = generate(tmp_path, universes=["u1"]) - text = snakefile.read_text() - # The external source becomes a Snakemake input slot… - assert 'union21_table="data/table.txt"' in text - # …and threads through to run_rule's inputs dict so write_manifest - # records it under the raw declared id. - assert '"union21_table": Path(input.union21_table)' in text - - - - -# --------------------------------------------------------------------------- -# _git_remote() URL normalisation -# --------------------------------------------------------------------------- - - -class TestGitRemote: - """Probe ``_git_remote`` indirectly via subprocess monkeypatching. - - The function shells out to ``git config --get remote.origin.url``; - we capture the stdout it sees and check the normalisation. - """ - - @pytest.fixture - def git_remote(self, monkeypatch: pytest.MonkeyPatch): - import subprocess as sp - - from lightcone.engine import snakefile as sf - - def factory(stdout: str, returncode: int = 0): - class _R: - pass - r = _R() - r.stdout = stdout - r.returncode = returncode - r.stderr = "" - - def fake_run(*args: Any, **kwargs: Any): - return r - monkeypatch.setattr(sp, "run", fake_run) - return sf._git_remote(Path(".")) - - return factory - - def test_https_url_preserved(self, git_remote) -> None: - assert git_remote("https://github.com/dkn16/test\n") \ - == "https://github.com/dkn16/test" - - def test_ssh_url_normalised(self, git_remote) -> None: - assert git_remote("git@github.com:dkn16/test.git\n") \ - == "https://github.com/dkn16/test" - - def test_strips_dot_git_suffix(self, git_remote) -> None: - assert git_remote("https://gitlab.com/team/proj.git\n") \ - == "https://gitlab.com/team/proj" - - def test_returns_none_when_no_remote(self, git_remote) -> None: - assert git_remote("", returncode=1) is None - - def test_returns_none_when_empty_url(self, git_remote) -> None: - assert git_remote("\n") is None - - -def test_generate_kubernetes_runtime_unwrapped_with_registry_ref( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On the kubernetes runtime the worker pod runs the project image: - recipes stay unwrapped, and the image identity that flows into - code_version is the registry ref, so a Containerfile edit (new ref) - still triggers reruns.""" - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - _spec( - tmp_path, - { - "name": "proj", - "container": "Containerfile", - "outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}], - }, - ) - monkeypatch.setenv( - "LIGHTCONE_REGISTRY", "europe-west1-docker.pkg.dev/hub/images" - ) - _, cfg_path = generate(tmp_path, universes=["u1"], runtime="kubernetes") - entry = json.loads(cfg_path.read_text())["foo"]["u1"] - - assert "docker run" not in entry["shell_command"] - assert "podman" not in entry["shell_command"] - assert "echo foo" in entry["shell_command"] - # The declared spec is what the manifest records — unchanged. - assert entry["container_image"] == "Containerfile" - - # Same spec, docker runtime (local tag, no registry): different - # image identity → different code_version. - monkeypatch.delenv("LIGHTCONE_REGISTRY") - _, cfg_path2 = generate(tmp_path, universes=["u1"], runtime="docker") - entry2 = json.loads(cfg_path2.read_text())["foo"]["u1"] - assert entry["code_version"] != entry2["code_version"] diff --git a/tests/test_status.py b/tests/test_status.py deleted file mode 100644 index afcd3078..00000000 --- a/tests/test_status.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for engine/status.py — manifest-driven status walker.""" -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import pytest -import yaml - -from lightcone.engine.manifest import code_version, write_manifest -from lightcone.engine.status import OutputStatus, get_output_status - - -def _write_spec(project_root: Path, spec: dict[str, Any]) -> None: - project_root.mkdir(parents=True, exist_ok=True) - (project_root / "astra.yaml").write_text(yaml.safe_dump(spec)) - - -def _materialize( - project_root: Path, - output_id: str, - universe_id: str, - *, - recipe: str, - decisions: dict[str, Any] | None = None, - container_image: str | None = None, -) -> Path: - out = project_root / "results" / universe_id / output_id - out.mkdir(parents=True, exist_ok=True) - (out / "data.txt").write_text("output bytes") - cv = code_version( - recipe=recipe, - container_image=container_image, - decisions=decisions or {}, - ) - write_manifest( - output_dir=out, - inputs={}, - cfg={ - "output_id": output_id, - "universe_id": universe_id, - "recipe": recipe, - "container_image": container_image, - "decisions": decisions or {}, - "code_version": cv, - "git_sha": "abc", - "lc_version": "0.0", - }, - ) - return out - - -def test_status_missing_when_nothing_materialized(tmp_path: Path) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - ] - }, - ) - statuses = list(get_output_status(tmp_path, universe_id="u1")) - assert len(statuses) == 1 - assert statuses[0].output_id == "foo" - assert statuses[0].status == "missing" - - -def test_status_ok_when_manifest_matches(tmp_path: Path) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - ] - }, - ) - _materialize(tmp_path, "foo", "u1", recipe="echo foo") - statuses = list(get_output_status(tmp_path, universe_id="u1")) - assert statuses[0].status == "ok" - assert statuses[0].manifest is not None - - -def test_status_stale_when_recipe_changed(tmp_path: Path) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo NEW"}}, - ] - }, - ) - # Materialize with the OLD recipe - _materialize(tmp_path, "foo", "u1", recipe="echo old") - statuses = list(get_output_status(tmp_path, universe_id="u1")) - assert statuses[0].status == "stale" - - -def test_status_no_recipe_for_alias(tmp_path: Path) -> None: - """Outputs declared with `from:` (no recipe) are aliases — they are - materialized as a side-effect of their upstream and have no own status. - """ - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "alias_out", "from": "sub.real_out"}, - ], - "analyses": { - "sub": { - "outputs": [ - {"id": "real_out", "recipe": {"command": "echo r"}}, - ] - } - }, - }, - ) - statuses = {s.output_id: s for s in get_output_status(tmp_path, universe_id="u1")} - assert "alias_out" in statuses - assert statuses["alias_out"].status == "alias" - - -def test_status_walks_subanalyses(tmp_path: Path) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "root_out", "recipe": {"command": "echo r"}}, - ], - "analyses": { - "sub": { - "outputs": [ - {"id": "sub_out", "recipe": {"command": "echo s"}}, - ] - } - }, - }, - ) - statuses = list(get_output_status(tmp_path, universe_id="u1")) - ids = {s.output_id for s in statuses} - assert ids == {"root_out", "sub_out"} - - -def test_status_outputstatus_dataclass(tmp_path: Path) -> None: - """OutputStatus is the public dataclass; check its shape.""" - _write_spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, - ) - _materialize(tmp_path, "foo", "u1", recipe="echo foo") - [s] = get_output_status(tmp_path, universe_id="u1") - assert isinstance(s, OutputStatus) - assert s.output_id == "foo" - assert s.universe_id == "u1" - assert s.status == "ok" - assert s.output_dir.exists() - - -def test_status_universe_specific(tmp_path: Path) -> None: - """Asking for one universe must not pick up materializations from another.""" - _write_spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, - ) - _materialize(tmp_path, "foo", "u1", recipe="echo foo") - statuses = list(get_output_status(tmp_path, universe_id="u2")) - assert statuses[0].status == "missing" - - -def test_status_ok_on_kubernetes_deployment( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """On a hub, `lc run` hashes the registry ref into code_version; - status must resolve the image identity the same way or every - freshly materialized output reads as stale (live-hub regression).""" - from lightcone.engine.container import registry_image_ref - - monkeypatch.setattr(Path, "home", lambda: tmp_path / "home") - monkeypatch.setenv("DASK_GATEWAY__ADDRESS", "http://proxy/services/dask-gateway") - monkeypatch.setenv( - "LIGHTCONE_REGISTRY", "europe-west1-docker.pkg.dev/hub/images" - ) - - _write_spec( - tmp_path, - { - "name": "proj", - "container": "Containerfile", - "outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}], - }, - ) - (tmp_path / "Containerfile").write_text("FROM python:3.12-slim\n") - ref = registry_image_ref( - "proj", - tmp_path / "Containerfile", - tmp_path, - registry="europe-west1-docker.pkg.dev/hub/images", - ) - _materialize( - tmp_path, "foo", "baseline", recipe="echo foo", container_image=ref - ) - - statuses = list(get_output_status(tmp_path, universe_id="baseline")) - assert [s.status for s in statuses] == ["ok"] diff --git a/tests/test_templates.py b/tests/test_templates.py new file mode 100644 index 00000000..50685498 --- /dev/null +++ b/tests/test_templates.py @@ -0,0 +1,96 @@ +"""Tests for `lightcone.engine.templates` — the scaffold's file templates.""" + +from __future__ import annotations + +import pytest + +from lightcone.engine import templates + + +def test_every_declared_template_is_shipped() -> None: + """The templates are package *data*, so a packaging slip would only + show up at `lc init` time. Read them all here instead.""" + for name in templates.TEMPLATE_NAMES: + assert templates.read(name).strip(), f"{name} is empty" + + +def test_unknown_template_fails_loudly() -> None: + with pytest.raises(KeyError, match="unknown template"): + templates.read("nope.tmpl") + + +def test_pyproject_renders_every_placeholder() -> None: + from lightcone.engine.constants import DEFAULT_PYTHON_FLOOR, MIN_UV_VERSION + + rendered = templates.pyproject(name="my-analysis") + assert "$" not in rendered + assert 'name = "my-analysis"' in rendered + assert f'requires-python = ">={DEFAULT_PYTHON_FLOOR}"' in rendered + assert f'required-version = ">={MIN_UV_VERSION}"' in rendered + assert templates.lightcone_requirement() in rendered + + +def test_pyproject_is_virtual() -> None: + """Containerized mode builds `--no-install-project`, so a packaged + project's own import would fail inside its image (spec §1).""" + assert "[build-system]" not in templates.pyproject(name="x") + + +def test_lightcone_requirement_pins_the_running_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Driver and project stay in lockstep — except for dev builds, whose + versions aren't published, so pinning one would make the project's lock + unsolvable.""" + import importlib.metadata + + monkeypatch.setattr(importlib.metadata, "version", lambda _name: "1.2.3") + assert templates.lightcone_requirement() == "lightcone-cli==1.2.3" + + monkeypatch.setattr(importlib.metadata, "version", lambda _name: "1.2.3.dev4+gabc") + assert templates.lightcone_requirement() == "lightcone-cli" + + def _missing(_name: str) -> str: + raise importlib.metadata.PackageNotFoundError + + monkeypatch.setattr(importlib.metadata, "version", _missing) + assert templates.lightcone_requirement() == "lightcone-cli" + + +def test_index_md_renders_the_title_and_keeps_myst_roles() -> None: + """The report body legitimately contains braces (`{astra}` roles) — + which is why substitution is `string.Template`, not `str.format`.""" + rendered = templates.index_md(title="My Analysis") + assert rendered.startswith("# My Analysis\n") + assert "{astra}`decisions.example_method`" in rendered + assert "{astra:value}`outputs.main_result`" in rendered + assert "$" not in rendered + + +def test_gitignore_entries_are_the_patterns_only() -> None: + """Convergence compares patterns, so comments and blanks must not leak + into the set — a comment treated as an entry would be re-appended + forever.""" + entries = templates.gitignore_entries() + assert not any(e.startswith("#") or not e.strip() for e in entries) + assert ".venv/" in entries + assert "results/*" in entries + assert "!results/README.md" in entries + + +def test_gitignore_entries_keep_template_order() -> None: + """`!results/README.md` only works after the `results/*` it negates.""" + entries = templates.gitignore_entries() + assert entries.index("results/*") < entries.index("!results/README.md") + + +def test_missing_entries_ignores_comments_in_the_target() -> None: + """A pattern that appears only inside a comment is not present.""" + assert "results/*" in templates.missing_gitignore_entries("# results/*\n") + assert templates.missing_gitignore_entries(templates.gitignore()) == [] + + +def test_results_readme_explains_the_output_layout() -> None: + """`results/` is git-ignored and starts empty, so the README is the + only thing a clone shows for it.""" + assert "results///" in templates.results_readme() diff --git a/tests/test_tree.py b/tests/test_tree.py deleted file mode 100644 index 662404d0..00000000 --- a/tests/test_tree.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Tests for analysis tree helpers — pure spec walking, no orchestrator.""" -from __future__ import annotations - -from pathlib import Path - -import pytest -from astra.helpers import load_yaml, resolve_analysis_tree - -from lightcone.engine.tree import ( - collect_tree_outputs, - resolve_output_path, -) - - -@pytest.fixture -def project(tmp_path: Path) -> Path: - (tmp_path / "astra.yaml").write_text( - """ -version: "1.0" -name: "Test" -inputs: - - id: raw - type: data - source: /tmp/raw.csv -outputs: - - id: root_out - recipe: - command: echo r -analyses: - feat: - outputs: - - id: features - recipe: - command: echo f -""" - ) - return tmp_path - - -def test_collect_tree_outputs(project: Path) -> None: - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - outs = collect_tree_outputs(spec) - ids = {(o.analysis_id, o.output_id) for o in outs} - assert (None, "root_out") in ids - assert ("feat", "features") in ids - - -def test_resolve_output_path_root(project: Path) -> None: - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - [root] = [o for o in collect_tree_outputs(spec) if o.analysis_id is None] - p = resolve_output_path(project, root, "u1") - # Root outputs land under /results// - assert p == project / "results" / "u1" - - -def test_resolve_output_path_sub_analysis_inline(project: Path) -> None: - """Sub-analyses without an explicit `path:` share the root results dir.""" - spec = resolve_analysis_tree(load_yaml(project / "astra.yaml"), project) - [sub] = [o for o in collect_tree_outputs(spec) if o.analysis_id == "feat"] - p = resolve_output_path(project, sub, "u1") - assert p == project / "results" / "u1" diff --git a/tests/test_validation.py b/tests/test_validation.py deleted file mode 100644 index 58a3797a..00000000 --- a/tests/test_validation.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Tests for post-materialization result file validation.""" -from __future__ import annotations - -import json -import math -from pathlib import Path - -from lightcone.engine.validation import validate_output - - -class TestValidateOutputCommon: - def test_missing_directory_warns(self, tmp_path: Path) -> None: - warnings = validate_output(tmp_path / "nonexistent", "metric", "my_output") - assert len(warnings) == 1 - assert "missing" in warnings[0] - - def test_empty_directory_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "output" - out_dir.mkdir() - warnings = validate_output(out_dir, "metric", "my_output") - assert len(warnings) == 1 - assert "empty" in warnings[0] - - def test_unknown_type_skips_content_check(self, tmp_path: Path) -> None: - out_dir = tmp_path / "output" - out_dir.mkdir() - (out_dir / "data.bin").write_bytes(b"\x00" * 100) - assert validate_output(out_dir, "data", "my_output") == [] - - def test_none_type_skips_content_check(self, tmp_path: Path) -> None: - out_dir = tmp_path / "output" - out_dir.mkdir() - (out_dir / "data.bin").write_bytes(b"\x00" * 100) - assert validate_output(out_dir, None, "my_output") == [] - - -class TestValidateMetric: - def test_valid_json_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text(json.dumps({"value": 0.95, "count": 100})) - assert validate_output(out_dir, "metric", "result") == [] - - def test_missing_json_file_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "output.txt").write_text("some text") - warnings = validate_output(out_dir, "metric", "result") - assert len(warnings) == 1 - assert "no JSON files" in warnings[0] - - def test_invalid_json_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text("not valid json {{{") - warnings = validate_output(out_dir, "metric", "result") - assert len(warnings) == 1 - assert "not valid JSON" in warnings[0] - - def test_all_null_dict_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text(json.dumps({"a": None, "b": None})) - warnings = validate_output(out_dir, "metric", "result") - assert len(warnings) == 1 - assert "null/NaN" in warnings[0] - - def test_null_scalar_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text("null") - warnings = validate_output(out_dir, "metric", "result") - assert len(warnings) == 1 - assert "null/NaN" in warnings[0] - - def test_partial_null_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text(json.dumps({"a": None, "b": 1.0})) - assert validate_output(out_dir, "metric", "result") == [] - - def test_empty_dict_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text("{}") - assert validate_output(out_dir, "metric", "result") == [] - - def test_nested_all_null_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "metric.json").write_text(json.dumps({"a": {"x": None, "y": None}})) - warnings = validate_output(out_dir, "metric", "result") - assert len(warnings) == 1 - assert "null/NaN" in warnings[0] - - -class TestValidateTable: - def test_valid_csv_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "table.csv").write_text("col_a,col_b\n1.0,2.0\n3.0,4.0\n") - assert validate_output(out_dir, "table", "result") == [] - - def test_missing_csv_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "output.txt").write_text("some text") - warnings = validate_output(out_dir, "table", "result") - assert len(warnings) == 1 - assert "no CSV files" in warnings[0] - - def test_all_nan_all_columns_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - nan = math.nan - (out_dir / "table.csv").write_text(f"col_a,col_b\n{nan},{nan}\n{nan},{nan}\n") - warnings = validate_output(out_dir, "table", "result") - assert len(warnings) == 1 - assert "all-NaN" in warnings[0] - assert "every numeric column" in warnings[0] - - def test_all_nan_single_column_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - nan = math.nan - (out_dir / "table.csv").write_text(f"col_a,col_b\n1.0,{nan}\n2.0,{nan}\n") - warnings = validate_output(out_dir, "table", "result") - assert len(warnings) == 1 - assert "col_b" in warnings[0] - assert "every numeric column" not in warnings[0] - - def test_empty_csv_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "table.csv").write_text("col_a,col_b\n") - warnings = validate_output(out_dir, "table", "result") - assert len(warnings) == 1 - assert "no data rows" in warnings[0] - - def test_non_numeric_columns_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "table.csv").write_text("name,label\nfoo,bar\nbaz,qux\n") - assert validate_output(out_dir, "table", "result") == [] - - def test_mixed_valid_and_nan_rows_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - nan = math.nan - (out_dir / "table.csv").write_text(f"col_a,col_b\n1.0,{nan}\n{nan},4.0\n") - assert validate_output(out_dir, "table", "result") == [] - - -class TestValidateFigure: - def test_valid_png_no_warning(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "figure.png").write_bytes(b"\x89PNG fake content") - assert validate_output(out_dir, "figure", "result") == [] - - def test_missing_image_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "output.txt").write_text("some text") - warnings = validate_output(out_dir, "figure", "result") - assert len(warnings) == 1 - assert "no image files" in warnings[0] - - def test_empty_image_file_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "figure.png").write_bytes(b"") - warnings = validate_output(out_dir, "figure", "result") - assert len(warnings) == 1 - assert "empty" in warnings[0] - assert "0 bytes" in warnings[0] - - def test_svg_accepted(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "figure.svg").write_text("content") - assert validate_output(out_dir, "figure", "result") == [] - - def test_pdf_accepted(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "figure.pdf").write_bytes(b"%PDF-1.4 fake content") - assert validate_output(out_dir, "figure", "result") == [] - - def test_multiple_figures_partial_empty_warns(self, tmp_path: Path) -> None: - out_dir = tmp_path / "result" - out_dir.mkdir() - (out_dir / "fig1.png").write_bytes(b"\x89PNG fake content") - (out_dir / "fig2.png").write_bytes(b"") - warnings = validate_output(out_dir, "figure", "result") - assert len(warnings) == 1 - assert "fig2.png" in warnings[0] diff --git a/tests/test_verify.py b/tests/test_verify.py deleted file mode 100644 index 37ef6dca..00000000 --- a/tests/test_verify.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Tests for engine/verify.py — the integrity checker.""" -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import yaml - -from lightcone.engine.manifest import ( - MANIFEST_FILENAME, - code_version, - write_manifest, -) -from lightcone.engine.verify import VerifyResult, verify_outputs - - -def _spec(project_root: Path, spec: dict[str, Any]) -> None: - project_root.mkdir(parents=True, exist_ok=True) - (project_root / "astra.yaml").write_text(yaml.safe_dump(spec)) - - -def _materialize( - project_root: Path, - output_id: str, - universe_id: str, - *, - recipe: str = "echo hi", - inputs: dict[str, Path] | None = None, -) -> Path: - out = project_root / "results" / universe_id / output_id - out.mkdir(parents=True, exist_ok=True) - (out / "data.txt").write_text(f"output of {output_id}") - write_manifest( - output_dir=out, - inputs=inputs or {}, - cfg={ - "output_id": output_id, - "universe_id": universe_id, - "recipe": recipe, - "container_image": None, - "decisions": {}, - "code_version": code_version( - recipe=recipe, container_image=None, decisions={} - ), - "git_sha": "g", - "lc_version": "0.0", - }, - ) - return out - - -def test_verify_clean_chain_passes(tmp_path: Path) -> None: - _spec( - tmp_path, - { - "outputs": [ - {"id": "upstream", "recipe": {"command": "echo u"}}, - { - "id": "downstream", - "inputs": ["upstream"], - "recipe": {"command": "echo d"}, - }, - ] - }, - ) - up = _materialize(tmp_path, "upstream", "u1", recipe="echo u") - _materialize(tmp_path, "downstream", "u1", recipe="echo d", inputs={"upstream": up}) - - results = list(verify_outputs(tmp_path, universe_id="u1")) - assert all(r.passed for r in results), [r for r in results if not r.passed] - assert {r.output_id for r in results} == {"upstream", "downstream"} - - -def test_verify_detects_tampered_data(tmp_path: Path) -> None: - _spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo f"}}]}, - ) - out = _materialize(tmp_path, "foo", "u1", recipe="echo f") - # Tamper with the output AFTER the manifest was written. - (out / "data.txt").write_text("agent forged this") - [r] = list(verify_outputs(tmp_path, universe_id="u1")) - assert not r.passed - assert r.failure == "tampered_data" - - -def test_verify_detects_broken_chain(tmp_path: Path) -> None: - _spec( - tmp_path, - { - "outputs": [ - {"id": "upstream", "recipe": {"command": "echo u"}}, - { - "id": "downstream", - "inputs": ["upstream"], - "recipe": {"command": "echo d"}, - }, - ] - }, - ) - up = _materialize(tmp_path, "upstream", "u1", recipe="echo u") - _materialize(tmp_path, "downstream", "u1", recipe="echo d", inputs={"upstream": up}) - - # Re-materialize upstream (gives it a NEW data_version) without - # re-running downstream. Downstream's recorded input_version no longer - # matches upstream's current data_version → broken chain. - (up / "data.txt").write_text("upstream changed") - write_manifest( - output_dir=up, - inputs={}, - cfg={ - "output_id": "upstream", - "universe_id": "u1", - "recipe": "echo u", - "container_image": None, - "decisions": {}, - "code_version": code_version( - recipe="echo u", container_image=None, decisions={} - ), - "git_sha": "g", - "lc_version": "0.0", - }, - ) - - results = {r.output_id: r for r in verify_outputs(tmp_path, universe_id="u1")} - assert results["upstream"].passed - assert not results["downstream"].passed - assert results["downstream"].failure == "broken_chain" - - -def test_verify_detects_missing_manifest(tmp_path: Path) -> None: - _spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo f"}}]}, - ) - # Drop a fake output without going through write_manifest. This is the - # agent-forged-file scenario. - out = tmp_path / "results" / "u1" / "foo" - out.mkdir(parents=True, exist_ok=True) - (out / "data.txt").write_text("forged") - - [r] = list(verify_outputs(tmp_path, universe_id="u1")) - assert not r.passed - assert r.failure == "missing_manifest" - - -def test_verify_detects_corrupt_manifest(tmp_path: Path) -> None: - _spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo f"}}]}, - ) - out = _materialize(tmp_path, "foo", "u1", recipe="echo f") - (out / MANIFEST_FILENAME).write_text("not json") - [r] = list(verify_outputs(tmp_path, universe_id="u1")) - assert not r.passed - assert r.failure == "missing_manifest" - - -def test_verify_skips_aliases(tmp_path: Path) -> None: - """Outputs declared with `from:` (no recipe) are aliases. They have no - own materialization, so verify must not flag them as failures.""" - _spec( - tmp_path, - { - "outputs": [{"id": "alias", "from": "sub.real"}], - "analyses": { - "sub": {"outputs": [{"id": "real", "recipe": {"command": "echo r"}}]} - }, - }, - ) - _materialize(tmp_path, "real", "u1", recipe="echo r") - results = list(verify_outputs(tmp_path, universe_id="u1")) - ids = {r.output_id for r in results} - # Only the real output is verified; aliases are skipped. - assert "real" in ids - assert "alias" not in ids - - -def test_verify_detects_broken_chain_for_qualified_input(tmp_path: Path) -> None: - """``Output.inputs`` referencing a sub-analysis output by qualified - id (``sub.real``) must resolve through to the producing manifest so - a drifted upstream surfaces as ``broken_chain``. - """ - sub_dir = tmp_path / "sub" - (sub_dir / "results" / "u1" / "real").mkdir(parents=True, exist_ok=True) - _spec( - tmp_path, - { - "outputs": [ - { - "id": "downstream", - "inputs": ["sub.real"], - "recipe": {"command": "echo d"}, - } - ], - "analyses": { - "sub": { - "path": "./sub", - "outputs": [{"id": "real", "recipe": {"command": "echo r"}}], - } - }, - }, - ) - - # Materialize the upstream sub-analysis output at its real on-disk path. - up = sub_dir / "results" / "u1" / "real" - (up / "data.txt").write_text("upstream v1") - write_manifest( - output_dir=up, - inputs={}, - cfg={ - "output_id": "real", - "universe_id": "u1", - "recipe": "echo r", - "container_image": None, - "decisions": {}, - "code_version": code_version( - recipe="echo r", container_image=None, decisions={} - ), - "git_sha": "g", - "lc_version": "0.0", - }, - ) - # Materialize downstream; its recorded input_version chains to up. - _materialize( - tmp_path, "downstream", "u1", recipe="echo d", inputs={"sub.real": up} - ) - - # Now mutate the upstream and rewrite its manifest so its - # ``data_version`` drifts. Downstream's chain must detect this. - (up / "data.txt").write_text("upstream v2") - write_manifest( - output_dir=up, - inputs={}, - cfg={ - "output_id": "real", - "universe_id": "u1", - "recipe": "echo r", - "container_image": None, - "decisions": {}, - "code_version": code_version( - recipe="echo r", container_image=None, decisions={} - ), - "git_sha": "g", - "lc_version": "0.0", - }, - ) - - results = {r.output_id: r for r in verify_outputs(tmp_path, universe_id="u1")} - assert not results["downstream"].passed - assert results["downstream"].failure == "broken_chain", ( - "qualified-id (sub.real) inputs must be resolved through the " - "tree, not silently treated as external." - ) - - -def test_verify_passes_with_external_input(tmp_path: Path) -> None: - """An output whose declared inputs include an analysis-level Input - (resolved to a file via ``source:``) must verify cleanly: the - Snakefile generator threads the external path into ``run_rule``'s - ``inputs`` dict, so write_manifest fingerprints it. Without that - plumbing, verify reports broken_chain. Regression for issue #90.""" - src = tmp_path / "data" / "table.txt" - src.parent.mkdir() - src.write_text("z mu\n0.1 38.0\n") - _spec( - tmp_path, - { - "inputs": [ - {"id": "union21_table", "type": "data", "source": "data/table.txt"}, - ], - "outputs": [ - { - "id": "map_fit", - "inputs": ["union21_table"], - "recipe": {"command": "cp"}, - } - ], - }, - ) - _materialize( - tmp_path, "map_fit", "u1", recipe="cp", inputs={"union21_table": src} - ) - - [r] = list(verify_outputs(tmp_path, universe_id="u1")) - assert r.passed, f"unexpected failure: {r.failure} — {r.detail}" - - -def test_verifyresult_dataclass(tmp_path: Path) -> None: - _spec(tmp_path, {"outputs": [{"id": "foo", "recipe": {"command": "echo f"}}]}) - _materialize(tmp_path, "foo", "u1", recipe="echo f") - [r] = list(verify_outputs(tmp_path, universe_id="u1")) - assert isinstance(r, VerifyResult) - assert r.output_id == "foo" - assert r.passed - assert r.failure is None diff --git a/tests/test_wrroc.py b/tests/test_wrroc.py deleted file mode 100644 index 733fadda..00000000 --- a/tests/test_wrroc.py +++ /dev/null @@ -1,785 +0,0 @@ -"""Tests for engine/wrroc.py — Workflow Run RO-Crate exporter.""" -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import pytest -import yaml -from click.testing import CliRunner - -from lightcone.cli.commands import main -from lightcone.engine.manifest import code_version, write_manifest -from lightcone.engine.wrroc import ( - PROVENANCE_RUN_CRATE_PROFILE, - ExportResult, - export_wrroc, -) - -# --------------------------------------------------------------------------- -# Fixtures: tiny project + materialized outputs -# --------------------------------------------------------------------------- - - -def _write_spec(project: Path, spec: dict[str, Any]) -> None: - project.mkdir(parents=True, exist_ok=True) - (project / "astra.yaml").write_text(yaml.safe_dump(spec)) - - -def _write_universe(project: Path, universe_id: str, decisions: dict[str, Any]) -> None: - udir = project / "universes" - udir.mkdir(parents=True, exist_ok=True) - (udir / f"{universe_id}.yaml").write_text( - yaml.safe_dump({"decisions": decisions}) - ) - - -def _materialize( - project: Path, - output_id: str, - universe_id: str, - *, - recipe: str, - decisions: dict[str, Any] | None = None, - container_image: str | None = None, - inputs: dict[str, Path] | None = None, - body: str = "output bytes", -) -> Path: - out = project / "results" / universe_id / output_id - out.mkdir(parents=True, exist_ok=True) - (out / "data.txt").write_text(body) - cv = code_version( - recipe=recipe, - container_image=container_image, - decisions=decisions or {}, - ) - write_manifest( - output_dir=out, - inputs=inputs or {}, - cfg={ - "output_id": output_id, - "universe_id": universe_id, - "recipe": recipe, - "container_image": container_image, - "decisions": decisions or {}, - "code_version": cv, - "git_sha": "abc1234", - "lc_version": "0.0.1", - }, - ) - return out - - -@pytest.fixture -def minimal_project(tmp_path: Path) -> Path: - """A project with one universe and one materialized output.""" - _write_spec( - tmp_path, - { - "name": "minimal", - "description": "test", - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo > data.txt"}}, - ], - }, - ) - _write_universe(tmp_path, "baseline", {}) - _materialize(tmp_path, "foo", "baseline", recipe="echo foo > data.txt") - return tmp_path - - -@pytest.fixture -def chained_project(tmp_path: Path) -> Path: - """Two-step DAG: step_b depends on step_a.""" - _write_spec( - tmp_path, - { - "name": "chained", - "description": "Two-step chained DAG for WRROC tests.", - # ASTRA's decisions schema: dict keyed by decision id, with - # options also a dict keyed by option id. - "decisions": { - "method": { - "label": "Method", - "default": "A", - "options": { - "A": {"label": "Option A"}, - "B": {"label": "Option B"}, - }, - }, - }, - "outputs": [ - {"id": "step_a", "recipe": {"command": "echo a > data.txt"}}, - { - "id": "step_b", - "inputs": ["step_a"], - "recipe": {"command": "cat data/step_a/data.txt > data.txt"}, - }, - ], - }, - ) - _write_universe(tmp_path, "baseline", {"method": "A"}) - - out_a = _materialize( - tmp_path, "step_a", "baseline", - recipe="echo a > data.txt", - decisions={"method": "A"}, - ) - _materialize( - tmp_path, "step_b", "baseline", - recipe="cat data/step_a/data.txt > data.txt", - decisions={"method": "A"}, - inputs={"step_a": out_a}, - ) - return tmp_path - - -# --------------------------------------------------------------------------- -# Module-level tests -# --------------------------------------------------------------------------- - - -class TestMinimalExport: - def test_returns_export_result(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - result = export_wrroc(minimal_project, out, author="Tester ") - assert isinstance(result, ExportResult) - assert result.bundle_path == out - assert result.runs_included == 1 - assert result.universes_included == ["baseline"] - assert result.is_zip is False - - def test_bundle_has_metadata_file(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="Tester ") - assert (out / "ro-crate-metadata.json").is_file() - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - assert "@context" in meta - assert "@graph" in meta - - def test_bundle_includes_astra_yaml(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="Tester ") - assert (out / "astra.yaml").is_file() - - def test_root_conforms_to_wrroc_profiles(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="Tester ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - root = next(g for g in meta["@graph"] if g["@id"] == "./") - conforms_ids = [c["@id"] for c in root["conformsTo"]] - assert PROVENANCE_RUN_CRATE_PROFILE in conforms_ids - - -class TestChainPreserved: - def test_step_b_object_references_step_a_dataset( - self, chained_project: Path - ) -> None: - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="Tester ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - - # Find step_b's CreateAction - actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] - step_b_action = next(a for a in actions if "step_b" in a["@id"]) - - # Its `object` list should include step_a's dataset @id - object_ids = [o["@id"] for o in step_b_action["object"]] - assert "results/baseline/step_a/" in object_ids - - def test_both_steps_have_create_actions(self, chained_project: Path) -> None: - out = chained_project / "wrroc" - result = export_wrroc(chained_project, out, author="X ") - assert result.runs_included == 2 - - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] - ids = {a["@id"] for a in actions} - assert any("step_a" in i for i in ids) - assert any("step_b" in i for i in ids) - - -class TestDecisionsAttached: - def test_decisions_emitted_as_property_values( - self, chained_project: Path - ) -> None: - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - - pvs = [g for g in meta["@graph"] if g.get("@type") == "PropertyValue"] - method_pvs = [p for p in pvs if p.get("name") == "method"] - assert len(method_pvs) >= 1 - assert all(p["value"] == "A" for p in method_pvs) - - def test_complex_decision_value_is_serialized(self, tmp_path: Path) -> None: - """Non-primitive decision values must be JSON-serialized for - PropertyValue.value compatibility. - """ - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - ] - }, - ) - _write_universe(tmp_path, "u1", {"opts": {"a": 1, "b": [2, 3]}}) - _materialize( - tmp_path, "foo", "u1", - recipe="echo foo", - decisions={"opts": {"a": 1, "b": [2, 3]}}, - ) - - export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") - meta = json.loads((tmp_path / "wrroc" / "ro-crate-metadata.json").read_text()) - opts_pv = next( - g for g in meta["@graph"] - if g.get("@type") == "PropertyValue" and g.get("name") == "opts" - ) - # Coerced to a JSON string - assert isinstance(opts_pv["value"], str) - assert json.loads(opts_pv["value"]) == {"a": 1, "b": [2, 3]} - - -class TestRoundTrip: - def test_load_via_rocrate_py(self, chained_project: Path) -> None: - """A bundle we wrote must be loadable by rocrate.Crate(path).""" - from rocrate.rocrate import ROCrate - - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="X ") - - crate = ROCrate(out) - assert crate.name == "chained" - actions = crate.get_by_type("CreateAction") - assert len(actions) == 2 - - def test_workflow_is_main_entity(self, chained_project: Path) -> None: - from rocrate.rocrate import ROCrate - - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="X ") - crate = ROCrate(out) - assert crate.mainEntity is not None - assert crate.mainEntity.id == "astra.yaml" - - -class TestMetadataOnly: - def test_skips_data_files(self, chained_project: Path) -> None: - out = chained_project / "wrroc" - export_wrroc( - chained_project, out, author="X ", include_data=False, - ) - # data.txt files should NOT be copied - assert not (out / "results" / "baseline" / "step_a" / "data.txt").exists() - # but manifests SHOULD be - assert ( - out / "results" / "baseline" / "step_a" / ".lightcone-manifest.json" - ).is_file() - - def test_chain_still_valid_in_metadata(self, chained_project: Path) -> None: - """Even without data files, the @id chain must still link upstream.""" - out = chained_project / "wrroc" - export_wrroc( - chained_project, out, author="X ", include_data=False, - ) - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] - step_b = next(a for a in actions if "step_b" in a["@id"]) - ids = [o["@id"] for o in step_b["object"]] - assert "results/baseline/step_a/" in ids - - -class TestZipBundle: - def test_produces_zip(self, minimal_project: Path) -> None: - zip_path = minimal_project / "bundle.zip" - result = export_wrroc( - minimal_project, zip_path, author="X ", zip_bundle=True, - ) - assert result.is_zip is True - assert zip_path.is_file() - # Contains ro-crate-metadata.json - import zipfile - with zipfile.ZipFile(zip_path) as zf: - names = zf.namelist() - assert any("ro-crate-metadata.json" in n for n in names) - - def test_zip_raises_on_existing_directory(self, minimal_project: Path) -> None: - dir_path = minimal_project / "existing_dir" - dir_path.mkdir() - with pytest.raises(FileExistsError, match="existing directory"): - export_wrroc(minimal_project, dir_path, author="X ", zip_bundle=True) - - -class TestAuthor: - def test_explicit_author_overrides(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="Alice ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - persons = [g for g in meta["@graph"] if g.get("@type") == "Person"] - assert len(persons) == 1 - assert persons[0]["name"] == "Alice" - assert persons[0]["email"] == "a@b.c" - - def test_author_used_as_action_agent(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="Alice ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - action = next(g for g in meta["@graph"] if g.get("@type") == "CreateAction") - assert action["agent"]["@id"] == "#author-a_at_b.c" - - -class TestUniverseFilter: - def test_restricts_to_listed_universes(self, tmp_path: Path) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - ] - }, - ) - _write_universe(tmp_path, "u1", {}) - _write_universe(tmp_path, "u2", {}) - _materialize(tmp_path, "foo", "u1", recipe="echo foo") - _materialize(tmp_path, "foo", "u2", recipe="echo foo") - - result = export_wrroc( - tmp_path, tmp_path / "wrroc", - universes=["u1"], author="X ", - ) - assert result.universes_included == ["u1"] - assert result.runs_included == 1 - - -class TestEmptyProject: - def test_no_materializations_warns_but_succeeds( - self, tmp_path: Path, - ) -> None: - _write_spec( - tmp_path, - { - "outputs": [ - {"id": "foo", "recipe": {"command": "echo foo"}}, - ] - }, - ) - _write_universe(tmp_path, "u1", {}) - # No materialization - - result = export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") - assert result.runs_included == 0 - # Bundle still has the workflow definition - assert (tmp_path / "wrroc" / "astra.yaml").is_file() - - -class TestRefuseClobber: - def test_non_empty_target_dir_errors(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - out.mkdir() - (out / "existing.txt").write_text("hi") - with pytest.raises(FileExistsError): - export_wrroc(minimal_project, out, author="X ") - - -class TestSubAnalyses: - """Sub-analysis outputs must be captured with the correct path-rooted - `@id`s and the chain back to root outputs preserved. - """ - - @pytest.fixture - def subanalysis_project(self, tmp_path: Path) -> Path: - # Root project declares a sub-analysis at analyses/sub/ - _write_spec( - tmp_path, - { - "name": "with-subs", - "description": "Project with one sub-analysis.", - "outputs": [ - {"id": "root_out", "recipe": {"command": "echo r"}}, - ], - "analyses": { - "sub": {"path": "./analyses/sub"}, - }, - }, - ) - # Sub-analysis has its own astra.yaml. - sub_dir = tmp_path / "analyses" / "sub" - sub_dir.mkdir(parents=True) - (sub_dir / "astra.yaml").write_text(yaml.safe_dump({ - "name": "sub", - "description": "Sub-analysis.", - "outputs": [ - {"id": "sub_out", "recipe": {"command": "echo s"}}, - ], - })) - _write_universe(tmp_path, "baseline", {}) - - # Materialize root output at /results/baseline/root_out/ - _materialize(tmp_path, "root_out", "baseline", recipe="echo r") - - # Materialize sub-analysis output at /results/baseline/sub_out/ - sub_out_dir = sub_dir / "results" / "baseline" / "sub_out" - sub_out_dir.mkdir(parents=True) - (sub_out_dir / "data.txt").write_text("sub bytes") - cv = code_version(recipe="echo s", container_image=None, decisions={}) - write_manifest( - output_dir=sub_out_dir, - inputs={}, - cfg={"output_id": "sub_out", "universe_id": "baseline", - "recipe": "echo s", "container_image": None, - "decisions": {}, "code_version": cv, - "git_sha": "abc", "lc_version": "0.0.1"}, - ) - return tmp_path - - def test_both_root_and_sub_outputs_captured( - self, subanalysis_project: Path, - ) -> None: - out = subanalysis_project / "wrroc" - result = export_wrroc(subanalysis_project, out, author="X ") - assert result.runs_included == 2 - - def test_sub_dataset_id_includes_sub_path( - self, subanalysis_project: Path, - ) -> None: - out = subanalysis_project / "wrroc" - export_wrroc(subanalysis_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - ids = {g["@id"] for g in meta["@graph"]} - # Root dataset uses results/// - assert "results/baseline/root_out/" in ids - # Sub-analysis dataset uses /results/// - assert "analyses/sub/results/baseline/sub_out/" in ids - - def test_sub_create_action_id_qualified( - self, subanalysis_project: Path, - ) -> None: - """CreateAction @ids include the analysis_id qualifier so sub - and root outputs with the same id never collide. - """ - out = subanalysis_project / "wrroc" - export_wrroc(subanalysis_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - action_ids = { - g["@id"] for g in meta["@graph"] - if g.get("@type") == "CreateAction" - } - assert "#run-baseline-root_out" in action_ids - assert "#run-baseline-sub.sub_out" in action_ids - - def test_sub_data_files_bundled( - self, subanalysis_project: Path, - ) -> None: - out = subanalysis_project / "wrroc" - export_wrroc(subanalysis_project, out, author="X ") - # Sub-analysis data file should be copied at the corresponding - # path inside the bundle. - assert ( - out / "analyses" / "sub" / "results" / "baseline" - / "sub_out" / "data.txt" - ).is_file() - - -class TestToolName: - """SoftwareApplication.name resolution: tool_name > heuristic > output_id.""" - - def _project_with_recipe( - self, - tmp_path: Path, - *, - recipe_command: str, - tool_name: str | None = None, - ) -> Path: - recipe: dict[str, Any] = {"command": recipe_command} - if tool_name: - recipe["tool_name"] = tool_name - _write_spec( - tmp_path, - { - "outputs": [{"id": "foo", "recipe": recipe}], - }, - ) - _write_universe(tmp_path, "u1", {}) - _materialize(tmp_path, "foo", "u1", recipe=recipe_command) - return tmp_path - - def _software_app(self, bundle: Path) -> dict[str, Any]: - meta = json.loads((bundle / "ro-crate-metadata.json").read_text()) - return next( - g for g in meta["@graph"] - if g.get("@type") == "SoftwareApplication" - and g["@id"].startswith("#recipe") - ) - - def test_explicit_tool_name_wins(self, tmp_path: Path) -> None: - project = self._project_with_recipe( - tmp_path, - recipe_command="python scripts/analyze.py --x 1", - tool_name="analyze (chi-squared)", - ) - out = project / "wrroc" - export_wrroc(project, out, author="X ") - sw = self._software_app(out) - assert sw["name"] == "analyze (chi-squared)" - assert sw["description"] == "python scripts/analyze.py --x 1" - - def test_heuristic_extracts_script_path(self, tmp_path: Path) -> None: - project = self._project_with_recipe( - tmp_path, - recipe_command="python scripts/analyze.py --x 1", - ) - out = project / "wrroc" - export_wrroc(project, out, author="X ") - sw = self._software_app(out) - assert sw["name"] == "scripts/analyze.py" - - def test_falls_back_to_output_id(self, tmp_path: Path) -> None: - # Recipe with no script-like token in it - project = self._project_with_recipe( - tmp_path, - recipe_command="echo hello", - ) - out = project / "wrroc" - export_wrroc(project, out, author="X ") - sw = self._software_app(out) - assert sw["name"] == "foo" # the output id - - -class TestGitRemote: - def test_emits_code_repository_entity(self, tmp_path: Path) -> None: - """When manifests carry git_remote, the bundle gets a CodeRepository.""" - _write_spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, - ) - _write_universe(tmp_path, "u1", {}) - out = tmp_path / "results" / "u1" / "foo" - out.mkdir(parents=True) - (out / "data.txt").write_text("bytes") - cv = code_version(recipe="echo foo", container_image=None, decisions={}) - write_manifest( - output_dir=out, inputs={}, - cfg={ - "output_id": "foo", "universe_id": "u1", - "recipe": "echo foo", "container_image": None, - "decisions": {}, "code_version": cv, - "git_sha": "abc", - "git_remote": "https://github.com/dkn16/test-repo", - "lc_version": "0.0.1", - }, - ) - - bundle = tmp_path / "wrroc" - export_wrroc(tmp_path, bundle, author="X ") - meta = json.loads((bundle / "ro-crate-metadata.json").read_text()) - - repos = [ - g for g in meta["@graph"] - if "CodeRepository" in ( - g["@type"] if isinstance(g["@type"], list) else [g["@type"]] - ) - ] - assert len(repos) == 1 - assert repos[0]["@id"] == "https://github.com/dkn16/test-repo" - assert repos[0]["url"] == "https://github.com/dkn16/test-repo" - - wf = next( - g for g in meta["@graph"] - if "ComputationalWorkflow" in ( - g["@type"] if isinstance(g["@type"], list) else [g["@type"]] - ) - ) - assert wf.get("codeRepository", {}).get("@id") == \ - "https://github.com/dkn16/test-repo" - - def test_no_git_remote_no_repo_entity(self, minimal_project: Path) -> None: - """Without git_remote in manifests, no CodeRepository is emitted.""" - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - for g in meta["@graph"]: - t = g.get("@type") - tlist = t if isinstance(t, list) else [t] - assert "CodeRepository" not in tlist - - -class TestUnreadableManifest: - def test_skips_permission_denied( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - ) -> None: - """If a manifest read raises OSError (permission, broken symlink), - the exporter must warn and skip rather than abort the whole run. - Mirrors what happens with cross-user symlinked results dirs. - """ - from lightcone.engine import wrroc as wrroc_mod - - _write_spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, - ) - _write_universe(tmp_path, "u1", {}) - # No manifest exists, but inject one that raises PermissionError. - from lightcone.engine import manifest as manifest_mod - - def boom(out_dir: Path) -> dict[str, Any] | None: - raise PermissionError(f"denied: {out_dir}") - - monkeypatch.setattr(manifest_mod, "read_manifest", boom) - # wrroc.py imports read_manifest by name, so patch there too. - monkeypatch.setattr(wrroc_mod, "read_manifest", boom) - - # Should not raise; should warn and produce a (mostly empty) bundle. - result = export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") - assert result.runs_included == 0 - - -class TestProfileConformance: - """The bundle's @graph must declare profile CreativeWork entities, - set a license, and include FormalParameter additionalType — the - Provenance Run Crate 0.5 validator's REQUIRED checks all hinge on - these. - """ - - def test_root_has_license(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - root = next(g for g in meta["@graph"] if g["@id"] == "./") - assert "license" in root - assert root["license"]["@id"].startswith("http") - - def test_explicit_license_passed_through(self, minimal_project: Path) -> None: - out = minimal_project / "wrroc" - export_wrroc( - minimal_project, out, author="X ", - license="https://opensource.org/licenses/MIT", - ) - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - root = next(g for g in meta["@graph"] if g["@id"] == "./") - assert root["license"]["@id"] == "https://opensource.org/licenses/MIT" - - def test_profile_creativework_entities_declared( - self, minimal_project: Path, - ) -> None: - """conformsTo profile URLs must each have a CreativeWork entity.""" - out = minimal_project / "wrroc" - export_wrroc(minimal_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - ids = {g["@id"] for g in meta["@graph"]} - assert PROVENANCE_RUN_CRATE_PROFILE in ids - - def test_formal_parameters_have_additional_type( - self, chained_project: Path, - ) -> None: - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - params = [g for g in meta["@graph"] if g.get("@type") == "FormalParameter"] - assert len(params) >= 1 - for p in params: - assert "additionalType" in p - assert p["additionalType"]["@id"].startswith("http://schema.org/") - - def test_workflow_haspart_recipes(self, chained_project: Path) -> None: - """ComputationalWorkflow MUST link recipes via hasPart.""" - out = chained_project / "wrroc" - export_wrroc(chained_project, out, author="X ") - meta = json.loads((out / "ro-crate-metadata.json").read_text()) - wf = next( - g for g in meta["@graph"] - if "ComputationalWorkflow" in ( - g["@type"] if isinstance(g["@type"], list) else [g["@type"]] - ) - ) - has_part = wf.get("hasPart") or [] - recipe_refs = [hp["@id"] for hp in has_part if hp["@id"].startswith("#recipe-")] - assert len(recipe_refs) >= 2 # both step_a and step_b recipes - - -# --------------------------------------------------------------------------- -# CLI tests -# --------------------------------------------------------------------------- - - -class TestCli: - def test_export_wrroc_help(self) -> None: - runner = CliRunner() - result = runner.invoke(main, ["export", "wrroc", "--help"]) - assert result.exit_code == 0 - assert "WRROC" in result.output - assert "--zip" in result.output - assert "--metadata-only" in result.output - - def test_export_wrroc_runs( - self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.chdir(minimal_project) - runner = CliRunner() - result = runner.invoke( - main, - [ - "export", "wrroc", - "-o", "out-dir", - "--author", "Tester ", - ], - ) - assert result.exit_code == 0, result.output - assert (minimal_project / "out-dir" / "ro-crate-metadata.json").is_file() - assert "Wrote WRROC" in result.output - - def test_export_wrroc_zip( - self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.chdir(minimal_project) - runner = CliRunner() - result = runner.invoke( - main, - [ - "export", "wrroc", - "-o", "bundle.zip", - "--zip", - "--author", "Tester ", - ], - ) - assert result.exit_code == 0, result.output - assert (minimal_project / "bundle.zip").is_file() - - def test_export_wrroc_metadata_only( - self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, - ) -> None: - monkeypatch.chdir(minimal_project) - runner = CliRunner() - result = runner.invoke( - main, - [ - "export", "wrroc", - "-o", "meta", - "--metadata-only", - "--author", "Tester ", - ], - ) - assert result.exit_code == 0, result.output - # Manifest yes, data no - out_dir = minimal_project / "meta" / "results" / "baseline" / "foo" - assert (out_dir / ".lightcone-manifest.json").is_file() - assert not (out_dir / "data.txt").exists() - - def test_export_wrroc_no_runs_warns( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - ) -> None: - _write_spec( - tmp_path, - {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, - ) - _write_universe(tmp_path, "u1", {}) - monkeypatch.chdir(tmp_path) - runner = CliRunner() - result = runner.invoke( - main, - ["export", "wrroc", "-o", "out", "--author", "X "], - ) - assert result.exit_code == 0, result.output - assert "no materialized outputs" in result.output.lower() From 16d4fb2524f8e549e12bf7477bf7de7a1305513f Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Mon, 17 Aug 2026 16:05:42 +0200 Subject: [PATCH 02/23] streamline pyproject --- src/lightcone/engine/templates/files/pyproject.toml.tmpl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lightcone/engine/templates/files/pyproject.toml.tmpl b/src/lightcone/engine/templates/files/pyproject.toml.tmpl index 48bfac78..f63c6a17 100644 --- a/src/lightcone/engine/templates/files/pyproject.toml.tmpl +++ b/src/lightcone/engine/templates/files/pyproject.toml.tmpl @@ -2,9 +2,6 @@ name = "${name}" version = "0.0.1" requires-python = ">=${python_floor}" -# Analysis dependencies: add with `uv add ` — never install into -# another environment by hand. The engine (lightcone-cli) is a normal -# locked dependency: the experiment pins its own execution layer. dependencies = [ "${lc_requirement}", ] From 43b80321b0afbafbf2294859d3b3b743f8245004 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Mon, 17 Aug 2026 17:20:19 +0200 Subject: [PATCH 03/23] minor changes --- CLAUDE.md | 155 ++++-- src/lightcone/cli/__init__.py | 10 - src/lightcone/cli/commands.py | 138 +++-- src/lightcone/engine/__init__.py | 18 +- src/lightcone/engine/constants.py | 26 - src/lightcone/engine/project.py | 477 +++++++++------- src/lightcone/engine/templates/__init__.py | 110 +++- .../templates/files/pyproject.toml.tmpl | 4 +- tests/conftest.py | 63 ++- tests/test_cli.py | 80 ++- tests/test_project.py | 517 ++++++++++++------ tests/test_templates.py | 111 +++- 12 files changed, 1074 insertions(+), 635 deletions(-) delete mode 100644 src/lightcone/engine/constants.py diff --git a/CLAUDE.md b/CLAUDE.md index 4229909c..64740a59 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -107,10 +107,9 @@ src/lightcone/ # namespace — NO __init__.py │ ├── __init__.py # exposes main(); the launcher hooks in here (layer 3) │ └── commands.py # lc init — the only verb so far └── engine/ - ├── __init__.py # lc_version() - ├── constants.py # engine constants (interpreter pin, uv floor) - ├── project.py # what a project IS: discovery + convergence - └── templates/ # the scaffold's file templates + ├── __init__.py # docstring only + ├── project.py # how a directory converges into a project + └── templates/ # the scaffold's file content ├── __init__.py # loader + one renderer per scaffolded file └── files/*.tmpl # the templates themselves, as real files @@ -141,16 +140,32 @@ in `engine/project.py`: | Symbol | Role | |---|---| -| `find_root(start)` | Discovery: nearest ancestor holding an `astra.yaml`, else `None` | +| `converge(dir, *, write)` | The whole scaffold operation | +| `ConvergenceReport` | `created` / `repaired` / `unchanged` / `blocked` / `warnings`, plus `.converged` and `.as_dict()` | | `project_name(dir)` | PEP 503-ish name from the directory name | -| `converge(dir, *, write, git, sync)` | The whole scaffold operation | -| `ConvergenceReport` | `created` / `repaired` / `unchanged` / `warnings`, `.converged`, `.as_dict()` | | `ProjectError` | The one engine exception; `_EngineErrorGroup` in the CLI turns it into a clean `ClickException` | The engine never imports click and never prints. `converge(write=False)` -is check mode — the *same* decision path with side effects switched off -(via `_Converger.item` / `.file`), which is what keeps `--check` honest -rather than a second implementation. +is check mode — the *same* decision path with side effects switched off, +which is what keeps `--check` honest rather than a second implementation. +Every item therefore routes through `_Converger.item`, `.file`, or +`.blocked`; **nothing writes or records outside that mechanism.** `.file` +takes a *thunk*, so check mode renders no template at all. + +Two report distinctions that matter: a **warning** is something +convergence can see but must not fix (advisory — never affects +`converged`); a **blocked** item is one convergence cannot complete, and it +does count, so a report can never claim a project is converged while +something it owns is absent. + +**Project discovery** — the `astra.yaml` walk-up — arrives with the +launcher (layer 3), the first thing that needs it. `lc init` is handed its +directory. + +**CLI startup stays cheap.** `commands.py` imports the engine *inside* the +command callbacks and builds the rich console lazily, so `lc --help` and +shell completion pay for neither. Keep this up as verbs land: a module-scope +engine import would make every invocation pay for the heaviest layer. **Templates are files, not string literals.** `engine/templates/files/*.tmpl` are package data, loaded through `importlib.resources`; `templates/__init__.py` @@ -159,6 +174,14 @@ exposes one function per scaffolded file. Placeholders are templates legitimately contain braces (TOML tables, MyST `{astra}` roles). Substitution is strict, so a missing key raises. +**No engine constants for the environment.** The scaffolded +`.python-version` is the interpreter `lc` itself is running on, and +`requires-python` is lightcone-cli's own `Requires-Python`, verbatim. These +can't conflict — lc only runs on an interpreter satisfying its own bound — +and neither is a number to maintain. Identity follows the project's files +from there: `env_version` hashes `.python-version`'s bytes, not anything in +the engine (spec §3). + **`.gitignore` converges entry-wise, not by marker.** `templates.gitignore_entries()` is the template minus comments and blanks; `missing_gitignore_entries(text)` is what a repair appends, in template @@ -173,30 +196,97 @@ user owns: | Path | Role | |---|---| -| `astra.yaml`, `universes/baseline.yaml`, `src/` | ASTRA spec scaffold (from astra's boilerplate; the `container:` key is stripped — ASTRA carries no environment) | +| `astra.yaml` + `universes/baseline.yaml` | astra's boilerplate spec, verbatim, as **one item keyed on `astra.yaml`** — the baseline references the boilerplate's example decision, so it must never land beside a user-authored spec. Its `container:` key is ignored outright — see Recorded decisions | | `pyproject.toml` | The uv project: **virtual** (no `[build-system]`), `lightcone-cli` pinned in its own dependencies — the engine lives inside the experiment's lock | -| `.python-version` | Exact interpreter patch (`engine.constants.DEFAULT_PYTHON`) | -| `uv.lock`, `.venv` | `uv lock`, then `uv sync --locked --exact --compile-bytecode` | +| `.python-version` | The exact patch of the interpreter `lc` is running on | +| `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | | `.gitignore` | One managed block of patterns; convergence ensures each is present | | `results/` + `README.md` | Where outputs land; the README states the materialize-don't-hand-write contract | | `myst.yml`, `index.md` | Template MyST report referencing `astra.yaml` *by path* | +- **Only what git can carry is converged.** No `src/`, and no empty + `universes/`: git does not track empty directories, so converging one + reports drift on every fresh clone, forever. astra dropped `src/` for the + same reason (astra-tools#100) — where analysis code lives is the user's + layout, and the boilerplate's `python src/main.py` is a placeholder. + Universes are discovered by `glob("*.yaml")`, which is empty-not-error on + a missing directory. `tests/test_project.py::test_a_clone_of_a_converged_project_is_converged` + pins this: a clone must need nothing but `.venv`. - **Convergence, not scaffolding.** Each item is created if missing, offered to a conservative `repair(text) -> str | None` hook otherwise, and left alone when the hook returns `None`. `--check` computes the same report without writing (exit 1 when not converged); `--json` - emits `{converged, created, repaired, unchanged, warnings}`. -- **An authored `Containerfile` is refused**, before any write: images - are generated from the lock, so a hand-written one would be silently - ignored. The user's own file operation is the consent to migrate — no - flag substitutes for it (spec §8). -- **uv is required** and is invoked through a single seam - (`commands._run_uv`) that tests monkeypatch. Every uv invocation - carries an explicit `--project` — uv's own walk-up discovery is never - trusted (spec §4). -- **Project discovery** is one rule (`engine.project.find_root`): the - nearest ancestor, including the start directory, holding an - `astra.yaml`. + emits `{converged, created, repaired, unchanged, blocked, warnings}`. +- **Derived artifacts converge by correctness, not existence.** `uv.lock` + and `.venv` go through `_Converger.derived`, whose predicate is uv's own + no-write verification (`uv lock --check`, `uv sync --locked --exact + --check` — both confirmed read-only against uv 0.12.3). A lock that no + longer matches `pyproject.toml`, or an environment that no longer matches + the lock, is exactly as unconverged as a missing one and reports as + `repaired`. Existence alone made `converge()` a no-op on drift, which + layer 3's launcher — which converges on every invocation precisely to + guarantee the environment matches the lock — would have silently + inherited. + - The probe is skipped when the artifact is absent (nothing to ask), so a + fresh project costs none and the created/repaired split falls out of the + same check. + - Check mode may *probe* but never mutates; `test_check_mode_only_probes` + pins that every uv call it makes carries `--check`. + - **Honest limitation** (measured, uv 0.12.3): `uv sync --check` catches + packages the lock requires and the environment lacks, but not *extras* — + a hand-installed package leaves it reporting "would make no changes". + Set-level, not byte-level, as spec §3 already accepts; what bounds what + a recipe can import is the sandbox, not this probe. +- **An authored `Containerfile` is left alone.** Spec §8 has `lc init` + refuse one; we don't. Images are generated from the lock, so a + hand-written file is simply not an input to anything — treating it as a + blocking half-state was more ceremony than the situation earns. Revisit + if the container hatch (layer 6) turns out to need the disambiguation. +- **Nothing about uv's caching or linking is overridden**, because uv's + defaults are already right: it clones (copy-on-write) or hard links + package content out of a global cache, so projects share it. Measured + here: two environments cost 284 MB together rather than 432 MB. The docs + discourage forcing `symlink` mode (it couples every environment to the + cache's survival), and **`--system-site-packages` must never be used** — + it would make packages outside the lock importable, which is exactly what + the environment model exists to prevent (spec §7, G6). + - The sharing silently stops working when the cache and the project are + on **different filesystems** (uv falls back to full copies). uv warns; + `tool_warnings()` lifts that warning out of uv's progress output into + the report, so it reaches both the console and `--json`. This is why + the site registry supplies `UV_CACHE_DIR` on Perlmutter (spec §4). + - `--compile-bytecode` is the one genuinely per-project cost: bytecode is + generated into the venv, never linked (~55 MB of 216 MB here). It is a + deliberate trade — the environment is read-only at execution time, so + without it every recipe run re-compiles. +- **Every external tool goes through one seam**, `project._run`, which + tests monkeypatch — so the suite never shells out, and every call is + inspectable. `_check_call` turns a nonzero exit into a `ProjectError`: + nothing convergence invokes is allowed to fail silently. Every uv + invocation carries an explicit `--project` — uv's own walk-up discovery + is never trusted (spec §4). +- **uv is required; git is optional.** An absent uv is a refusal (it is + the environment substrate); an absent git is simply nothing to converge, + since a project without version control is still a valid project. +- **`git init` checks for an *enclosing* work tree**, not just a `.git` in + the directory, so `lc init subdir/` inside a repository can't create a + nested one. (`.git` may be a file — linked worktree or submodule — so the + test is `exists`, not `is_dir`.) +- **`lc init` has exactly two flags**, `--check` and `--json`. `--no-git` + and `--no-sync` were deleted: neither had a caller outside the test suite, + `--no-git` was a workaround for the missing enclosing-repo check, and + `--no-sync`'s real home is containerized mode (layer 6), where the host + `.venv` is inert. Don't add a flag whose only user is a test — stub + `project._run` instead. +### Recorded decisions + +- **The ASTRA `container:` directive is ignored, entirely.** astra's + boilerplate writes `container: python:3.12-slim` into `astra.yaml`, and + lightcone-cli does nothing with it: not read, not stripped, not + validated, not migrated. The environment is `pyproject.toml` + `uv.lock` + (spec §2), so a scaffolded project simply carries a key no code path + consults. Don't "fix" this by reconciling the two — a later layer will + decide whether the key is dropped upstream, refused, or migrated. ### Recorded deviations from the spec @@ -215,23 +305,26 @@ user owns: |---|---|---| | Add the next layer | the spec (§11 = the layer ordering) | Land code + tests + deps together; update the layer table above. Docs are deliberately deferred | | Change what a scaffolded file contains | `src/lightcone/engine/templates/files/` | Edit the `.tmpl`; add new ones to `TEMPLATE_NAMES` + a renderer | +| Add a value to the scaffold | `src/lightcone/engine/templates/__init__.py` | Derive it from the environment or our own metadata before introducing a constant | | Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | -| Add an engine constant | `src/lightcone/engine/constants.py` | Constants ship inside the locked engine — never resolved at run time | ## Test Patterns -- `tests/conftest.py` — the `fake_uv` autouse fixture stubs the uv seam - (`engine.project._run_uv`), emulating `uv lock` writing `uv.lock` and - `uv sync` making `.venv`, plus an isolated `$HOME`. Tests are hermetic: - no network, no real resolution. +- `tests/conftest.py` — the `tools` autouse fixture stubs + `engine.project._run`, emulating each tool's observable effect (`uv lock` + writes `uv.lock`, `uv sync` makes `.venv`, `git init` makes `.git`) and + recording every argv. `uv_calls(tools)` narrows it to uv. Tests are + hermetic: no network, no resolution, no subprocesses. - `tests/test_project.py` — discovery and convergence semantics, called directly. This is where scaffold behavior is tested. - `tests/test_cli.py` — the CLI surface only: flags reaching the engine, rendering, exit codes, error translation. Rich wraps output at terminal width, so assert on short unwrappable fragments. - `tests/test_templates.py` — template loading (guards the packaging of - package data), strict substitution, marker presence. + package data), strict substitution, and the `.gitignore` entry/repair + logic. Content assertions live here; `test_project.py` asserts only that + the file written *is* the template, so a template edit touches one file. ## Conventions diff --git a/src/lightcone/cli/__init__.py b/src/lightcone/cli/__init__.py index 599ba55f..1fc431b3 100644 --- a/src/lightcone/cli/__init__.py +++ b/src/lightcone/cli/__init__.py @@ -2,18 +2,8 @@ from __future__ import annotations -try: - from importlib.metadata import version - - __version__ = version("lightcone-cli") -except Exception: - __version__ = "0.0.0.dev" - def main() -> None: - # The launcher (discover → mode-detect → scrub UV_* → converge → - # delegate into the project-locked engine, spec §4) hooks in here when - # its layer lands. Until then `lc` dispatches directly. from lightcone.cli.commands import main as _main _main() diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 0e91cc32..bb33a9a8 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -1,34 +1,35 @@ """Command-line interface for lightcone-cli — the ASTRA execution layer. - -This is a clean rebuild against the normative design spec, re-added layer -by layer. - -**Layer 1 — project scaffolding.** The only verb is ``lc init``. What a -project *is* and how one is converged lives in -:mod:`lightcone.engine.project`; the file templates live in -:mod:`lightcone.engine.templates`. This module owns only the CLI: flags, -console rendering, and exit codes. - -The remaining verbs — ``lc materialize``, ``lc run``, ``lc status``, -``lc verify``, ``lc build`` — arrive with their layers. """ from __future__ import annotations +import functools import json import logging import sys from pathlib import Path +from typing import TYPE_CHECKING import click -from rich.console import Console -from lightcone.engine.project import ConvergenceReport, ProjectError, converge +if TYPE_CHECKING: + from rich.console import Console + + from lightcone.engine.project import ConvergenceReport -console = Console() logger = logging.getLogger(__name__) +@functools.cache +def _console() -> Console: + """The rich console, built on first use to avoid startup cost at each + invocation of the cli even when the console is not needed. + """ + from rich.console import Console + + return Console() + + class _EngineErrorGroup(click.Group): """Render engine errors as clean CLI errors instead of tracebacks. @@ -39,6 +40,8 @@ class _EngineErrorGroup(click.Group): """ def invoke(self, ctx: click.Context) -> object: + from lightcone.engine.project import ProjectError + try: return super().invoke(ctx) except ProjectError as e: @@ -47,10 +50,8 @@ def invoke(self, ctx: click.Context) -> object: @click.group(cls=_EngineErrorGroup) @click.version_option(package_name="lightcone-cli") -@click.pass_context -def main(ctx: click.Context) -> None: +def main() -> None: """lightcone-cli — execution layer for ASTRA projects.""" - ctx.ensure_object(dict) # ============================================================================= @@ -66,12 +67,6 @@ def main(ctx: click.Context) -> None: @main.command() @click.argument("directory", type=click.Path(file_okay=False, path_type=Path), default=".") -@click.option("--no-git", is_flag=True, help="Skip git init") -@click.option( - "--no-sync", - is_flag=True, - help="Skip materializing the .venv (uv lock still runs).", -) @click.option( "--check", "check_only", @@ -87,19 +82,11 @@ def main(ctx: click.Context) -> None: is_flag=True, help="Emit the convergence report as JSON on stdout.", ) -def init( - directory: Path, - no_git: bool, - no_sync: bool, - check_only: bool, - as_json: bool, -) -> None: - """Converge DIRECTORY into an ASTRA project (idempotent). +def init(directory: Path, check_only: bool, as_json: bool) -> None: + """Converge DIRECTORY into a standard Lightcone project (idempotent). Safe to re-run at any time: creates whatever is missing, repairs the - pieces lightcone manages, and never overwrites files you own. A - directory that already holds an ``astra.yaml`` is adopted, not - rejected. + pieces lightcone manages, and never overwrites files you own. The spec scaffold (``astra.yaml``, ``universes/baseline.yaml``) follows the ``astra init`` boilerplate; on top of it sit the uv @@ -108,59 +95,60 @@ def init( ``results/``, and a template MyST report (``myst.yml`` + ``index.md``). """ + from lightcone.engine.project import converge + directory = directory.resolve() write = not check_only if write and not as_json: - console.print(f"[cyan]{_LIGHTCONE}[/cyan]") + _console().print(f"[cyan]{_LIGHTCONE}[/cyan]") - report = converge(directory, write=write, git=not no_git, sync=not no_sync) + report = converge(directory, write=write) if as_json: click.echo(json.dumps(report.as_dict(), indent=2)) - elif check_only: - _render_check(report, directory) else: - _render_run(report, directory) + _render_init_output(report, directory, dry_run=check_only) if check_only and not report.converged: sys.exit(1) -def _render_check(report: ConvergenceReport, directory: Path) -> None: - if report.converged: - console.print(f"[green]✓[/green] {directory} is converged — nothing to do") - else: - for item in report.created: - console.print(f" [yellow]would create[/yellow] {item}") - for item in report.repaired: - console.print(f" [yellow]would repair[/yellow] {item}") - for warning in report.warnings: - console.print(f" [yellow]![/yellow] {warning}") - - -def _render_run(report: ConvergenceReport, directory: Path) -> None: - for item in report.created: - console.print(f"[green]✓[/green] created {item}") - for item in report.repaired: - console.print(f"[green]✓[/green] repaired {item}") - for warning in report.warnings: - console.print(f"[yellow]![/yellow] {warning}") +def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: bool) -> None: + """Print a convergence report: the items, then the verdict. + + ``--check`` and a real run print the *same* report — they disagree + only about tense — so ``dry_run`` selects the mood and nothing else, + mirroring the ``write`` flag convergence itself takes. Blocked items + are listed with created and repaired ones because they count the same + way: they are what lets a report say "not converged" (see + :class:`~lightcone.engine.project.ConvergenceReport`). Their reason + arrives separately, as a warning. + """ + mark, style = ("·", "yellow") if dry_run else ("✓", "green") + + lines: list[str] = [] + for items, label, item_mark, item_style in ( + (report.created, "would create" if dry_run else "created", mark, style), + (report.repaired, "would repair" if dry_run else "repaired", mark, style), + (report.blocked, "blocked", "✗", "red"), + ): + lines += [f" [{item_style}]{item_mark}[/{item_style}] {label} {item}" for item in items] + lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + if report.converged: - console.print(f"\n[green]Project already converged at[/green] {directory}") + # A dry run over a converged project finds nothing to do because + # there is nothing to do — one line serves both moods. + verdict = f"[green]✓[/green] {directory} is already converged — nothing to do" + elif report.blocked: + # A write run that left an item blocked did not converge the + # project either; only a dry run gets to be neutral about it. + verdict = f"[red]✗[/red] {directory} is not converged" + elif dry_run: + verdict = f"[yellow]![/yellow] {directory} is not converged" else: - console.print(f"\n[green]Project converged at[/green] {directory}") - - # Next steps only make sense for a freshly scaffolded spec. - if "astra.yaml" in report.created: - console.print("\nNext steps:") - console.print(f" • Go to the newly created directory [cyan]cd {directory}[/cyan]") - console.print( - " • Add analysis dependencies with [cyan]uv add[/cyan] " - "(e.g. [cyan]uv add numpy astropy[/cyan])" - ) - console.print(" • Describe your analysis in [cyan]astra.yaml[/cyan]") - console.print( - " • Preview the report with [cyan]myst start[/cyan] " - "(requires the MyST CLI: [cyan]npm i -g mystmd[/cyan])" - ) + verdict = f"[green]✓[/green] Project converged at {directory}" + + if lines: + lines.append("") # space the verdict off the list + _console().print("\n".join([*lines, verdict])) diff --git a/src/lightcone/engine/__init__.py b/src/lightcone/engine/__init__.py index 0725493b..7e897028 100644 --- a/src/lightcone/engine/__init__.py +++ b/src/lightcone/engine/__init__.py @@ -1,19 +1,9 @@ """The lightcone execution engine. -Rebuild in progress: the engine is being re-added layer by layer on top -of the normative design spec (``docs/design/execution-environment.md``). -Layer 1 is project scaffolding only — :mod:`lightcone.engine.project` -(discovery) and :mod:`lightcone.engine.constants`. +Rebuild in progress: the engine is being re-added layer by layer on top of +the normative design spec. Layer 1 is project scaffolding only — +:mod:`lightcone.engine.project` (convergence) and +:mod:`lightcone.engine.templates` (the scaffolded files). """ from __future__ import annotations - - -def lc_version() -> str: - """The running lightcone-cli version ("unknown" for broken installs).""" - try: - from importlib.metadata import version - - return version("lightcone-cli") - except Exception: - return "unknown" diff --git a/src/lightcone/engine/constants.py b/src/lightcone/engine/constants.py deleted file mode 100644 index 4a960b86..00000000 --- a/src/lightcone/engine/constants.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Engine constants. - -Constants ship *inside* the locked engine, never resolved at run time: -a new value reaches a project only through an engine release plus a -relock, which is what keeps environment identity a pure function of the -repo plus the engine (spec §3). - -The image layer's own pinned digests (default base, uv distribution) -arrive with the container hatch; layer 1 needs only the interpreter pin. -""" - -from __future__ import annotations - -#: Exact interpreter patch scaffolded into ``.python-version`` by -#: ``lc init``. Projects may pin a different one — environment identity -#: follows the file, not this constant. -DEFAULT_PYTHON = "3.12.12" - -#: The floor written into the scaffolded ``requires-python``, derived -#: from :data:`DEFAULT_PYTHON` so the two can never drift. -DEFAULT_PYTHON_FLOOR = ".".join(DEFAULT_PYTHON.split(".")[:2]) - -#: Minimum uv scaffolded into ``[tool.uv] required-version``. 0.12 is -#: the release the spec's empirical evidence pass was run against -#: (spec §13). -MIN_UV_VERSION = "0.12" diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 0bd62bc3..8233d0a2 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -1,20 +1,16 @@ """What a project *is*, and how to converge one. -Two responsibilities, both about the project as an object rather than -about the CLI: - -- **discovery** — :func:`find_root`. One rule, shared by every verb (and, - once it lands, the launcher): the project root is the nearest ancestor, - including the start directory, that contains an ``astra.yaml``. uv's own - walk-up discovery is never trusted — every uv invocation downstream - carries an explicit ``--project `` (spec §4). -- **convergence** — :func:`converge`. Bring a directory to the state a - lightcone project is defined to have: an ASTRA spec sitting on a uv - project. Idempotent, and never destructive to files the user owns. +:func:`converge` brings a directory to the state a lightcone project is +defined to have: an ASTRA spec sitting on a uv project. Idempotent, and +never destructive to files the user owns. Nothing here knows about Click or the console: convergence returns a :class:`ConvergenceReport` and raises :class:`ProjectError`; rendering and exit codes are the CLI's business. + +Project *discovery* — the ``astra.yaml`` walk-up that finds a root from a +working directory — arrives with the launcher, the first thing that needs +it. ``lc init`` is handed its directory explicitly. """ from __future__ import annotations @@ -23,37 +19,16 @@ import shutil import subprocess from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from pathlib import Path from lightcone.engine import templates -from lightcone.engine.constants import DEFAULT_PYTHON SPEC_FILENAME = "astra.yaml" class ProjectError(Exception): - """A project cannot be discovered, read, or converged.""" - - -# ============================================================================= -# Discovery -# ============================================================================= - - -def find_root(start: Path | None = None) -> Path | None: - """Walk up from *start* (default: cwd) to the project root, or ``None``.""" - p = (start or Path.cwd()).resolve() - for parent in [p, *p.parents]: - if (parent / SPEC_FILENAME).is_file(): - return parent - return None - - -def project_name(directory: Path) -> str: - """A PEP 503-ish project name derived from the directory name.""" - name = re.sub(r"[^A-Za-z0-9._-]+", "-", directory.name).strip("-._").lower() - return name or "analysis" + """A project cannot be read or converged.""" # ============================================================================= @@ -66,35 +41,46 @@ class ConvergenceReport: """What a convergence did, or — in check mode — would have done. ``warnings`` carries problems convergence can *see* but must not fix; - they never affect whether the project counts as converged. + they are advisory and never affect ``converged``. ``blocked`` is + different: an item convergence cannot complete at all, which does + count — a report must never claim a project is converged while + something convergence is responsible for is absent. """ created: list[str] = field(default_factory=list) repaired: list[str] = field(default_factory=list) unchanged: list[str] = field(default_factory=list) + blocked: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @property def converged(self) -> bool: - return not self.created and not self.repaired + """Whether the project needed nothing done to it. + + Note the tense: after a write run that created files this is + ``False``. It reports what convergence *found* — which is the + question check mode asks — not "the project is now good". + """ + return not self.created and not self.repaired and not self.blocked def as_dict(self) -> dict[str, object]: - return { - "converged": self.converged, - "created": self.created, - "repaired": self.repaired, - "unchanged": self.unchanged, - "warnings": self.warnings, - } + """The report as JSON-ready data, ``converged`` first. + + Built from :func:`dataclasses.asdict` so a field added to the + report cannot silently go missing from ``lc init --json``, which is + the agent-facing contract. + """ + return {"converged": self.converged, **asdict(self)} class _Converger: """Decide-then-maybe-write, so check mode reuses the real decisions. - Every item routes through :meth:`item` or :meth:`file`, which record - the outcome first and only then apply it when ``write`` is set. That - is what keeps ``lc init --check`` honest: it is the same code path with - side effects switched off, not a second implementation. + Every item routes through :meth:`item`, :meth:`file`, or + :meth:`blocked`, which record the outcome first and only then apply it + when ``write`` is set. That is what keeps ``lc init --check`` honest: it + is the same code path with side effects switched off, not a second + implementation. """ def __init__(self, *, write: bool) -> None: @@ -114,19 +100,26 @@ def file( self, name: str, path: Path, - template: str, + template: Callable[[], str], repair: Callable[[str], str | None] | None = None, ) -> None: """Create *path* from *template* if missing; else offer it to *repair*. + *template* is a thunk rather than a string, so check mode renders + nothing at all and a steady-state run renders only the files it + actually writes. Parent directories are created, so a managed file + never depends on an earlier item having made its directory first. + ``repair`` receives the current text and returns the fixed text, or ``None`` when the file is already fine. Repairs must be - conservative by construction — see :func:`_repair_gitignore`. + conservative by construction — see + :func:`~lightcone.engine.templates.gitignore_repair`. """ if not path.exists(): self.report.created.append(name) if self.write: - path.write_text(template) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(template()) elif repair is not None and (fixed := repair(path.read_text())) is not None: self.report.repaired.append(name) if self.write: @@ -134,7 +127,49 @@ def file( else: self.report.unchanged.append(name) + def derived( + self, + name: str, + path: Path, + *, + is_current: Callable[[], bool], + apply: Callable[[], object], + ) -> None: + """Converge a *derived* artifact — one whose agreement with its + inputs, not its mere existence, is the question. + + ``uv.lock`` and ``.venv`` are derived: a lock that no longer matches + ``pyproject.toml``, or an environment that no longer matches the + lock, is exactly as unconverged as a missing one. Existence alone + would make convergence a no-op on drift — and layer 3's launcher + converges on every invocation precisely to guarantee the environment + matches the lock. + + ``is_current`` is only consulted when the artifact exists, so a + fresh project pays no probe, and the created/repaired distinction + falls out of the same check. + """ + if not path.exists(): + self.report.created.append(name) + elif is_current(): + self.report.unchanged.append(name) + return + else: + self.report.repaired.append(name) + if self.write: + apply() + + def blocked(self, name: str, reason: str) -> None: + """Record an item convergence cannot complete, and why. + + Stronger than :meth:`warn`: the project is not converged, so + ``--check`` fails instead of reporting a file that isn't there. + """ + self.report.blocked.append(name) + self.report.warnings.append(reason) + def warn(self, message: str) -> None: + """Record something convergence can see but must not fix.""" self.report.warnings.append(message) @@ -143,13 +178,7 @@ def warn(self, message: str) -> None: # ============================================================================= -def converge( - directory: Path, - *, - write: bool = True, - git: bool = True, - sync: bool = True, -) -> ConvergenceReport: +def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: """Converge *directory* into an ASTRA project. Idempotent. Creates whatever is missing, repairs the pieces lightcone manages, and @@ -160,13 +189,11 @@ def converge( directory itself — and the returned report describes what a real run would have done. - Raises :class:`ProjectError` for the two states convergence must not - paper over: an authored ``Containerfile``, and a missing or failing - uv. + Raises :class:`ProjectError` if uv is missing or fails — it is the + environment substrate, so there is no useful project without it. """ directory = directory.resolve() - _refuse_authored_containerfile(directory) _require_uv() c = _Converger(write=write) @@ -174,44 +201,47 @@ def converge( if write: directory.mkdir(parents=True, exist_ok=True) - _converge_spec(c, directory) + # The spec is one item, keyed on astra.yaml: astra treats it and + # universes/baseline.yaml as a unit, because the baseline references the + # boilerplate's example decision and must never land beside a + # user-authored spec. Nothing else about the layout is converged — + # `src/` and an empty `universes/` are directories git cannot track, so + # converging them would report drift on every fresh clone; where + # analysis code lives is the user's layout, not ours (astra-tools#100). + c.item( + "astra.yaml", + (directory / SPEC_FILENAME).exists(), + lambda: _scaffold_spec(directory), + ) _converge_uv_project(c, directory) - _converge_managed_files(c, directory) + c.file( + ".gitignore", + directory / ".gitignore", + templates.gitignore, + repair=templates.gitignore_repair, + ) _converge_results(c, directory) _converge_report_template(c, directory) - - if git: - c.item( - ".git", - (directory / ".git").exists(), - lambda: subprocess.run(["git", "init", "-q"], cwd=directory, check=False), - ) + _converge_git(c, directory) # Lock and sync last: they are the expensive steps, and they are # meaningless until pyproject.toml and .python-version exist. - c.item("uv.lock", (directory / "uv.lock").exists(), lambda: _uv_lock(directory)) - if sync: - c.item(".venv", (directory / ".venv").exists(), lambda: _uv_sync(directory)) + c.derived( + "uv.lock", + directory / "uv.lock", + is_current=lambda: _lock_is_current(directory), + apply=lambda: _uv_lock(c, directory), + ) + c.derived( + ".venv", + directory / ".venv", + is_current=lambda: _env_is_current(directory), + apply=lambda: _uv_sync(c, directory), + ) return c.report -def _refuse_authored_containerfile(directory: Path) -> None: - """Refuse before any write. - - Images are generated from the lock, so a hand-authored Containerfile - would be silently ignored — a half-state the user has to resolve. The - user's own file operation is the consent to migrate; no flag can - substitute for it (spec §8). - """ - if (directory / "Containerfile").is_file(): - raise ProjectError( - f"{directory}/Containerfile: images are generated from the " - "lock — delete or rename it, then re-run `lc init`. Declare " - "system dependencies in [tool.lightcone.image] instead." - ) - - def _require_uv() -> None: if shutil.which("uv") is None: raise ProjectError( @@ -220,171 +250,218 @@ def _require_uv() -> None: ) -def _converge_spec(c: _Converger, directory: Path) -> None: - """The ASTRA spec scaffold: astra.yaml + universes/ + src/.""" - c.item("astra.yaml", (directory / SPEC_FILENAME).exists(), lambda: _scaffold_spec(directory)) - - def _scaffold_spec(directory: Path) -> None: - """Write astra's boilerplate, then make it environment-free. - - astra's ``init`` *command* refuses non-empty directories and - overwrites ``.gitignore`` — both wrong for convergence — so this uses - the bare scaffold API and manages ``.gitignore`` separately. The - public ``create_boilerplate`` (LightconeResearch/astra-tools#99) is - preferred, with the private helper as the fallback on releases that - predate it; resolved by ``getattr`` rather than an import guard so the - code type-checks under either. + """Write astra's boilerplate spec: ``astra.yaml`` + ``universes/``. + + ``create_boilerplate`` is astra's public scaffold API, split out + (astra-tools#99) so downstream tools can scaffold into existing + directories under their own conventions: it writes the spec and nothing + else — no ``.gitignore``, no git init, no emptiness checks. ``astra + init`` delegates to the same function, so this is the shared path, not + a back door. + + Its boilerplate carries a ``container:`` key. **lightcone-cli ignores + the ASTRA ``container:`` directive entirely** — not stripped, not + validated, not migrated. The environment is ``pyproject.toml`` + + ``uv.lock``, and no code path here reads that key. """ - from astra import cli as astra_cli - - create_boilerplate = getattr(astra_cli, "create_boilerplate", None) - if create_boilerplate is not None: - create_boilerplate(directory) - else: - (directory / "universes").mkdir(exist_ok=True) - astra_cli._create_boilerplate_astra_yaml(directory) + from astra.cli import create_boilerplate - # The boilerplate recipes reference scripts under src/ (e.g. - # ``python src/main.py``); astra's own init creates the directory, so - # the scaffold must too. - (directory / "src").mkdir(exist_ok=True) - - # ASTRA carries analysis structure only — the environment lives in - # pyproject.toml + uv.lock. Strip the ``container:`` line the - # boilerplate ships. - spec_path = directory / SPEC_FILENAME - stripped = re.sub(r"(?m)^container:.*\n?", "", spec_path.read_text(), count=1) - spec_path.write_text(stripped) + create_boilerplate(directory) def _converge_uv_project(c: _Converger, directory: Path) -> None: """pyproject.toml + .python-version — the environment definition. An existing ``pyproject.toml`` is the user's: read, possibly warned - about, never edited. + about, never edited. The warning is decided against the *pre-existing* + file, so one we just wrote — which always names the engine — can never + trigger it. """ pyproject_path = directory / "pyproject.toml" - if not pyproject_path.exists(): - c.report.created.append("pyproject.toml") - if c.write: - pyproject_path.write_text(templates.pyproject(name=project_name(directory))) - else: - c.report.unchanged.append("pyproject.toml") - if "lightcone-cli" not in pyproject_path.read_text(): - c.warn( - "pyproject.toml does not depend on lightcone-cli — the " - "engine should live inside the experiment's lock: " - "`uv add lightcone-cli`." - ) - - c.file(".python-version", directory / ".python-version", f"{DEFAULT_PYTHON}\n") - - -def _converge_managed_files(c: _Converger, directory: Path) -> None: + adopted = pyproject_path.exists() c.file( - ".gitignore", - directory / ".gitignore", - templates.gitignore(), - repair=_repair_gitignore, + "pyproject.toml", + pyproject_path, + lambda: templates.pyproject(name=project_name(directory)), ) + if adopted and "lightcone-cli" not in pyproject_path.read_text(): + c.warn( + "pyproject.toml does not depend on lightcone-cli — the " + "engine should live inside the experiment's lock: " + "`uv add lightcone-cli`." + ) + + c.file(".python-version", directory / ".python-version", templates.python_version) def _converge_results(c: _Converger, directory: Path) -> None: results_dir = directory / "results" if results_dir.exists() and not results_dir.is_dir(): - c.report.unchanged.extend(["results/", "results/README.md"]) - c.warn( + c.blocked( + "results/", "results exists but is not a directory; outputs cannot " - "materialize until it is one." + "materialize until it is one.", ) return c.item("results/", results_dir.is_dir(), results_dir.mkdir) - c.file("results/README.md", results_dir / "README.md", templates.results_readme()) + c.file("results/README.md", results_dir / "README.md", templates.results_readme) def _converge_report_template(c: _Converger, directory: Path) -> None: """The MyST report. Recommended add-on on top of the spec, not part of it — which is why it is scaffolded here and not by ``astra init``.""" - c.file("myst.yml", directory / "myst.yml", templates.myst_yml()) + c.file("myst.yml", directory / "myst.yml", templates.myst_yml) c.file( "index.md", directory / "index.md", - templates.index_md(title=directory.name or "My Analysis"), + lambda: templates.index_md(title=directory.name or "My Analysis"), ) +def _converge_git(c: _Converger, directory: Path) -> None: + """``git init``, unless the project is already under version control. + + Whether the directory *itself* holds a ``.git`` is the wrong question: + ``lc init subdir/`` inside an existing repository must not create a + nested one, so the check walks up. (``.git`` can be a file — a linked + worktree or submodule — hence ``exists`` rather than ``is_dir``.) + + git is optional, unlike uv: a project without it is perfectly valid, so + an absent git is silently nothing to converge rather than a warning. + """ + if shutil.which("git") is None: + return + already = any((p / ".git").exists() for p in [directory, *directory.parents]) + c.item(".git", already, lambda: _check_call(["git", "init", "-q"], cwd=directory)) + + +def project_name(directory: Path) -> str: + """A PEP 503-ish project name derived from the directory name.""" + name = re.sub(r"[^A-Za-z0-9._-]+", "-", directory.name).strip("-._").lower() + return name or "analysis" + + # ============================================================================= -# Repairs — append behind a marker, never rewrite +# The external-tool seam # ============================================================================= -def _repair_gitignore(text: str) -> str | None: - """Ensure every pattern lightcone manages is present, appending only - the ones that are missing. +def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + """One seam for every external tool convergence invokes. + + Tests monkeypatch this, so the suite never shells out and every call is + inspectable. + """ + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True, check=False) + - Entry-wise rather than marker-based: idempotency is then structural — - a pattern already in the file is never added again, whoever put it - there — and a pattern introduced by a later lc release still reaches - projects that already have a ``.gitignore``. +def _check_call(argv: list[str], *, cwd: Path) -> list[str]: + """Run a tool, surfacing a nonzero exit as :class:`ProjectError`. - The append preserves template order, which is what keeps - ``!results/README.md`` after the ``results/*`` it negates. (A file - holding the negation *without* ``results/*`` would end up with them - inverted; that state can only be hand-written, and re-ordering - someone's ignores to fix it would be the more surprising behavior.) + Nothing convergence shells out to is allowed to fail silently: a + broken lock, or a ``git init`` that didn't happen, would surface later + as something far more confusing. Returns the tool's own warnings. """ - missing = templates.missing_gitignore_entries(text) - if not missing: - return None + proc = _run(argv, cwd=cwd) + if proc.returncode != 0: + raise ProjectError(f"`{' '.join(argv)}` failed:\n{proc.stderr.strip()}") + return tool_warnings(proc.stderr) - block = "\n".join(missing) + "\n" - # The header is cosmetic, so add it only when it isn't already there; - # a second repair later appends bare patterns under the first one. - if templates.GITIGNORE_HEADER not in text: - block = templates.GITIGNORE_HEADER + "\n" + block - if not text.strip(): - return block - return text.rstrip("\n") + "\n\n" + block +def tool_warnings(stderr: str) -> list[str]: + """A tool's warnings, lifted out of its progress output. -# ============================================================================= -# The uv seam -# ============================================================================= + uv interleaves warnings with progress on stderr, so relaying the whole + stream would bury them under a line per installed package. A warning is + a line starting ``warning:`` plus any indented continuation. + Worth relaying at all because of one warning in particular: when the uv + cache and the project are on different filesystems, uv cannot link and + silently falls back to copying every package. That is the difference + between a project costing a few MB and costing the whole environment + again (measured: ~148 MB of a 216 MB environment is shared when linking + works), and nothing else would tell the user. + """ + found: list[str] = [] + for line in stderr.splitlines(): + if line.startswith("warning:"): + found.append(line.removeprefix("warning:").strip()) + elif found and line.startswith((" ", "\t")) and line.strip(): + found[-1] += " " + line.strip() + return found + + +def _probe(argv: list[str], *, cwd: Path) -> bool: + """Ask a tool a yes/no question. + + A nonzero exit is the answer "no", not a failure — so unlike + :func:`_check_call` this never raises. Whatever the tool wrote to stderr + is its explanation of the "no", which the caller is about to act on + anyway. + """ + return _run(argv, cwd=cwd).returncode == 0 -def _run_uv(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: - """One seam for every uv invocation (tests monkeypatch it).""" - return subprocess.run(["uv", *args], cwd=cwd, capture_output=True, text=True, check=False) +def _lock_is_current(directory: Path) -> bool: + """Whether ``uv.lock`` still agrees with ``pyproject.toml``. -def _uv(args: list[str], *, directory: Path, what: str) -> None: - """Run uv, surfacing a failure as :class:`ProjectError`. + uv's own no-write verification, so the answer is uv's rather than a + heuristic of ours (verified read-only against uv 0.12.3). + """ + return _probe(["uv", "lock", "--check", "--project", str(directory)], cwd=directory) + + +def _env_is_current(directory: Path) -> bool: + """Whether ``.venv`` still satisfies ``uv.lock``. + + **Honest limitation** (uv 0.12.3, measured): this catches packages the + lock requires and the environment lacks, but *not* extras — a package + installed by hand into the environment leaves ``--check`` reporting + "would make no changes", even though a real ``--exact`` sync would + remove it. So a hand-polluted environment still reads as current. The + design already accepts set-level rather than byte-level environment + checks (spec §3); what ultimately bounds what a recipe can import is + the sandbox, not this probe. + """ + argv = ["uv", "sync", "--locked", "--exact", "--check", "--project", str(directory)] + return _probe(argv, cwd=directory) + + +def _uv(c: _Converger, args: list[str], *, directory: Path) -> None: + """Run uv against *directory*, recording anything it warns about. Every invocation carries an explicit ``--project``: uv's native - walk-up discovery is never trusted (spec §4). A failure surfaces here - with uv's own stderr, because a silently broken lock would fail every - later verb more confusingly. + walk-up discovery is never trusted (spec §4). + + Nothing about linking or caching is overridden. uv's defaults already + share package content between projects — it clones (copy-on-write) or + hard links out of a global cache — and the docs discourage forcing + ``symlink`` mode, which would couple every environment to the cache's + survival. `--system-site-packages` is likewise never used: it would + make packages outside the lock importable, which is precisely what the + environment model exists to prevent (spec §7 G6). """ - proc = _run_uv([*args, "--project", str(directory)], cwd=directory) - if proc.returncode != 0: - raise ProjectError(f"`uv {what}` failed:\n{proc.stderr.strip()}") + for warning in _check_call(["uv", *args, "--project", str(directory)], cwd=directory): + c.warn(f"uv: {warning}") -def _uv_lock(directory: Path) -> None: - _uv(["lock"], directory=directory, what="lock") +def _uv_lock(c: _Converger, directory: Path) -> None: + _uv(c, ["lock"], directory=directory) -def _uv_sync(directory: Path) -> None: +def _uv_sync(c: _Converger, directory: Path) -> None: """Converge the environment, with the flags the spec fixes. - ``--locked`` so a drifted lock is an error rather than a silent - relock; ``--exact`` because a plain sync is additive; and - ``--compile-bytecode`` because the environment is read-only at - execution time (spec §4, §6). + ``--locked`` so a drifted lock is an error rather than a silent relock; + ``--exact`` because a plain sync is additive; and ``--compile-bytecode`` + because the environment is read-only at execution time, so a recipe + cannot write ``.pyc`` files as it imports (spec §4, §6, §7). + + ``--compile-bytecode`` is the environment's one genuinely per-project + cost: bytecode is generated into the venv rather than linked from the + cache, ~55 MB of a 216 MB environment here. The alternative is paying + compilation on every recipe run instead, since the sandbox denies the + write — so this is a deliberate trade, not an oversight. """ - _uv( - ["sync", "--locked", "--exact", "--compile-bytecode"], - directory=directory, - what="sync", - ) + _uv(c, ["sync", "--locked", "--exact", "--compile-bytecode"], directory=directory) diff --git a/src/lightcone/engine/templates/__init__.py b/src/lightcone/engine/templates/__init__.py index 448e014f..e6c1b79e 100644 --- a/src/lightcone/engine/templates/__init__.py +++ b/src/lightcone/engine/templates/__init__.py @@ -11,23 +11,18 @@ (TOML tables, MyST roles like ``{astra}``) that ``format`` would try to interpret. Substitution is strict: a missing key raises rather than silently emitting a placeholder. + +This module owns file *content*, including how content merges into a file +the user already owns (:func:`gitignore_repair`). It knows nothing about +convergence bookkeeping or the console. """ from __future__ import annotations +import sys from importlib import resources from string import Template -from lightcone.engine.constants import ( - DEFAULT_PYTHON_FLOOR, - MIN_UV_VERSION, -) - -#: Comment introducing lightcone's entries in a ``.gitignore`` it did not -#: author. Cosmetic only — what makes a repair idempotent is the entry set -#: (:func:`gitignore_entries`), not this line. -GITIGNORE_HEADER = "# lightcone-cli" - #: Every template shipped, by file name. The loader checks membership so a #: typo fails loudly at the call site instead of as a packaging mystery. TEMPLATE_NAMES = frozenset( @@ -68,12 +63,51 @@ def pyproject(*, name: str) -> str: return _render( "pyproject.toml.tmpl", name=name, - python_floor=DEFAULT_PYTHON_FLOOR, - min_uv=MIN_UV_VERSION, + requires_python=requires_python(), lc_requirement=lightcone_requirement(), ) +def python_version() -> str: + """``.python-version`` — the exact interpreter patch, taken from the + interpreter ``lc`` is running on. + + Deliberately not an engine constant: a new project pins the python the + researcher actually has, rather than one lc would have to download to + honor a number baked into a release. Identity follows this file from + here on — ``env_version`` hashes its bytes, not anything in the engine + (spec §3) — so a project is free to repin it afterwards. + """ + v = sys.version_info + return f"{v.major}.{v.minor}.{v.micro}\n" + + +def requires_python() -> str: + """The scaffolded ``requires-python``, taken verbatim from + lightcone-cli's own ``Requires-Python``. + + A scaffolded project depends on the engine, so uv enforces this bound + during resolution regardless; declaring the same specifier states it + rather than inventing a second, unrelated one. Verbatim, so a compound + specifier carries over intact. + + It cannot conflict with :func:`python_version`: lc is only *able* to + run on an interpreter satisfying this specifier, so the ambient pin + always satisfies it. The fallback for a metadata-less install is the + running interpreter's own minor version, which holds that property too. + """ + from importlib.metadata import PackageNotFoundError, metadata + + try: + # Message-style lookup: absent keys come back as None, not KeyError. + declared = metadata("lightcone-cli")["Requires-Python"] + except PackageNotFoundError: + declared = None + if declared: + return str(declared) + return f">={sys.version_info.major}.{sys.version_info.minor}" + + def lightcone_requirement() -> str: """The ``lightcone-cli`` requirement pinned into the scaffold. @@ -93,7 +127,7 @@ def lightcone_requirement() -> str: # ============================================================================= -# Managed files +# .gitignore — converged entry-wise, not by marker # ============================================================================= @@ -102,6 +136,16 @@ def gitignore() -> str: return read("gitignore.tmpl") +def gitignore_header() -> str: + """The template's own leading comment. + + Read back out of the template rather than duplicated as a constant, so + rewording it there can never leave :func:`gitignore_repair` appending a + second header to a file that already carries the first. + """ + return read("gitignore.tmpl").splitlines()[0] + + def gitignore_entries() -> tuple[str, ...]: """The patterns lightcone manages, in template order. @@ -129,6 +173,40 @@ def missing_gitignore_entries(text: str) -> list[str]: return [e for e in gitignore_entries() if e not in present] +def gitignore_repair(text: str) -> str | None: + """*text* with every managed pattern present, or ``None`` if it already + carries them all. + + Only ever appends, and only what is missing — so idempotency is + structural: a pattern already in the file is never added again, whoever + put it there. + + The append preserves template order, which is what keeps + ``!results/README.md`` after the ``results/*`` it negates. (A file + holding the negation *without* ``results/*`` would end up with them + inverted; that state can only be hand-written, and re-ordering + someone's ignores to fix it would be the more surprising behavior.) + """ + missing = missing_gitignore_entries(text) + if not missing: + return None + + block = "\n".join(missing) + "\n" + # The header is cosmetic, so add it only when it isn't already there; + # a later repair then appends bare patterns under the first one. + header = gitignore_header() + if header not in text: + block = header + "\n" + block + if not text.strip(): + return block + return text.rstrip("\n") + "\n\n" + block + + +# ============================================================================= +# results/ and the MyST report +# ============================================================================= + + def results_readme() -> str: """``results/README.md`` — where outputs land. @@ -139,12 +217,8 @@ def results_readme() -> str: return read("results-README.md.tmpl") -# ============================================================================= -# The MyST report -# ============================================================================= - - def myst_yml() -> str: + """``myst.yml`` — the report's MyST configuration.""" return read("myst.yml.tmpl") diff --git a/src/lightcone/engine/templates/files/pyproject.toml.tmpl b/src/lightcone/engine/templates/files/pyproject.toml.tmpl index f63c6a17..f851d29e 100644 --- a/src/lightcone/engine/templates/files/pyproject.toml.tmpl +++ b/src/lightcone/engine/templates/files/pyproject.toml.tmpl @@ -1,10 +1,10 @@ [project] name = "${name}" version = "0.0.1" -requires-python = ">=${python_floor}" +requires-python = "${requires_python}" dependencies = [ "${lc_requirement}", ] [tool.uv] -required-version = ">=${min_uv}" +required-version = ">=0.12" diff --git a/tests/conftest.py b/tests/conftest.py index e7617194..5ff37cce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,10 +8,6 @@ import pytest from click.testing import CliRunner -#: Minimal lockfile body the faked ``uv lock`` writes. Its only job is to -#: exist and be non-empty — nothing in layer 1 parses a lock. -UV_LOCK_MIN = 'version = 1\nrequires-python = ">=3.12"\n' - @pytest.fixture def runner() -> CliRunner: @@ -19,36 +15,53 @@ def runner() -> CliRunner: @pytest.fixture(autouse=True) -def _isolated_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Redirect ``~`` to a temp dir so tests can't touch the real home.""" - fake_home = tmp_path / "_home" - fake_home.mkdir() - monkeypatch.setattr(Path, "home", lambda: fake_home) - return fake_home - +def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """Fake every external tool convergence shells out to, so the suite is + hermetic — no network, no real resolution, no subprocesses. -@pytest.fixture(autouse=True) -def fake_uv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: - """Fake the uv seam so convergence tests are hermetic — no network, no - real resolution. Emulates the observable effects: ``uv lock`` writes - ``uv.lock``, ``uv sync`` materializes ``.venv``. + Models each tool's observable effect: ``uv lock`` writes ``uv.lock``, + ``uv sync`` materializes ``.venv``, ``git init`` makes ``.git``. The + ``--check`` probes answer from existence — enough for the convergence + tests; drift is exercised by tests that stub ``_run`` themselves, since + only uv can really tell a stale lock from a current one. - Returns the recorded argv lists, so a test can assert on *which* uv - commands ran with *which* flags. + Returns the recorded argv lists, so a test can assert on *which* tools + ran with *which* flags. :func:`uv_calls` narrows that to uv. """ calls: list[list[str]] = [] - def fake_run_uv(args: list[str], *, cwd: Path) -> MagicMock: - calls.append(list(args)) - project = Path(args[args.index("--project") + 1]) - if args[0] == "lock": - (project / "uv.lock").write_text(UV_LOCK_MIN) - elif args[0] == "sync": + def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: + calls.append(list(argv)) + project = _project(argv) if "--project" in argv else cwd + if argv[0] == "uv" and "--check" in argv: + artifact = "uv.lock" if argv[1] == "lock" else ".venv" + return MagicMock(returncode=0 if (project / artifact).exists() else 1) + if argv[:2] == ["uv", "lock"]: + # Only has to exist and be non-empty — layer 1 parses no lock. + (project / "uv.lock").write_text("version = 1\n") + elif argv[:2] == ["uv", "sync"]: (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) + elif argv[:2] == ["git", "init"]: + (cwd / ".git").mkdir(exist_ok=True) return MagicMock(returncode=0, stdout="", stderr="") from lightcone.engine import project - monkeypatch.setattr(project, "_run_uv", fake_run_uv) + monkeypatch.setattr(project, "_run", fake_run) monkeypatch.setattr(project.shutil, "which", lambda name, path=None: f"/usr/bin/{name}") return calls + + +def _project(argv: list[str]) -> Path: + """The project root a uv invocation was pointed at.""" + return Path(argv[argv.index("--project") + 1]) + + +def uv_calls(calls: list[list[str]]) -> list[list[str]]: + """Just the uv invocations, with the leading ``uv`` stripped.""" + return [c[1:] for c in calls if c[0] == "uv"] + + +def probes(calls: list[list[str]]) -> list[list[str]]: + """Just the read-only ``--check`` probes.""" + return [c for c in uv_calls(calls) if "--check" in c] diff --git a/tests/test_cli.py b/tests/test_cli.py index 618e1c01..834052e7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -32,18 +32,19 @@ def test_help_does_not_advertise_unbuilt_verbs(runner: CliRunner) -> None: assert f" {verb}" not in result.output -def test_engine_errors_render_cleanly(runner: CliRunner, tmp_path: Path) -> None: +def test_engine_errors_render_cleanly( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """A ProjectError from anywhere in the engine surfaces as a one-line CLI error (exit 1), not a traceback — the group boundary translates it.""" - project = tmp_path / "proj" - project.mkdir() - (project / "Containerfile").write_text("FROM python:3.12-slim\n") + from lightcone.engine import project + + monkeypatch.setattr(project.shutil, "which", lambda name, path=None: None) - result = runner.invoke(main, ["init", str(project), "--no-git"]) + result = runner.invoke(main, ["init", str(tmp_path / "proj")]) assert result.exit_code == 1 - assert "delete or rename it" in result.output - assert "[tool.lightcone.image]" in result.output + assert "uv is required" in result.output assert "Traceback" not in result.output @@ -52,28 +53,18 @@ def test_engine_errors_render_cleanly(runner: CliRunner, tmp_path: Path) -> None def test_init_creates_a_project(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git"]) + result = runner.invoke(main, ["init", str(project)]) assert result.exit_code == 0, result.output assert (project / "astra.yaml").exists() assert (project / ".venv").exists() - assert not (project / ".git").exists() - - -def test_init_no_sync_skips_the_venv( - runner: CliRunner, tmp_path: Path, fake_uv: list[list[str]] -) -> None: - project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--no-sync"]) - assert result.exit_code == 0 - assert [c[0] for c in fake_uv] == ["lock"] - assert not (project / ".venv").exists() + assert (project / ".git").exists() def test_init_defaults_to_the_current_directory( runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.chdir(tmp_path) - result = runner.invoke(main, ["init", "--no-git"]) + result = runner.invoke(main, ["init"]) assert result.exit_code == 0 assert (tmp_path / "astra.yaml").exists() @@ -82,23 +73,33 @@ def test_init_defaults_to_the_current_directory( def test_run_reports_what_it_created(runner: CliRunner, tmp_path: Path) -> None: - result = runner.invoke(main, ["init", str(tmp_path / "proj"), "--no-git"]) + result = runner.invoke(main, ["init", str(tmp_path / "proj")]) assert "created astra.yaml" in result.output assert "Project converged at" in result.output - # Next steps only make sense for a freshly scaffolded spec. - assert "Next steps" in result.output -def test_run_on_a_converged_project_says_so_and_skips_next_steps( - runner: CliRunner, tmp_path: Path -) -> None: +def test_run_on_a_converged_project_says_so(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - assert runner.invoke(main, ["init", str(project), "--no-git"]).exit_code == 0 + assert runner.invoke(main, ["init", str(project)]).exit_code == 0 - result = runner.invoke(main, ["init", str(project), "--no-git"]) + result = runner.invoke(main, ["init", str(project)]) assert result.exit_code == 0 assert "already converged" in result.output - assert "Next steps" not in result.output + + +def test_blocked_items_are_rendered(runner: CliRunner, tmp_path: Path) -> None: + """A blocked item is why a run can end unconverged, so it has to be + visible — its reason alone (carried as a warning) doesn't say which + item is missing.""" + project = tmp_path / "proj" + project.mkdir() + (project / "results").write_text("not a directory\n") + + result = runner.invoke(main, ["init", str(project)]) + assert result.exit_code == 0 + assert "blocked results/" in result.output + # Rich wraps on the terminal width, so assert on an unwrappable fragment. + assert "✗" in result.output def test_run_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: @@ -106,7 +107,7 @@ def test_run_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: project.mkdir() (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') - result = runner.invoke(main, ["init", str(project), "--no-git"]) + result = runner.invoke(main, ["init", str(project)]) assert result.exit_code == 0 assert "does not depend on lightcone-cli" in result.output @@ -116,7 +117,7 @@ def test_run_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: def test_check_reports_drift_without_writing(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--check"]) + result = runner.invoke(main, ["init", str(project), "--check"]) assert result.exit_code == 1 assert "would create" in result.output assert not project.exists() @@ -124,9 +125,9 @@ def test_check_reports_drift_without_writing(runner: CliRunner, tmp_path: Path) def test_check_passes_on_a_converged_project(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - assert runner.invoke(main, ["init", str(project), "--no-git"]).exit_code == 0 + assert runner.invoke(main, ["init", str(project)]).exit_code == 0 - result = runner.invoke(main, ["init", str(project), "--no-git", "--check"]) + result = runner.invoke(main, ["init", str(project), "--check"]) assert result.exit_code == 0 # Rich wraps on the terminal width, so assert on an unwrappable fragment. assert "nothing to do" in result.output @@ -134,32 +135,27 @@ def test_check_passes_on_a_converged_project(runner: CliRunner, tmp_path: Path) def test_json_report_is_machine_readable(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--json"]) + result = runner.invoke(main, ["init", str(project), "--json"]) assert result.exit_code == 0 + # Parsed straight off stdout: --json suppresses the banner. payload = json.loads(result.output) assert payload["converged"] is False assert "astra.yaml" in payload["created"] assert payload["warnings"] == [] - payload = json.loads(runner.invoke(main, ["init", str(project), "--no-git", "--json"]).output) + payload = json.loads(runner.invoke(main, ["init", str(project), "--json"]).output) assert payload["converged"] is True assert payload["created"] == [] and payload["repaired"] == [] -def test_json_suppresses_the_banner(runner: CliRunner, tmp_path: Path) -> None: - """The report has to be parseable straight off stdout.""" - result = runner.invoke(main, ["init", str(tmp_path / "proj"), "--no-git", "--json"]) - json.loads(result.output) - - def test_check_json_writes_nothing_and_exits_nonzero( runner: CliRunner, tmp_path: Path ) -> None: """`--check --json` together are the agent form: a drift report with no side effects and an exit code to branch on.""" project = tmp_path / "proj" - result = runner.invoke(main, ["init", str(project), "--no-git", "--check", "--json"]) + result = runner.invoke(main, ["init", str(project), "--check", "--json"]) assert result.exit_code == 1 assert json.loads(result.output)["converged"] is False assert not project.exists() diff --git a/tests/test_project.py b/tests/test_project.py index e8334e77..6b2eb3fb 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,18 +1,20 @@ -"""Tests for `lightcone.engine.project` — what a project is, and how one -converges.""" +"""Tests for `lightcone.engine.project` — how a directory converges into a +project.""" from __future__ import annotations +import shutil from pathlib import Path import pytest +from conftest import probes, uv_calls +from lightcone.engine import templates from lightcone.engine.project import ( SPEC_FILENAME, ConvergenceReport, ProjectError, converge, - find_root, project_name, ) @@ -20,7 +22,6 @@ SCAFFOLD = ( "astra.yaml", "universes/baseline.yaml", - "src", "pyproject.toml", ".python-version", ".gitignore", @@ -32,37 +33,7 @@ ) -# ---- discovery ------------------------------------------------------------ - - -def test_find_root_finds_the_spec_in_place(tmp_path: Path) -> None: - (tmp_path / SPEC_FILENAME).write_text("name: x\n") - assert find_root(tmp_path) == tmp_path.resolve() - - -def test_find_root_walks_up(tmp_path: Path) -> None: - (tmp_path / SPEC_FILENAME).write_text("name: x\n") - deep = tmp_path / "a" / "b" / "c" - deep.mkdir(parents=True) - assert find_root(deep) == tmp_path.resolve() - - -def test_find_root_returns_the_nearest_ancestor(tmp_path: Path) -> None: - """A sub-analysis nested inside a project is its own root.""" - (tmp_path / SPEC_FILENAME).write_text("name: outer\n") - inner = tmp_path / "sub" - inner.mkdir() - (inner / SPEC_FILENAME).write_text("name: inner\n") - assert find_root(inner) == inner.resolve() - - -def test_find_root_is_none_outside_a_project(tmp_path: Path) -> None: - assert find_root(tmp_path) is None - - -def test_find_root_ignores_a_directory_named_like_the_spec(tmp_path: Path) -> None: - (tmp_path / SPEC_FILENAME).mkdir() - assert find_root(tmp_path) is None +# ---- naming --------------------------------------------------------------- @pytest.mark.parametrize( @@ -88,10 +59,28 @@ def test_report_is_converged_only_when_nothing_changed() -> None: assert not ConvergenceReport(repaired=[".gitignore"]).converged -def test_warnings_do_not_make_a_project_unconverged() -> None: - """Warnings are things convergence can see but must not fix — they - must not turn a converged project into a failing `--check`.""" +def test_warnings_are_advisory_but_blocked_items_are_not() -> None: + """Warnings are things convergence can see but must not fix, so they + must not fail `--check`. A blocked item is different: something + convergence is responsible for is absent.""" assert ConvergenceReport(unchanged=["astra.yaml"], warnings=["heads up"]).converged + assert not ConvergenceReport(unchanged=["astra.yaml"], blocked=["results/"]).converged + + +def test_as_dict_carries_every_field() -> None: + """`--json` is the agent-facing contract, so it is built from the + dataclass rather than a hand-written field list.""" + payload = ConvergenceReport(created=["astra.yaml"]).as_dict() + assert payload["converged"] is False + assert payload["created"] == ["astra.yaml"] + assert set(payload) == { + "converged", + "created", + "repaired", + "unchanged", + "blocked", + "warnings", + } # ---- the scaffold --------------------------------------------------------- @@ -99,56 +88,54 @@ def test_warnings_do_not_make_a_project_unconverged() -> None: def test_converge_creates_the_whole_scaffold(tmp_path: Path) -> None: project = tmp_path / "proj" - report = converge(project, git=False) + report = converge(project) assert not report.converged for rel in SCAFFOLD: assert (project / rel).exists(), f"missing {rel}" -def test_converge_scaffolds_a_uv_project(tmp_path: Path) -> None: - """The environment is the uv project: a *virtual* pyproject (no - [build-system]) carrying the engine in its own dependency list, plus - the exact interpreter pin (spec §2).""" - from lightcone.engine.constants import DEFAULT_PYTHON, DEFAULT_PYTHON_FLOOR - +def test_converge_writes_the_templates_verbatim(tmp_path: Path) -> None: + """Convergence's claim is "the file written is the template, rendered + for this directory" — the content itself is `test_templates.py`'s.""" project = tmp_path / "proj" - converge(project, git=False) + converge(project) - pyproject = (project / "pyproject.toml").read_text() - assert 'name = "proj"' in pyproject - assert "lightcone-cli" in pyproject - assert f'requires-python = ">={DEFAULT_PYTHON_FLOOR}"' in pyproject - assert "[build-system]" not in pyproject - assert (project / ".python-version").read_text().strip() == DEFAULT_PYTHON + assert (project / "pyproject.toml").read_text() == templates.pyproject(name="proj") + assert (project / ".python-version").read_text() == templates.python_version() + assert (project / ".gitignore").read_text() == templates.gitignore() + assert (project / "results/README.md").read_text() == templates.results_readme() + assert (project / "myst.yml").read_text() == templates.myst_yml() + assert (project / "index.md").read_text() == templates.index_md(title="proj") -def test_converge_scaffolds_no_container_declaration(tmp_path: Path) -> None: - """Containerized mode is *derived* from a declaration the user makes, - never scaffolded — and ASTRA carries nothing about the environment, so - the boilerplate's `container:` key is stripped (spec §1, §2).""" +def test_converge_scaffolds_no_environment_escalation(tmp_path: Path) -> None: + """Containerized mode is *derived* from a declaration the user makes, so + the scaffold never writes one (spec §1). What astra's boilerplate puts + in `astra.yaml` is astra's business — reconciling that with the + environment model belongs to the environment layer.""" project = tmp_path / "proj" - converge(project, git=False) + converge(project) assert not (project / "Containerfile").exists() - assert "container:" not in (project / SPEC_FILENAME).read_text() assert "[tool.lightcone.image]" not in (project / "pyproject.toml").read_text() -def test_converge_gitignores_results_but_keeps_its_readme(tmp_path: Path) -> None: +def test_converge_scaffolds_no_src_directory(tmp_path: Path) -> None: + """astra stopped creating it (astra-tools#100) and so do we: the + boilerplate's `python src/main.py` is a TODO placeholder, and git drops + the empty directory from every clone anyway.""" project = tmp_path / "proj" - converge(project, git=False) + converge(project) + assert not (project / "src").exists() - gitignore = (project / ".gitignore").read_text() - assert ".venv/" in gitignore - assert "results/*" in gitignore - assert "!results/README.md" in gitignore - -def test_converge_titles_the_report_after_the_directory(tmp_path: Path) -> None: +def test_converge_writes_no_agent_notes(tmp_path: Path) -> None: + """`AGENTS.md` is deliberately not scaffolded — see the deviation note + in CLAUDE.md.""" project = tmp_path / "proj" - converge(project, git=False) - assert (project / "index.md").read_text().startswith("# proj\n") + converge(project) + assert not (project / "AGENTS.md").exists() # ---- idempotency and adoption -------------------------------------------- @@ -156,122 +143,113 @@ def test_converge_titles_the_report_after_the_directory(tmp_path: Path) -> None: def test_converge_is_idempotent(tmp_path: Path) -> None: project = tmp_path / "proj" - converge(project, git=False) + converge(project) before = {p: p.read_bytes() for p in project.rglob("*") if p.is_file()} - report = converge(project, git=False) + report = converge(project) assert report.converged assert {p: p.read_bytes() for p in project.rglob("*") if p.is_file()} == before -def test_converge_adopts_an_existing_project(tmp_path: Path) -> None: +def test_converge_adopts_an_existing_spec_without_touching_it(tmp_path: Path) -> None: """A directory that already holds an astra.yaml is adopted, not rejected — and the user's own file is never overwritten.""" project = tmp_path / "proj" project.mkdir() (project / SPEC_FILENAME).write_text('name: "mine"\n') - report = converge(project, git=False) + report = converge(project) assert (project / SPEC_FILENAME).read_text() == 'name: "mine"\n' assert "astra.yaml" in report.unchanged assert (project / "pyproject.toml").exists() -def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: +def test_adoption_adds_no_layout_of_its_own(tmp_path: Path) -> None: + """Neither `src/` nor an empty `universes/` is created beside a + user-authored spec: git cannot track an empty directory, so converging + one would report drift on every fresh clone. Where analysis code lives + is the user's layout (astra-tools#100), and universes are discovered by + glob — a missing directory yields no universes, not an error.""" project = tmp_path / "proj" - converge(project, git=False) - (project / "myst.yml").unlink() - - report = converge(project, git=False) - assert report.created == ["myst.yml"] - assert (project / "myst.yml").exists() + project.mkdir() + (project / SPEC_FILENAME).write_text('name: "mine"\n') + converge(project) + assert not (project / "src").exists() + assert not (project / "universes").exists() + assert converge(project).converged -def test_gitignore_repair_preserves_user_content_and_adds_the_entries( - tmp_path: Path, -) -> None: - """A repair only ever appends: the user's own ignores survive intact.""" - from lightcone.engine import templates +def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: + """The property the removed `src/` item broke: everything convergence + tracks has to be something git can carry, or a fresh clone reports + drift forever. `.venv` is exempt — it is git-ignored and rebuilt.""" project = tmp_path / "proj" - project.mkdir() - (project / ".gitignore").write_text("mine.txt\nbuild/\n") + converge(project) - report = converge(project, git=False) - assert ".gitignore" in report.repaired + # What a clone carries: tracked files and a repo — no empty directories, + # and no .venv. + clone = tmp_path / "clone" + for src in project.rglob("*"): + if src.is_file() and not {".venv", ".git"} & set(src.parts): + dst = clone / src.relative_to(project) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_bytes(src.read_bytes()) + (clone / ".git").mkdir() - text = (project / ".gitignore").read_text() - assert text.startswith("mine.txt\nbuild/\n") - assert templates.missing_gitignore_entries(text) == [] + report = converge(clone, write=False) + assert report.created == [".venv"], report.created -def test_gitignore_repair_appends_only_what_is_missing(tmp_path: Path) -> None: - """Entry-wise convergence, so a pattern the user already has is never - duplicated — whoever put it there.""" +def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: project = tmp_path / "proj" - project.mkdir() - (project / ".gitignore").write_text("__pycache__/\n.venv/\n") + converge(project) + (project / "myst.yml").unlink() - converge(project, git=False) - text = (project / ".gitignore").read_text() - assert text.count("__pycache__/") == 1 - assert text.count(".venv/") == 1 - assert "results/*" in text + report = converge(project) + assert report.created == ["myst.yml"] + assert (project / "myst.yml").exists() -def test_gitignore_repair_is_idempotent(tmp_path: Path) -> None: +def test_converge_recreates_a_nested_managed_file(tmp_path: Path) -> None: + """`results/README.md` must not depend on an earlier item having made + its directory — `file()` creates parents itself.""" project = tmp_path / "proj" - project.mkdir() - (project / ".gitignore").write_text("mine.txt\n") + converge(project) + shutil.rmtree(project / "results") - converge(project, git=False) - once = (project / ".gitignore").read_text() - report = converge(project, git=False) + converge(project) + assert (project / "results" / "README.md").exists() - assert ".gitignore" in report.unchanged - assert (project / ".gitignore").read_text() == once +# ---- .gitignore ---------------------------------------------------------- -def test_gitignore_repair_reaches_a_file_that_has_the_header_already( - tmp_path: Path, -) -> None: - """The header is cosmetic: a project carrying it but missing an entry - — an older lc wrote it, or a hand edit removed a line — still - converges. This is the case a marker check would have skipped.""" - from lightcone.engine import templates +def test_gitignore_repair_preserves_user_content(tmp_path: Path) -> None: + """A repair only ever appends: the user's own ignores survive intact.""" project = tmp_path / "proj" project.mkdir() - (project / ".gitignore").write_text(f"{templates.GITIGNORE_HEADER}\n.venv/\n") - - report = converge(project, git=False) - text = (project / ".gitignore").read_text() + (project / ".gitignore").write_text("mine.txt\nbuild/\n") + report = converge(project) assert ".gitignore" in report.repaired - assert templates.missing_gitignore_entries(text) == [] - # Cosmetic, so still exactly one. - assert text.count(templates.GITIGNORE_HEADER) == 1 + text = (project / ".gitignore").read_text() + assert text.startswith("mine.txt\nbuild/\n") + assert templates.missing_gitignore_entries(text) == [] -def test_repair_of_an_empty_gitignore_adds_no_leading_blank_lines( - tmp_path: Path, -) -> None: - from lightcone.engine import templates +def test_gitignore_repair_is_idempotent(tmp_path: Path) -> None: project = tmp_path / "proj" project.mkdir() - (project / ".gitignore").write_text("") - - converge(project, git=False) - assert (project / ".gitignore").read_text() == templates.gitignore() + (project / ".gitignore").write_text("mine.txt\n") + converge(project) + once = (project / ".gitignore").read_text() + report = converge(project) -def test_converge_writes_no_agent_notes(tmp_path: Path) -> None: - """`AGENTS.md` is deliberately not scaffolded — see the deviation note - in CLAUDE.md.""" - project = tmp_path / "proj" - converge(project, git=False) - assert not (project / "AGENTS.md").exists() + assert ".gitignore" in report.unchanged + assert (project / ".gitignore").read_text() == once # ---- write=False (check mode) -------------------------------------------- @@ -281,7 +259,7 @@ def test_check_mode_writes_nothing_at_all(tmp_path: Path) -> None: """Not even the project directory — a drift report must be free of side effects.""" project = tmp_path / "proj" - report = converge(project, write=False, git=False) + report = converge(project, write=False) assert not report.converged assert "astra.yaml" in report.created @@ -292,14 +270,14 @@ def test_check_mode_agrees_with_a_real_run(tmp_path: Path) -> None: """Check mode is the same decisions with side effects off, so its report must match what a real run reports.""" project = tmp_path / "proj" - dry = converge(project, write=False, git=False) - wet = converge(project, git=False) + dry = converge(project, write=False) + wet = converge(project) assert dry.as_dict() == wet.as_dict() - assert converge(project, write=False, git=False).converged + assert converge(project, write=False).converged -# ---- warnings ------------------------------------------------------------- +# ---- warnings and blocked items ------------------------------------------ def test_warns_when_pyproject_lacks_the_engine(tmp_path: Path) -> None: @@ -309,36 +287,104 @@ def test_warns_when_pyproject_lacks_the_engine(tmp_path: Path) -> None: project.mkdir() (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') - report = converge(project, git=False) + report = converge(project) assert any("does not depend on lightcone-cli" in w for w in report.warnings) assert 'name = "mine"' in (project / "pyproject.toml").read_text() -def test_warns_when_results_is_not_a_directory(tmp_path: Path) -> None: +def test_no_engine_warning_for_a_pyproject_we_wrote(tmp_path: Path) -> None: + """The warning is decided against the pre-existing file, so a scaffolded + one — which always names the engine — can never trigger it.""" + report = converge(tmp_path / "proj") + assert report.warnings == [] + + +def test_results_that_is_not_a_directory_blocks_convergence(tmp_path: Path) -> None: + """Reporting `results/README.md` as unchanged when it cannot exist would + let `--check --json` call a broken project converged.""" project = tmp_path / "proj" project.mkdir() (project / "results").write_text("not a directory\n") - report = converge(project, git=False) - assert any("results exists but is not a directory" in w for w in report.warnings) + report = converge(project) + assert report.blocked == ["results/"] + assert any("not a directory" in w for w in report.warnings) + assert "results/README.md" not in report.unchanged assert (project / "results").read_text() == "not a directory\n" + # And it stays unconverged on re-run, rather than settling into a lie. + assert not converge(project).converged -# ---- refusals ------------------------------------------------------------- +# ---- git ------------------------------------------------------------------ -def test_refuses_an_authored_containerfile(tmp_path: Path) -> None: - """Images are generated from the lock; a hand-written Containerfile - would be silently ignored. Refuse — before any write (spec §8).""" + +def test_initializes_a_repository(tmp_path: Path, tools: list[list[str]]) -> None: project = tmp_path / "proj" - project.mkdir() - (project / "Containerfile").write_text("FROM python:3.12-slim\n") + report = converge(project) + + assert ["git", "init", "-q"] in tools + assert ".git" in report.created + + +def test_does_not_nest_a_repository_inside_one(tmp_path: Path, tools: list[list[str]]) -> None: + """`lc init subdir/` inside an existing repository must not create a + nested one — the check walks up, it doesn't just look in the directory.""" + (tmp_path / ".git").mkdir() + report = converge(tmp_path / "nested" / "proj") + + assert not any(c[0] == "git" for c in tools) + assert ".git" in report.unchanged + assert not (tmp_path / "nested" / "proj" / ".git").exists() + + +def test_a_linked_worktree_counts_as_version_controlled(tmp_path: Path) -> None: + """In a linked worktree or submodule `.git` is a *file*, not a + directory.""" + (tmp_path / ".git").write_text("gitdir: /elsewhere/.git/worktrees/wt\n") + assert ".git" in converge(tmp_path / "proj").unchanged + + +def test_git_absent_is_not_a_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, tools: list[list[str]] +) -> None: + """git is optional, unlike uv: a project without it is valid, so an + absent git is nothing to converge rather than a warning.""" + from lightcone.engine import project as project_mod + + monkeypatch.setattr( + project_mod.shutil, + "which", + lambda name, path=None: None if name == "git" else f"/usr/bin/{name}", + ) + report = converge(tmp_path / "proj") + + assert not any(c[0] == "git" for c in tools) + assert ".git" not in report.created + report.unchanged + assert report.warnings == [] + + +def test_surfaces_a_git_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Nothing convergence shells out to gets to fail silently — a `.git` + reported as created but never made is the kind of lie the report exists + to prevent.""" + from unittest.mock import MagicMock + + from lightcone.engine import project as project_mod - with pytest.raises(ProjectError, match="delete or rename it"): - converge(project, git=False) + def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[0] == "git": + return MagicMock(returncode=128, stdout="", stderr="fatal: cannot mkdir") + if argv[:2] == ["uv", "lock"]: + (cwd / "uv.lock").write_text("version = 1\n") + return MagicMock(returncode=0, stdout="", stderr="") - assert not (project / SPEC_FILENAME).exists() - assert not (project / "pyproject.toml").exists() + monkeypatch.setattr(project_mod, "_run", fake_run) + with pytest.raises(ProjectError, match="cannot mkdir"): + converge(tmp_path / "proj") + + +# ---- refusals ------------------------------------------------------------- def test_requires_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -346,39 +392,164 @@ def test_requires_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(project_mod.shutil, "which", lambda name, path=None: None) with pytest.raises(ProjectError, match="uv is required"): - converge(tmp_path / "proj", git=False) + converge(tmp_path / "proj") # ---- the uv seam ---------------------------------------------------------- -def test_locks_then_syncs_exactly(tmp_path: Path, fake_uv: list[list[str]]) -> None: +def test_locks_then_syncs_exactly(tmp_path: Path, tools: list[list[str]]) -> None: """Converge once, with the flags the spec fixes: ``--locked --exact`` (no drift, no additive sync) and ``--compile-bytecode`` (§4, §6).""" project = tmp_path / "proj" - converge(project, git=False) + converge(project) - assert [c[0] for c in fake_uv] == ["lock", "sync"] - sync = fake_uv[1] + lock, sync = uv_calls(tools) + assert lock[0] == "lock" + assert sync[0] == "sync" for flag in ("--locked", "--exact", "--compile-bytecode"): assert flag in sync # uv's own walk-up discovery is never trusted (spec §4). - assert fake_uv[0][-2:] == ["--project", str(project.resolve())] + assert lock[-2:] == ["--project", str(project.resolve())] assert sync[-2:] == ["--project", str(project.resolve())] -def test_sync_false_still_locks(tmp_path: Path, fake_uv: list[list[str]]) -> None: +def test_a_fresh_project_is_not_probed(tmp_path: Path, tools: list[list[str]]) -> None: + """Nothing exists, so nothing needs asking — the created/repaired split + comes from existence, and only an existing artifact costs a probe.""" + converge(tmp_path / "proj") + assert probes(tools) == [] + + +def test_a_converged_project_is_verified_not_assumed( + tmp_path: Path, tools: list[list[str]] +) -> None: + """The whole point: a second run asks uv whether the lock and the + environment still agree with their inputs, rather than seeing two paths + and declaring victory.""" + project = tmp_path / "proj" + converge(project) + tools.clear() + + report = converge(project) + assert report.converged + assert [p[0] for p in probes(tools)] == ["lock", "sync"] + # Verification only — the mutating forms never ran. + assert all("--check" in c for c in uv_calls(tools)) + + +def test_check_mode_only_probes(tmp_path: Path, tools: list[list[str]]) -> None: + """Check mode may ask uv questions — the probes are read-only (verified + against uv 0.12.3) — but must never run a mutating form.""" project = tmp_path / "proj" - converge(project, git=False, sync=False) + converge(project) + tools.clear() + + converge(project, write=False) + assert uv_calls(tools), "expected the probes to run" + assert all("--check" in c for c in uv_calls(tools)) + + +def test_check_mode_on_a_fresh_project_runs_nothing( + tmp_path: Path, tools: list[list[str]] +) -> None: + converge(tmp_path / "proj", write=False) + assert tools == [] + + +def test_a_stale_lock_is_repaired_not_ignored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Converging by existence made this a no-op: `uv.lock` and `.venv` are + *derived*, so a lock that no longer matches `pyproject.toml` is exactly + as unconverged as a missing one.""" + from unittest.mock import MagicMock + + from lightcone.engine import project as project_mod + + project = tmp_path / "proj" + converge(project) + + # uv now reports the lock as stale; the environment still matches it. + real_run = project_mod._run + + def drifted(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:3] == ["uv", "lock", "--check"]: + return MagicMock(returncode=1, stdout="", stderr="") + return real_run(argv, cwd=cwd) # type: ignore[no-any-return] + + monkeypatch.setattr(project_mod, "_run", drifted) + + report = converge(project, write=False) + assert not report.converged + assert report.repaired == ["uv.lock"] + assert "uv.lock" not in report.unchanged + + +def test_a_stale_environment_is_repaired_not_ignored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The case layer 3's launcher depends on: an environment that no longer + satisfies the lock must be re-synced, not waved through because `.venv` + happens to be a directory.""" + from unittest.mock import MagicMock + + from lightcone.engine import project as project_mod + + project = tmp_path / "proj" + converge(project) + real_run = project_mod._run + + def drifted(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:2] == ["uv", "sync"] and "--check" in argv: + return MagicMock(returncode=1, stdout="", stderr="") + return real_run(argv, cwd=cwd) # type: ignore[no-any-return] + + monkeypatch.setattr(project_mod, "_run", drifted) + tools_before = converge(project) + + assert tools_before.repaired == [".venv"] + assert tools_before.unchanged and "uv.lock" in tools_before.unchanged + + +def test_relays_uv_warnings_into_the_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The one that matters: a cache on a different filesystem means uv + copies every package instead of linking it, and nothing else would tell + the user their environment costs full price.""" + from unittest.mock import MagicMock + + from lightcone.engine import project as project_mod + + stderr = ( + "warning: Failed to hardlink files; falling back to full copy.\n" + " If the cache and target directories are on different " + "filesystems, hardlinking may not be supported.\n" + "Installed 117 packages in 34ms\n" + ) + + def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:2] == ["uv", "lock"]: + (cwd / "uv.lock").write_text("version = 1\n") + return MagicMock(returncode=0, stdout="", stderr=stderr) + + monkeypatch.setattr(project_mod, "_run", fake_run) + report = converge(tmp_path / "proj") + + assert any("falling back to full copy" in w for w in report.warnings) + # The continuation line is folded in, the progress line is not. + assert any("different filesystems" in w for w in report.warnings) + assert not any("Installed 117" in w for w in report.warnings) + # Advisory: a heavy venv is still a converged project. + assert not report.blocked - assert [c[0] for c in fake_uv] == ["lock"] - assert (project / "uv.lock").exists() - assert not (project / ".venv").exists() +def test_tool_warnings_ignores_progress_output() -> None: + from lightcone.engine.project import tool_warnings -def test_check_mode_runs_no_uv(tmp_path: Path, fake_uv: list[list[str]]) -> None: - converge(tmp_path / "proj", write=False, git=False) - assert fake_uv == [] + assert tool_warnings("Installed 3 packages\n + click==8.4\n") == [] + assert tool_warnings("warning: a\nwarning: b\n") == ["a", "b"] def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -390,8 +561,8 @@ def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch monkeypatch.setattr( project_mod, - "_run_uv", - lambda args, *, cwd: MagicMock(returncode=1, stdout="", stderr="no solution found"), + "_run", + lambda argv, *, cwd: MagicMock(returncode=1, stdout="", stderr="no solution found"), ) with pytest.raises(ProjectError, match="no solution found"): - converge(tmp_path / "proj", git=False) + converge(tmp_path / "proj") diff --git a/tests/test_templates.py b/tests/test_templates.py index 50685498..4d7be65f 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -2,6 +2,8 @@ from __future__ import annotations +import sys + import pytest from lightcone.engine import templates @@ -19,21 +21,54 @@ def test_unknown_template_fails_loudly() -> None: templates.read("nope.tmpl") -def test_pyproject_renders_every_placeholder() -> None: - from lightcone.engine.constants import DEFAULT_PYTHON_FLOOR, MIN_UV_VERSION +# ---- the uv project ------------------------------------------------------- + +def test_pyproject_renders_every_placeholder() -> None: rendered = templates.pyproject(name="my-analysis") assert "$" not in rendered assert 'name = "my-analysis"' in rendered - assert f'requires-python = ">={DEFAULT_PYTHON_FLOOR}"' in rendered - assert f'required-version = ">={MIN_UV_VERSION}"' in rendered + assert f'requires-python = "{templates.requires_python()}"' in rendered assert templates.lightcone_requirement() in rendered + # Virtual by design: containerized mode builds `--no-install-project`, + # so a packaged project's own import would fail inside its image (§1). + assert "[build-system]" not in rendered + + +def test_python_version_pins_the_running_interpreter() -> None: + """No engine constant — a new project pins the python the researcher + actually has, rather than one lc would have to download.""" + v = sys.version_info + assert templates.python_version() == f"{v.major}.{v.minor}.{v.micro}\n" + +def test_requires_python_comes_from_our_own_metadata() -> None: + """Declaring the engine's own bound rather than inventing a second one.""" + from importlib.metadata import metadata -def test_pyproject_is_virtual() -> None: - """Containerized mode builds `--no-install-project`, so a packaged - project's own import would fail inside its image (spec §1).""" - assert "[build-system]" not in templates.pyproject(name="x") + assert templates.requires_python() == metadata("lightcone-cli")["Requires-Python"] + + +def test_the_ambient_pin_always_satisfies_the_declared_floor() -> None: + """Not a coincidence: lc can only run on an interpreter meeting its own + Requires-Python, so the two can never conflict.""" + from packaging.specifiers import SpecifierSet + + assert templates.python_version().strip() in SpecifierSet(templates.requires_python()) + + +def test_requires_python_falls_back_to_the_running_minor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A metadata-less install still gets a bound the ambient pin meets.""" + import importlib.metadata + + def _missing(_name: str) -> object: + raise importlib.metadata.PackageNotFoundError + + monkeypatch.setattr(importlib.metadata, "metadata", _missing) + v = sys.version_info + assert templates.requires_python() == f">={v.major}.{v.minor}" def test_lightcone_requirement_pins_the_running_version( @@ -57,14 +92,7 @@ def _missing(_name: str) -> str: assert templates.lightcone_requirement() == "lightcone-cli" -def test_index_md_renders_the_title_and_keeps_myst_roles() -> None: - """The report body legitimately contains braces (`{astra}` roles) — - which is why substitution is `string.Template`, not `str.format`.""" - rendered = templates.index_md(title="My Analysis") - assert rendered.startswith("# My Analysis\n") - assert "{astra}`decisions.example_method`" in rendered - assert "{astra:value}`outputs.main_result`" in rendered - assert "$" not in rendered +# ---- .gitignore ----------------------------------------------------------- def test_gitignore_entries_are_the_patterns_only() -> None: @@ -84,10 +112,55 @@ def test_gitignore_entries_keep_template_order() -> None: assert entries.index("results/*") < entries.index("!results/README.md") -def test_missing_entries_ignores_comments_in_the_target() -> None: - """A pattern that appears only inside a comment is not present.""" - assert "results/*" in templates.missing_gitignore_entries("# results/*\n") +def test_gitignore_header_is_the_templates_own_first_line() -> None: + """Derived, not duplicated: rewording the template's comment can't leave + the repair appending a second header.""" + assert templates.gitignore().startswith(templates.gitignore_header() + "\n") + + +def test_repair_is_none_when_nothing_is_missing() -> None: assert templates.missing_gitignore_entries(templates.gitignore()) == [] + assert templates.gitignore_repair(templates.gitignore()) is None + + +def test_repair_of_an_empty_file_is_just_the_template() -> None: + assert templates.gitignore_repair("") == templates.gitignore() + + +def test_repair_appends_only_what_is_missing_behind_the_header() -> None: + repaired = templates.gitignore_repair("mine.txt\n.venv/\n") + assert repaired is not None + assert repaired.startswith("mine.txt\n.venv/\n\n") + assert repaired.count(".venv/") == 1 + assert templates.gitignore_header() in repaired + assert templates.missing_gitignore_entries(repaired) == [] + + +def test_repair_does_not_add_a_second_header() -> None: + """The case a marker check would have skipped: header present, entries + missing.""" + header = templates.gitignore_header() + repaired = templates.gitignore_repair(f"{header}\n.venv/\n") + assert repaired is not None + assert repaired.count(header) == 1 + assert templates.missing_gitignore_entries(repaired) == [] + + +def test_a_pattern_inside_a_comment_does_not_count_as_present() -> None: + assert "results/*" in templates.missing_gitignore_entries("# results/*\n") + + +# ---- the rest ------------------------------------------------------------- + + +def test_index_md_renders_the_title_and_keeps_myst_roles() -> None: + """The report body legitimately contains braces (`{astra}` roles) — + which is why substitution is `string.Template`, not `str.format`.""" + rendered = templates.index_md(title="My Analysis") + assert rendered.startswith("# My Analysis\n") + assert "{astra}`decisions.example_method`" in rendered + assert "{astra:value}`outputs.main_result`" in rendered + assert "$" not in rendered def test_results_readme_explains_the_output_layout() -> None: From c0bf65b036ba1c81314a757a90c0696b83f6b4ee Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Tue, 18 Aug 2026 07:58:25 +0200 Subject: [PATCH 04/23] minor clean-up --- CLAUDE.md | 7 +- src/lightcone/engine/project.py | 240 +++++++++++--------------------- 2 files changed, 87 insertions(+), 160 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 64740a59..5f40139b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,9 +218,10 @@ user owns: same report without writing (exit 1 when not converged); `--json` emits `{converged, created, repaired, unchanged, blocked, warnings}`. - **Derived artifacts converge by correctness, not existence.** `uv.lock` - and `.venv` go through `_Converger.derived`, whose predicate is uv's own - no-write verification (`uv lock --check`, `uv sync --locked --exact - --check` — both confirmed read-only against uv 0.12.3). A lock that no + and `.venv` go through `_Converger.item`'s optional `is_current=` + predicate, which is uv's own no-write verification (`uv lock --check`, + `uv sync --locked --exact --check` — both confirmed read-only against uv + 0.12.3). A lock that no longer matches `pyproject.toml`, or an environment that no longer matches the lock, is exactly as unconverged as a missing one and reports as `repaired`. Existence alone made `converge()` a no-op on drift, which diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 8233d0a2..8c5ee4ce 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -1,16 +1,4 @@ -"""What a project *is*, and how to converge one. - -:func:`converge` brings a directory to the state a lightcone project is -defined to have: an ASTRA spec sitting on a uv project. Idempotent, and -never destructive to files the user owns. - -Nothing here knows about Click or the console: convergence returns a -:class:`ConvergenceReport` and raises :class:`ProjectError`; rendering and -exit codes are the CLI's business. - -Project *discovery* — the ``astra.yaml`` walk-up that finds a root from a -working directory — arrives with the launcher, the first thing that needs -it. ``lc init`` is handed its directory explicitly. +"""Utilities to manage a Lightcone project. """ from __future__ import annotations @@ -87,14 +75,32 @@ def __init__(self, *, write: bool) -> None: self.write = write self.report = ConvergenceReport() - def item(self, name: str, present: bool, apply: Callable[[], object]) -> None: - """Converge something whose *presence* is the whole question.""" - if present: + def item( + self, + name: str, + present: bool, + apply: Callable[[], object], + *, + is_current: Callable[[], bool] | None = None, + ) -> None: + """Converge something whose *presence* is the question. + + ``is_current`` makes it a *derived* artifact instead, one whose + agreement with its inputs is the question: a ``uv.lock`` that no + longer matches ``pyproject.toml``, or a ``.venv`` that no longer + matches the lock, is exactly as unconverged as a missing one, and + reports as ``repaired``. It is consulted only when the item is + present, so a fresh project pays no probe. + """ + if not present: + self.report.created.append(name) + elif is_current is None or is_current(): self.report.unchanged.append(name) + return else: - self.report.created.append(name) - if self.write: - apply() + self.report.repaired.append(name) + if self.write: + apply() def file( self, @@ -127,38 +133,6 @@ def file( else: self.report.unchanged.append(name) - def derived( - self, - name: str, - path: Path, - *, - is_current: Callable[[], bool], - apply: Callable[[], object], - ) -> None: - """Converge a *derived* artifact — one whose agreement with its - inputs, not its mere existence, is the question. - - ``uv.lock`` and ``.venv`` are derived: a lock that no longer matches - ``pyproject.toml``, or an environment that no longer matches the - lock, is exactly as unconverged as a missing one. Existence alone - would make convergence a no-op on drift — and layer 3's launcher - converges on every invocation precisely to guarantee the environment - matches the lock. - - ``is_current`` is only consulted when the artifact exists, so a - fresh project pays no probe, and the created/repaired distinction - falls out of the same check. - """ - if not path.exists(): - self.report.created.append(name) - elif is_current(): - self.report.unchanged.append(name) - return - else: - self.report.repaired.append(name) - if self.write: - apply() - def blocked(self, name: str, reason: str) -> None: """Record an item convergence cannot complete, and why. @@ -192,6 +166,12 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: Raises :class:`ProjectError` if uv is missing or fails — it is the environment substrate, so there is no useful project without it. """ + # Imported here rather than at module scope because `astra.cli` costs + # ~0.5 s to import — Click, Rich, and the validation stack, which pulls + # linkml_runtime. astra-tools#102 moves the scaffold to a stdlib-only + # `astra.scaffold`; once that lands this can go back up top. + from astra.cli import create_boilerplate + directory = directory.resolve() _require_uv() @@ -201,17 +181,15 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: if write: directory.mkdir(parents=True, exist_ok=True) - # The spec is one item, keyed on astra.yaml: astra treats it and - # universes/baseline.yaml as a unit, because the baseline references the - # boilerplate's example decision and must never land beside a - # user-authored spec. Nothing else about the layout is converged — - # `src/` and an empty `universes/` are directories git cannot track, so - # converging them would report drift on every fresh clone; where - # analysis code lives is the user's layout, not ours (astra-tools#100). + # `create_boilerplate` is astra's public scaffold API, the same one + # `astra init` delegates to: it writes the spec — astra.yaml plus + # universes/baseline.yaml, which converge as one item because the + # baseline references the boilerplate's example decision — and nothing + # else. c.item( "astra.yaml", (directory / SPEC_FILENAME).exists(), - lambda: _scaffold_spec(directory), + lambda: create_boilerplate(directory), ) _converge_uv_project(c, directory) c.file( @@ -221,22 +199,29 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: repair=templates.gitignore_repair, ) _converge_results(c, directory) - _converge_report_template(c, directory) + # The MyST report is a recommended add-on on top of the spec, not part + # of it — which is why it is scaffolded here and not by `astra init`. + c.file("myst.yml", directory / "myst.yml", templates.myst_yml) + c.file( + "index.md", + directory / "index.md", + lambda: templates.index_md(title=directory.name or "My Analysis"), + ) _converge_git(c, directory) # Lock and sync last: they are the expensive steps, and they are # meaningless until pyproject.toml and .python-version exist. - c.derived( + c.item( "uv.lock", - directory / "uv.lock", + (directory / "uv.lock").exists(), + lambda: _uv(c, ["lock"], directory=directory), is_current=lambda: _lock_is_current(directory), - apply=lambda: _uv_lock(c, directory), ) - c.derived( + c.item( ".venv", - directory / ".venv", + (directory / ".venv").exists(), + lambda: _uv(c, _SYNC_ARGS, directory=directory), is_current=lambda: _env_is_current(directory), - apply=lambda: _uv_sync(c, directory), ) return c.report @@ -250,26 +235,6 @@ def _require_uv() -> None: ) -def _scaffold_spec(directory: Path) -> None: - """Write astra's boilerplate spec: ``astra.yaml`` + ``universes/``. - - ``create_boilerplate`` is astra's public scaffold API, split out - (astra-tools#99) so downstream tools can scaffold into existing - directories under their own conventions: it writes the spec and nothing - else — no ``.gitignore``, no git init, no emptiness checks. ``astra - init`` delegates to the same function, so this is the shared path, not - a back door. - - Its boilerplate carries a ``container:`` key. **lightcone-cli ignores - the ASTRA ``container:`` directive entirely** — not stripped, not - validated, not migrated. The environment is ``pyproject.toml`` + - ``uv.lock``, and no code path here reads that key. - """ - from astra.cli import create_boilerplate - - create_boilerplate(directory) - - def _converge_uv_project(c: _Converger, directory: Path) -> None: """pyproject.toml + .python-version — the environment definition. @@ -308,17 +273,6 @@ def _converge_results(c: _Converger, directory: Path) -> None: c.file("results/README.md", results_dir / "README.md", templates.results_readme) -def _converge_report_template(c: _Converger, directory: Path) -> None: - """The MyST report. Recommended add-on on top of the spec, not part of - it — which is why it is scaffolded here and not by ``astra init``.""" - c.file("myst.yml", directory / "myst.yml", templates.myst_yml) - c.file( - "index.md", - directory / "index.md", - lambda: templates.index_md(title=directory.name or "My Analysis"), - ) - - def _converge_git(c: _Converger, directory: Path) -> None: """``git init``, unless the project is already under version control. @@ -376,12 +330,10 @@ def tool_warnings(stderr: str) -> list[str]: stream would bury them under a line per installed package. A warning is a line starting ``warning:`` plus any indented continuation. - Worth relaying at all because of one warning in particular: when the uv - cache and the project are on different filesystems, uv cannot link and - silently falls back to copying every package. That is the difference - between a project costing a few MB and costing the whole environment - again (measured: ~148 MB of a 216 MB environment is shared when linking - works), and nothing else would tell the user. + The one that has to reach the user: when the uv cache and the project + are on different filesystems, uv cannot link and silently falls back to + copying every package — most of an environment stops being shared, and + nothing else would say so. """ found: list[str] = [] for line in stderr.splitlines(): @@ -392,76 +344,50 @@ def tool_warnings(stderr: str) -> list[str]: return found -def _probe(argv: list[str], *, cwd: Path) -> bool: - """Ask a tool a yes/no question. +def _is_current(argv: list[str], *, directory: Path) -> bool: + """Ask uv whether an artifact still agrees with its inputs. - A nonzero exit is the answer "no", not a failure — so unlike - :func:`_check_call` this never raises. Whatever the tool wrote to stderr - is its explanation of the "no", which the caller is about to act on - anyway. + A nonzero exit is the answer "no", not a failure, so unlike + :func:`_check_call` this never raises. Both probes are uv's own + no-write verification (confirmed read-only against uv 0.12.3), so the + answer is uv's rather than a heuristic of ours. """ - return _run(argv, cwd=cwd).returncode == 0 + return _run(["uv", *argv, "--project", str(directory)], cwd=directory).returncode == 0 def _lock_is_current(directory: Path) -> bool: - """Whether ``uv.lock`` still agrees with ``pyproject.toml``. - - uv's own no-write verification, so the answer is uv's rather than a - heuristic of ours (verified read-only against uv 0.12.3). - """ - return _probe(["uv", "lock", "--check", "--project", str(directory)], cwd=directory) + """Whether ``uv.lock`` still agrees with ``pyproject.toml``.""" + return _is_current(["lock", "--check"], directory=directory) def _env_is_current(directory: Path) -> bool: """Whether ``.venv`` still satisfies ``uv.lock``. - **Honest limitation** (uv 0.12.3, measured): this catches packages the - lock requires and the environment lacks, but *not* extras — a package - installed by hand into the environment leaves ``--check`` reporting - "would make no changes", even though a real ``--exact`` sync would - remove it. So a hand-polluted environment still reads as current. The - design already accepts set-level rather than byte-level environment - checks (spec §3); what ultimately bounds what a recipe can import is - the sandbox, not this probe. + Set-level, not byte-level, as spec §3 accepts: uv catches packages the + lock requires and the environment lacks, but not extras installed by + hand, which leave ``--check`` reporting "would make no changes". What + bounds what a recipe can import is the sandbox, not this probe. """ - argv = ["uv", "sync", "--locked", "--exact", "--check", "--project", str(directory)] - return _probe(argv, cwd=directory) + return _is_current(["sync", "--locked", "--exact", "--check"], directory=directory) + + +# --locked so a drifted lock is an error rather than a silent relock; +# --exact because a plain sync is additive; --compile-bytecode because the +# environment is read-only at execution time, so a recipe cannot write .pyc +# as it imports — paying compilation once here instead of on every recipe +# run (spec §4, §6, §7). +_SYNC_ARGS = ["sync", "--locked", "--exact", "--compile-bytecode"] def _uv(c: _Converger, args: list[str], *, directory: Path) -> None: """Run uv against *directory*, recording anything it warns about. - Every invocation carries an explicit ``--project``: uv's native - walk-up discovery is never trusted (spec §4). - - Nothing about linking or caching is overridden. uv's defaults already - share package content between projects — it clones (copy-on-write) or - hard links out of a global cache — and the docs discourage forcing - ``symlink`` mode, which would couple every environment to the cache's - survival. `--system-site-packages` is likewise never used: it would - make packages outside the lock importable, which is precisely what the - environment model exists to prevent (spec §7 G6). + Every invocation carries an explicit ``--project``: uv's own walk-up + discovery is never trusted (spec §4). Nothing about linking or caching + is overridden — uv's defaults already share package content between + projects — and ``--system-site-packages`` is never used, since it would + make packages outside the lock importable, which is what the + environment model exists to prevent (spec §7, G6). """ for warning in _check_call(["uv", *args, "--project", str(directory)], cwd=directory): c.warn(f"uv: {warning}") - - -def _uv_lock(c: _Converger, directory: Path) -> None: - _uv(c, ["lock"], directory=directory) - - -def _uv_sync(c: _Converger, directory: Path) -> None: - """Converge the environment, with the flags the spec fixes. - - ``--locked`` so a drifted lock is an error rather than a silent relock; - ``--exact`` because a plain sync is additive; and ``--compile-bytecode`` - because the environment is read-only at execution time, so a recipe - cannot write ``.pyc`` files as it imports (spec §4, §6, §7). - - ``--compile-bytecode`` is the environment's one genuinely per-project - cost: bytecode is generated into the venv rather than linked from the - cache, ~55 MB of a 216 MB environment here. The alternative is paying - compilation on every recipe run instead, since the sandbox denies the - write — so this is a deliberate trade, not an oversight. - """ - _uv(c, ["sync", "--locked", "--exact", "--compile-bytecode"], directory=directory) From 9921c76dacf9b6d22746ea2c792ddd0fdda97bdf Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Tue, 18 Aug 2026 08:25:05 +0200 Subject: [PATCH 05/23] fixing minor surprise --- src/lightcone/engine/project.py | 26 +++++++++++++++++++++--- tests/test_project.py | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 8c5ee4ce..f03d86e4 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import re import shutil import subprocess @@ -306,8 +307,23 @@ def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: Tests monkeypatch this, so the suite never shells out and every call is inspectable. + + """ + return subprocess.run( + argv, cwd=cwd, capture_output=True, text=True, check=False, env=child_env() + ) + + +def child_env() -> dict[str, str]: + """The environment external tools run in: ours, minus ``VIRTUAL_ENV``. + + Every uv invocation names its project explicitly, so an activated + environment elsewhere is never what we mean — uv agrees, ignoring it and + warning that it did, once per invocation, which would otherwise land in + the report and in ``--json``. Explicit flags beat ambient variables + (spec §13); the launcher's broader ``UV_*`` scrub lands with layer 3. """ - return subprocess.run(argv, cwd=cwd, capture_output=True, text=True, check=False) + return {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} def _check_call(argv: list[str], *, cwd: Path) -> list[str]: @@ -328,7 +344,11 @@ def tool_warnings(stderr: str) -> list[str]: uv interleaves warnings with progress on stderr, so relaying the whole stream would bury them under a line per installed package. A warning is - a line starting ``warning:`` plus any indented continuation. + a line starting ``warning:`` plus its continuations, which uv aligns + under the 9-character ``warning: `` prefix. The two-space floor is what + separates those from uv's own change list (`` + pkg==1.0``), which is + indented by exactly one — folding those in swallowed the whole install + list into the warning text. The one that has to reach the user: when the uv cache and the project are on different filesystems, uv cannot link and silently falls back to @@ -339,7 +359,7 @@ def tool_warnings(stderr: str) -> list[str]: for line in stderr.splitlines(): if line.startswith("warning:"): found.append(line.removeprefix("warning:").strip()) - elif found and line.startswith((" ", "\t")) and line.strip(): + elif found and line.startswith((" ", "\t")) and line.strip(): found[-1] += " " + line.strip() return found diff --git a/tests/test_project.py b/tests/test_project.py index 6b2eb3fb..421cc796 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -512,6 +512,22 @@ def drifted(argv: list[str], *, cwd: Path) -> MagicMock: assert tools_before.unchanged and "uv.lock" in tools_before.unchanged +def test_ambient_virtualenv_is_not_passed_to_tools( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Every uv call names its project explicitly, so an activated + environment elsewhere is never what we mean — and leaving it set makes + uv warn, once per invocation, into a report agents read.""" + from lightcone.engine.project import child_env + + monkeypatch.setenv("VIRTUAL_ENV", "/somewhere/else/.venv") + monkeypatch.setenv("LC_TEST_CANARY", "kept") + + env = child_env() + assert "VIRTUAL_ENV" not in env + assert env["LC_TEST_CANARY"] == "kept", "the rest of the environment is untouched" + + def test_relays_uv_warnings_into_the_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -552,6 +568,25 @@ def test_tool_warnings_ignores_progress_output() -> None: assert tool_warnings("warning: a\nwarning: b\n") == ["a", "b"] +def test_tool_warnings_does_not_swallow_the_change_list() -> None: + """uv indents its change list by one space and its warning + continuations by nine, so the continuation rule has to tell them + apart — otherwise a hundred `+ pkg==ver` lines land inside the + warning text.""" + from lightcone.engine.project import tool_warnings + + found = tool_warnings( + "warning: first\n" + "Resolved 114 packages in 12ms\n" + " + aiohttp==3.14.3\n" + " - six==1.17.0\n" + " ~ click==8.4.2\n" + "warning: second\n" + " continued here\n" + ) + assert found == ["first", "second continued here"] + + def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A silently broken lock would fail every later verb more confusingly — so the failure surfaces here, with uv's own stderr.""" From e391564dfb352cd1310b3b5c6e21fffac6a8b2b9 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Tue, 18 Aug 2026 12:54:52 +0200 Subject: [PATCH 06/23] Layer 5: the exec boundary and `lc run` (#173) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lands the sandbox layer and the one verb that exercises it. Per spec §4, `lc run CMD` is the **probe** verb: byte-for-byte the recipe environment — same lock, same `.venv` — under the same boundary recipes will get. A probe has no output, so it needs neither manifests (layer 2) nor the fabric (layer 4). That is what let layer 5 land ahead of them, and it makes it the smallest honest consumer of the boundary that layer 4 will plug recipes into. No new dependencies. ## The abstraction **The seam is a pure argv rewrite.** Every mechanism is `Backend.wrap(policy, argv) -> argv'` — a function turning a command into *a different command that sandboxes itself*. ``` lc run CMD │ engine/run.py ── mode check · rename guard · uv hop │ sandbox.run(backend, policy, argv, prefix=uv…) │ backend = detect() ← the ONLY sys.platform branch argv = backend.wrap(…) ← pure Popen(…) ← mechanism-blind │ ┌───────────────────┼───────────────────┐ LandlockBackend SeatbeltBackend Unavailable [python, -m [sandbox-exec, argv unchanged lightcone. -p , _sandbox_exec, -DREAD_0=…, --, --policy , *argv] --, *argv] ``` Seatbelt is natively that shape; **Landlock is not** — it is a self-restriction, which is precisely what `lightcone/_sandbox_exec.py` exists to fix. Once both reduce to argv, nothing above the seam branches on platform, and **every backend is testable on a host that cannot run it**: `tests/test_sandbox_wrap.py` checks the Landlock wrap *and* the generated macOS SBPL on Linux CI, with no privileges and nothing spawned. Three types stay deliberately distinct — `Policy` (what we will enforce), `Capability` (what this host can do), `Attestation` (what was actually enforced, spec §7's manifest field). Collapsing any two is how a sandbox starts lying. Prior art surveyed: OpenAI's codex CLI and this design's own earlier implementation on `redesign_prototype`. They agree on the shape and disagree on one mechanism; this takes codex's answer. ## Deviations from the spec, all recorded in CLAUDE.md - **The Landlock policy travels as JSON on argv, not as an inherited ruleset FD.** §7 specifies building the ruleset before fork and passing it with `pass_fds`. The shim builds and applies it itself instead — codex's shape. This makes `wrap()` pure and closes §11's own *blocking* spike ("does the Landlock FD survive `uv run`'s spawn/exec chain? an FD cannot be reopened") by making the question moot. It also survives into a container later, where a host FD cannot. - **Network is not controlled on either platform**, by decision. The generated SBPL explicitly allows it and both platforms attest `network: allowed` — rather than §7's matrix value of `denied`. Nothing pretends to a control it does not apply. - **`--require-sandbox` is a bare flag.** §7's `=declared-fs` form would be the same flag today, since every existing mechanism scopes the filesystem. It arrives with the first one that doesn't. - **The denial's system-layer remedy is trimmed** to `uv add` + the ASTRA input declaration. §7's `[tool.lightcone.image]` TOML arrives with the container hatch; printing it now would be a copy-pasteable fix that does nothing. - **No manifest is written** — a probe has no output. `Attestation.to_manifest()` exists and is tested; layer 4 is its first consumer. - **One tightening §7 does not contemplate:** `/tmp` leaves the write scope when the project lives under it. Granting it unconditionally would make such a project's tree writable and silently void the read-only-tree guarantee. `TMPDIR` points into the private scope regardless, so `tempfile` works either way. ## Three grants that turned out to be load-bearing Each found by a real failure while building this, each now pinned by a test: | Grant | Without it | |---|---| | the realpath'd **ELF loader** in the exec set | *every* dynamically linked binary fails EACCES — bash and python included | | **read** on the uv-managed interpreter's install root | `Failed to import encodings module` — the stdlib sits beside the binary, outside the project and outside `/usr` | | **read** on `/dev/urandom` | `failed to get random numbers` — CPython seeds hash randomization before `main` | ## Verification `170 passed`, ruff clean, mypy strict clean. The suite splits along the seam: - **Pure, every OS** — policy construction, both backends' argv, the golden SBPL, the denial renderer. - **Real subprocess, no kernel needed** — the shim's contract: its argv, and the reserved exit code `97` that means "lc's own setup broke" and never "the command ran unsandboxed". - **Real enforcement, no fakes** — `test_sandbox_landlock.py`, module-level `skipif` on `abi() == 0`. A mocked sandbox proves nothing, so where Landlock is absent the file skips whole. Policies there are hand-built so each test grants exactly what it is testing. - **Pre-exec decisions** — discovery, the containerized refusal, the rename guard, declared inputs, the uv hop. Exercised by hand end to end against a real scaffolded project (kernel 7.1, Landlock ABI 9): runs in the project `.venv`; project reads allowed; in-tree writes denied *with the file verifiably unchanged*; declared `astra.yaml` inputs become readable; undeclared tools and files denied with their respective remedies; `$HOME` redirected to a private scope; no in-tree `__pycache__`; `--sandbox-debug`, `--no-sandbox`, `--require-sandbox`, and the rename guard all behaving. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DqTza2GqRLZ4yY5ywAusQH --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/lint.yml | 4 + .github/workflows/tests.yml | 51 +- CLAUDE.md | 369 ++++++++++++- src/lightcone/_sandbox_exec.py | 313 +++++++++++ src/lightcone/cli/commands.py | 34 +- src/lightcone/engine/project.py | 38 +- src/lightcone/engine/run.py | 101 ++++ src/lightcone/engine/sandbox/__init__.py | 36 ++ src/lightcone/engine/sandbox/boundary.py | 250 +++++++++ src/lightcone/engine/sandbox/denial.py | 188 +++++++ src/lightcone/engine/sandbox/landlock.py | 95 ++++ src/lightcone/engine/sandbox/model.py | 122 +++++ src/lightcone/engine/sandbox/policy.py | 357 +++++++++++++ .../engine/sandbox/profiles/base.sbpl | 139 +++++ .../engine/sandbox/profiles/network.sbpl | 48 ++ .../sandbox/profiles/platform-defaults.sbpl | 211 ++++++++ src/lightcone/engine/sandbox/seatbelt.py | 231 ++++++++ src/lightcone/engine/templates/__init__.py | 12 +- tests/conftest.py | 20 +- tests/test_cli.py | 168 +++++- tests/test_project.py | 8 +- tests/test_run.py | 131 +++++ tests/test_sandbox_denial.py | 159 ++++++ tests/test_sandbox_enforcement.py | 491 ++++++++++++++++++ tests/test_sandbox_policy.py | 406 +++++++++++++++ tests/test_sandbox_shim.py | 132 +++++ tests/test_sandbox_wrap.py | 294 +++++++++++ tests/test_templates.py | 2 +- 28 files changed, 4370 insertions(+), 40 deletions(-) create mode 100644 src/lightcone/_sandbox_exec.py create mode 100644 src/lightcone/engine/run.py create mode 100644 src/lightcone/engine/sandbox/__init__.py create mode 100644 src/lightcone/engine/sandbox/boundary.py create mode 100644 src/lightcone/engine/sandbox/denial.py create mode 100644 src/lightcone/engine/sandbox/landlock.py create mode 100644 src/lightcone/engine/sandbox/model.py create mode 100644 src/lightcone/engine/sandbox/policy.py create mode 100644 src/lightcone/engine/sandbox/profiles/base.sbpl create mode 100644 src/lightcone/engine/sandbox/profiles/network.sbpl create mode 100644 src/lightcone/engine/sandbox/profiles/platform-defaults.sbpl create mode 100644 src/lightcone/engine/sandbox/seatbelt.py create mode 100644 tests/test_run.py create mode 100644 tests/test_sandbox_denial.py create mode 100644 tests/test_sandbox_enforcement.py create mode 100644 tests/test_sandbox_policy.py create mode 100644 tests/test_sandbox_shim.py create mode 100644 tests/test_sandbox_wrap.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24fa006b..a7181621 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,6 +4,10 @@ on: push: branches: [main] pull_request: + # `ready_for_review` is not in the default set, and this job is gated + # on `draft == false` — so without it a PR opened as a draft and then + # marked ready never runs at all until someone happens to push again. + types: [opened, synchronize, reopened, ready_for_review] jobs: lint: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 57f91804..c8903ee7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,14 +4,35 @@ on: push: branches: [main] pull_request: + # `ready_for_review` is not in the default set, and this job is gated + # on `draft == false` — so without it a PR opened as a draft and then + # marked ready never runs at all until someone happens to push again. + types: [opened, synchronize, reopened, ready_for_review] jobs: test: if: github.event_name == 'push' || github.event.pull_request.draft == false - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: + # Both mechanisms are enforced by the same suite, so a failure on one + # OS is exactly the information the other run is there to provide. + fail-fast: false matrix: + os: [ubuntu-latest] python-version: ["3.11", "3.12", "3.13"] + include: + # macOS is where Seatbelt is the mechanism, and the only place + # the generated SBPL is ever executed. One interpreter is + # enough: what differs here is the kernel, not the python. + - os: macos-latest + python-version: "3.13" + + env: + # Enforcement tests skip themselves where no mechanism exists. On CI + # that must be a failure instead: a runner that quietly loses + # Landlock or Seatbelt would otherwise report green while testing + # nothing at all. + LC_SANDBOX_TESTS_REQUIRED: "1" steps: - uses: actions/checkout@v4 @@ -26,5 +47,33 @@ jobs: - name: Install dependencies run: uv sync --group dev + - name: Report the sandbox this runner can build + # Printed before the suite so a failure downstream can be read + # against what this host could actually enforce, and against the + # exec set it resolved. The allowlist resolves off a fixed search + # path, so where a tool lives differs per OS and a missing one is + # otherwise a very confusing "Operation not permitted". + run: | + python -c " + import shutil, tempfile + from pathlib import Path + from lightcone.engine.sandbox import detect + from lightcone.engine.sandbox.policy import ( + EXEC_ALLOWLIST, _UTILITY_PATH, probe_policy, + ) + c = detect().capability + print(f'mechanism: {c.kind} abi: {c.landlock_abi} {c.detail}') + print(f'utility search path: {_UTILITY_PATH}') + policy = probe_policy(Path(tempfile.mkdtemp()) / 'proj') + granted = set(policy.execute) + for name in EXEC_ALLOWLIST: + found = shutil.which(name, path=_UTILITY_PATH) + if found is None: + print(f' {name:10} not on this host') + elif Path(found).resolve() not in granted: + print(f' {name:10} {found} -> {Path(found).resolve()} NOT GRANTED') + print(f'exec set: {len(policy.execute)} paths, write: {len(policy.write)}') + " + - name: Run tests run: pytest diff --git a/CLAUDE.md b/CLAUDE.md index 5f40139b..3164833b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,11 +42,17 @@ speculatively. | 2 | Environment layer — `env_version`, lock scan, manifest schema | ⬜ | | 3 | The `lc` entrypoint — launcher: discover → mode-detect → scrub `UV_*` → converge → delegate | ⬜ | | 4 | Fabric — `lc materialize`, worker sequence, mid-run relock gate | ⬜ | -| 5 | Sandbox layer — Landlock / Seatbelt, exec-shim, denial UX | ⬜ | +| 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | | 6 | Container hatch — `[tool.lightcone.image]`, generated Containerfile, `lc build` | ⬜ | | 7 | Venues — Perlmutter, hub/GKE, Cloud Build | ⬜ | | 8 | `lc status`, `lc verify`, WRROC export | ⬜ | +Layer 5 landed **out of order**, ahead of 2–4: `lc run` is the spec's +*probe* verb, and a probe has no output, so it needs neither manifests +(layer 2) nor the fabric (layer 4) — only project discovery, which came +with it. That makes it the smallest honest consumer of the exec boundary, +and the boundary is what layer 4 will then plug recipes into. + The spec's §11 (Migration) is the authoritative ordering; the table above is the working map. **Layer boundaries are also dependency boundaries** — a dependency enters `pyproject.toml` with the layer that needs it, not @@ -54,6 +60,33 @@ before. ### Rules while rebuilding +Each of these has been asked for in review at least once; none is optional. + +- **Never reference the design spec in code or comments.** No `spec §7`, + no section numbers, no "the spec says". Code and its comments must + stand on their own; design rationale lives in the design documents. + (This file is the exception — it is *about* the design.) +- **No backward-compatibility code.** Nothing exists to honor the + behavior of an older CLI, an older wire format, or trained fingers. + If old behavior isn't promised, don't guard, version, or migrate it. +- **No foreshadowing.** No code, comment, flag, or user-facing message + may mention a verb, layer, or feature that does not exist yet. The + codebase is consistent with the project *at this point in time*. +- **No escape hatches around guarantees.** A feature that enforces + something ships without a flag to turn the enforcement off. +- **Prefer literal behavior over invented convenience.** The current + directory is the project root; erroring beats walking up or guessing. +- **Nothing waits on a human.** A verb is run by an agent more often + than by a person, so no interactive prompt and no interactive shell — + either is a hang, not a UX choice. +- **Streamline before shipping.** No small helper functions or + rendering layers where a few inline lines read fine; consolidate. +- **Be honest about provenance.** Third-party material we adapt is + "inspired from" upstream, clearly marked, with its license named — + never passed off as verbatim, never left unattributed. +- **Leave working files alone.** Don't edit files that are fine just + because a change nearby made them look touchable. + - **No dead code.** If nothing in the current layer calls it, it doesn't land yet. `lc --help` advertises only verbs that work. - **`docs/` is frozen at its pre-rebuild state, by decision.** It still @@ -103,12 +136,22 @@ Any new `lightcone-*` package must: ``` src/lightcone/ # namespace — NO __init__.py +├── _sandbox_exec.py # the Landlock shim — stdlib only, zero lightcone imports ├── cli/ # the CLI only: flags, rendering, exit codes │ ├── __init__.py # exposes main(); the launcher hooks in here (layer 3) -│ └── commands.py # lc init — the only verb so far +│ └── commands.py # lc init, lc run └── engine/ ├── __init__.py # docstring only - ├── project.py # how a directory converges into a project + ├── project.py # what a project is: convergence + discovery + ├── run.py # what `lc run` is: the probe + the uv hop + ├── sandbox/ # the exec boundary + │ ├── __init__.py # the public surface (detect, run, scope, the types) + │ ├── model.py # Policy · Capability · Attestation · Backend protocol + │ ├── policy.py # what a probe may touch + │ ├── boundary.py # detect() + run(): the mechanism-blind half + │ ├── landlock.py # Linux backend + │ ├── seatbelt.py # macOS backend + │ └── denial.py # the denial UX └── templates/ # the scaffold's file content ├── __init__.py # loader + one renderer per scaffolded file └── files/*.tmpl # the templates themselves, as real files @@ -158,9 +201,12 @@ convergence can see but must not fix (advisory — never affects does count, so a report can never claim a project is converged while something it owns is absent. -**Project discovery** — the `astra.yaml` walk-up — arrives with the -launcher (layer 3), the first thing that needs it. `lc init` is handed its -directory. +**There is no project discovery, by decision.** `lc init` is handed its +directory (defaulting to `.`); `lc run` assumes the current directory is +the project root — `project.current_project()` checks only that the +environment is there (`pyproject.toml`, `uv.lock`, `.venv`) and does not +require an `astra.yaml`, so any uv project can be probed. No walk-up: +the directory you invoke from is the project, or it is a clean error. **CLI startup stays cheap.** `commands.py` imports the engine *inside* the command callbacks and builds the rich console lazily, so `lc --help` and @@ -279,8 +325,213 @@ user owns: `--no-sync`'s real home is containerized mode (layer 6), where the host `.venv` is inert. Don't add a flag whose only user is a test — stub `project._run` instead. +## Key Invariants (layer 5) + +**The seam is a pure argv rewrite.** Every mechanism is a +`Backend.wrap(policy, argv) -> argv'` — a function turning a command into +*a different command that sandboxes itself*. Seatbelt is natively that +shape; Landlock is not (it is a self-restriction), which is exactly what +`lightcone/_sandbox_exec.py` exists to fix. Because both reduce to argv, +`boundary.run` never branches on platform, and **every backend is fully +testable on a host that cannot run it** — `tests/test_sandbox_wrap.py` +checks the Landlock wrap and the generated SBPL on whatever machine is +running, with no privileges. Keep `wrap` pure: no temp files, no file +descriptors, no global state. `tests/test_sandbox_wrap.py::test_wrap_is_pure` +pins it. + +**`detect()` in `sandbox/boundary.py` holds the only `sys.platform` +branch.** Everything downstream branches on `Capability.kind`, a *value*. +Adding bubblewrap or podman later is one module plus one line there. + +**Three types, deliberately distinct** (`sandbox/model.py`): `Policy` is +*what we will enforce* (mechanism-free path sets), `Capability` is *what +this host can do* (the probe's answer), `Attestation` is *what was +actually enforced* (spec §7's manifest field, derived from the flags +applied — never from what the mechanism matrix says should have +happened). Collapsing any two of them is how a sandbox starts lying. + +**`Unavailable` is a real backend, not a special case.** It satisfies the +protocol, wraps to the argv it was given, and attests `fs: open`. Saying +so is the caller's job; pretending is nobody's. There is deliberately +**no flag surface around the sandbox** — no opt-out, no require, no +debug dump. Enforcement always happens where a mechanism exists, and a +host without one gets the downgrade note, not a choice. + +**A denial is never invisible.** `denial.explain()` is a best-guess +heuristic over the child's stderr and is *allowed to return nothing* — +a command can swallow the `PermissionError` or rewrap it. That is why +`denial.trailer()` fires on **every** nonzero sandboxed exit, +unconditionally. Both fallbacks are pinned in `tests/test_sandbox_denial.py`. + +**The exec tier is per-file, and read ≠ execute.** `/usr` is readable (the +dynamic linker needs it) and never executable — `/usr/bin` holds `bash` +and `latex` alike, so a directory grant there would admit every +undeclared tool on the host and leave the layer enforcing nothing. + +**Three things are load-bearing and non-obvious**, each found by a real +failure and each now pinned by a test: + +| Grant | Without it | +|---|---| +| the realpath'd **ELF loader** in the exec set | *every* dynamically linked binary fails EACCES — bash and python included | +| **read** on the uv-managed interpreter's install root | `Failed to import encodings module` — the stdlib sits beside the binary, outside the project and outside `/usr` | +| **read** on `/dev/urandom` | `failed to get random numbers` — CPython seeds hash randomization before `main` | + +**Never grant EXECUTE on a directory that could be a system prefix.** +Landlock unions rights over ancestors, so a single EXECUTE grant on +`/usr` silently outranks the entire per-file allowlist and leaves the +layer enforcing nothing — while every test still passes, because the +allowlisted binaries are exactly the ones that were going to work. This +shipped once: the venv interpreter's *install root* was granted EXECUTE +so the resolved symlink target would be runnable, which is `/usr` for +any venv built on a system python (`uv venv --python-preference +only-system`, most CI images, HPC site pythons). The grant is on the +interpreter **file**; only READ goes to the install root, for the stdlib. +`test_a_system_interpreter_does_not_make_the_whole_prefix_executable` +pins it. + +**The env overlay belongs to the seam, not to the parent and not to each +backend.** `boundary.env_argv()` composes `policy.env` into an `env K=V …` +prefix *inside* the wrap, once, for every mechanism — never merged into +the `subprocess` environment, because everything *outside* the rewrite +must keep the real one. `uv` resolves its cache from `XDG_CACHE_HOME` and +its managed interpreters from `XDG_DATA_HOME`; overlaying those for the +`uv run` hop points it at a throwaway `mkdtemp` that `scope()` then +deletes, so every probe re-downloads the world, air-gapped hosts fail +outright, and `~/.config/uv/uv.toml` and `~/.netrc` go missing. + +Two rules, and the second was learned the hard way. **If `prefix` is +outside the boundary, nothing the boundary imposes may reach it.** And +**anything every backend must do belongs to the seam, not to the +backends** — while each applied its own overlay, `Unavailable` applied +none, so an unenforced run silently got a different environment as well +as no enforcement. A mechanism added later cannot forget what it never +had to remember. + +**The shim stays alone.** `lightcone/_sandbox_exec.py` imports nothing but +the stdlib and nothing from lightcone — `lightcone` is a namespace package +with no `__init__`, so `python -m lightcone._sandbox_exec` executes that +file and nothing else. It runs on every sandboxed exec; an engine import +there would put click, rich, and the astra stack on that path. Two tests +pin it. Its setup failures exit the reserved code **97**, distinguishable +from anything a command could return, and it never falls through to +running the command unsandboxed. + +**Layer boundaries showed up as parameters we did *not* add.** +`writable_project`, output-dir write scope, scratch dirs, `in_container()`, +and the `podman*`/`pod*` attestation branches all belong to layers 4 and 6 +and are absent, not stubbed. + +**The macOS profile is vendored, not authored.** +`sandbox/profiles/{base,network,platform-defaults}.sbpl` come from the codex CLI +(itself Chrome-derived), with a provenance header naming the upstream +commit and a single `LIGHTCONE DELTA`. The macOS read baseline is not +derivable from first principles — it is a list of things that break, +found one production failure at a time: `/dev/dtracehelper`, the +`/dev/fd` and pty regexes, firmlink-parent traversal under +`/System/Volumes/Data`, the `opendirectoryd.libinfo` lookup without which +`getpwuid()` raises `KeyError`, `/opt/homebrew/lib`. Keep them +near-verbatim so `diff` against upstream stays the re-sync tool; put our +own rules in the generator, not in the vendored text. The one delta — +upstream's blanket `(allow process-exec)`, which exists because codex +does not restrict exec — is pinned by a test so a re-sync cannot silently +restore it. + +**Not controlling the network takes more than allowing sockets.** On +macOS, name resolution and TLS trust go through *mach services* +(`SystemConfiguration.DNSConfiguration`, `configd`, `SecurityServer`, +`trustd.agent`, `ocspd`), which the base's `(deny default)` blocks — +`(allow network*)` opens the socket families and nothing else. That is +why `network.sbpl` is emitted unconditionally: without it the +attestation reads `network: allowed` on a host where every lookup fails, +which is worse than either honest answer. Linux needs no equivalent +because Landlock gates the filesystem only. + +**One resolution rule for the utility allowlist**, `policy.utility()`. +Anything that hardcodes a tool's path instead is a second answer to the +same question, and the two diverge the moment a host keeps its copy +somewhere else. `env` is the load-bearing case: the seam execs it to +apply the overlay, so a stale `/usr/bin/env` is not a missing +convenience but a denial on the first exec of *every* run — `/usr` is +readable and never executable. Pinned by +`test_the_env_the_seam_execs_is_one_the_policy_granted`, which points +the search path at a copy, because a test against the real path passes +on this host and ships the bug. + +**A derived read root may never be `$HOME` or above** (`_stdlib_root`). +The interpreter's install root is granted READ for the stdlib beside it, +and for an interpreter installed straight into `~/bin` that root *is* +the home directory — silently undoing the private-`$HOME` design, which +is the whole point of the environment overlay. Reading `pyvenv.cfg`'s +`base-prefix` instead does not help: it reports the same directory. The +guard is the fix; failing loudly on a layout nobody uses beats voiding +the guarantee for the people who do. + +**SBPL is last-match-wins; Landlock unions.** This asymmetry decides +where a rule can live, and it cuts both ways. A later `(deny …)` can take +back an earlier allow, which is how `_read_only_guard` claws back the +write the vendored defaults grant on `/tmp`; Landlock has no equivalent — +a narrower rule only ever *adds* — so the Linux side solves that by +leaving the root out of the policy (`policy._write_roots`). + +But the same asymmetry means SBPL does **not** give nesting for free. +Landlock unions, so a writable output directory inside a readable project +tree just works. In SBPL the guard's `(deny file-write* PROJECT)` would +revoke it — which is why `generate_profile` restates the **write tier +last**, after the guard. Get that order wrong and layer 4 materializes on +Linux and refuses on macOS, with the golden test still green because it +only checks that the guard is present. Verified empirically, not assumed. + ### Recorded decisions +- **Reads stay restricted, and the OS baseline is ours to maintain.** + Codex restricts reads too now, but its Linux read baseline is a *mount + table*, not a path list — there is nothing to adopt there. Keeping the + FHS allowlist is a deliberate choice about which way the failure + points: a missing allowlist entry is a **loud** `EACCES` that gets + reported and fixed once for everyone, while any exclusion-based scheme + fails **silently**, in exactly the thing the layer exists to prevent. + Worth remembering when the list next comes up short: none of the three + bugs we hit (ELF loader, interpreter root, `/dev/urandom`) came from + the list — two were derived paths and one was already in it. +- **Devices, `/proc`, and `/sys` are granted generously, on purpose.** + Writable: the terminal set (`/dev/tty`, `/dev/pts`, `/dev/ptmx`), the + discard devices (`/dev/null`, `/dev/zero`, `/dev/full`), and `/proc` + and `/sys` whole. The threat model is accidental leakage, not a hostile + recipe (spec §7), and **none of these is a channel undeclared *inputs* + arrive through** — that is the test to apply when the next one comes + up. Tightening them buys nothing and costs real failures: + `pty.openpty()`, `/dev/full` ENOSPC handling, `oom_score_adj`, MPI and + CUDA runtimes poking `/sys`. + It also reads more permissive than it is — Landlock only ever *removes* + access, never adds it, so ordinary Unix permissions remain the real + gate (devpts hands each pty to its allocating user at 0620; nearly all + of `/proc` and `/sys` is root-owned). Granting them just stops lc + adding a second, more confusing denial on top of the OS's own. + The line is drawn at `/dev/urandom` and `/dev/random`, which stay + read-only: writing those seeds the *host's* pool — a side effect on + the machine rather than on the run. + The general rule, learned here: **a grant whose absence produces an + unattributable error is one to make freely.** Without devpts, + `pty.openpty()` raises `OSError: out of pty devices` — naming neither + a path nor the sandbox, so `denial.explain()` can extract nothing and + only the trailer fires. A denial the user cannot act on is worse than + the access it withheld. +- **Landlock stays the Linux mechanism; bubblewrap is not adopted.** + Codex moved its Linux default to bwrap+seccomp (Landlock is now + `--use-legacy-landlock`) because it needs rights *subtraction*: + `read_only_subpaths` inside a writable root, to stop an agent writing + `.git/hooks` and escalating. Landlock cannot express that, and their + own older code silently dropped the carve-out on Linux as a result. + **That requirement is not ours** — spec §7's threat model is accidental + leakage, not a hostile recipe, and our policy's exceptions always + *widen* (a writable output dir inside a readable tree), which is + exactly the direction union semantics handle for free. The cost would + be real: a bundled `bwrap` binary, a user-namespace probe, a WSL1 + refusal, and the Ubuntu 24.04 AppArmor wall that made + `hermeticity-enforcement.md` §3 call bwrap "an opportunistic upgrade, + never the requirement". Re-add triggers: a policy shape that genuinely + needs subtraction, or ambient `bwrap` becoming universal. - **The ASTRA `container:` directive is ignored, entirely.** astra's boilerplate writes `container: python:3.12-slim` into `astra.yaml`, and lightcone-cli does nothing with it: not read, not stripped, not @@ -299,6 +550,59 @@ user owns: pins the absence, so re-adding it is a deliberate act. - Scaffolded file bodies stay descriptive of what works today — see the trimmed `results/README.md` for the same reason. +- **The Landlock policy travels as JSON on argv, not as an inherited + ruleset FD** (spec §7 specifies the FD, built before fork and passed + with `pass_fds`). The shim builds and applies the ruleset itself, which + is also what the codex CLI does. This makes `wrap()` a pure function and + closes §11's own blocking spike — *"does the Landlock FD survive + `uv run`'s spawn/exec chain? an FD cannot be reopened"* — by making the + question moot. It also survives into a container later, where a host FD + cannot. The document is deliberately *not* versioned: the wrap always + invokes the shim on lc's own interpreter, so writer and reader are the + same lightcone-cli by construction, and a compatibility field would be + backward-compat machinery with no consumer. +- **Network is not controlled, on either platform**, by decision. §7's + matrix has Seatbelt record `denied`; the generated SBPL explicitly + allows network and both platforms attest `network: allowed`. Symmetric + and honest — nothing pretends to a control it does not apply. (codex + ships a seccomp denylist for this; adding one is a live option, not a + gap we are hiding.) +- **`lc run` has no rename guard and no sandbox flags.** §4's guard + against `lc run ` existed only for muscle memory from the + pre-rebuild CLI — backward compatibility we do not promise — and §7's + `--require-sandbox` / `--no-sandbox` / `--sandbox-debug` are all + absent: there is no hatch to escape the sandbox, so there is nothing + for the flags to switch. The verb takes a command and nothing else. +- **`lc run` does not detect containerized mode.** The + `[tool.lightcone.image]` refusal was removed with the rest of the + future-facing surface; a system-layer declaration is currently inert, + like `astra.yaml`'s `container:` key. +- **The denial's remedies are only what works today** — `uv add` for a + Python package, plus the ASTRA input declaration. Nothing in a denial + message names a verb, flag, or declaration that does not exist. +- **No manifest is written, and no serializer for one.** A probe has no + output (§4), so the `Attestation` is returned and printed rather than + persisted. An earlier draft carried a `to_manifest()` with no caller — + deleted, because "no dead code" applies to this layer's own + conveniences too. It lands with layer 4, which is what needs it. +- **`/tmp` is dropped from the write scope when the project lives under + it**, which §7 does not contemplate. Granting `/tmp` unconditionally + would make the tree of a `/tmp`-hosted project writable and silently + void the read-only-tree guarantee. `TMPDIR` points into the private + scope regardless, so `tempfile` works either way. The drop is + Linux-only in effect: on macOS the vendored defaults grant + `/private/tmp` write unconditionally and the guard only claws back our + *read* roots, so the surrounding `/tmp` stays writable there. The + project itself is protected on both — it is a read root, so the guard + names it — and `/tmp` writability is not a channel undeclared inputs + arrive through, so the asymmetry is recorded rather than closed. +- **`/run` is granted whole**, where codex names only + `/run/current-system/sw`. It reaches `/run/user/$UID` — dconf, the + gnupg and keyring sockets, portal state. Kept deliberately: the test + is whether undeclared *inputs* arrive through a path, and runtime + sockets are not something a recipe accidentally reads data from. + `/etc/resolv.conf` is a symlink into `/run` wherever systemd-resolved + is in use, so the grant also keeps DNS working. ## Extending the Codebase @@ -309,6 +613,9 @@ user owns: | Add a value to the scaffold | `src/lightcone/engine/templates/__init__.py` | Derive it from the environment or our own metadata before introducing a constant | | Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | +| Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | +| Change what a sandboxed command may touch | `sandbox/policy.py` + `tests/test_sandbox_policy.py` | Path sets only — no mechanism ever leaks in here | +| Change a denial message | `sandbox/denial.py` + `tests/test_sandbox_denial.py` | Remedies must be copy-pasteable and real *today*; the trailer stays unconditional | ## Test Patterns @@ -327,6 +634,56 @@ user owns: logic. Content assertions live here; `test_project.py` asserts only that the file written *is* the template, so a template edit touches one file. +The sandbox suite splits along the seam, which is what makes it cheap: + +- `tests/test_sandbox_policy.py`, `test_sandbox_wrap.py`, + `test_sandbox_denial.py` — **pure, and run on every OS.** The policy is + data, `wrap` is a function, and the denial renderer is a function, so + the Landlock wrap and the macOS SBPL are both checked on Linux CI with + no privileges and nothing spawned. +- `tests/test_sandbox_shim.py` — the shim as a **real subprocess**, + because its contract *is* its argv and exit codes. Needs no kernel + support: every case here is a setup failure or a pure-function check. +- `tests/test_sandbox_enforcement.py` — **the kernel's answer**, written + once for both mechanisms. See below; this is the one that matters. +- `tests/test_run.py` — what `lc run` decides *before* it execs: the + current-directory project check, declared inputs, the uv hop. Nothing + spawns. + +Note the autouse `tools` fixture stubs `engine.project._run` only — +sandbox tests spawn real processes deliberately, and are the one place in +the suite that does. + +### The enforcement suite, and why it is shaped that way + +`test_sandbox_enforcement.py` is the only file that can tell you the +layer works. Four properties, each of which it would be easy to lose: + +1. **One suite, both mechanisms.** The same tests run Landlock on Linux + and Seatbelt on macOS, parameterised by `detect()` alone. That is the + seam paying rent — and it is the only way the two stay honest, since + *a leak only Linux catches is a leak*. macOS is in the CI matrix for + exactly this: it is the sole place the generated SBPL is ever + executed. +2. **The real policy.** It runs against `probe_policy` — what an actual + `lc run` gets — never a policy hand-built to make the point. A test + that grants exactly what it is testing cannot discover that the + *shipped* policy grants something else. (This is how `/usr` sat in the + exec set through a full green suite.) +3. **Real leaks, tried literally.** Undeclared *tools* are executed, + undeclared *libraries* are `dlopen`ed, undeclared *data* is read — + the three channels of spec §7. Assertions about path sets belong in + `test_sandbox_policy.py`; this file runs the command. +4. **It cannot pass by not running.** `LC_SANDBOX_TESTS_REQUIRED=1` is + set in CI, which turns "no mechanism here, skip" into a hard failure. + Two tests cover the guard itself, because an unfailing guard is worse + than none. + +**When you add an enforcement test, mutation-check it**: run the same +command through `Unavailable()` and confirm it *succeeds*. A denial test +that would pass unsandboxed is testing nothing, and the failure mode is +silent. Every leak case here was checked that way. + ## Conventions - Ruff for linting (E, F, I, N, W, UP), line length 100, target Python 3.11 diff --git a/src/lightcone/_sandbox_exec.py b/src/lightcone/_sandbox_exec.py new file mode 100644 index 00000000..233c1717 --- /dev/null +++ b/src/lightcone/_sandbox_exec.py @@ -0,0 +1,313 @@ +"""The Landlock exec shim — ``python -m lightcone._sandbox_exec``. + +Landlock is a *self*-restriction: a process narrows its own access rights +and can never widen them again, and the restriction is inherited by +everything it execs. There is no way to restrict *another* process, so +there is no ``sandbox-exec``-shaped command to wrap a recipe in. This +module is that missing command: it reads a policy, restricts itself, +and ``execvp``'s the real command. That is what lets the engine treat +Landlock and Seatbelt as the same thing — a pure argv rewrite (see +:mod:`lightcone.engine.sandbox`). + +The policy arrives as **JSON on argv**, not as an inherited ruleset FD — +the shape the codex CLI uses. A Landlock FD cannot be reopened, so +whether one would survive ``uv run``'s spawn/exec chain is a question +serializing the policy never has to answer, and it is what keeps the +engine's argv rewrite a pure function. + +Two properties of this module are load-bearing and pinned by tests: + +- **Stdlib only, and no lightcone imports.** ``lightcone`` is a PEP 420 + namespace package with no ``__init__``, so ``-m lightcone._sandbox_exec`` + executes this file and nothing else. It runs on every sandboxed exec; + it must not drag the engine in. +- **It never proceeds unsandboxed.** Every setup failure exits + :data:`SETUP_FAILURE_EXIT` with a distinguishable message, so a broken + sandbox can never be mistaken for a working one (the "silent + best-effort" trap this design exists to avoid). +""" + +from __future__ import annotations + +import ctypes +import errno +import functools +import json +import os +import stat +import sys + +# --- the kernel interface -------------------------------------------------- + +# asm-generic syscall numbers — identical on x86_64 and aarch64. +SYS_LANDLOCK_CREATE_RULESET = 444 +SYS_LANDLOCK_ADD_RULE = 445 +SYS_LANDLOCK_RESTRICT_SELF = 446 + +_LANDLOCK_CREATE_RULESET_VERSION = 1 << 0 +_RULE_PATH_BENEATH = 1 +_PR_SET_NO_NEW_PRIVS = 38 + +# Access-right bits, grouped by the ABI that introduced them. +ACCESS_FS_EXECUTE = 1 << 0 +ACCESS_FS_WRITE_FILE = 1 << 1 +ACCESS_FS_READ_FILE = 1 << 2 +ACCESS_FS_READ_DIR = 1 << 3 +_ABI1_ALL = (1 << 13) - 1 # bits 0..12: read, write, and every make/remove +ACCESS_FS_REFER = 1 << 13 # ABI ≥ 2 +ACCESS_FS_TRUNCATE = 1 << 14 # ABI ≥ 3 + +#: Rights the kernel accepts on a rule whose parent is a regular file. +#: Everything else is directory-only and makes ``add_rule`` fail EINVAL. +_FILE_ONLY_BITS = ( + ACCESS_FS_EXECUTE | ACCESS_FS_WRITE_FILE | ACCESS_FS_READ_FILE | ACCESS_FS_TRUNCATE +) + +READ_BITS = ACCESS_FS_READ_FILE | ACCESS_FS_READ_DIR +EXEC_BITS = ACCESS_FS_EXECUTE | ACCESS_FS_READ_FILE + +#: Landlock's syscall numbers are asm-generic, but the ABI is not defined +#: for every architecture; refuse rather than issue syscall 444 blind. +_SUPPORTED_ARCHES = frozenset({"x86_64", "aarch64", "arm64"}) + +#: Reserved exit code for a failure to *set up* the sandbox — never a +#: command's own. The engine renders it as an lc problem, not a recipe's. +SETUP_FAILURE_EXIT = 97 + +class _RulesetAttr(ctypes.Structure): + _fields_ = [ + ("handled_access_fs", ctypes.c_uint64), + ("handled_access_net", ctypes.c_uint64), + ] + + +class _PathBeneathAttr(ctypes.Structure): + # The kernel struct is packed (u64 followed by s32). `_layout_` is the + # 3.14 spelling and is ignored by older ctypes, which honor `_pack_`. + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("allowed_access", ctypes.c_uint64), + ("parent_fd", ctypes.c_int32), + ] + + +@functools.cache +def _libc() -> ctypes.CDLL: + """The already-loaded libc. + + Cached, and `CDLL(None)` rather than `find_library("c")`: this is + called once per rule, and `find_library` forks `ldconfig -p` every + time — 74 subprocesses for a 71-path policy, which measured at ~740 ms + of the shim's ~780 ms. `dlopen(NULL)` reaches the libc already mapped + into this process, needs no search, and drops `ctypes.util` (and the + `tempfile`/`random` chain behind it) off the import path. + """ + return ctypes.CDLL(None, use_errno=True) + + +def abi() -> int: + """The Landlock ABI this kernel supports, or 0 if it supports none. + + A version query, so it is cheap and side-effect free. Every failure + mode — kernel < 5.13, the syscall blocked by seccomp, an + architecture we do not vouch for — answers 0 rather than raising: + the caller's question is "can I sandbox here", and "no" is a valid + answer to it. Recording *which* ABI answered is the caller's job. + """ + if os.uname().machine not in _SUPPORTED_ARCHES: + return 0 + try: + result = _libc().syscall( + SYS_LANDLOCK_CREATE_RULESET, None, ctypes.c_size_t(0), ctypes.c_uint32( + _LANDLOCK_CREATE_RULESET_VERSION + ) + ) + except OSError: # pragma: no cover - no libc to load + return 0 + return int(result) if result > 0 else 0 + + +def handled_access(abi_level: int) -> int: + """The rights the ruleset declares it governs, for *abi_level*. + + Unknown bits make ``landlock_create_ruleset`` fail EINVAL, so this + only ever widens with the ABI. Widening matters in both directions: + a right the ruleset does not *handle* is one the kernel does not + check at all, so ABI 1 silently permits ``ftruncate`` — while a + handled right must also be granted somewhere or nothing can use it. + + ``REFER`` is the case worth naming: when a ruleset does not handle + it, the kernel denies *every* cross-directory rename and link, which + is the ABI-1 EXDEV the denial classifier knows about. Handling it + (and granting it on the write roots) is what lets a recipe rename its + own temporary files. + + ``IOCTL_DEV`` (ABI 5) is deliberately left unhandled: ioctls on + device nodes are outside the accidental-leakage threat model, and + handling it would break anything opening a terminal afresh. + """ + handled = _ABI1_ALL + if abi_level >= 2: + handled |= ACCESS_FS_REFER + if abi_level >= 3: + handled |= ACCESS_FS_TRUNCATE + return handled + + +def write_bits(abi_level: int) -> int: + """The rights a writable root is granted: everything but EXECUTE. + + Write implies read — a directory you may create files in but not + list is not a useful grant — so the read bits are included. + """ + return (handled_access(abi_level) & ~ACCESS_FS_EXECUTE) | READ_BITS + + +def create_ruleset(handled_fs: int) -> int: + """A new, empty Landlock ruleset FD governing *handled_fs*.""" + attr = _RulesetAttr(handled_fs, 0) + fd = _libc().syscall( + SYS_LANDLOCK_CREATE_RULESET, + ctypes.byref(attr), + ctypes.c_size_t(ctypes.sizeof(attr)), + ctypes.c_uint32(0), + ) + if fd < 0: + raise OSError(ctypes.get_errno(), "landlock_create_ruleset failed") + return int(fd) + + +def add_path_rule(ruleset_fd: int, path: str, access: int) -> None: + """Grant *access* beneath *path*. + + Rights are masked down for a regular-file rule, because the kernel + rejects directory-only rights on one. That is what makes a per-file + EXECUTE grant work — and per-file is the point: ``/usr/bin`` holds + both the utility allowlist and every undeclared tool on the host, so + granting the directory would defeat the layer. + """ + parent_fd = os.open(path, os.O_PATH | os.O_CLOEXEC) + try: + # `stat.S_ISDIR`, not a bitwise test: S_IFSOCK (0o140000) and + # S_IFBLK (0o060000) both carry the S_IFDIR bit, so a hand-rolled + # mask calls a socket a directory, leaves directory-only rights + # unmasked, and turns a declared unix-socket input into EINVAL — + # surfacing as a sandbox *setup* failure rather than a grant. + if not stat.S_ISDIR(os.fstat(parent_fd).st_mode): + access &= _FILE_ONLY_BITS + if not access: + return + attr = _PathBeneathAttr(access, parent_fd) + result = _libc().syscall( + SYS_LANDLOCK_ADD_RULE, + ctypes.c_int(ruleset_fd), + ctypes.c_uint32(_RULE_PATH_BENEATH), + ctypes.byref(attr), + ctypes.c_uint32(0), + ) + if result != 0: + code = ctypes.get_errno() + raise OSError(code, f"landlock_add_rule({path}): {errno.errorcode.get(code, code)}") + finally: + os.close(parent_fd) + + +def build_ruleset(policy: dict[str, object], abi_level: int) -> int: + """A ruleset FD realizing *policy*, or raise :class:`OSError`. + + A path that has vanished between policy construction and here grants + nothing, which is safe, so it is skipped rather than fatal. + """ + fd = create_ruleset(handled_access(abi_level)) + try: + for key, access in ( + ("read", READ_BITS), + ("write", write_bits(abi_level)), + ("execute", EXEC_BITS), + ): + for path in _paths(policy, key): + try: + add_path_rule(fd, path, access) + except FileNotFoundError: + continue + except BaseException: + os.close(fd) + raise + return fd + + +def restrict_self(ruleset_fd: int) -> None: + """Apply *ruleset_fd* to this process, irreversibly. + + ``PR_SET_NO_NEW_PRIVS`` first: the kernel refuses an unprivileged + ``landlock_restrict_self`` without it, and it is also what stops a + setuid binary from escaping the domain. + """ + libc = _libc() + if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_NO_NEW_PRIVS) failed") + if libc.syscall(SYS_LANDLOCK_RESTRICT_SELF, ctypes.c_int(ruleset_fd), ctypes.c_uint32(0)) != 0: + raise OSError(ctypes.get_errno(), "landlock_restrict_self failed") + + +# --- the entry point ------------------------------------------------------- + + +def _paths(policy: dict[str, object], key: str) -> list[str]: + value = policy.get(key, []) + if not isinstance(value, list) or not all(isinstance(p, str) for p in value): + raise ValueError(f"policy field {key!r} must be a list of strings") + return list(value) + + +def _fail(message: str) -> None: + sys.stderr.write(f"lc sandbox setup failed: {message}\n") + raise SystemExit(SETUP_FAILURE_EXIT) + + +def main(argv: list[str] | None = None) -> None: + """``--policy -- ...`` — restrict, then become it. + + Hand-parsed rather than argparse'd: this runs before every sandboxed + exec, and everything after ``--`` belongs to the command, including + arguments that look like our own options. + """ + args = list(sys.argv[1:] if argv is None else argv) + if len(args) < 2 or args[0] != "--policy": + _fail("usage: -m lightcone._sandbox_exec --policy -- ...") + raw = args[1] + rest = args[2:] + if not rest or rest[0] != "--": + _fail("missing `--` before the command") + command = rest[1:] + if not command: + _fail("no command after `--`") + + try: + policy = json.loads(raw) + if not isinstance(policy, dict): + raise ValueError("policy must be a JSON object") + abi_level = abi() + if abi_level == 0: + raise ValueError("landlock unavailable (kernel < 5.13, or blocked by seccomp)") + fd = build_ruleset(policy, abi_level) + restrict_self(fd) + os.close(fd) + except (ValueError, OSError) as e: + _fail(str(e)) + + try: + os.execvp(command[0], command) + except OSError as e: + # Not a setup failure: we are already restricted, so this is very + # likely the sandbox denying the exec. Report it the way a shell + # would — 126 "cannot execute", 127 "not found" — so the denial + # classifier sees a message and an exit code it recognizes, and + # exit 97 keeps meaning "lc's own sandbox setup broke". + sys.stderr.write(f"lc sandbox: {command[0]}: {e.strerror}\n") + raise SystemExit(127 if e.errno == errno.ENOENT else 126) from e + + +if __name__ == "__main__": + main() diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index bb33a9a8..252d1d58 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -23,8 +23,7 @@ @functools.cache def _console() -> Console: """The rich console, built on first use to avoid startup cost at each - invocation of the cli even when the console is not needed. - """ + invocation of the cli even when the console is not needed.""" from rich.console import Console return Console() @@ -152,3 +151,34 @@ def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: if lines: lines.append("") # space the verdict off the list _console().print("\n".join([*lines, verdict])) + + +# ============================================================================= +# lc run +# ============================================================================= + + +@main.command(context_settings={"ignore_unknown_options": True, "allow_interspersed_args": False}) +@click.argument("command", nargs=-1, required=True, type=click.UNPROCESSED) +def run(command: tuple[str, ...]) -> None: + """Run COMMAND in the project environment, inside the sandbox. + + Byte-for-byte the environment recipes get — same lock, same + ``.venv``, same boundary — so a probe that works means a recipe + will. Reads the project tree and the inputs declared in + ``astra.yaml``; writes nowhere but its own temporary scope. + """ + from lightcone.engine import run as engine_run + from lightcone.engine.project import current_project + + outcome = engine_run.probe(current_project(), command) + if outcome.notes: + # click.echo, not rich: these lines are built to be pasted, and + # reflowing a `uv add` remedy would break the one thing the + # denial message is for. + click.echo("\n".join(["", *outcome.notes]), err=True) + # `Popen.returncode` is negative for a signal, and `sys.exit(-9)` + # truncates to 247. `lc run` is a proxy for the command it runs, so + # an OOM-killed probe comes back as the shell's conventional 128+N. + code = outcome.returncode + sys.exit(128 - code if code < 0 else code) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index f03d86e4..528bd815 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -175,7 +175,7 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: directory = directory.resolve() - _require_uv() + require_uv() c = _Converger(write=write) @@ -228,7 +228,8 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: return c.report -def _require_uv() -> None: +def require_uv() -> None: + """Refuse early when uv is absent — it is the environment substrate.""" if shutil.which("uv") is None: raise ProjectError( "uv is required (the environment substrate). Install it: " @@ -297,6 +298,28 @@ def project_name(directory: Path) -> str: return name or "analysis" +def current_project(directory: Path | None = None) -> Path: + """*directory* (default: the working directory), taken as the project root. + + ``lc run`` assumes it is invoked from the root, so the only check is + that the environment is actually there: ``pyproject.toml``, + ``uv.lock``, ``.venv``. An ``astra.yaml`` is deliberately not + required — a command can be probed in any uv project, spec or no + spec. There is no walk-up: the directory you are in is the directory + that is used, or it is an error. + """ + directory = (directory or Path.cwd()).resolve() + missing = [ + name for name in ("pyproject.toml", "uv.lock", ".venv") if not (directory / name).exists() + ] + if missing: + raise ProjectError( + f"{directory} is missing {', '.join(missing)} — not a project " + "root. Run `lc init` to converge one." + ) + return directory + + # ============================================================================= # The external-tool seam # ============================================================================= @@ -320,8 +343,7 @@ def child_env() -> dict[str, str]: Every uv invocation names its project explicitly, so an activated environment elsewhere is never what we mean — uv agrees, ignoring it and warning that it did, once per invocation, which would otherwise land in - the report and in ``--json``. Explicit flags beat ambient variables - (spec §13); the launcher's broader ``UV_*`` scrub lands with layer 3. + the report and in ``--json``. Explicit flags beat ambient variables. """ return {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} @@ -383,7 +405,7 @@ def _lock_is_current(directory: Path) -> bool: def _env_is_current(directory: Path) -> bool: """Whether ``.venv`` still satisfies ``uv.lock``. - Set-level, not byte-level, as spec §3 accepts: uv catches packages the + Set-level, not byte-level: uv catches packages the lock requires and the environment lacks, but not extras installed by hand, which leave ``--check`` reporting "would make no changes". What bounds what a recipe can import is the sandbox, not this probe. @@ -395,7 +417,7 @@ def _env_is_current(directory: Path) -> bool: # --exact because a plain sync is additive; --compile-bytecode because the # environment is read-only at execution time, so a recipe cannot write .pyc # as it imports — paying compilation once here instead of on every recipe -# run (spec §4, §6, §7). +# run. _SYNC_ARGS = ["sync", "--locked", "--exact", "--compile-bytecode"] @@ -403,11 +425,11 @@ def _uv(c: _Converger, args: list[str], *, directory: Path) -> None: """Run uv against *directory*, recording anything it warns about. Every invocation carries an explicit ``--project``: uv's own walk-up - discovery is never trusted (spec §4). Nothing about linking or caching + discovery is never trusted. Nothing about linking or caching is overridden — uv's defaults already share package content between projects — and ``--system-site-packages`` is never used, since it would make packages outside the lock importable, which is what the - environment model exists to prevent (spec §7, G6). + environment model exists to prevent. """ for warning in _check_call(["uv", *args, "--project", str(directory)], cwd=directory): c.warn(f"uv: {warning}") diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py new file mode 100644 index 00000000..108c1d41 --- /dev/null +++ b/src/lightcone/engine/run.py @@ -0,0 +1,101 @@ +"""``lc run`` executes a command inside the reproducible environment. + +Byte-for-byte the environment recipes will get — the same lock, the same +converged ``.venv`` — under the same sandbox. That equivalence is the +point: if a probe works, the recipe will, and if a probe +is denied, the recipe would have been. + +A probe has no output, which is what makes it the strictest consumer of +the boundary: it may read the project and the declared inputs, and write +only its own tmp scope. Nothing it does can land in the project tree. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from lightcone.engine import sandbox +from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv + + +def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: + """Run *command* in the project environment, inside the boundary. + + *command* is required, and there is deliberately no bare-``lc run`` + shell. A probe is run far more often by an agent than by a person, + and an agent that opens an interactive shell waits forever for input + nobody is going to type. + """ + require_uv() + spec = read_spec(project) + + with sandbox.scope(project, read_paths=input_paths(project, spec)) as policy: + return sandbox.run( + sandbox.detect(), + policy, + list(command), + cwd=project, + prefix=uv_prefix(project), + # Same reason as convergence: this uv invocation names its + # project explicitly, so an environment activated elsewhere + # is never what we mean — and uv says so, once per run, in + # the middle of the probe's own output. + env=child_env(), + ) + + +def uv_prefix(project: Path) -> list[str]: + """``uv run``, pinned to *project* and refusing to drift. + + ``--locked`` makes a stale lock uv's loud error rather than a silent + relock, and ``--exact`` keeps a previously-installed extra from + surviving into the environment being probed. Always an explicit + ``--project``: uv's own walk-up discovery is never trusted. + """ + return ["uv", "run", "--locked", "--exact", "--project", str(project), "--"] + + +def read_spec(project: Path) -> dict[str, Any]: + """The project's ``astra.yaml``, with sub-analyses merged in if possible. + + Best-effort by design: a probe exists to debug a project, and a spec + whose sub-analysis references are stale is exactly when someone runs + one. A tree that will not resolve degrades to the top-level document + rather than making the verb unusable — and no spec at all is simply + an empty one, with no declared inputs. + """ + from astra.helpers import load_yaml, resolve_analysis_tree + + spec_path = project / SPEC_FILENAME + if not spec_path.exists(): + return {} + data: dict[str, Any] = load_yaml(spec_path) + try: + return dict(resolve_analysis_tree(data, project)) + except Exception: + return data + + +def input_paths(project: Path, spec: dict[str, Any]) -> list[Path]: + """The declared inputs that are filesystem paths, resolved. + + A probe's read allowlist is the union of *all* declared inputs — it + has no output, so it has no narrower set to use. ASTRA's ``source`` + is free-form (a URI, a dotted name, a path), so the test for "is + this a path" is whether it resolves to something that exists. + Anything else is somebody else's input kind. + """ + from astra.helpers import get_inputs + + found: list[Path] = [] + for declared in get_inputs(spec): + source = declared.get("source") + if not isinstance(source, str) or not source: + continue + candidate = Path(source) + resolved = candidate if candidate.is_absolute() else project / candidate + if resolved.exists(): + found.append(resolved.resolve()) + return found diff --git a/src/lightcone/engine/sandbox/__init__.py b/src/lightcone/engine/sandbox/__init__.py new file mode 100644 index 00000000..07bb5ff0 --- /dev/null +++ b/src/lightcone/engine/sandbox/__init__.py @@ -0,0 +1,36 @@ +"""The exec boundary: what a command may touch, and how that is enforced. + +Guarantees that a command cannot use executables or files outside the +declared set wherever a mechanism exists, and what actually enforced it +is always recorded. + +This includes three components. :mod:`policy` decides *what* is allowed, in +plain path sets that mention no mechanism. A :class:`~model.Backend` +turns that into an argv rewrite — Landlock through +:mod:`lightcone._sandbox_exec`, Seatbelt through ``sandbox-exec``, and +:class:`~boundary.Unavailable` through no change at all. :mod:`boundary` +picks one, runs it, and reports what it enforced. +""" + +from __future__ import annotations + +from lightcone.engine.sandbox.boundary import ( + Outcome, + Unavailable, + detect, + run, + scope, +) +from lightcone.engine.sandbox.model import Attestation, Backend, Capability, Policy + +__all__ = [ + "Attestation", + "Backend", + "Capability", + "Outcome", + "Policy", + "Unavailable", + "detect", + "run", + "scope", +] diff --git a/src/lightcone/engine/sandbox/boundary.py b/src/lightcone/engine/sandbox/boundary.py new file mode 100644 index 00000000..fdb44bfd --- /dev/null +++ b/src/lightcone/engine/sandbox/boundary.py @@ -0,0 +1,250 @@ +"""Selecting a mechanism, and running a command through it. + +This is the mechanism-blind half of the layer. It picks a backend, asks +it to rewrite the argv, runs the result, and turns whatever came back +into an :class:`Outcome`. It contains the only ``sys.platform`` branch in +the codebase, in :func:`detect`; everything else here would read the same +if a third mechanism landed tomorrow. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import threading +from collections import deque +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import IO + +from lightcone.engine.sandbox import policy as policy_module +from lightcone.engine.sandbox.model import Attestation, Backend, Capability, Policy + +#: How much of the child's stderr to keep for the denial classifier. The +#: denial is in the last few lines of a traceback, and a recipe that +#: prints megabytes must not be buffered whole. +_STDERR_TAIL_BYTES = 64 * 1024 + +#: Set inside the boundary so a nested lc — or a test — can tell it is +#: already sandboxed. Neither mechanism nests: `sandbox-exec` refuses, +#: and a Landlock domain can only be tightened. +SANDBOX_ENV = "LC_SANDBOX" + + +@dataclass(frozen=True) +class Unavailable: + """The honest null backend: no rewrite, and it says so. + + Not a special case for callers to branch on — it satisfies the same + protocol, wraps to the same argv it was given, and attests + ``fs: open``. Telling the user is the caller's job; pretending is + nobody's. + """ + + capability: Capability = field(default_factory=lambda: Capability(kind="none")) + + def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + return list(argv) + + def attest(self, policy: Policy) -> Attestation: + return Attestation(mechanism="none", fs="open") + + +@dataclass(frozen=True) +class Outcome: + """What one trip through the boundary produced.""" + + returncode: int + attestation: Attestation + #: Console lines the caller prints verbatim: the downgrade notice, + #: the denial explanation, the failure trailer. + notes: tuple[str, ...] = () + + +def detect() -> Backend: + """The best mechanism this host can offer. + + The single platform branch. A backend that probes unavailable falls + through to the next candidate and finally to :class:`Unavailable`, + so adding another mechanism is one import and one line. + """ + if sys.platform == "linux": + from lightcone.engine.sandbox.landlock import LandlockBackend, capability + + found = capability() + if found.kind == "landlock": + return LandlockBackend(capability=found) + return Unavailable(capability=found) + if sys.platform == "darwin": + from lightcone.engine.sandbox.seatbelt import SeatbeltBackend, capability + + found = capability() + if found.kind == "seatbelt": + return SeatbeltBackend(capability=found) + return Unavailable(capability=found) + return Unavailable( + capability=Capability(kind="none", detail=f"no sandbox mechanism on {sys.platform}") + ) + + +@contextmanager +def scope(project: Path, *, read_paths: Sequence[Path] = ()) -> Iterator[Policy]: + """A probe policy, with its per-run HOME cleaned up afterwards. + + The policy owns a real directory on disk (the private ``$HOME``), so + building one is not free and leaking one is a real cost on a machine + that runs many probes. A context manager makes the lifetime the + caller's, visibly. + """ + built = policy_module.probe_policy(project, read_paths=read_paths) + try: + yield built + finally: + shutil.rmtree(built.tmp_home, ignore_errors=True) + + +def run( + backend: Backend, + policy: Policy, + argv: Sequence[str], + *, + cwd: Path, + env: dict[str, str], + prefix: Sequence[str] = (), +) -> Outcome: + """Run *argv* through *backend*, and explain it if it fails. + + *prefix* is spawned **outside** the rewrite — it is how the caller + says "wrap the command, not this". ``lc run`` uses it for the + ``uv run`` hop, because uv, its config, and its caches are trusted + plumbing that must stay outside the boundary. + + stdout is inherited untouched, so a probe stays a probe — output + arrives live. stderr is teed: written through as it arrives *and* + retained, because the denial classifier needs text and the user + needs immediacy. (A denial printed only to stdout is therefore + missed; the trailer still fires.) + """ + wrapped = [*prefix, *backend.wrap(policy, [*env_argv(policy), *argv])] + attestation = backend.attest(policy) + # `policy.env` is deliberately **not** merged here: it went inside + # the wrap, above, via :func:`env_argv`. Everything *outside* the + # rewrite has to keep the real environment — `uv` resolves its cache + # from `XDG_CACHE_HOME` and its interpreters from `XDG_DATA_HOME`, so + # overlaying those for the `uv run` prefix would point it at a + # throwaway directory `scope()` then deletes. + child_env = {**env, SANDBOX_ENV: attestation.mechanism} + + notes: list[str] = [] + if backend.capability.kind == "none": + notes.append(_downgrade_note(backend.capability)) + + proc = subprocess.Popen( + wrapped, + cwd=cwd, + env=child_env, + stderr=subprocess.PIPE, + text=True, + errors="replace", + ) + assert proc.stderr is not None # Popen was given PIPE + tail = _Tail(proc.stderr) + tail.start() + returncode = proc.wait() + tail.join(timeout=5) + + # Imported here, not at module scope: `sandbox/__init__` loads this + # module eagerly, and the shim drags ctypes in for one integer. + from lightcone._sandbox_exec import SETUP_FAILURE_EXIT + + if returncode == SETUP_FAILURE_EXIT: + # The shim's reserved code: the sandbox could not be *built*. Say + # so plainly — the trailer would otherwise point the user at their + # own command's permissions for a failure that is entirely ours. + notes.append( + "lc could not set up the sandbox (see above) — this is an lc " + "problem, not your command's" + ) + elif returncode != 0 and attestation.mechanism != "none": + from lightcone.engine.sandbox import denial + + explanation = denial.explain(tail.text(), policy, cwd=cwd) + notes.extend([*explanation, ""] if explanation else []) + notes.append(denial.trailer(attestation.mechanism)) + + return Outcome( + returncode=returncode, + attestation=attestation, + notes=tuple(notes), + ) + + +def env_argv(policy: Policy) -> list[str]: + """``env K=V …``, prefixed to the command *inside* the wrap. + + One place, every backend — including :class:`Unavailable`, which + would otherwise silently run in a different environment than a + sandboxed run. + Composed inside the wrap rather than around it so the ``prefix`` + (the ``uv run`` hop) keeps the real environment: uv resolves its + cache from ``XDG_CACHE_HOME`` and its interpreters from + ``XDG_DATA_HOME``. + + ``env`` is resolved the same way the exec set resolved it, not by a + literal path: the set grants whatever + :func:`~lightcone.engine.sandbox.policy.utility` found, so a hardcoded + ``/usr/bin/env`` is a denial on the first exec of every run on any + host that keeps its copy elsewhere. + """ + if not policy.env: + return [] + found = policy_module.utility("env") + if found is None: # pragma: no cover - no `env` on the search path + raise RuntimeError(f"`env` not found on {policy_module._UTILITY_PATH}") + return [str(found), *(f"{k}={v}" for k, v in sorted(policy.env.items()))] + + +def _downgrade_note(capability: Capability) -> str: + """The line a user must see when they were not actually sandboxed. + + Never silent: finishing a run believing you were sandboxed when you + were not is the failure this design exists to prevent, and it is the + one shipped implementations are cited for. + """ + reason = f" — {capability.detail}" if capability.detail else "" + return f"not sandboxed on this host{reason}; recorded as `fs: open`" + + +class _Tail(threading.Thread): + """Pumps the child's stderr through to ours, keeping a bounded tail. + + Concurrent by necessity: the pipe has to be drained while the child + runs, or a chatty command blocks on a full buffer. Bounded because + the denial classifier only needs the last few lines of a traceback + and a recipe that prints megabytes must not be held whole. + """ + + def __init__(self, stream: IO[str]) -> None: + super().__init__(daemon=True) + self._stream = stream + # A deque because the bound is on *bytes*, so it cannot be + # delegated to `maxlen` — but eviction is from the left, and + # `list.pop(0)` is O(n) under exactly the load the bound exists + # to survive. + self._chunks: deque[str] = deque() + self._size = 0 + + def run(self) -> None: + for line in self._stream: + sys.stderr.write(line) + self._chunks.append(line) + self._size += len(line) + while self._size > _STDERR_TAIL_BYTES and len(self._chunks) > 1: + self._size -= len(self._chunks.popleft()) + sys.stderr.flush() + + def text(self) -> str: + return "".join(self._chunks) diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py new file mode 100644 index 00000000..6b43ff41 --- /dev/null +++ b/src/lightcone/engine/sandbox/denial.py @@ -0,0 +1,188 @@ +"""The denial UX — the sandbox's primary user interface. + +A sandbox that only says "permission denied" trains people to disable +it. So when a sandboxed command fails, this module tries to name *what* +was blocked, guess whether it was a tool or a data file, and lead with +the remedy that matches — each as something the reader can paste. + +The classification is a best-guess heuristic over the child's own error +text, and it is allowed to come up empty: a recipe can swallow the +``PermissionError``, or rewrap it past recognition. That is why +:func:`trailer` is unconditional. Between them the guarantee is that a +denial is never *invisible*, even when it cannot be explained. +""" + +from __future__ import annotations + +import os +import re +import shutil +from pathlib import Path + +from lightcone.engine.sandbox.model import Policy + +#: Ways a blocked path shows up in a child's stderr. Ordered by how +#: specific they are, all applied — the first *confirmed* hit is +#: rendered. +_CANDIDATE_PATTERNS = ( + # Python: PermissionError: [Errno 13] Permission denied: '/path' + re.compile(r"(?:PermissionError|FileNotFoundError|OSError).*?['\"]([^'\"]+)['\"]"), + # Our own shim, when the exec itself was denied. + re.compile(r"lc sandbox: (\S+): "), + # bash/sh: `line 1: /path: Permission denied`, `cmd: command not found` + re.compile( + r"(?:bash|sh)(?::\s*line \d+)?: ([^\s:]+): " + # macOS reports a Seatbelt denial as EPERM, Linux as EACCES. + r"(?:Permission denied|Operation not permitted|command not found)" + ), + # Landlock ABI 1 has no REFER, so a cross-directory rename is EXDEV. + re.compile(r"Invalid cross-device link.*?['\"]([^'\"]+)['\"]"), +) + +#: Directories whose contents are almost certainly programs, used when +#: the exec bit alone is ambiguous. +_BIN_DIR_HINTS = ("/bin", "/sbin", "/Library/TeX", "/opt") + + +def explain(stderr: str, policy: Policy, *, cwd: Path) -> list[str]: + """Lines explaining the first confirmed denial in *stderr*, or ``[]``. + + "Confirmed" is doing real work here. A candidate is dropped when the + path does not exist (an ordinary missing-file bug, not a denial) or + when the policy fully grants it (someone else's permission problem). + Both checks run in the unsandboxed parent, where ``stat`` sees + everything — which is the whole reason the parent does the + explaining rather than the child. + + The three outcomes are three different mistakes, and conflating them + is how a sandbox message becomes useless: + + - a path granted for **neither** access — an undeclared tool or an + undeclared input; + - a path granted for **read but not write** — which, since reading + it was allowed, can only have been a write attempt into the + read-only project tree. + """ + # Access-aware, and that distinction is what keeps the message + # honest: every allowlisted binary lives under `/usr`, which the read + # baseline grants, so an executable being *readable* says nothing + # about whether running it was allowed. + for raw in _candidates(stderr): + path = _resolve(raw, cwd) + if path is None or not path.exists(): + continue + if _classify(path) == "tool": + if policy.grants(path, policy.execute): + continue + return _render_tool(path) + if policy.grants(path, policy.write): + continue + if policy.grants(path, policy.read): + return _render_write(path) + return _render_data(path) + return [] + + +def trailer(mechanism: str) -> str: + """The one line printed after *every* failed sandboxed run. + + Unconditional by design: :func:`explain` fires only when it can + extract and confirm a path, and the cases where it cannot — a recipe + that catches the ``PermissionError`` and exits with something else — + are exactly the cases where someone would otherwise spend an hour + fighting an invisible wall. + """ + return ( + f"this ran under the lc sandbox ({mechanism}) — a permissions or " + "missing-file error can mean the command reached for something " + "outside the declared environment" + ) + + +def _candidates(stderr: str) -> list[str]: + """Every path-ish string the patterns find, de-duplicated, in order.""" + found: dict[str, None] = {} + for pattern in _CANDIDATE_PATTERNS: + for match in pattern.finditer(stderr): + found.setdefault(match.group(1), None) + return list(found) + + +def _resolve(raw: str, cwd: Path) -> Path | None: + """Turn a path-ish fragment from an error message into a real path. + + Three shapes reach us. An absolute path is itself. A relative one is + relative to where the command ran — a recipe that says + ``open('astra.yaml')`` reports exactly that string. A bare name from + ``command not found`` is resolved against the *host* PATH, which is + what turns "latex: command not found" into "you have latex, it just + isn't declared" — the difference between a useful message and a + confusing one. + """ + if raw.startswith("/"): + return Path(raw) + relative = cwd / raw + if relative.exists(): + return relative + found = shutil.which(raw) + return Path(found) if found else None + + +def _classify(path: Path) -> str: + """``"tool"`` or ``"data"`` — which kind of denial this looks like.""" + if path.is_file() and os.access(path, os.X_OK): + return "tool" + if str(path.parent).endswith(_BIN_DIR_HINTS): + return "tool" + return "data" + + +def _render_tool(path: Path) -> list[str]: + return _message( + f"cannot execute {path}", + [ + " if this is a tool the command needs, declare it in the environment:", + f" uv add ", + ], + ) + + +def _render_data(path: Path) -> list[str]: + return _message( + f"cannot read {path}", + [ + " if this is a data file, declare it as an input in astra.yaml:", + " inputs:", + " - id: ", + " type: data", + f" source: {path}", + ], + ) + + +def _render_write(path: Path) -> list[str]: + """The project tree is readable but not writable — say so plainly. + + Reading it was allowed, so this can only have been a write. There is + no declaration that makes the tree writable to a probe: a probe has + no output, and that is the answer rather than a limitation to work + around. + """ + return _message( + f"cannot write {path}", + [ + " the project tree is read-only in here — nothing a probe does can", + " land in it. Write somewhere scratch instead:", + " import tempfile; tempfile.mkdtemp() # or $TMPDIR, /tmp", + ], + ) + + +def _message(headline: str, remedy: list[str]) -> list[str]: + """One shape for every denial: what, why, then the fix.""" + return [ + f"blocked by lc sandbox: {headline} —", + "not part of the declared environment.", + "", + *remedy, + ] diff --git a/src/lightcone/engine/sandbox/landlock.py b/src/lightcone/engine/sandbox/landlock.py new file mode 100644 index 00000000..6b36d16c --- /dev/null +++ b/src/lightcone/engine/sandbox/landlock.py @@ -0,0 +1,95 @@ +"""The Linux backend: Landlock, reached through the exec shim. + +All the kernel work lives in :mod:`lightcone._sandbox_exec`, which has to +be stdlib-only and importable on its own. What is left here is the two +things the engine needs: probing whether this host can enforce, and the +argv rewrite that makes a self-restricting mechanism look like a wrapper +command. +""" + +from __future__ import annotations + +import functools +import json +import sys +from collections.abc import Sequence +from dataclasses import dataclass, field + +from lightcone import _sandbox_exec +from lightcone.engine.sandbox.model import ( + EXEC_ALLOWLIST_VERSION, + Attestation, + Capability, + Policy, +) + + +@functools.cache +def capability() -> Capability: + """Whether this kernel can enforce, and at which ABI. + + Cached: it is a syscall whose answer cannot change inside one + process. The ABI is *recorded*, not merely gated on — a "best + effort" that succeeded against a kernel with no Landlock at all is + exactly the silent degradation this layer exists to make impossible. + """ + abi = _sandbox_exec.abi() + if abi > 0: + return Capability(kind="landlock", landlock_abi=abi) + return Capability( + kind="none", + detail="landlock unavailable (kernel < 5.13, blocked by seccomp, or unsupported arch)", + ) + + +def _document(policy: Policy) -> dict[str, object]: + """*policy* as the shim's ``--policy`` JSON. + + The shim's wire format, so it lives with the backend that speaks it + rather than on the mechanism-free :class:`Policy`. Strings only — + the shim parses it with nothing but the stdlib. + """ + return { + "read": [str(p) for p in policy.read], + "write": [str(p) for p in policy.write], + "execute": [str(p) for p in policy.execute], + } + + +@dataclass(frozen=True) +class LandlockBackend: + """Landlock, expressed as an argv rewrite. + + The command becomes `` -m lightcone._sandbox_exec --policy + -- ``: the shim restricts itself and then *becomes* + the command, so the restriction is inherited by everything below it + and can never be shed. + + The interpreter is **lc's own**, not the project's. It only has to + live long enough to issue three syscalls before ``execvp``, and using + ours means the shim is always the same lightcone-cli as the engine + that wrote the policy. + """ + + capability: Capability = field(default_factory=capability) + interpreter: str = sys.executable + + def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + document = json.dumps(_document(policy), separators=(",", ":"), sort_keys=True) + return [ + self.interpreter, + "-m", + "lightcone._sandbox_exec", + "--policy", + document, + "--", + *argv, + ] + + def attest(self, policy: Policy) -> Attestation: + return Attestation( + mechanism="landlock", + fs="declared", + landlock_abi=self.capability.landlock_abi, + exec_allowlist_version=EXEC_ALLOWLIST_VERSION, + ) diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py new file mode 100644 index 00000000..05ba1f67 --- /dev/null +++ b/src/lightcone/engine/sandbox/model.py @@ -0,0 +1,122 @@ +"""The sandbox layer's data types, and the seam every mechanism meets. + +Three types, and keeping them distinct is the whole design: + +- :class:`Policy` — *what we will enforce*. Mechanism-free path sets. +- :class:`Capability` — *what this host can do*. The probe's answer. +- :class:`Attestation` — *what was actually enforced*. Recorded with + every run, and derived from the flags actually applied — never a + paraphrase of what should have happened. + +:class:`Backend` is the seam. Every mechanism reduces to one pure +function, ``wrap(policy, argv) -> argv``: turn a command into a +*different command that sandboxes itself*. Seatbelt is natively that +shape (``sandbox-exec`` is a wrapper command); Landlock is not — it is a +self-restriction — which is what +:mod:`lightcone._sandbox_exec` exists to fix. Once both are argv +rewrites, everything above the seam is mechanism-blind and every backend +is testable on any OS with no privileges, by asserting on the argv it +emits. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Protocol + +#: Bumped when the meaning of the exec allowlist changes. It is recorded +#: in the attestation, so a run stays interpretable after the list +#: grows — the allowlist is a maintained policy surface. +EXEC_ALLOWLIST_VERSION = 1 + + +@dataclass(frozen=True) +class Policy: + """What a sandboxed command may touch — one policy, every mechanism. + + Every path is realpath'd at construction: macOS resolves ``/tmp`` to + ``/private/tmp`` and Landlock evaluates the resolved path, so an + unresolved path is a rule that silently matches nothing. + + ``write`` implies read. ``execute`` does not imply either beyond the + file itself — shared libraries are covered by the read baseline. + """ + + read: tuple[Path, ...] + write: tuple[Path, ...] + execute: tuple[Path, ...] + #: The fresh per-run directory that becomes ``$HOME``. Lives under the + #: write scope; the caller owns removing it. + tmp_home: Path + #: Environment the boundary overlays: HOME, the XDG trio, + #: MPLCONFIGDIR, PYTHONPYCACHEPREFIX. + env: dict[str, str] = field(default_factory=dict) + + def grants(self, path: Path, roots: tuple[Path, ...]) -> bool: + """Whether *path* lies under any of *roots*. + + The one containment predicate for the layer. ``is_relative_to`` is + reflexive, so a root grants itself — spelling it `p == r or + p.is_relative_to(r)` is not just redundant, it teaches the next + reader that the stdlib does not do the obvious thing. + """ + resolved = path.resolve() + return any(resolved.is_relative_to(root) for root in roots) + + +@dataclass(frozen=True) +class Capability: + """What enforcement this host can provide, as probed.""" + + kind: Literal["landlock", "seatbelt", "none"] + landlock_abi: int | None = None + #: Why, when ``kind`` is ``none``. Reaches the user — a downgrade is + #: never silent. + detail: str = "" + + +@dataclass(frozen=True) +class Attestation: + """The hermeticity record for one exec. + + Derived from the flags actually applied, never from the mechanism + matrix's expectations. ``network`` is ``allowed`` on every path here + by recorded decision: lc applies no network restriction, and saying + so is the honest value. + """ + + mechanism: Literal["landlock", "seatbelt", "none"] + fs: Literal["declared", "open"] + network: Literal["allowed"] = "allowed" + landlock_abi: int | None = None + exec_allowlist_version: int | None = None + + +class Backend(Protocol): + """One sandbox mechanism, reduced to an argv rewrite. + + Implementations must keep :meth:`wrap` pure — no temporary files, no + file descriptors, no global state. That is what lets the execution + path below it stay mechanism-blind, and what makes a backend + testable on a host that cannot run it. + """ + + @property + def capability(self) -> Capability: + """What this backend probed on this host. + + A read-only property rather than a bare attribute, so a frozen + dataclass satisfies the protocol — an immutable backend is the + point, since ``wrap`` must be pure. + """ + ... + + def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + """*argv*, rewritten into a command that sandboxes itself.""" + ... + + def attest(self, policy: Policy) -> Attestation: + """What :meth:`wrap`'s command will actually have enforced.""" + ... diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py new file mode 100644 index 00000000..08df5035 --- /dev/null +++ b/src/lightcone/engine/sandbox/policy.py @@ -0,0 +1,357 @@ +"""Building the policy a sandboxed command runs under. + +One policy, both mechanisms. This module decides *what* a command may +touch; :mod:`~lightcone.engine.sandbox.landlock` and +:mod:`~lightcone.engine.sandbox.seatbelt` only decide how to say it. + +The shape it encodes is the design's filesystem policy: read the project +and the declared inputs and the OS baseline; write only a private +scratch scope; execute only the environment plus a versioned utility +allowlist. Everything a recipe reaches for outside that set fails +loudly, which is the point. +""" + +from __future__ import annotations + +import functools +import os +import shutil +import tempfile +from collections.abc import Iterable, Sequence +from fnmatch import fnmatch +from pathlib import Path + +from lightcone.engine.sandbox.model import Policy + +#: The utility tier of the exec allowlist. A maintained policy +#: surface, versioned by ``EXEC_ALLOWLIST_VERSION`` — recipes routinely +#: shell out to these, and none of them is a scientific dependency in +#: disguise, so admitting them costs nothing the design cares about. +EXEC_ALLOWLIST: tuple[str, ...] = ( + "sh", "bash", "env", + "grep", "egrep", "fgrep", "sed", "awk", "gawk", "mawk", + "tar", "gzip", "gunzip", "zcat", "bzip2", "xz", + "cat", "head", "tail", "ls", "cp", "mv", "rm", "mkdir", "rmdir", "ln", + "chmod", "touch", "date", "sort", "uniq", "cut", "tr", "wc", "tee", + "find", "xargs", "mktemp", "readlink", "realpath", "dirname", "basename", + "echo", "printf", "sleep", "true", "false", "test", +) # fmt: skip + +#: Where the allowlist is resolved from — deliberately *not* the ambient +#: ``$PATH``. A user's PATH may front `/usr/bin` with a directory full of +#: undeclared tools, and resolving through it would quietly admit them. +#: NixOS keeps none of the allowlist under FHS paths, so its system +#: profile is listed too or nothing but `/bin/sh` ever resolves there. +_UTILITY_PATH = "/usr/local/bin:/usr/bin:/bin:/run/current-system/sw/bin" + + +def utility(name: str) -> Path | None: + """Where *name* resolves from the fixed search path, if it is there. + + The **only** way anything works out where an allowlisted tool lives. + Anywhere that hardcodes a path instead is a second answer to the same + question, and the two disagree the moment a host keeps its copy + somewhere else: the exec set would grant one file and the caller + would run the other, which is a denial on every single run. + """ + found = shutil.which(name, path=_UTILITY_PATH) + return Path(found) if found else None + +#: Readable everywhere: the OS the interpreter and its libraries live in. +#: Read, never execute — being able to *read* /usr is what lets the +#: dynamic linker work; being able to *run* what is in it is the leak. +#: +#: System-level paths only, deliberately: this list must never widen to +#: reach user data, which is what the project and declared-input grants +#: are for. `/nix/store` and `/run/current-system/sw` are here because on +#: NixOS *everything* — interpreter, libraries, the utility allowlist — +#: resolves into them, so without them the sandbox is unusable there +#: rather than merely incomplete. (Both entries are taken from codex's +#: own `LINUX_PLATFORM_DEFAULT_READ_ROOTS`.) +#: +#: `/run` is granted whole, where codex names only `/run/current-system/sw`. +#: It reaches `/run/user/$UID` — sockets, dconf, portal state — and that +#: is fine here: the test is whether undeclared *inputs* arrive through a +#: path, and runtime sockets are not a channel a recipe accidentally +#: reads data from. `/etc/resolv.conf` is a symlink into `/run` wherever +#: systemd-resolved is in use, so the grant also keeps DNS working. +_OS_READ_BASELINE = ( + "/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt", "/run", + "/nix/store", "/run/current-system/sw", + "/dev/urandom", "/dev/random", +) # fmt: skip + +#: Writable everywhere: the scratch surfaces and device nodes a command +#: legitimately uses. +#: +#: The `/dev` entries mirror what bubblewrap's `--dev` primitive +#: materializes — `null, zero, full, random, urandom, tty`, plus a devpts +#: mount and `ptmx`. Landlock has no device-tree primitive, so where bwrap +#: gets them from one flag we enumerate them. `/dev/tty` is what lets +#: anything open the controlling terminal afresh (`lc run`'s own shell +#: included); `/dev/pts` and `/dev/ptmx` are what let a command allocate a +#: *new* pty, which pexpect, pytest's capture, and any subprocess wanting +#: a terminal all do. +#: +#: Granting the whole devpts directory is deliberate. The threat model is +#: accidental leakage, not a hostile recipe, and terminals are +#: not a channel undeclared *inputs* arrive through. It is also less +#: permissive than it reads: Landlock only ever removes access, never adds +#: it, so ordinary Unix permissions still apply — devpts gives each pty to +#: its allocating user at mode 0620. +#: +#: `/dev/zero` and `/dev/full` are here rather than in the read baseline +#: because writes to them are discard-by-construction — that is what the +#: devices *are* — so read-only buys nothing and costs the tools that use +#: `/dev/full` to exercise ENOSPC handling. Only the entropy sources stay +#: read-only, since writing to those seeds the host's pool. +#: +#: `/proc` and `/sys` are unrestricted for the same reason. Real tools do +#: write them — `/proc/self/oom_score_adj`, `coredump_filter`, MPI and +#: CUDA runtimes poking `/sys` — and none of that is a channel undeclared +#: *inputs* arrive through. Landlock only ever removes access, so the +#: kernel's own permissions are still the real gate here: almost all of +#: both trees is root-owned, and this simply stops lc adding a second, +#: more confusing denial on top of the one the OS already enforces. +_WRITE_BASELINE = ( + "/tmp", "/var/tmp", "/dev/shm", "/proc", "/sys", + "/dev/null", "/dev/zero", "/dev/full", "/dev/tty", "/dev/pts", "/dev/ptmx", +) # fmt: skip + +#: The ELF interpreter. Landlock checks EXECUTE on the *loader's* open, +#: so without these every dynamically linked binary — bash and python +#: included — fails EACCES and the sandbox is unusable. Globbed rather +#: than hardcoded: the path differs +#: across glibc/musl and architectures. +_ELF_LOADER_GLOBS = ( + "/lib64/ld-linux-*.so.*", + "/lib/ld-linux*.so.*", + "/lib/ld-musl-*.so.*", + "/usr/lib/ld-linux*.so.*", + "/usr/lib64/ld-linux-*.so.*", +) + +#: Prefixes shared with the rest of the host. An interpreter installed +#: into one of these does not bring its own tree with it, so only the +#: binary itself may be granted EXECUTE — granting the prefix would make +#: every tool on the machine runnable, since Landlock unions rights over +#: ancestors. Anything else — a uv-managed store, a framework version +#: directory, a Homebrew Cellar — *is* the interpreter's own tree. +_SHARED_PREFIXES = frozenset( + {"/", "/usr", "/usr/local", "/opt", "/opt/homebrew", "/opt/local", "/System", "/Library"} +) + +#: The redirected environment, as ``variable -> subdirectory of HOME``. +#: One mapping, so the directories that get created and the variables +#: that point at them cannot drift apart. +_HOME_LAYOUT = { + "XDG_CONFIG_HOME": ".config", + "XDG_CACHE_HOME": ".cache", + "XDG_DATA_HOME": ".local/share", + "MPLCONFIGDIR": ".mplconfig", + "PYTHONPYCACHEPREFIX": ".pycache", + "TMPDIR": ".tmp", +} + + +def probe_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: + """The policy for ``lc run`` — a probe. + + A probe has no output, so it gets **no in-tree write scope at all**: + it may read the project and the declared inputs, and write only to + its own tmp scope. That is stricter than a recipe's policy, and it is + what stops a probe from quietly materializing something. + + Creates the per-run HOME on disk as a side effect; the caller owns + removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). + """ + tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-")).resolve() + for sub in _HOME_LAYOUT.values(): + (tmp_home / sub).mkdir(parents=True, exist_ok=True) + + python = _venv_python(project) + # EXECUTE on the interpreter *file*; READ on the install root beside + # it, for the stdlib. See :func:`_venv_python` and :func:`_stdlib_root`. + stdlib = _stdlib_root(python) + write = _existing([tmp_home, *_write_roots(project)]) + read = _existing([project, *read_paths, *stdlib, *(Path(p) for p in _OS_READ_BASELINE)]) + + return Policy( + read=read, + write=write, + execute=_existing(_exec_set(project, python)), + tmp_home=tmp_home, + env=home_overlay(tmp_home, project), + ) + + +def _write_roots(project: Path) -> list[Path]: + """The write baseline, minus any root that would swallow the project. + + ``/tmp`` is writable by design — but a project that + *lives* under it would then be writable too, which would silently + void the read-only-tree guarantee for exactly the people who keep + scratch analyses in ``/tmp``. Dropping the offending root is safe + because ``TMPDIR`` points at the private scope regardless, so + ``tempfile`` keeps working either way. The device entries can never + contain a project, so the filter is a no-op for them. + """ + resolved = project.resolve() + roots = tuple(Path(root).resolve() for root in _WRITE_BASELINE) + return [root for root in roots if not resolved.is_relative_to(root)] + + +def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: + """HOME and friends, pointed at a fresh directory. + + The real ``$HOME`` is neither readable nor writable inside the + boundary. Left alone, that breaks matplotlib, astropy, and R on first + import — and the obvious patch, mounting ``$HOME`` read-only, would + reopen the dotfile-steering channel the whole layer exists to close. + Giving them a private HOME instead is the Bazel/nix move: they work, + and they cannot be steered. + + ``PATH`` is set for a different reason, but the same one at heart: + what the command resolves has to be what the policy granted. + + ``PYTHONPYCACHEPREFIX`` is here for the same reason at the other end: + the project tree is read-only inside the boundary, so without it + every ``import`` of an in-tree module fails to write its + ``__pycache__``. ``TMPDIR`` points inside too, so ``tempfile`` works + even where the shared ``/tmp`` had to leave the write set. + """ + return { + "HOME": str(tmp_home), + # The search path *is* the exec set. Without this the command + # resolves tools through the host's ambient PATH while the policy + # granted whatever `_UTILITY_PATH` resolved — so on a machine + # whose PATH fronts another copy (homebrew's bash on macOS, say) + # the sandbox denies `bash` itself, and the message blames the + # user's command for lc's own incoherence. + "PATH": os.pathsep.join([str(project / ".venv" / "bin"), _UTILITY_PATH]), + **{k: str(tmp_home / v) for k, v in _HOME_LAYOUT.items()}, + } + + +def _venv_python(project: Path) -> Path | None: + """The realpath of the venv's interpreter, if there is one. + + Resolved, because ``.venv/bin/python`` is a symlink and Landlock + evaluates the target. What gets granted on it is + :func:`_exec_set`'s decision, and its install root is separately a + read root (:func:`probe_policy`) for the standard library beside it. + """ + python = project / ".venv" / "bin" / "python" + return python.resolve() if python.exists() else None + + +def _stdlib_root(python: Path | None) -> list[Path]: + """The install root to grant READ on, for the standard library. + + The stdlib sits beside the interpreter, outside the project and + outside `/usr` for a managed build, so without this grant the child + dies with ``Failed to import encodings module``. + + Refused when the root is ``$HOME`` or an ancestor of it, which is + what an interpreter installed straight into ``~/bin`` produces. That + grant would make the real home readable and silently undo the + private-``$HOME`` design — the one thing the environment overlay + exists to guarantee. Failing loudly on a layout nobody uses beats + voiding the guarantee for the people who do. (Reading the base + prefix out of ``pyvenv.cfg`` instead would not help: it reports the + same directory.) + """ + if python is None: + return [] + root = python.parent.parent + return [] if Path.home().resolve().is_relative_to(root) else [root] + + +def _exec_set(project: Path, python: Path | None) -> list[Path]: + """The two exec tiers: the environment, and the utility allowlist. + + Grants are per *file* for the utilities, never per directory: + ``/usr/bin`` holds ``bash`` and ``latex`` alike, so a directory grant + there would admit every undeclared tool on the host and leave the + layer enforcing nothing. + + The interpreter is the one place that judgement is not enough, and + the rule is narrower than either extreme. Its **own tree** is granted + — a uv-managed store, a framework version directory, a Homebrew + Cellar — because a framework build does not exec the binary on PATH + at all: it re-execs itself into + ``Resources/Python.app/Contents/MacOS/Python``, and a grant on the + launcher alone leaves it unable to start. A **shared** prefix is not + granted (:data:`_SHARED_PREFIXES`): a venv built against the system + python roots at ``/usr``, and since Landlock unions rights over + ancestors, that single grant would make every binary on the host + runnable and silently outrank this whole allowlist. + """ + paths: list[Path] = [] + bin_dir = project / ".venv" / "bin" + if bin_dir.is_dir(): + paths.append(bin_dir) + if python is not None: + paths.append(python) + # macOS framework builds `posix_spawn` themselves into + # `Resources/Python.app/Contents/MacOS/Python`, a *different* + # file from the one on PATH — so granting the launcher alone + # leaves the interpreter unable to start itself. + install_root = python.parent.parent + if str(install_root) not in _SHARED_PREFIXES: + paths.append(install_root) + for name in EXEC_ALLOWLIST: + found = utility(name) + if found is not None: + paths.append(found) + paths.extend(elf_loaders()) + return paths + + +@functools.cache +def elf_loaders() -> tuple[Path, ...]: + """The dynamic loaders present on this host, realpath'd. + + Scans each distinct directory once rather than globbing five + patterns: on a merged-``/usr`` system all five resolve to the same + directory, and `glob` re-lists and re-matches its ~8000 entries per + pattern — which measured as 95% of the whole policy build. Cached + because the answer cannot change while the process runs. + """ + found: set[Path] = set() + for directory, patterns in _loader_patterns().items(): + try: + entries = list(os.scandir(directory)) + except OSError: + continue + for entry in entries: + # Cheap prefix reject before fnmatch: almost nothing in a + # library directory starts with `ld-`. + if entry.name.startswith("ld-") and any( + fnmatch(entry.name, pattern) for pattern in patterns + ): + found.add(Path(entry.path).resolve()) + return tuple(sorted(found)) + + +def _loader_patterns() -> dict[str, set[str]]: + """The loader globs, grouped by the real directory they name.""" + grouped: dict[str, set[str]] = {} + for pattern in _ELF_LOADER_GLOBS: + directory, _, name = pattern.rpartition("/") + grouped.setdefault(os.path.realpath(directory), set()).add(name) + return grouped + + +def _existing(paths: Iterable[Path]) -> tuple[Path, ...]: + """Resolve, drop what is not there, de-duplicate, keep order. + + A rule for a path that does not exist cannot be added, and a + baseline entry missing on this OS is normal rather than an error. + """ + resolved: dict[Path, None] = {} + for path in paths: + candidate = Path(path).resolve() + if candidate.exists(): + resolved.setdefault(candidate, None) + return tuple(resolved) diff --git a/src/lightcone/engine/sandbox/profiles/base.sbpl b/src/lightcone/engine/sandbox/profiles/base.sbpl new file mode 100644 index 00000000..ce449b04 --- /dev/null +++ b/src/lightcone/engine/sandbox/profiles/base.sbpl @@ -0,0 +1,139 @@ +; --------------------------------------------------------------------------- +; Inspired from openai/codex — codex-rs/sandboxing/src/seatbelt_base_policy.sbpl +; upstream commit f5e9d66851 (retrieved 2026-08-18) +; +; Copyright OpenAI, licensed under the Apache License 2.0 +; https://github.com/openai/codex/blob/main/LICENSE +; Upstream is itself derived from Chromium's macOS sandbox policy +; (Copyright The Chromium Authors, BSD 3-Clause). +; +; Kept close to the upstream text so it can be diffed when re-syncing. +; Local changes are marked `LIGHTCONE DELTA`. +; --------------------------------------------------------------------------- + +(version 1) + +; inspired by Chrome's sandbox policy: +; https://source.chromium.org/chromium/chromium/src/+/main:sandbox/policy/mac/common.sb;l=273-319;drc=7b3962fe2e5fc9e2ee58000dc8fbf3429d84d3bd +; https://source.chromium.org/chromium/chromium/src/+/main:sandbox/policy/mac/renderer.sb;l=64;drc=7b3962fe2e5fc9e2ee58000dc8fbf3429d84d3bd + +; start with closed-by-default +(deny default) + +; child processes inherit the policy of their parent +; LIGHTCONE DELTA: upstream allows `process-exec` unconditionally because it +; does not restrict exec. Restricting exec is our whole guarantee (G6, the +; two-tier exec set of spec §7), so the blanket allow is removed and the +; generator emits per-path `process-exec*` grants instead. +;(allow process-exec) +(allow process-fork) +(allow signal (target same-sandbox)) + +; process-info +(allow process-info* (target same-sandbox)) + +(allow file-write-data + (require-all + (path "/dev/null") + (vnode-type CHARACTER-DEVICE))) + +; sysctls permitted. +(allow sysctl-read + (sysctl-name "hw.activecpu") + (sysctl-name "hw.busfrequency_compat") + (sysctl-name "hw.byteorder") + (sysctl-name "hw.cacheconfig") + (sysctl-name "hw.cachelinesize_compat") + (sysctl-name "hw.cpufamily") + (sysctl-name "hw.cpufrequency_compat") + (sysctl-name "hw.cputype") + (sysctl-name "hw.l1dcachesize_compat") + (sysctl-name "hw.l1icachesize_compat") + (sysctl-name "hw.l2cachesize_compat") + (sysctl-name "hw.l3cachesize_compat") + (sysctl-name "hw.logicalcpu_max") + (sysctl-name "hw.machine") + (sysctl-name "hw.model") + (sysctl-name "hw.memsize") + (sysctl-name "hw.ncpu") + (sysctl-name "hw.nperflevels") + ; Chrome locks these CPU feature detection down a bit more tightly, + ; but mostly for fingerprinting concerns which isn't an issue for codex. + (sysctl-name-prefix "hw.optional.arm.") + (sysctl-name-prefix "hw.optional.armv8_") + (sysctl-name "hw.packages") + (sysctl-name "hw.pagesize_compat") + (sysctl-name "hw.pagesize") + (sysctl-name "hw.physicalcpu") + (sysctl-name "hw.physicalcpu_max") + (sysctl-name "hw.logicalcpu") + (sysctl-name "hw.cpufrequency") + (sysctl-name "hw.tbfrequency_compat") + (sysctl-name "hw.vectorunit") + (sysctl-name "machdep.cpu.brand_string") + (sysctl-name "kern.argmax") + (sysctl-name "kern.hostname") + (sysctl-name "kern.maxfilesperproc") + (sysctl-name "kern.maxproc") + (sysctl-name "kern.osproductversion") + (sysctl-name "kern.osrelease") + (sysctl-name "kern.ostype") + (sysctl-name "kern.osvariant_status") + (sysctl-name "kern.osversion") + (sysctl-name "kern.secure_kernel") + (sysctl-name "kern.usrstack64") + (sysctl-name "kern.version") + (sysctl-name "sysctl.proc_cputype") + (sysctl-name "vm.loadavg") + (sysctl-name-prefix "hw.perflevel") + (sysctl-name-prefix "kern.proc.pgrp.") + (sysctl-name-prefix "kern.proc.pid.") + (sysctl-name-prefix "net.routetable.") +) + +; Allow Java to read some CPU info. This is misclassified as a "write" because +; userspace passes a memory buffer to the sysctl, but conceptually it is a read. +(allow sysctl-write + (sysctl-name "kern.grade_cputype")) + +; IOKit +(allow iokit-open + (iokit-registry-entry-class "RootDomainUserClient") +) + +; needed to look up user info, see https://crbug.com/792228 +(allow mach-lookup + (global-name "com.apple.system.opendirectoryd.libinfo") +) + +; Needed for python multiprocessing on MacOS for the SemLock +(allow ipc-posix-sem) + +; Needed for PyTorch/libomp on macOS to register OpenMP runtimes. +(allow ipc-posix-shm-read-data + ipc-posix-shm-write-create + ipc-posix-shm-write-unlink + (ipc-posix-name-regex #"^/__KMP_REGISTERED_LIB_[0-9]+$")) + +(allow mach-lookup + (global-name "com.apple.PowerManagement.control") +) + +; allow openpty() +(allow pseudo-tty) +(allow file-read* file-write* file-ioctl (literal "/dev/ptmx")) +(allow file-read* file-write* + (require-all + (regex #"^/dev/ttys[0-9]+") + (extension "com.apple.sandbox.pty"))) +; PTYs created before entering seatbelt may lack the extension; allow ioctl +; on those slave ttys so interactive shells detect a TTY and remain functional. +(allow file-ioctl (regex #"^/dev/ttys[0-9]+")) + +; allow readonly user preferences +(allow ipc-posix-shm-read* (ipc-posix-name-prefix "apple.cfprefs.")) +(allow mach-lookup + (global-name "com.apple.cfprefsd.daemon") + (global-name "com.apple.cfprefsd.agent") + (local-name "com.apple.cfprefsd.agent")) +(allow user-preference-read) diff --git a/src/lightcone/engine/sandbox/profiles/network.sbpl b/src/lightcone/engine/sandbox/profiles/network.sbpl new file mode 100644 index 00000000..588749ae --- /dev/null +++ b/src/lightcone/engine/sandbox/profiles/network.sbpl @@ -0,0 +1,48 @@ +; --------------------------------------------------------------------------- +; Inspired from openai/codex — codex-rs/sandboxing/src/seatbelt_network_policy.sbpl +; upstream commit f5e9d66851 (retrieved 2026-08-18) +; +; Copyright OpenAI, licensed under the Apache License 2.0 +; https://github.com/openai/codex/blob/main/LICENSE +; Upstream is itself derived from Chromium's macOS sandbox policy +; (Copyright The Chromium Authors, BSD 3-Clause). +; +; Kept close to the upstream text so it can be diffed when re-syncing. +; Local changes are marked `LIGHTCONE DELTA`. +; +; lc controls no network on either platform, so this fragment is always +; emitted. The socket families are opened by the generator; what lives +; here are the mach services DNS resolution and TLS certificate checking +; go through, which `(deny default)` in the base would otherwise block — +; leaving the attestation claiming `network: allowed` on macOS while +; every name lookup failed. +; --------------------------------------------------------------------------- + + +; allow only safe AF_SYSTEM sockets used for local platform services. +(allow system-socket + (require-all + (socket-domain AF_SYSTEM) + (socket-protocol 2) + ) +) + +(allow mach-lookup + ; Used by platform helpers that resolve user directory locations. + (global-name "com.apple.bsd.dirhelper") + (global-name "com.apple.system.opendirectoryd.membership") + + ; Communicate with the security server for TLS certificate information. + (global-name "com.apple.SecurityServer") + (global-name "com.apple.networkd") + (global-name "com.apple.ocspd") + (global-name "com.apple.trustd.agent") + + ; Read network configuration. + (global-name "com.apple.SystemConfiguration.DNSConfiguration") + (global-name "com.apple.SystemConfiguration.configd") +) + +(allow sysctl-read + (sysctl-name-regex #"^net.routetable") +) diff --git a/src/lightcone/engine/sandbox/profiles/platform-defaults.sbpl b/src/lightcone/engine/sandbox/profiles/platform-defaults.sbpl new file mode 100644 index 00000000..1e4898f2 --- /dev/null +++ b/src/lightcone/engine/sandbox/profiles/platform-defaults.sbpl @@ -0,0 +1,211 @@ +; --------------------------------------------------------------------------- +; Inspired from openai/codex — codex-rs/sandboxing/src/restricted_read_only_platform_defaults.sbpl +; upstream commit f5e9d66851 (retrieved 2026-08-18) +; +; Copyright OpenAI, licensed under the Apache License 2.0 +; https://github.com/openai/codex/blob/main/LICENSE +; Upstream is itself derived from Chromium's macOS sandbox policy +; (Copyright The Chromium Authors, BSD 3-Clause). +; +; Kept close to the upstream text so it can be diffed when re-syncing. +; Local changes are marked `LIGHTCONE DELTA`. +; --------------------------------------------------------------------------- + +; macOS platform defaults included when a split filesystem policy requests `:minimal`. + +; Read access to standard system paths +(allow file-read* file-test-existence + (subpath "/Library/Apple") + (subpath "/Library/Filesystems/NetFSPlugins") + (subpath "/Library/Preferences/Logging") + (subpath "/private/var/db/DarwinDirectory/local/recordStore.data") + (subpath "/private/var/db/timezone") + (subpath "/usr/lib") + (subpath "/usr/share") + (subpath "/Library/Preferences") + (subpath "/var/db") + (subpath "/private/var/db")) + +; Map system frameworks + dylibs for loader. +(allow file-map-executable + (subpath "/Library/Apple/System/Library/Frameworks") + (subpath "/Library/Apple/System/Library/PrivateFrameworks") + (subpath "/Library/Apple/usr/lib") + (subpath "/System/Library/Extensions") + (subpath "/System/Library/Frameworks") + (subpath "/System/Library/PrivateFrameworks") + (subpath "/System/Library/SubFrameworks") + (subpath "/System/iOSSupport/System/Library/Frameworks") + (subpath "/System/iOSSupport/System/Library/PrivateFrameworks") + (subpath "/System/iOSSupport/System/Library/SubFrameworks") + (subpath "/usr/lib")) + +; System Framework and AppKit resources +(allow file-read* file-test-existence + (subpath "/Library/Apple/System/Library/Frameworks") + (subpath "/Library/Apple/System/Library/PrivateFrameworks") + (subpath "/Library/Apple/usr/lib") + (subpath "/System/Library/Frameworks") + (subpath "/System/Library/PrivateFrameworks") + (subpath "/System/Library/SubFrameworks") + (subpath "/System/iOSSupport/System/Library/Frameworks") + (subpath "/System/iOSSupport/System/Library/PrivateFrameworks") + (subpath "/System/iOSSupport/System/Library/SubFrameworks") + (subpath "/usr/lib")) + +; Allow guarded vnodes. +(allow system-mac-syscall (mac-policy-name "vnguard")) + +; Determine whether a container is expected. +(allow system-mac-syscall + (require-all + (mac-policy-name "Sandbox") + (mac-syscall-number 67))) + +; Allow resolution of standard system symlinks. +(allow file-read-metadata file-test-existence + (literal "/etc") + (literal "/tmp") + (literal "/var") + (literal "/private/etc/localtime")) + +; Allow stat'ing of firmlink parent path components. +(allow file-read-metadata file-test-existence + (path-ancestors "/System/Volumes/Data/private")) + +; Allow processes to get their current working directory. +(allow file-read* file-test-existence + (literal "/")) + +; Allow FSIOC_CAS_BSDFLAGS as alternate chflags. +(allow system-fsctl (fsctl-command FSIOC_CAS_BSDFLAGS)) + +; Allow access to standard special files. +(allow file-read* file-test-existence + (literal "/dev/autofs_nowait") + (literal "/dev/random") + (literal "/dev/urandom") + (literal "/private/etc/master.passwd") + (literal "/private/etc/passwd") + (literal "/private/etc/protocols") + (literal "/private/etc/services")) + +; Allow null/zero read/write. +(allow file-read* file-test-existence file-write-data + (literal "/dev/null") + (literal "/dev/zero")) + +; Allow read/write access to the file descriptors. +(allow file-read-data file-test-existence file-write-data + (subpath "/dev/fd")) + +; Provide access to debugger helpers. +(allow file-read* file-test-existence file-write-data file-ioctl + (literal "/dev/dtracehelper")) + +; Scratch space so tools can create temp files. +(allow file-read* file-test-existence file-write* (subpath "/tmp")) +(allow file-read* file-write* (subpath "/private/tmp")) +(allow file-read* file-write* (subpath "/var/tmp")) +(allow file-read* file-write* (subpath "/private/var/tmp")) + +; Allow reading standard config directories. +(allow file-read* (subpath "/etc")) +(allow file-read* (subpath "/private/etc")) + +(allow file-read* file-test-existence + (literal "/System/Library/CoreServices") + (literal "/System/Library/CoreServices/.SystemVersionPlatform.plist") + (literal "/System/Library/CoreServices/SystemVersion.plist")) + +; Some processes read /var metadata during startup. +(allow file-read-metadata (subpath "/var")) +(allow file-read-metadata (subpath "/private/var")) + +; IOKit access for root domain services. +(allow iokit-open + (iokit-registry-entry-class "RootDomainUserClient")) + +; macOS Standard library queries opendirectoryd at startup +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Allow IPC to analytics, logging, trust, and other system agents. +(allow mach-lookup + (global-name "com.apple.analyticsd") + (global-name "com.apple.analyticsd.messagetracer") + (global-name "com.apple.appsleep") + (global-name "com.apple.bsd.dirhelper") + (global-name "com.apple.cfprefsd.agent") + (global-name "com.apple.cfprefsd.daemon") + (global-name "com.apple.diagnosticd") + (global-name "com.apple.dt.automationmode.reader") + (global-name "com.apple.espd") + (global-name "com.apple.logd") + (global-name "com.apple.logd.events") + (global-name "com.apple.runningboard") + (global-name "com.apple.secinitd") + (global-name "com.apple.system.DirectoryService.libinfo_v1") + (global-name "com.apple.system.logger") + (global-name "com.apple.system.notification_center") + (global-name "com.apple.system.opendirectoryd.membership") + (global-name "com.apple.trustd") + (global-name "com.apple.trustd.agent") + (global-name "com.apple.xpc.activity.unmanaged") + (local-name "com.apple.cfprefsd.agent")) + +; Allow IPC to the syslog socket for logging. +(allow network-outbound (literal "/private/var/run/syslog")) + +; macOS Notifications +(allow ipc-posix-shm-read* + (ipc-posix-name "apple.shm.notification_center")) + +; Regulatory domain support. +(allow file-read* + (literal "/private/var/db/eligibilityd/eligibility.plist")) + +; Audio and power management services. +(allow mach-lookup (global-name "com.apple.audio.audiohald")) +(allow mach-lookup (global-name "com.apple.audio.AudioComponentRegistrar")) +(allow mach-lookup (global-name "com.apple.PowerManagement.control")) + +; Allow reading the minimum system runtime so exec works. +(allow file-read-data (subpath "/bin")) +(allow file-read-metadata (subpath "/bin")) +(allow file-read-data (subpath "/sbin")) +(allow file-read-metadata (subpath "/sbin")) +(allow file-read-data (subpath "/usr/bin")) +(allow file-read-metadata (subpath "/usr/bin")) +(allow file-read-data (subpath "/usr/sbin")) +(allow file-read-metadata (subpath "/usr/sbin")) +(allow file-read-data (subpath "/usr/libexec")) +(allow file-read-metadata (subpath "/usr/libexec")) + +(allow file-read* (subpath "/Library/Preferences")) +(allow file-read* (subpath "/opt/homebrew/lib")) +(allow file-read* (subpath "/usr/local/lib")) + +; Terminal basics and device handles. +(allow file-read* (regex "^/dev/fd/(0|1|2)$")) +(allow file-write* (regex "^/dev/fd/(1|2)$")) +(allow file-read* file-write* (literal "/dev/null")) +(allow file-read* file-write* (literal "/dev/tty")) +(allow file-read-metadata (literal "/dev")) +(allow file-read-metadata (regex "^/dev/.*$")) +(allow file-read-metadata (literal "/dev/stdin")) +(allow file-read-metadata (literal "/dev/stdout")) +(allow file-read-metadata (literal "/dev/stderr")) +(allow file-read-metadata (regex "^/dev/tty[^/]*$")) +(allow file-read-metadata (regex "^/dev/pty[^/]*$")) +(allow file-read* file-write* (regex "^/dev/ttys[0-9]+$")) +(allow file-read* file-write* (literal "/dev/ptmx")) +(allow file-ioctl (regex "^/dev/ttys[0-9]+$")) + +; Allow metadata traversal for firmlink parents. +(allow file-read-metadata (literal "/System/Volumes") (vnode-type DIRECTORY)) +(allow file-read-metadata (literal "/System/Volumes/Data") (vnode-type DIRECTORY)) +(allow file-read-metadata (literal "/System/Volumes/Data/Users") (vnode-type DIRECTORY)) + +; App sandbox extensions +(allow file-read* (extension "com.apple.app-sandbox.read")) +(allow file-read* file-write* (extension "com.apple.app-sandbox.read-write")) diff --git a/src/lightcone/engine/sandbox/seatbelt.py b/src/lightcone/engine/sandbox/seatbelt.py new file mode 100644 index 00000000..13a654ec --- /dev/null +++ b/src/lightcone/engine/sandbox/seatbelt.py @@ -0,0 +1,231 @@ +"""The macOS backend: a generated Seatbelt profile. + +``sandbox-exec`` is already the shape the seam wants — a wrapper command +that confines everything it execs — so there is no shim on this side. +The work is generating the SBPL, which is ordinary string building and +therefore testable (and golden-tested) on any OS. + +**Most of the profile is not ours.** ``profiles/base.sbpl`` and +``profiles/platform-defaults.sbpl`` are adapted from the codex CLI +(itself derived from Chrome's macOS sandbox policy), because the macOS +read baseline is not something to derive from first principles — it is a +list of things that break, discovered one production failure at a time. +It carries entries no one would guess: ``/dev/dtracehelper``, the +``/dev/fd`` and pty regexes, firmlink-parent traversal under +``/System/Volumes/Data``, the ``opendirectoryd.libinfo`` lookup without +which ``getpwuid()`` raises ``KeyError``, ``cfprefsd``, and +``/opt/homebrew/lib``. They are kept near-verbatim so they can be diffed +against upstream; the single local delta is marked in the file. + +A third fragment, ``profiles/network.sbpl``, is the mach half of *not* +controlling the network: lc restricts none, and on macOS saying so takes +more than opening the socket families. + +What *is* ours is the three-tier policy on top: the project and declared +inputs readable, the private scope writable, and the environment plus the +utility allowlist executable. Two rules, both learned from shipped +implementations: + +- **Paths never go into the profile text.** They are referenced as + ``(param "READ_0")`` and supplied as ``-DREAD_0=`` on argv, so + no path can ever be quoted wrong or close a form early. +- **Realpath everything** before emitting, which :class:`Policy` + guarantees: ``/tmp`` is a symlink to ``/private/tmp`` on macOS, and a + rule naming the symlink matches nothing at all. +""" + +from __future__ import annotations + +import functools +import os +from collections.abc import Sequence +from dataclasses import dataclass, field +from importlib import resources + +from lightcone.engine.sandbox.model import ( + EXEC_ALLOWLIST_VERSION, + Attestation, + Capability, + Policy, +) + +SANDBOX_EXEC = "/usr/bin/sandbox-exec" + +#: The adapted upstream fragments, in the order they are concatenated. +BASE_PROFILE = "base.sbpl" +NETWORK = "network.sbpl" +PLATFORM_DEFAULTS = "platform-defaults.sbpl" + + +@functools.cache +def read_profile(name: str) -> str: + """The raw text of an SBPL fragment, cached — it is immutable + package data read on every ``wrap``.""" + if name not in (BASE_PROFILE, NETWORK, PLATFORM_DEFAULTS): + raise KeyError(f"unknown profile fragment: {name!r}") + return (resources.files(__package__) / "profiles" / name).read_text(encoding="utf-8") + + +@functools.cache +def capability() -> Capability: + """Whether ``sandbox-exec`` is present and usable on this host. + + Seatbelt has been "deprecated" since 2012 and has never been given a + removal date, but the profile dialect does drift across releases — + so this runs a live canary rather than trusting the file's presence. + + Cached like its Landlock twin: the canary is a subprocess, and the + answer cannot change inside one process. + """ + if os.name != "posix" or not os.path.isfile(SANDBOX_EXEC): + return Capability(kind="none", detail=f"{SANDBOX_EXEC} not present") + import subprocess + + try: + canary = subprocess.run( + [SANDBOX_EXEC, "-p", "(version 1)(allow default)", "/usr/bin/true"], + capture_output=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as e: + return Capability(kind="none", detail=f"sandbox-exec canary failed: {e}") + if canary.returncode != 0: + return Capability(kind="none", detail=f"sandbox-exec canary exited {canary.returncode}") + return Capability(kind="seatbelt") + + +@dataclass(frozen=True) +class SeatbeltBackend: + """Seatbelt, expressed as an argv rewrite.""" + + capability: Capability = field(default_factory=lambda: Capability(kind="seatbelt")) + + def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + return [ + SANDBOX_EXEC, + "-p", + generate_profile(policy), + *(f"-D{name}={value}" for name, value in profile_params(policy)), + "--", + *argv, + ] + + def attest(self, policy: Policy) -> Attestation: + return Attestation( + mechanism="seatbelt", + fs="declared", + exec_allowlist_version=EXEC_ALLOWLIST_VERSION, + ) + + +def profile_params(policy: Policy) -> list[tuple[str, str]]: + """The ``-D`` bindings the generated profile refers to, in order.""" + return [ + (f"{prefix}_{index}", str(path)) + for prefix, paths in ( + ("READ", policy.read), + ("WRITE", policy.write), + ("EXEC", policy.execute), + ) + for index, path in enumerate(paths) + ] + + +def generate_profile(policy: Policy) -> str: + """The SBPL for *policy*: upstream base, our tiers, upstream defaults. + + Order is load-bearing, because **SBPL is last-match-wins**. That is + why ``(deny default)`` can lead the base and still be overridden, why + the read-only guard can take back the write the upstream defaults + hand out on ``/tmp`` — and why the write tier is restated *after* the + guard. + + That last one is not cosmetic. Landlock unions rights, so a narrower + grant only ever widens: a writable directory nested inside a readable + tree works there for free. Reproducing that here means the write set + must have the final word, or the guard's ``(deny file-write* …)`` + over the read roots would silently revoke any nested writable path — + Linux would allow and macOS would refuse the same policy. + """ + return "\n".join( + [ + read_profile(BASE_PROFILE), + _tier( + # `file-map-executable` rides with read, not with exec. + # A venv's compiled extension modules (`.so`/`.dylib`) + # live under site-packages — inside the *project*, so in + # the read tier — and macOS gates `dlopen` on mapping + # rather than on exec. Without this, `import numpy` + # fails on macOS alone. Landlock does not gate mmap at + # all, so read already implies it there: granting it here + # is what makes the two platforms mean the same thing. + "read: project tree + declared inputs", + "(allow file-read* file-test-existence file-map-executable", + "READ", + len(policy.read), + ), + _tier( + "execute: the environment + the versioned utility allowlist", + "(allow process-exec* file-map-executable", + "EXEC", + len(policy.execute), + ), + ";; network: not controlled by lc, on any platform (recorded as `allowed`)", + # Separate forms: `network*` and `system-socket` are distinct + # operation families, and one malformed form voids the whole + # profile rather than just its own line. + "(allow network*)", + "(allow system-socket)", + # Sockets alone do not make the network usable: name lookup + # and TLS go through mach services, which the base's + # `(deny default)` blocks. Without this the attestation would + # say `allowed` on macOS while every resolution failed. + read_profile(NETWORK), + "", + read_profile(PLATFORM_DEFAULTS), + _read_only_guard(policy), + # Last, so a nested writable path beats the guard above it. + _tier( + "write: the per-run private scope and scratch — never the project tree", + "(allow file-read* file-write*", + "WRITE", + len(policy.write), + ), + ] + ) + + +def _tier(comment: str, opener: str, prefix: str, count: int) -> str: + """One `(allow …)` form over a parameterised path list.""" + if not count: + return f";; {comment} — empty" + body = "\n".join(f' (subpath (param "{prefix}_{i}"))' for i in range(count)) + return f";; {comment}\n{opener}\n{body}\n)\n" + + +def _read_only_guard(policy: Policy) -> str: + """Take back writes on everything readable that is not also writable. + + The upstream defaults grant write to shared scratch (``/tmp``, + ``/var/tmp``) unconditionally, which would otherwise reopen exactly + the hole :func:`~lightcone.engine.sandbox.policy._write_roots` closes + on Linux — a project living under ``/tmp`` becoming writable. It is + only expressible because SBPL is last-match-wins; Landlock, which + unions rights, has to solve the same problem by leaving the root out + of the policy instead. + + Deliberately narrow: it names *our* read roots and nothing else, so + the device and pty writes the upstream defaults grant survive. And it + is emitted before the write tier, which restates the grants that must + win — see :func:`generate_profile`. + """ + unwritable = [ + index + for index, path in enumerate(policy.read) + if not policy.grants(path, policy.write) + ] + if not unwritable: + return "" + body = "\n".join(f' (subpath (param "READ_{i}"))' for i in unwritable) + return f";; readable, and deliberately not writable\n(deny file-write*\n{body}\n)\n" diff --git a/src/lightcone/engine/templates/__init__.py b/src/lightcone/engine/templates/__init__.py index e6c1b79e..f5f97218 100644 --- a/src/lightcone/engine/templates/__init__.py +++ b/src/lightcone/engine/templates/__init__.py @@ -53,13 +53,7 @@ def _render(name: str, /, **values: str) -> str: def pyproject(*, name: str) -> str: - """The scaffolded ``pyproject.toml`` for a project called *name*. - - Deliberately **virtual** — no ``[build-system]``. Containerized mode - builds the image ``--no-install-project`` (project code never enters - an image), so a packaged project's ``import my_analysis`` would fail - inside its own container (spec §1). - """ + """The scaffolded ``pyproject.toml`` for a project called *name*.""" return _render( "pyproject.toml.tmpl", name=name, @@ -74,9 +68,7 @@ def python_version() -> str: Deliberately not an engine constant: a new project pins the python the researcher actually has, rather than one lc would have to download to - honor a number baked into a release. Identity follows this file from - here on — ``env_version`` hashes its bytes, not anything in the engine - (spec §3) — so a project is free to repin it afterwards. + honor a number baked into a release. """ v = sys.version_info return f"{v.major}.{v.minor}.{v.micro}\n" diff --git a/tests/conftest.py b/tests/conftest.py index 5ff37cce..3dc99f57 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil from pathlib import Path from unittest.mock import MagicMock @@ -48,7 +49,24 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: from lightcone.engine import project monkeypatch.setattr(project, "_run", fake_run) - monkeypatch.setattr(project.shutil, "which", lambda name, path=None: f"/usr/bin/{name}") + # Narrowly, for two reasons. `project.shutil` *is* the global + # `shutil` module, so a blanket fake here patches `shutil.which` for + # the whole suite — it did, and the sandbox tests built their exec + # set from it, every tool resolving to a path that exists on Linux + # and not on macOS, so the enforcement suite tested a policy no user + # would ever get. And the answer is never invented: convergence asks + # only *whether* uv and git exist, and `_run` is faked too, so + # nothing ever execs what comes back. Where the tool is really + # installed, that is what is returned; where it is not, the stub + # says so rather than naming a plausible path that isn't there. + real_which = shutil.which + + def fake_which(name: str, path: str | None = None) -> str | None: + if name in ("uv", "git"): + return real_which(name) or f"/stub/{name}" + return real_which(name, path=path) + + monkeypatch.setattr(project.shutil, "which", fake_which) return calls diff --git a/tests/test_cli.py b/tests/test_cli.py index 834052e7..311cff86 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,9 +24,8 @@ def test_help_lists_the_implemented_verb(runner: CliRunner) -> None: def test_help_does_not_advertise_unbuilt_verbs(runner: CliRunner) -> None: - """Layer 1 ships `lc init` alone. The other verbs return with their - layers — advertising them before they work would be a lie the whole - rebuild is meant to avoid.""" + """`lc --help` advertises only verbs that work — advertising others + before they do would be a lie.""" result = runner.invoke(main, ["--help"]) for verb in ("materialize", "status", "verify", "build", "export"): assert f" {verb}" not in result.output @@ -72,19 +71,21 @@ def test_init_defaults_to_the_current_directory( # ---- rendering ------------------------------------------------------------ -def test_run_reports_what_it_created(runner: CliRunner, tmp_path: Path) -> None: +def test_init_reports_what_it_created(runner: CliRunner, tmp_path: Path) -> None: result = runner.invoke(main, ["init", str(tmp_path / "proj")]) assert "created astra.yaml" in result.output assert "Project converged at" in result.output -def test_run_on_a_converged_project_says_so(runner: CliRunner, tmp_path: Path) -> None: +def test_init_on_a_converged_project_says_so(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" assert runner.invoke(main, ["init", str(project)]).exit_code == 0 result = runner.invoke(main, ["init", str(project)]) assert result.exit_code == 0 - assert "already converged" in result.output + # "already converged" straddles rich's wrap point once the tmp path is + # long enough — assert on a fragment that cannot wrap. + assert "nothing to do" in result.output def test_blocked_items_are_rendered(runner: CliRunner, tmp_path: Path) -> None: @@ -102,7 +103,7 @@ def test_blocked_items_are_rendered(runner: CliRunner, tmp_path: Path) -> None: assert "✗" in result.output -def test_run_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: +def test_init_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: project = tmp_path / "proj" project.mkdir() (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') @@ -159,3 +160,156 @@ def test_check_json_writes_nothing_and_exits_nonzero( assert result.exit_code == 1 assert json.loads(result.output)["converged"] is False assert not project.exists() + + +# ---- lc run --------------------------------------------------------------- + + +@pytest.fixture +def project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """A minimal project, with the CLI's cwd pointed at its root.""" + root = tmp_path / "proj" + root.mkdir() + (root / "pyproject.toml").write_text('[project]\nname = "proj"\n') + (root / "uv.lock").write_text("version = 1\n") + (root / ".venv").mkdir() + monkeypatch.chdir(root) + return root + + +@pytest.fixture +def spawned(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, object]]: + """Record what the CLI asks the engine to do, without running anything.""" + from lightcone.engine import run as engine_run + from lightcone.engine.sandbox import Outcome + from lightcone.engine.sandbox.model import Attestation + + calls: list[dict[str, object]] = [] + + def fake_probe(project, command, **kwargs): # type: ignore[no-untyped-def] + calls.append({"project": project, "command": list(command), **kwargs}) + return Outcome( + returncode=0, + attestation=Attestation(mechanism="landlock", fs="declared", landlock_abi=4), + ) + + monkeypatch.setattr(engine_run, "probe", fake_probe) + return calls + + +def test_run_is_advertised(runner: CliRunner) -> None: + result = runner.invoke(main, ["--help"]) + assert "run" in result.output + + +def test_run_passes_the_command_through_untouched( + runner: CliRunner, project: Path, spawned: list[dict[str, object]] +) -> None: + """A probe's command has its own flags, and they belong to it — not + to us. `--help` after the command must reach the command.""" + result = runner.invoke(main, ["run", "python", "-c", "print(1)", "--help"]) + assert result.exit_code == 0 + assert spawned[0]["command"] == ["python", "-c", "print(1)", "--help"] + assert spawned[0]["project"] == project.resolve() + + +def test_the_childs_exit_code_is_the_cli_s( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A probe is a proxy for the command; swallowing its exit code would + make `lc run` useless in a script.""" + from lightcone.engine import run as engine_run + from lightcone.engine.sandbox import Outcome + from lightcone.engine.sandbox.model import Attestation + + monkeypatch.setattr( + engine_run, + "probe", + lambda *a, **k: Outcome( + returncode=42, attestation=Attestation(mechanism="none", fs="open") + ), + ) + assert runner.invoke(main, ["run", "false"]).exit_code == 42 + + +def test_a_signal_killed_command_reports_the_conventional_status( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`Popen.returncode` is negative for a signal and `sys.exit(-9)` + truncates to 247. A script testing for 137 (SIGKILL) has to see 137.""" + from lightcone.engine import run as engine_run + from lightcone.engine.sandbox import Outcome + from lightcone.engine.sandbox.model import Attestation + + monkeypatch.setattr( + engine_run, + "probe", + lambda *a, **k: Outcome( + returncode=-9, attestation=Attestation(mechanism="landlock", fs="declared") + ), + ) + assert runner.invoke(main, ["run", "sleep"]).exit_code == 137 + + +def test_notes_are_rendered( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from lightcone.engine import run as engine_run + from lightcone.engine.sandbox import Outcome + from lightcone.engine.sandbox.model import Attestation + + monkeypatch.setattr( + engine_run, + "probe", + lambda *a, **k: Outcome( + returncode=1, + attestation=Attestation(mechanism="landlock", fs="declared"), + notes=("blocked by lc sandbox: cannot execute /usr/bin/latex —",), + ), + ) + result = runner.invoke(main, ["run", "latex"]) + assert "blocked by lc sandbox" in result.output + + +def test_a_bare_run_is_refused_rather_than_opening_a_shell( + runner: CliRunner, project: Path, spawned: list[dict[str, object]] +) -> None: + """A probe is run by an agent far more often than by a person, and an + agent handed an interactive shell waits forever for input nobody is + going to type. Refusing is the only outcome that cannot hang.""" + result = runner.invoke(main, ["run"]) + assert result.exit_code != 0 + assert spawned == [] + + +def test_outside_a_project_is_a_clean_error( + runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + result = runner.invoke(main, ["run", "true"]) + assert result.exit_code == 1 + assert "lc init" in result.output + + +def test_a_subdirectory_of_a_project_is_not_the_project( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """No walk-up: `lc run` uses the directory it is invoked from, or + errors.""" + nested = project / "sub" + nested.mkdir() + monkeypatch.chdir(nested) + result = runner.invoke(main, ["run", "true"]) + assert result.exit_code == 1 + assert "not a project" in result.output + + +def test_run_needs_no_spec_file( + runner: CliRunner, project: Path, spawned: list[dict[str, object]] +) -> None: + """The environment is what a probe needs; `astra.yaml` is not + required to run a command in it.""" + assert not (project / "astra.yaml").exists() + result = runner.invoke(main, ["run", "true"]) + assert result.exit_code == 0 + assert spawned[0]["project"] == project.resolve() diff --git a/tests/test_project.py b/tests/test_project.py index 421cc796..ff14baed 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -111,7 +111,7 @@ def test_converge_writes_the_templates_verbatim(tmp_path: Path) -> None: def test_converge_scaffolds_no_environment_escalation(tmp_path: Path) -> None: """Containerized mode is *derived* from a declaration the user makes, so - the scaffold never writes one (spec §1). What astra's boilerplate puts + the scaffold never writes one. What astra's boilerplate puts in `astra.yaml` is astra's business — reconciling that with the environment model belongs to the environment layer.""" project = tmp_path / "proj" @@ -282,7 +282,7 @@ def test_check_mode_agrees_with_a_real_run(tmp_path: Path) -> None: def test_warns_when_pyproject_lacks_the_engine(tmp_path: Path) -> None: """The engine belongs inside the experiment's lock; a pyproject we - didn't write is the user's, so warn rather than edit (spec §2).""" + didn't write is the user's, so warn rather than edit.""" project = tmp_path / "proj" project.mkdir() (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') @@ -400,7 +400,7 @@ def test_requires_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_locks_then_syncs_exactly(tmp_path: Path, tools: list[list[str]]) -> None: """Converge once, with the flags the spec fixes: ``--locked --exact`` - (no drift, no additive sync) and ``--compile-bytecode`` (§4, §6).""" + (no drift, no additive sync) and ``--compile-bytecode``.""" project = tmp_path / "proj" converge(project) @@ -409,7 +409,7 @@ def test_locks_then_syncs_exactly(tmp_path: Path, tools: list[list[str]]) -> Non assert sync[0] == "sync" for flag in ("--locked", "--exact", "--compile-bytecode"): assert flag in sync - # uv's own walk-up discovery is never trusted (spec §4). + # uv's own walk-up discovery is never trusted. assert lock[-2:] == ["--project", str(project.resolve())] assert sync[-2:] == ["--project", str(project.resolve())] diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 00000000..96e1ab59 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,131 @@ +"""Tests for `lightcone.engine.run` — what `lc run` decides before it execs. + +The project check, the declared inputs, and the uv hop. Nothing here +spawns a command; the boundary is tested in `test_sandbox_*`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from lightcone.engine import run as engine_run +from lightcone.engine.project import ProjectError, current_project + +SPEC = """\ +title: Test +inputs: + - id: local + type: data + source: data/local.csv + - id: remote + type: data + source: https://example.org/nope.csv +outputs: + - id: best_fit + type: metric +""" + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + root = tmp_path / "proj" + (root / "data").mkdir(parents=True) + (root / "data" / "local.csv").write_text("a,b\n") + (root / "astra.yaml").write_text(SPEC) + (root / "pyproject.toml").write_text('[project]\nname = "proj"\n') + (root / "uv.lock").write_text("version = 1\n") + (root / ".venv").mkdir() + return root + + +# ---- the project check ---------------------------------------------------- + + +def test_a_directory_with_an_environment_is_a_project(project: Path) -> None: + assert current_project(project) == project.resolve() + + +def test_the_spec_is_not_required(project: Path) -> None: + """`lc run` probes the *environment*; a uv project with no + `astra.yaml` is still runnable.""" + (project / "astra.yaml").unlink() + assert current_project(project) == project.resolve() + + +def test_a_subdirectory_is_not_the_project(project: Path) -> None: + """No walk-up, deliberately: the directory a verb is invoked from is + the directory that is used, or it is an error.""" + nested = project / "a" / "b" + nested.mkdir(parents=True) + with pytest.raises(ProjectError, match="not a project"): + current_project(nested) + + +def test_the_missing_pieces_are_named(project: Path) -> None: + (project / "uv.lock").unlink() + with pytest.raises(ProjectError, match="missing uv.lock"): + current_project(project) + + +def test_a_directory_without_an_environment_says_so(tmp_path: Path) -> None: + with pytest.raises(ProjectError, match="lc init"): + current_project(tmp_path) + + +def test_the_default_is_the_working_directory( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(project) + assert current_project() == project.resolve() + + +def test_a_missing_spec_reads_as_an_empty_one(project: Path) -> None: + (project / "astra.yaml").unlink() + assert engine_run.read_spec(project) == {} + assert engine_run.input_paths(project, {}) == [] + + +# ---- declared inputs ------------------------------------------------------ + + +def test_declared_file_inputs_become_read_paths(project: Path) -> None: + spec = engine_run.read_spec(project) + assert engine_run.input_paths(project, spec) == [(project / "data" / "local.csv").resolve()] + + +def test_a_source_that_is_not_a_path_is_left_alone(project: Path) -> None: + """ASTRA's `source` is free-form — a URI, a dotted name, a path — so + "is this a path" is answered by whether it resolves to something that + exists, not by parsing.""" + spec = engine_run.read_spec(project) + assert not any("example.org" in str(p) for p in engine_run.input_paths(project, spec)) + + +def test_a_spec_with_no_inputs_is_fine(tmp_path: Path) -> None: + root = tmp_path / "bare" + root.mkdir() + (root / "astra.yaml").write_text("title: Bare\n") + assert engine_run.input_paths(root, engine_run.read_spec(root)) == [] + + +def test_an_unresolvable_tree_degrades_to_the_top_level_document(project: Path) -> None: + """A probe exists to debug a project, and a spec whose sub-analysis + references are stale is exactly when someone reaches for one.""" + (project / "astra.yaml").write_text(SPEC + "analyses:\n - id: sub\n path: ../gone\n") + assert engine_run.read_spec(project)["inputs"] + + +# ---- the uv hop ----------------------------------------------------------- + + +def test_uv_is_pinned_to_the_project_and_refuses_to_drift(project: Path) -> None: + """uv's own walk-up discovery is never trusted, and a stale lock must + be uv's loud error rather than a silent relock.""" + prefix = engine_run.uv_prefix(project) + assert prefix[:2] == ["uv", "run"] + assert "--locked" in prefix + assert "--exact" in prefix + assert prefix[prefix.index("--project") + 1] == str(project) + assert prefix[-1] == "--" diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py new file mode 100644 index 00000000..3f06592a --- /dev/null +++ b/tests/test_sandbox_denial.py @@ -0,0 +1,159 @@ +"""Tests for the denial UX — the sandbox's primary user interface. + +Pure functions over a captured stderr and a policy, so these run +anywhere. The two cases that matter most are the ones where the +classifier *fails*: a command that swallows the PermissionError, and one +that rewraps it past recognition. Both must still leave the user knowing +a sandbox was there. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from lightcone.engine.sandbox import denial +from lightcone.engine.sandbox.model import Policy + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + root = tmp_path / "proj" + root.mkdir() + (root / "astra.yaml").write_text("inputs: []\n") + return root + + +@pytest.fixture +def policy(project: Path, tmp_path: Path) -> Policy: + scratch = tmp_path / "scratch" + scratch.mkdir() + venv_bin = project / ".venv" / "bin" + venv_bin.mkdir(parents=True) + return Policy( + read=(project, Path("/usr")), + write=(scratch,), + execute=(venv_bin,), + tmp_home=scratch, + env={}, + ) + + +# ---- the three denial kinds ----------------------------------------------- + + +def test_an_undeclared_tool_is_named_with_its_remedy(policy: Policy, project: Path) -> None: + """/usr/bin/id is *readable* under the baseline, so only an + access-aware check can tell that executing it was the denial.""" + tool = shutil.which("id") + if tool is None: # pragma: no cover - a host without coreutils + pytest.skip("no `id` on this host") + lines = denial.explain(f"bash: line 1: {tool}: Permission denied\n", policy, cwd=project) + joined = "\n".join(lines) + assert f"cannot execute {tool}" in joined + assert "uv add" in joined + + +def test_an_undeclared_data_file_gets_the_astra_snippet( + policy: Policy, project: Path, tmp_path: Path +) -> None: + """The remedy is copy-pasteable and matches ASTRA's real schema — + inputs carry `source`, not `path`.""" + external = tmp_path / "elsewhere.fits" + external.write_text("") + stderr = f"PermissionError: [Errno 13] Permission denied: '{external}'\n" + joined = "\n".join(denial.explain(stderr, policy, cwd=project)) + assert f"cannot read {external}" in joined + assert "inputs:" in joined + assert f"source: {external}" in joined + + +def test_an_in_tree_write_is_its_own_kind_of_denial(policy: Policy, project: Path) -> None: + """Reading the project was allowed, so an EACCES on a project file + can only have been a write — and "declare it as an input" would be + exactly the wrong advice.""" + stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" + joined = "\n".join(denial.explain(stderr, policy, cwd=project)) + assert "cannot write" in joined + assert "read-only in here" in joined + assert "inputs:" not in joined + + +# ---- what must not be reported -------------------------------------------- + + +def test_a_granted_path_is_not_reported(policy: Policy, project: Path, tmp_path: Path) -> None: + """Somebody else's permission problem, inside a path we allow — not + ours to explain.""" + inside = tmp_path / "scratch" / "f.txt" + inside.write_text("") + stderr = f"PermissionError: [Errno 13] Permission denied: '{inside}'\n" + assert denial.explain(stderr, policy, cwd=project) == [] + + +def test_a_path_that_does_not_exist_is_not_reported(policy: Policy, project: Path) -> None: + """An ordinary missing-file bug. Claiming the sandbox blocked it + would send the reader after a file that was never there.""" + stderr = "FileNotFoundError: [Errno 2] No such file: '/nope/missing.txt'\n" + assert denial.explain(stderr, policy, cwd=project) == [] + + +def test_nothing_recognizable_explains_nothing(policy: Policy, project: Path) -> None: + assert denial.explain("Traceback: ValueError: bad fit\n", policy, cwd=project) == [] + + +# ---- the fallbacks -------------------------------------------------------- + + +def test_a_swallowed_permission_error_still_gets_the_trailer() -> None: + """The failure mode that produces hours of confusion: the command + caught the PermissionError and died of something unrelated, so + nothing in its output says a sandbox was involved.""" + text = denial.trailer("landlock") + assert "ran under the lc sandbox (landlock)" in text + + +def test_a_rewrapped_error_defeats_the_classifier_but_not_the_trailer( + policy: Policy, project: Path +) -> None: + stderr = "RuntimeError: could not open the calibration table (see log)\n" + assert denial.explain(stderr, policy, cwd=project) == [] + assert denial.trailer("seatbelt") + + +# ---- message shape -------------------------------------------------------- + + +def test_a_bare_command_name_is_resolved_against_the_host( + policy: Policy, project: Path +) -> None: + """`command not found` reports a name, not a location. Resolving it + outside the sandbox turns it into "you have this, it just isn't + declared".""" + if shutil.which("id") is None: # pragma: no cover + pytest.skip("no `id` on this host") + joined = "\n".join(denial.explain("bash: line 1: id: command not found\n", policy, cwd=project)) + assert "cannot execute" in joined + + +def test_a_relative_path_is_resolved_against_the_working_directory( + policy: Policy, project: Path +) -> None: + """Python reports the string the command passed, so an in-tree denial + arrives as `'astra.yaml'` with no directory at all.""" + stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" + assert denial.explain(stderr, policy, cwd=project) != [] + + +def test_no_escape_hatch_is_ever_offered(policy: Policy, project: Path) -> None: + """There is no way to run outside the sandbox, so no message may + suggest one.""" + joined = "\n".join( + denial.explain("PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n", policy, + cwd=project) + ) + assert "read-only in here" in joined + assert "--no-sandbox" not in joined + assert "sandbox-debug" not in joined diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py new file mode 100644 index 00000000..fb34e337 --- /dev/null +++ b/tests/test_sandbox_enforcement.py @@ -0,0 +1,491 @@ +"""Enforcement: does the kernel actually stop the leaks? + +Everything else in the suite checks what lc *says* it will enforce — the +policy it builds, the argv it emits, the profile it generates. This file +is the one that runs commands and looks at what happened, and it is +written once for **both** mechanisms: the same tests exercise Landlock on +Linux and Seatbelt on macOS, because the seam makes them the same shape. +That symmetry is the point. A leak that only Linux catches is a leak. + +Two deliberate choices: + +- **The real policy.** These run against + :func:`~lightcone.engine.sandbox.policy.probe_policy` — what an actual + ``lc run`` gets — not a policy hand-built to make a point. A test that + grants exactly what it is testing cannot discover that the shipped + policy grants something else. +- **Real leaks.** Undeclared *tools* are executed, undeclared *libraries* + are imported, undeclared *data* is read. Those are the three channels + the design exists to close, so they are attempted + literally rather than asserted about. + +Skipped whole where no mechanism exists — a mocked sandbox proves +nothing. But see :func:`_mechanism`: on CI that skip is a **failure**, +because a suite that silently skips its own subject is worse than no +suite. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import sysconfig +from collections.abc import Sequence +from pathlib import Path + +import pytest + +from lightcone.engine import sandbox +from lightcone.engine.sandbox.boundary import env_argv +from lightcone.engine.sandbox.policy import _UTILITY_PATH, EXEC_ALLOWLIST + +#: Set on CI. Turns "no sandbox here, skip" into a hard failure, so a +#: runner that quietly loses its mechanism cannot report green. +REQUIRED_ENV = "LC_SANDBOX_TESTS_REQUIRED" + + +def _mechanism() -> sandbox.Backend: + backend = sandbox.detect() + if backend.capability.kind == "none": + detail = backend.capability.detail or "no mechanism" + if os.environ.get(REQUIRED_ENV): + pytest.fail( + f"{REQUIRED_ENV} is set but this host cannot enforce: {detail}. " + "Enforcement tests must not be skipped on CI." + ) + pytest.skip(f"no sandbox mechanism here: {detail}") + return backend + + +@pytest.fixture(scope="module") +def backend() -> sandbox.Backend: + return _mechanism() + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + """A project shaped like a real one, minus the cost of a real venv. + + ``.venv/bin/python`` is a symlink to the running interpreter, which is + exactly the shape the policy cares about: it grants EXECUTE on the + *resolved* target and READ on the install root beside it. + """ + root = tmp_path / "proj" + (root / ".venv" / "bin").mkdir(parents=True) + (root / ".venv" / "bin" / "python").symlink_to(Path(sys.executable).resolve()) + (root / "astra.yaml").write_text("title: T\n") + (root / "data.txt").write_text("in-tree\n") + return root + + +@pytest.fixture +def outside(tmp_path: Path) -> Path: + """A directory the project never declares — the leak's other end.""" + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "secret.txt").write_text("undeclared\n") + (elsewhere / "sneaky.py").write_text("VALUE = 'undeclared import'\n") + return elsewhere + + +def run( + backend: sandbox.Backend, + policy: sandbox.Policy, + argv: Sequence[str], + *, + cwd: Path, +) -> subprocess.CompletedProcess[str]: + """Exactly what :func:`sandbox.run` spawns, captured instead of teed. + + Mirrors the boundary's composition — the env overlay inside the wrap, + no ``uv run`` prefix (these test the boundary, not the launcher) — so + what runs here is what runs in production. + """ + wrapped = backend.wrap(policy, [*env_argv(policy), *argv]) + return subprocess.run(wrapped, cwd=cwd, capture_output=True, text=True, check=False) + + +def shell( + backend: sandbox.Backend, policy: sandbox.Policy, script: str, *, cwd: Path +) -> subprocess.CompletedProcess[str]: + return run(backend, policy, ["bash", "-c", script], cwd=cwd) + + +def undeclared_tool() -> str: + """A real binary on this host that the exec allowlist does not name. + + Not hardcoded: the allowlist is a maintained surface and the runners + differ, so the test asks the host rather than assuming. + """ + for name in ("git", "curl", "openssl", "id", "who", "hostname"): + found = shutil.which(name, path=_UTILITY_PATH) + if found and name not in EXEC_ALLOWLIST: + return found + pytest.skip("no undeclared tool available to try") # pragma: no cover + + +# ---- the mechanism is what we think it is --------------------------------- + + +def test_the_expected_mechanism_is_in_use(backend: sandbox.Backend) -> None: + expected = {"linux": "landlock", "darwin": "seatbelt"}.get(sys.platform) + assert backend.capability.kind == expected + + +def test_a_sandboxed_run_attests_a_scoped_filesystem( + backend: sandbox.Backend, project: Path +) -> None: + with sandbox.scope(project) as policy: + attestation = backend.attest(policy) + assert attestation.fs == "declared" + assert attestation.mechanism == backend.capability.kind + + +# ---- leak channel 1: undeclared tools -------------------------------------- + + +def test_an_undeclared_host_tool_cannot_be_executed( + backend: sandbox.Backend, project: Path +) -> None: + """The #1 leakage channel: a recipe that works only because the + author happens to have some tool installed.""" + tool = undeclared_tool() + with sandbox.scope(project) as policy: + result = shell(backend, policy, f"{tool} --version", cwd=project) + assert result.returncode != 0, f"{tool} ran inside the sandbox" + + +def test_an_undeclared_tool_is_readable_but_still_not_executable( + backend: sandbox.Backend, project: Path +) -> None: + """The distinction the exec tier exists for. The OS baseline is + readable so the dynamic linker works, which means an undeclared tool + can be *seen* — running it is the leak, and only that is denied.""" + tool = undeclared_tool() + with sandbox.scope(project) as policy: + seen = shell(backend, policy, f"test -r {tool} && echo READABLE", cwd=project) + ran = shell(backend, policy, f"{tool} --version", cwd=project) + assert "READABLE" in seen.stdout + assert ran.returncode != 0 + + +def test_an_allowlisted_utility_is_executable( + backend: sandbox.Backend, project: Path +) -> None: + """The other half: the allowlist has to actually work, or every + recipe that pipes through sed breaks.""" + with sandbox.scope(project) as policy: + result = shell(backend, policy, "printf 'b\\na\\n' | sort | head -1", cwd=project) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "a" + + +def test_a_dynamically_linked_binary_runs_at_all( + backend: sandbox.Backend, project: Path +) -> None: + """Proves the loader tier — the ELF interpreter on Linux, dyld on + macOS. Without it *nothing* dynamically linked starts, bash included, + and every other test here would fail for the wrong reason.""" + with sandbox.scope(project) as policy: + result = shell(backend, policy, "echo LOADER-OK", cwd=project) + assert result.returncode == 0, result.stderr + assert "LOADER-OK" in result.stdout + + +def test_a_binary_dropped_into_the_writable_scope_cannot_be_run( + backend: sandbox.Backend, project: Path +) -> None: + """Write does not imply execute. Otherwise the allowlist is two lines + from being defeated: copy a tool into scratch, run it from there.""" + tool = undeclared_tool() + with sandbox.scope(project) as policy: + smuggled = policy.tmp_home / "smuggled" + result = shell( + backend, + policy, + f"cp {tool} {smuggled} && chmod +x {smuggled} && {smuggled} --version", + cwd=project, + ) + assert result.returncode != 0, "a binary copied into scratch was executable" + + +def test_the_projects_own_interpreter_runs_and_finds_its_stdlib( + backend: sandbox.Backend, project: Path +) -> None: + with sandbox.scope(project) as policy: + result = shell( + backend, policy, f"{project}/.venv/bin/python -c 'import json; print(json.dumps(1))'", + cwd=project, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "1" + + +# ---- leak channel 2: undeclared libraries ---------------------------------- + + +def test_an_undeclared_python_module_cannot_be_imported( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + """Python-level leakage: a module reachable on the host but not part + of the declared environment. Denied at *read*, so the import fails + however sys.path was arranged.""" + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"{project}/.venv/bin/python -c " + f"\"import sys; sys.path.insert(0, '{outside}'); import sneaky; print(sneaky.VALUE)\"", + cwd=project, + ) + assert result.returncode != 0 + assert "undeclared import" not in result.stdout + + +def test_a_compiled_extension_module_can_be_imported( + backend: sandbox.Backend, project: Path +) -> None: + """The declared environment has to keep working, and native code is + where the two mechanisms differ: Landlock does not gate `mmap`, so a + read grant is enough, while macOS gates `dlopen` on + `file-map-executable`. If the read tier lost that right, this is the + test that fails — on macOS only.""" + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"{project}/.venv/bin/python -c 'import ssl, zlib, _socket; print(\"EXT-OK\")'", + cwd=project, + ) + assert result.returncode == 0, result.stderr + assert "EXT-OK" in result.stdout + + +def test_an_undeclared_shared_library_cannot_be_loaded( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + """The dlopen channel, tried literally: a real native library sitting + at a path the project never declared. Denied at open on Linux, at + mapping on macOS — either way it must not load.""" + library = _a_compiled_extension() + smuggled = outside / library.name + shutil.copy(library, smuggled) + + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"{project}/.venv/bin/python -c " + f"\"import ctypes; ctypes.CDLL('{smuggled}'); print('LOADED')\"", + cwd=project, + ) + assert result.returncode != 0, "an undeclared shared library loaded" + assert "LOADED" not in result.stdout + + +def _a_compiled_extension() -> Path: + """Any real native library on this host, to smuggle somewhere undeclared. + + The interpreter's own stdlib extensions are the portable source: they + exist on both platforms and are genuine loadable objects, unlike + `/usr/lib/libz.dylib`, which on modern macOS is not a file at all. + """ + dynload = Path(sysconfig.get_paths()["stdlib"]) / "lib-dynload" + for extension in sorted(dynload.glob("*.so")) + sorted(dynload.glob("*.dylib")): + return extension + pytest.skip("no compiled extension module to copy") # pragma: no cover + + +# ---- leak channel 3: undeclared data --------------------------------------- + + +def test_an_undeclared_data_file_cannot_be_read( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + with sandbox.scope(project) as policy: + result = shell(backend, policy, f"cat {outside / 'secret.txt'}", cwd=project) + assert result.returncode != 0 + assert "undeclared" not in result.stdout + + +def test_a_declared_input_outside_the_project_can_be_read( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + """The same file, declared. This is what makes the denial actionable + rather than a wall: the remedy the message prints has to work.""" + declared = outside / "secret.txt" + with sandbox.scope(project, read_paths=[declared]) as policy: + result = shell(backend, policy, f"cat {declared}", cwd=project) + assert result.returncode == 0, result.stderr + assert "undeclared" in result.stdout + + +def test_the_project_tree_is_readable(backend: sandbox.Backend, project: Path) -> None: + with sandbox.scope(project) as policy: + result = shell(backend, policy, "cat data.txt", cwd=project) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "in-tree" + + +def test_the_real_home_is_not_readable(backend: sandbox.Backend, project: Path) -> None: + """The dotfile-steering channel. A private HOME is what lets + matplotlib and astropy work without it being reopened.""" + canary = Path.home() / ".lc-enforcement-canary" + canary.write_text("host home\n") + try: + with sandbox.scope(project) as policy: + result = shell(backend, policy, f"cat {canary}", cwd=project) + finally: + canary.unlink(missing_ok=True) + assert result.returncode != 0 + assert "host home" not in result.stdout + + +# ---- the write scope ------------------------------------------------------- + + +def test_the_project_tree_cannot_be_written( + backend: sandbox.Backend, project: Path +) -> None: + """A probe has no output, so nothing it does may land in + the tree — and the file has to be *unchanged*, not merely reported.""" + with sandbox.scope(project) as policy: + result = shell(backend, policy, "printf clobbered > data.txt", cwd=project) + assert result.returncode != 0 + assert (project / "data.txt").read_text() == "in-tree\n", "the file changed anyway" + + +def test_the_private_scope_is_writable(backend: sandbox.Backend, project: Path) -> None: + with sandbox.scope(project) as policy: + target = policy.tmp_home / "result.txt" + result = shell(backend, policy, f"printf ok > {target}", cwd=project) + wrote = target.read_text() if target.exists() else "" + assert result.returncode == 0, result.stderr + assert wrote == "ok" + + +def test_tempfile_works_inside_the_boundary( + backend: sandbox.Backend, project: Path +) -> None: + """TMPDIR points into the private scope, so the stdlib's own scratch + keeps working even where the shared /tmp left the write set.""" + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"{project}/.venv/bin/python -c " + "\"import tempfile; f=tempfile.NamedTemporaryFile(delete=False); " + "f.write(b'ok'); print('TEMP-OK')\"", + cwd=project, + ) + assert result.returncode == 0, result.stderr + assert "TEMP-OK" in result.stdout + + +def test_a_command_can_allocate_a_pty(backend: sandbox.Backend, project: Path) -> None: + """devpts and friends are granted, so pexpect and pytest's own + capture work. Without them `pty.openpty()` fails as "out of pty + devices" — a message naming neither a path nor the sandbox.""" + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"{project}/.venv/bin/python -c 'import pty; pty.openpty(); print(\"PTY-OK\")'", + cwd=project, + ) + assert result.returncode == 0, result.stderr + assert "PTY-OK" in result.stdout + + +# ---- the boundary cannot be shed ------------------------------------------- + + +def test_the_restriction_is_inherited_by_grandchildren( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + """Both mechanisms confine the whole descendant tree and neither can + be shed — which is why wrapping the outermost command is enough.""" + with sandbox.scope(project) as policy: + result = shell( + backend, + policy, + f"bash -c 'bash -c \"cat {outside / 'secret.txt'}\"'", + cwd=project, + ) + assert result.returncode != 0 + assert "undeclared" not in result.stdout + + +# ---- what the user is told ------------------------------------------------- + + +def test_a_denial_reaches_the_user_through_the_boundary( + backend: sandbox.Backend, project: Path +) -> None: + """The whole way through `sandbox.run`, not the test's own harness: + a real denial must produce the explanation *and* the trailer, or the + sandbox is an invisible wall.""" + tool = undeclared_tool() + with sandbox.scope(project) as policy: + outcome = sandbox.run( + backend, + policy, + ["bash", "-c", f"{tool} --version"], + cwd=project, + env=dict(os.environ), + ) + notes = "\n".join(outcome.notes) + assert outcome.returncode != 0 + assert "ran under the lc sandbox" in notes, notes + assert f"cannot execute {tool}" in notes, notes + + +# ---- the guard on this file itself ----------------------------------------- + + +def test_the_ci_guard_fails_rather_than_skipping(monkeypatch: pytest.MonkeyPatch) -> None: + """The one test here that must pass everywhere, including hosts with + no mechanism: it checks that CI cannot go green by skipping the rest. + + Without this, `LC_SANDBOX_TESTS_REQUIRED` is a comment. + """ + from lightcone.engine.sandbox.boundary import Unavailable + from lightcone.engine.sandbox.model import Capability + + monkeypatch.setenv(REQUIRED_ENV, "1") + monkeypatch.setattr( + sandbox, "detect", lambda: Unavailable(capability=Capability(kind="none", detail="pretend")) + ) + with pytest.raises(pytest.fail.Exception, match="must not be skipped"): + _mechanism() + + +def test_without_the_guard_a_mechanismless_host_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """And off CI it stays a skip — a laptop without Landlock should run + the rest of the suite, not fail it.""" + from lightcone.engine.sandbox.boundary import Unavailable + from lightcone.engine.sandbox.model import Capability + + monkeypatch.delenv(REQUIRED_ENV, raising=False) + monkeypatch.setattr( + sandbox, "detect", lambda: Unavailable(capability=Capability(kind="none", detail="pretend")) + ) + with pytest.raises(pytest.skip.Exception): + _mechanism() + + +def test_an_allowlisted_tool_resolves_to_the_copy_that_was_granted( + backend: sandbox.Backend, project: Path +) -> None: + """Found by the macOS runner: PATH inside the boundary has to be the + search path the exec set was built from. The runner's ambient PATH + fronts homebrew, so `env bash` resolved `/opt/homebrew/bin/bash` + while the policy had granted `/bin/bash` — and the sandbox denied + bash itself with "Operation not permitted".""" + with sandbox.scope(project) as policy: + result = shell(backend, policy, "command -v bash && echo RESOLVED", cwd=project) + assert result.returncode == 0, result.stderr + resolved = Path(result.stdout.splitlines()[0].strip()) + assert policy.grants(resolved, policy.execute), f"{resolved} is not in the exec set" diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py new file mode 100644 index 00000000..2ce44ee8 --- /dev/null +++ b/tests/test_sandbox_policy.py @@ -0,0 +1,406 @@ +"""Tests for `lightcone.engine.sandbox.policy` — what a probe may touch. + +Pure construction, no enforcement: these run on any OS. Whether the +kernel honors the policy is `test_sandbox_landlock.py`'s question. +""" + +from __future__ import annotations + +import os +import shutil +import sys +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from lightcone.engine.sandbox import policy as policy_module +from lightcone.engine.sandbox.boundary import scope +from lightcone.engine.sandbox.model import EXEC_ALLOWLIST_VERSION + + +@pytest.fixture +def built(tmp_path: Path) -> Iterator[policy_module.Policy]: + """A probe policy over a bare project directory. + + Through `boundary.scope`, which owns the per-run HOME's lifetime — + so the cleanup contract is exercised by the suite rather than + re-implemented seven times beside it. + """ + project = tmp_path / "proj" + project.mkdir() + with scope(project) as built: + yield built + + +# ---- the write scope ------------------------------------------------------ + + +def test_a_probe_never_writes_in_the_tree(built: policy_module.Policy, tmp_path: Path) -> None: + """The invariant that makes `lc run` safe to hand to an agent: a probe + has no output, so nothing it does can land in the project.""" + project = tmp_path / "proj" + assert not any(root == project or project.is_relative_to(root) for root in built.write) + + +def test_the_private_home_is_writable(built: policy_module.Policy) -> None: + assert built.tmp_home in built.write + + +def test_the_shared_tmp_is_writable_for_a_project_outside_it() -> None: + """`/tmp` specifically, not `gettempdir()`: on macOS the latter is the + per-user `$TMPDIR` under /var/folders, a different directory that is + not in the baseline at all.""" + with scope(Path.home() / ".lc-policy-test-project") as built: + assert Path("/tmp").resolve() in built.write + + +def test_a_project_living_under_tmp_does_not_become_writable() -> None: + """`/tmp` is writable by design, so a project that *lives* there would + otherwise be writable too — voiding the read-only-tree guarantee for + exactly the people who keep scratch analyses in /tmp.""" + shared = Path("/tmp").resolve() + with scope(shared / "lc-policy-under-tmp") as built: + assert shared not in built.write + + +def test_tmpdir_always_points_into_the_private_scope(built: policy_module.Policy) -> None: + """Which is what keeps `tempfile` working even when the shared /tmp + had to be dropped from the write set.""" + assert Path(built.env["TMPDIR"]).is_relative_to(built.tmp_home) + assert Path(built.env["TMPDIR"]).is_dir() + + +# ---- the read scope ------------------------------------------------------- + + +def test_the_project_is_readable(built: policy_module.Policy, tmp_path: Path) -> None: + assert (tmp_path / "proj").resolve() in built.read + + +def test_declared_inputs_join_the_read_scope(tmp_path: Path) -> None: + project = tmp_path / "proj" + project.mkdir() + external = tmp_path / "elsewhere" / "data.fits" + external.parent.mkdir() + external.touch() + + with scope(project, read_paths=[external]) as built: + assert external.resolve() in built.read + + +def test_the_os_baseline_is_readable_but_never_executable(built: policy_module.Policy) -> None: + """The distinction the whole layer rests on: /usr must be *readable* + or the dynamic linker cannot work, and must not be *executable* or + every undeclared tool on the host is admitted.""" + assert Path("/usr") in built.read + assert Path("/usr") not in built.execute + + +@pytest.mark.skipif(sys.platform == "win32", reason="no /dev on Windows") +def test_dev_urandom_is_readable(built: policy_module.Policy) -> None: + """Not decoration: CPython seeds hash randomization from it during + preinitialization, so without the grant the interpreter dies before + `main` with "failed to get random numbers".""" + assert Path("/dev/urandom") in built.read + + +def test_the_nix_roots_are_in_the_read_baseline() -> None: + """On NixOS *everything* — interpreter, libraries, the utility + allowlist — resolves into these, so omitting them makes the sandbox + unusable there rather than merely incomplete. Asserted against the + constant, since neither path exists on most hosts.""" + assert "/nix/store" in policy_module._OS_READ_BASELINE + assert "/run/current-system/sw" in policy_module._OS_READ_BASELINE + + +def test_the_read_baseline_is_system_paths_only() -> None: + """It must never widen to reach user data — that is what the project + and declared-input grants are for.""" + home = str(Path.home()) + for entry in policy_module._OS_READ_BASELINE: + assert not entry.startswith(home), entry + assert entry.startswith("/"), entry + + +def test_the_minimal_device_set_is_covered(built: policy_module.Policy) -> None: + """bubblewrap materializes `null, zero, full, random, urandom, tty` + from one `--dev` flag. Landlock has no device-tree primitive, so we + enumerate the same set, split by the access each needs — `/dev/tty` + writable, without which anything opening the controlling terminal + afresh fails, `lc run`'s own shell included.""" + granted = {*built.read, *built.write} + for node in ("null", "zero", "full", "random", "urandom", "tty"): + device = Path("/dev") / node + if device.exists(): + assert device in granted, device + # The terminal set specifically: writable, so anything opening the + # controlling terminal or allocating a pty works. That it *actually* + # works is `test_sandbox_enforcement.py`'s job. + for node in ("/dev/tty", "/dev/pts", "/dev/ptmx"): + device = Path(node) + if device.exists(): + assert device.resolve() in built.write, node + + +def test_discard_devices_are_writable(built: policy_module.Policy) -> None: + """Writes to these are discard-by-construction — that is what the + devices are — so read-only buys nothing and breaks tools that use + /dev/full to exercise ENOSPC handling.""" + for node in ("/dev/null", "/dev/zero", "/dev/full"): + device = Path(node) + if device.exists(): + assert device in built.write, node + + +def test_the_entropy_sources_stay_read_only(built: policy_module.Policy) -> None: + """The one place the permissive line is drawn: writing to these seeds + the *host's* pool, which is a side effect on the machine rather than + on the run.""" + for node in ("/dev/urandom", "/dev/random"): + device = Path(node) + if device.exists(): + assert device in built.read, node + assert device not in built.write, node + + +def test_proc_and_sys_are_not_restricted(built: policy_module.Policy) -> None: + """Real tools write them — /proc/self/oom_score_adj, coredump_filter, + MPI and CUDA runtimes poking /sys — and none of it is a channel + undeclared inputs arrive through. The kernel's own permissions stay + the real gate; Landlock only ever removes access, never adds it.""" + for node in ("/proc", "/sys"): + directory = Path(node) + if directory.exists(): + assert directory in built.write, node + + +# ---- the exec scope ------------------------------------------------------- + + +def test_allowlisted_utilities_are_granted_per_file(built: policy_module.Policy) -> None: + bash = shutil.which("bash", path=policy_module._UTILITY_PATH) + if bash is None: # pragma: no cover - a host without bash + pytest.skip("no bash in the utility path") + resolved = Path(bash).resolve() + assert resolved in built.execute, f"{bash} -> {resolved}; granted: {built.execute}" + assert resolved.parent not in built.execute + + +def test_the_allowlist_is_resolved_off_the_ambient_path( + built: policy_module.Policy, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A PATH fronted with a directory of undeclared tools must not widen + the exec set — which is why resolution uses a fixed search path.""" + shadow = tmp_path / "shadow" + shadow.mkdir() + impostor = shadow / "bash" + impostor.write_text("#!/bin/sh\n") + impostor.chmod(0o755) + monkeypatch.setenv("PATH", str(shadow)) + + project = tmp_path / "proj2" + project.mkdir() + with scope(project) as rebuilt: + assert impostor.resolve() not in rebuilt.execute + + +def test_the_env_the_seam_execs_is_one_the_policy_granted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The overlay is applied by exec'ing `env`, so it has to be in the + exec set — and the *same* `env`. + + The search path is pointed at a copy here on purpose. Two answers to + "where does env live" agree on a host whose copy sits at the usual + place and disagree everywhere else, so a test run against the real + path would pass while the bug shipped: `/usr` is readable and never + executable, making a stale answer a denial on the first exec of + every single run. + """ + from lightcone.engine.sandbox.boundary import env_argv + + elsewhere = tmp_path / "bin" + elsewhere.mkdir() + real = shutil.which("env", path=policy_module._UTILITY_PATH) + assert real is not None, "no `env` on the search path" + shutil.copy(real, elsewhere / "env") + monkeypatch.setattr(policy_module, "_UTILITY_PATH", str(elsewhere)) + + project = tmp_path / "proj" + project.mkdir() + with scope(project) as built: + spawned = Path(env_argv(built)[0]) + assert spawned == elsewhere / "env" + assert built.grants(spawned, built.execute) + + +@pytest.mark.skipif(sys.platform != "linux", reason="the ELF loader tier is Linux-only") +def test_the_elf_loader_is_in_the_exec_set(built: policy_module.Policy) -> None: + """Landlock checks EXECUTE on the loader's own open, so without this + every dynamically linked binary — bash and python included — fails + EACCES and the sandbox is unusable.""" + loaders = policy_module.elf_loaders() + assert loaders, "no ELF loader found on this host" + assert all(loader in built.execute for loader in loaders) + + +def test_the_venv_and_the_interpreter_behind_it_are_granted(tmp_path: Path) -> None: + """`.venv/bin/python` is a symlink and Landlock evaluates the resolved + path, so the target needs EXECUTE — as a *file* — and its install root + needs READ for the stdlib beside it.""" + project = tmp_path / "proj" + bin_dir = project / ".venv" / "bin" + bin_dir.mkdir(parents=True) + store = tmp_path / "store" / "cpython-3.13" / "bin" + store.mkdir(parents=True) + real = store / "python3" + real.write_text("") + real.chmod(0o755) + (bin_dir / "python").symlink_to(real) + + with scope(project) as built: + install_root = store.parent.resolve() + assert bin_dir.resolve() in built.execute + assert real.resolve() in built.execute + assert install_root in built.read + # Its own tree, so EXECUTE too: a framework build re-execs itself + # into `Resources/Python.app/Contents/MacOS/Python`, which the + # binary-only grant would miss. + assert install_root in built.execute + + +def test_a_system_interpreter_does_not_make_the_whole_prefix_executable( + tmp_path: Path, +) -> None: + """The other side of the same rule. A venv built against the system + python resolves to `/usr/bin/python3`, whose install root is `/usr` — + a prefix shared with the whole host. Granting EXECUTE there would + make every binary on the machine runnable, and because Landlock + unions rights over ancestors that one grant silently outranks the + entire per-file allowlist.""" + project = tmp_path / "proj" + bin_dir = project / ".venv" / "bin" + bin_dir.mkdir(parents=True) + system = Path("/usr/bin/python3") + if not system.exists(): # pragma: no cover - unusual host + pytest.skip("no system python3") + (bin_dir / "python").symlink_to(system) + + with scope(project) as built: + assert Path("/usr") not in built.execute + assert Path("/usr") in built.read + assert system.resolve() in built.execute + + +def test_an_interpreter_in_home_does_not_make_home_readable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The stdlib read root is derived from the interpreter's location, so + an interpreter installed straight into `~/bin` would hand the read + set `$HOME` itself — silently undoing the private-HOME design, which + is the one guarantee the environment overlay exists to make. Reading + the base prefix out of `pyvenv.cfg` would report the same directory, + so the guard is the fix, not a better lookup.""" + home = tmp_path / "home" + (home / "bin").mkdir(parents=True) + interpreter = home / "bin" / "python" + interpreter.write_text("#!/bin/sh\n") + interpreter.chmod(0o755) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: home)) + + project = tmp_path / "proj" + (project / ".venv" / "bin").mkdir(parents=True) + (project / ".venv" / "bin" / "python").symlink_to(interpreter) + + with scope(project) as built: + assert not built.grants(home, built.read), built.read + # The interpreter file itself is still runnable. + assert built.grants(interpreter, built.execute) + + +def test_the_utility_path_covers_the_nix_system_profile() -> None: + """The read baseline was widened for NixOS, but the allowlist is + resolved off a fixed search path — if that stays FHS-only, `bash` + never enters the exec set there and a bare `lc run` is denied with a + nonsense remedy.""" + assert "/run/current-system/sw/bin" in policy_module._UTILITY_PATH + + +# ---- HOME, XDG, and hygiene ----------------------------------------------- + + +def test_home_and_friends_point_into_the_write_scope(built: policy_module.Policy) -> None: + """matplotlib, astropy, and R all want a HOME. Giving them a private + one is what lets them work without the real `$HOME` being readable + — mounting `$HOME` RO instead would reopen the + dotfile-steering channel the layer exists to close.""" + for key in ("HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", "MPLCONFIGDIR"): + assert Path(built.env[key]).is_relative_to(built.tmp_home), key + + +def test_bytecode_is_redirected_out_of_the_read_only_tree(built: policy_module.Policy) -> None: + assert Path(built.env["PYTHONPYCACHEPREFIX"]).is_relative_to(built.tmp_home) + + +def test_the_home_subdirs_exist(built: policy_module.Policy) -> None: + """Created up front: a tool that wants `~/.config` should find it + rather than fail trying to make it.""" + assert (built.tmp_home / ".config").is_dir() + assert (built.tmp_home / ".cache").is_dir() + + +def test_every_path_is_realpathed(built: policy_module.Policy) -> None: + """macOS resolves /tmp to /private/tmp and Landlock evaluates the + resolved path, so an unresolved rule silently matches nothing.""" + for group in (built.read, built.write, built.execute): + for path in group: + assert path == path.resolve(), path + + +def test_nonexistent_paths_are_dropped(built: policy_module.Policy) -> None: + """A rule cannot be added for a path that is not there, and a + baseline entry missing on this OS is normal rather than fatal.""" + for group in (built.read, built.write, built.execute): + assert all(path.exists() for path in group) + + +def test_the_allowlist_version_reaches_the_attestation(built: policy_module.Policy) -> None: + """The exec allowlist is a maintained surface; an output stays + interpretable after it grows only because the version rode along into + the manifest.""" + from lightcone.engine.sandbox.landlock import LandlockBackend + from lightcone.engine.sandbox.model import Capability + + backend = LandlockBackend(capability=Capability(kind="landlock", landlock_abi=1)) + assert backend.attest(built).exec_allowlist_version == EXEC_ALLOWLIST_VERSION + + +def test_path_is_the_exec_search_path_we_granted( + built: policy_module.Policy, tmp_path: Path +) -> None: + """The command must resolve tools through the same list the policy + granted from. Otherwise a host whose ambient PATH fronts another copy + of an allowlisted tool — homebrew's bash on macOS — gets that copy + denied, and the denial blames the command for lc's own incoherence. + """ + entries = built.env["PATH"].split(os.pathsep) + assert entries[0] == str(tmp_path / "proj" / ".venv" / "bin"), "the project env comes first" + assert entries[1:] == policy_module._UTILITY_PATH.split(os.pathsep) + + +def test_the_allowlist_is_resolved_with_the_real_which() -> None: + """The autouse `tools` fixture fakes `shutil.which` — and because + `project.shutil` *is* the global module, a blanket fake silently + reached every test in the suite. The exec set became + `/usr/bin/` for every tool, which exists on Linux and does not + on macOS, so the enforcement tests ran against a policy no user could + ever have. The fake must cover uv and git and nothing else, and must + never invent a location for either. + """ + assert shutil.which("uv") is not None, "convergence's substrate check must pass" + # Anything else is the real `shutil.which`: a path that is really + # there, or None — never one the fixture made up. + found = shutil.which("sh") + assert found is not None and Path(found).exists(), found + assert shutil.which("lc-definitely-not-a-real-tool") is None diff --git a/tests/test_sandbox_shim.py b/tests/test_sandbox_shim.py new file mode 100644 index 00000000..81c7636d --- /dev/null +++ b/tests/test_sandbox_shim.py @@ -0,0 +1,132 @@ +"""Tests for `lightcone._sandbox_exec` — the Landlock exec shim. + +Run as a real subprocess, because that is the only way the properties +under test are real: the shim's contract is its argv, its exit codes, +and the fact that `python -m lightcone._sandbox_exec` pulls in nothing +but the stdlib. + +Only the paths that need no kernel support live here; enforcement itself +is `test_sandbox_landlock.py`. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from lightcone import _sandbox_exec + +SHIM = [sys.executable, "-m", "lightcone._sandbox_exec"] + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([*SHIM, *args], capture_output=True, text=True, check=False) + + +def _document(**overrides: object) -> str: + return json.dumps(dict(overrides)) + + +# ---- never proceed unsandboxed -------------------------------------------- + + +@pytest.mark.parametrize( + ("args", "because"), + [ + ((), "no arguments at all"), + (("--policy",), "a --policy with no document"), + (("--policy", _document(), "true"), "no `--` before the command"), + (("--policy", _document(), "--"), "nothing after the `--`"), + (("--policy", "{not json", "--", "true"), "a malformed document"), + (("--policy", "[]", "--", "true"), "a document that is not an object"), + (("--policy", _document(read="/usr"), "--", "true"), "a field that is not a list"), + ], +) +def test_setup_failures_use_the_reserved_exit_code(args: tuple[str, ...], because: str) -> None: + """A sandbox that cannot be set up must never fall through to running + the command anyway — the failure mode this whole layer exists to make + impossible. 97 is reserved so it is distinguishable from any exit a + command could produce itself.""" + result = _run(*args) + assert result.returncode == _sandbox_exec.SETUP_FAILURE_EXIT, because + assert "lc sandbox setup failed" in result.stderr, because + + +def test_the_command_never_runs_when_setup_fails(tmp_path: Path) -> None: + canary = tmp_path / "canary" + _run("--policy", "{bad", "--", "touch", str(canary)) + assert not canary.exists() + + +# ---- the module stays alone ----------------------------------------------- + + +def test_the_shim_imports_nothing_from_lightcone() -> None: + """`lightcone` is a namespace package with no `__init__`, so `-m + lightcone._sandbox_exec` executes this module and nothing else. The + shim runs on every sandboxed exec; dragging the engine in would put + click, rich, and the astra stack on that path.""" + probe = ( + "import sys, lightcone._sandbox_exec;" + "print([m for m in sys.modules if m.startswith('lightcone')])" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, check=True + ) + assert sorted(json.loads(result.stdout.replace("'", '"'))) == [ + "lightcone", + "lightcone._sandbox_exec", + ] + + +def test_the_shim_uses_only_the_standard_library() -> None: + """Nothing third-party may sit between `lc` and an exec.""" + source = Path(_sandbox_exec.__file__).read_text() + for line in source.splitlines(): + if line.startswith(("import ", "from ")) and "__future__" not in line: + module = line.split()[1].split(".")[0] + assert module in sys.stdlib_module_names, line + + +# ---- the ABI ladder ------------------------------------------------------- + + +def test_handled_rights_only_widen_with_the_abi() -> None: + """Unknown bits make `landlock_create_ruleset` fail EINVAL, so the + mask is built up rather than assumed.""" + for lower, higher in ((1, 2), (2, 3), (3, 4)): + assert ( + _sandbox_exec.handled_access(lower) & _sandbox_exec.handled_access(higher) + == _sandbox_exec.handled_access(lower) + ) + + +def test_refer_is_handled_from_abi_2() -> None: + """A ruleset that does not *handle* REFER makes the kernel deny every + cross-directory rename — the ABI-1 EXDEV the denial classifier knows + about. Handling it is what lets a recipe rename its own temp files.""" + assert not _sandbox_exec.handled_access(1) & _sandbox_exec.ACCESS_FS_REFER + assert _sandbox_exec.handled_access(2) & _sandbox_exec.ACCESS_FS_REFER + + +def test_a_writable_root_never_gets_execute() -> None: + """Otherwise a recipe could write a binary into its own scratch and + run it, which is the exec allowlist defeated in two lines.""" + for abi in (1, 2, 3, 4): + assert not _sandbox_exec.write_bits(abi) & _sandbox_exec.ACCESS_FS_EXECUTE + + +def test_write_implies_read() -> None: + """A directory you may create files in but cannot list is not a + useful grant.""" + assert _sandbox_exec.write_bits(1) & _sandbox_exec.READ_BITS == _sandbox_exec.READ_BITS + + +def test_abi_is_zero_or_a_real_version() -> None: + """Every failure — old kernel, seccomp, unsupported arch — answers 0 + rather than raising: "can I sandbox here" has "no" as a valid answer.""" + assert _sandbox_exec.abi() >= 0 diff --git a/tests/test_sandbox_wrap.py b/tests/test_sandbox_wrap.py new file mode 100644 index 00000000..d5de9a2c --- /dev/null +++ b/tests/test_sandbox_wrap.py @@ -0,0 +1,294 @@ +"""Tests for the seam: every backend is a pure argv rewrite. + +These are the tests that make the abstraction pay off — the Landlock +wrap and the Seatbelt profile are both checked here, on this host, +whichever host that is. Nothing is executed and no privilege is needed, +because `wrap` is a function from a policy and an argv to an argv. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from lightcone.engine.sandbox import policy as policy_module +from lightcone.engine.sandbox import seatbelt +from lightcone.engine.sandbox.boundary import Unavailable, env_argv +from lightcone.engine.sandbox.landlock import LandlockBackend +from lightcone.engine.sandbox.model import Capability, Policy +from lightcone.engine.sandbox.seatbelt import SeatbeltBackend + + +@pytest.fixture +def policy(tmp_path: Path) -> Policy: + return Policy( + read=(tmp_path / "proj",), + write=(tmp_path / "home",), + execute=(tmp_path / "proj" / ".venv" / "bin",), + tmp_home=tmp_path / "home", + env={"HOME": str(tmp_path / "home")}, + ) + + +def _backends(policy: Policy) -> list[tuple[str, object]]: + return [ + ("landlock", LandlockBackend(capability=Capability(kind="landlock", landlock_abi=4))), + ("seatbelt", SeatbeltBackend()), + ("none", Unavailable()), + ] + + +# ---- the shared contract -------------------------------------------------- + + +def test_every_backend_ends_with_the_command_verbatim(policy: Policy) -> None: + """The one property the whole design rests on: whatever a backend + prepends, the command it was handed is still the tail of the result. + A backend that rewrote the command would not be interchangeable.""" + argv = ["python", "-c", "print('--policy')", "--", "-x"] + for name, backend in _backends(policy): + wrapped = backend.wrap(policy, argv) # type: ignore[attr-defined] + assert wrapped[-len(argv) :] == argv, name + + +def test_wrap_is_pure(policy: Policy, tmp_path: Path) -> None: + """No temp files, no file descriptors, no global state — which is + what lets the execution path stay mechanism-blind and lets these + tests run on a host that cannot enforce anything.""" + before = set(tmp_path.rglob("*")) + for name, backend in _backends(policy): + first = backend.wrap(policy, ["true"]) # type: ignore[attr-defined] + second = backend.wrap(policy, ["true"]) # type: ignore[attr-defined] + assert first == second, name + assert set(tmp_path.rglob("*")) == before + + +# ---- Landlock ------------------------------------------------------------- + + +def test_landlock_wraps_through_the_shim(policy: Policy) -> None: + backend = LandlockBackend( + capability=Capability(kind="landlock", landlock_abi=4), interpreter="/usr/bin/python3" + ) + wrapped = backend.wrap(policy, ["echo", "hi"]) + assert wrapped[:4] == ["/usr/bin/python3", "-m", "lightcone._sandbox_exec", "--policy"] + assert wrapped[5] == "--" + + +def test_the_landlock_policy_travels_as_json_the_shim_understands(policy: Policy) -> None: + """Serialized rather than passed as an inherited ruleset FD: a + Landlock FD cannot be reopened, and whether one survives `uv run`'s + spawn chain is a question JSON never has to answer.""" + backend = LandlockBackend(capability=Capability(kind="landlock", landlock_abi=4)) + document = json.loads(backend.wrap(policy, ["true"])[4]) + + assert document["read"] == [str(p) for p in policy.read] + assert document["write"] == [str(p) for p in policy.write] + assert document["execute"] == [str(p) for p in policy.execute] + + +def test_the_overlay_is_the_seam_s_job_not_each_backend_s(policy: Policy) -> None: + """Composed once, inside the wrap, for *every* backend — including + the null one, which must run in the same environment as a sandboxed + run (same private `$HOME`, same PYTHONPYCACHEPREFIX) or the two stop + being comparable.""" + prefixed = [*env_argv(policy), "true"] + # Deliberately not a literal path: the exec set grants whatever the + # utility search path resolved, and a second answer to that question + # is a denial on the first exec of every run. + assert Path(prefixed[0]) == policy_module.utility("env") + assert f"HOME={policy.env['HOME']}" in prefixed + + for name, backend in _backends(policy): + wrapped = backend.wrap(policy, prefixed) # type: ignore[attr-defined] + assert wrapped[-len(prefixed) :] == prefixed, name + + +def test_the_overlay_stays_out_of_the_landlock_document(policy: Policy) -> None: + """It must not reach the `uv run` prefix, which is outside the + boundary: uv resolves its cache from XDG_CACHE_HOME and would be sent + to a throwaway directory that `scope()` then deletes.""" + backend = LandlockBackend(capability=Capability(kind="landlock", landlock_abi=4)) + assert "env" not in json.loads(backend.wrap(policy, ["true"])[4]) + + +def test_the_write_tier_is_restated_after_the_read_only_guard() -> None: + """Landlock unions rights, so a writable output directory nested in a + readable project tree works there for free. SBPL is last-match-wins, + so reproducing that needs the write set to have the final word — or + the guard's `(deny file-write* PROJECT)` revokes the output directory + layer 4's recipes write into, and macOS refuses what Linux allows.""" + project = Path("/tmp/proj") + built = Policy( + read=(project,), + write=(project / "results" / "u" / "o",), + execute=(), + tmp_home=project, + env={}, + ) + profile = seatbelt.generate_profile(built) + assert profile.rindex("(allow file-read* file-write*") > profile.rindex("(deny file-write*") + + +def test_landlock_attests_the_probed_abi(policy: Policy) -> None: + """Recording which ABI answered is what separates this from the + "best effort silently succeeded on a kernel with no Landlock" trap.""" + backend = LandlockBackend(capability=Capability(kind="landlock", landlock_abi=3)) + attestation = backend.attest(policy) + assert attestation.mechanism == "landlock" + assert attestation.fs == "declared" + assert attestation.landlock_abi == 3 + + +# ---- Seatbelt ------------------------------------------------------------- + + +def test_seatbelt_invokes_sandbox_exec_by_absolute_path(policy: Policy) -> None: + """Never resolved through PATH: the sandbox must not be something an + earlier PATH entry can replace.""" + wrapped = SeatbeltBackend().wrap(policy, ["echo", "hi"]) + assert wrapped[0] == "/usr/bin/sandbox-exec" + assert wrapped[1] == "-p" + + +def test_seatbelt_passes_paths_as_parameters_not_profile_text(policy: Policy) -> None: + """Paths are never interpolated into SBPL — they are bound with -D + and referenced as (param "..."), so no path can close a form early + or be quoted wrong.""" + wrapped = SeatbeltBackend().wrap(policy, ["true"]) + profile = wrapped[1] + for path in (*policy.read, *policy.write, *policy.execute): + assert str(path) not in profile + assert f"-DREAD_0={policy.read[0]}" in wrapped + assert f"-DWRITE_0={policy.write[0]}" in wrapped + assert f"-DEXEC_0={policy.execute[0]}" in wrapped + + +def test_the_generated_profile_denies_by_default(policy: Policy) -> None: + profile = seatbelt.generate_profile(policy) + assert "(version 1)" in profile + assert "(deny default)" in profile + assert profile.index("(deny default)") < profile.index("(allow file-read*") + + +def test_the_profile_references_every_policy_path_once(policy: Policy) -> None: + for prefix, paths in (("READ", policy.read), ("WRITE", policy.write), ("EXEC", policy.execute)): + for index in range(len(paths)): + assert f'(param "{prefix}_{index}")' in seatbelt.generate_profile(policy), prefix + + +def test_the_upstream_fragments_are_shipped() -> None: + """They are package *data*, so a packaging slip would only surface on a + macOS host at run time. Read them here instead.""" + for name in (seatbelt.BASE_PROFILE, seatbelt.NETWORK, seatbelt.PLATFORM_DEFAULTS): + assert "(allow" in seatbelt.read_profile(name), name + + +def test_an_unknown_fragment_fails_loudly() -> None: + with pytest.raises(KeyError, match="unknown profile fragment"): + seatbelt.read_profile("nope.sbpl") + + +def test_the_upstream_base_does_not_grant_blanket_exec() -> None: + """The one local delta. Upstream allows `process-exec` outright because + codex does not restrict exec; restricting it is our whole guarantee, so + the blanket allow must stay commented out however the file is re-synced.""" + base = seatbelt.read_profile(seatbelt.BASE_PROFILE) + assert "\n(allow process-exec)" not in base + assert "LIGHTCONE DELTA" in base + + +def test_the_platform_defaults_carry_the_hard_won_entries(policy: Policy) -> None: + """Spot-checks on entries nobody would derive from first principles — + the reason this file is adapted from upstream rather than written.""" + profile = seatbelt.generate_profile(policy) + for needle in ( + "/dev/dtracehelper", # debugger helpers + "com.apple.system.opendirectoryd.libinfo", # getpwuid() or KeyError + "/opt/homebrew/lib", # brew dylibs + "^/dev/ttys[0-9]+$", # pty handles + "/System/Volumes/Data/private", # firmlink parent traversal + ): + assert needle in profile, needle + + +def test_the_read_tier_can_map_executables(policy: Policy) -> None: + """A venv's compiled extension modules sit under site-packages — + inside the project, so in the *read* tier — and macOS gates `dlopen` + on mapping. Without this `import numpy` fails on macOS alone, while + Landlock (which does not gate mmap) is fine either way.""" + profile = seatbelt.generate_profile(policy) + read_form = profile[profile.index(";; read: project tree") :].splitlines()[1] + assert "file-map-executable" in read_form, read_form + + +def test_the_loader_can_map_system_libraries(policy: Policy) -> None: + """dyld is the mach-o twin of the ELF loader tier: without + `file-map-executable` on the system frameworks nothing dynamically + linked starts.""" + profile = seatbelt.generate_profile(policy) + assert "(allow file-map-executable" in profile + assert '(subpath "/System/Library/Frameworks")' in profile + + +def test_the_profile_does_not_restrict_the_network(policy: Policy) -> None: + """lc controls no network on any platform, so the profile says so + rather than denying what the attestation then calls `allowed`.""" + profile = seatbelt.generate_profile(policy) + assert "(allow network*)" in profile + assert "(deny network" not in profile + assert SeatbeltBackend().attest(policy).network == "allowed" + + +def test_the_network_the_profile_allows_is_actually_usable(policy: Policy) -> None: + """Opening the socket families is not enough on macOS: name lookup + and TLS go through mach services the base's `(deny default)` blocks. + Without them the attestation would claim `allowed` while every + resolution failed — worse than either honest answer.""" + profile = seatbelt.generate_profile(policy) + for service in ( + "com.apple.SystemConfiguration.DNSConfiguration", # resolver config + "com.apple.SystemConfiguration.configd", + "com.apple.SecurityServer", # TLS trust evaluation + "com.apple.trustd.agent", + "com.apple.ocspd", + ): + assert service in profile, service + # After `(deny default)`, or it grants nothing at all. + assert profile.index("(deny default)") < profile.index("com.apple.SecurityServer") + + +def test_readable_but_unwritable_paths_are_denied_write_last(tmp_path: Path) -> None: + """SBPL is last-match-wins, so the guard has to come after the upstream + defaults — which grant /tmp write unconditionally and would otherwise + reopen the hole the Linux side closes by omitting the root.""" + built = Policy( + read=(tmp_path / "proj", tmp_path / "scratch"), + write=(tmp_path / "scratch",), + execute=(), + tmp_home=tmp_path / "scratch", + env={}, + ) + profile = seatbelt.generate_profile(built) + guard = profile.index("(deny file-write*") + assert guard > profile.index("(allow file-read* file-write*") + assert guard > profile.index("/opt/homebrew/lib"), "the guard must follow the defaults" + # READ_0 is the project (not writable); READ_1 is also a write root. + tail = profile[guard:] + assert '(param "READ_0")' in tail + assert '(param "READ_1")' not in tail + + +# ---- the null backend ----------------------------------------------------- + + +def test_unavailable_changes_nothing_and_admits_it(policy: Policy) -> None: + """Not a special case callers branch on — it satisfies the same + protocol, and the honesty lives in the attestation.""" + backend = Unavailable() + assert backend.wrap(policy, ["echo", "hi"]) == ["echo", "hi"] + attestation = backend.attest(policy) + assert attestation.mechanism == "none" + assert attestation.fs == "open" diff --git a/tests/test_templates.py b/tests/test_templates.py index 4d7be65f..9c7e5cb1 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -31,7 +31,7 @@ def test_pyproject_renders_every_placeholder() -> None: assert f'requires-python = "{templates.requires_python()}"' in rendered assert templates.lightcone_requirement() in rendered # Virtual by design: containerized mode builds `--no-install-project`, - # so a packaged project's own import would fail inside its image (§1). + # so a packaged project's own import would fail inside its image. assert "[build-system]" not in rendered From 45ad94a65d00da1e3cf7328e705025ac0bd017f0 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Tue, 18 Aug 2026 13:02:45 +0200 Subject: [PATCH 07/23] Say "not a Lightcone project" and point elsewhere, not at `lc init` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standing in $HOME, `lc run` answered "/home/francois is missing pyproject.toml, uv.lock, .venv — not a project root. Run `lc init` to converge one." — a list of files nobody expected to be there, and advice to scaffold a project in your home directory. The two ways the check fails are different mistakes. A directory with no project markers is the wrong *place*: it now says so and says to cd to the root of one, with no mention of `lc init`. A directory that holds a pyproject.toml or an astra.yaml but lacks the built environment is the right place, unconverged — a fresh clone is exactly this, since git carries no .venv — and there `lc init` really is the answer, so the missing pieces are still named. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DqTza2GqRLZ4yY5ywAusQH --- src/lightcone/engine/project.py | 43 ++++++++++++++++++++++++--------- tests/test_cli.py | 4 +-- tests/test_run.py | 22 ++++++++++++----- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 528bd815..8f082ecf 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -298,26 +298,45 @@ def project_name(directory: Path) -> str: return name or "analysis" +#: What makes a directory a project root: the environment ``lc run`` +#: enters. ``astra.yaml`` is deliberately not among them — a command can +#: be probed in any uv project, spec or no spec. +_ENVIRONMENT_FILES = ("pyproject.toml", "uv.lock", ".venv") + + def current_project(directory: Path | None = None) -> Path: """*directory* (default: the working directory), taken as the project root. ``lc run`` assumes it is invoked from the root, so the only check is - that the environment is actually there: ``pyproject.toml``, - ``uv.lock``, ``.venv``. An ``astra.yaml`` is deliberately not - required — a command can be probed in any uv project, spec or no - spec. There is no walk-up: the directory you are in is the directory - that is used, or it is an error. + that the environment is actually there. There is no walk-up: the + directory you are in is the directory that is used, or it is an + error. + + The two ways that fails are different mistakes and get different + advice. A directory with no project markers at all is the wrong + *place* — the answer is to go to the right one, and telling someone + standing in ``$HOME`` to run ``lc init`` there would be telling them + to scaffold a project in their home directory. A directory that + holds a ``pyproject.toml`` or an ``astra.yaml`` but lacks the built + environment is the right place, not yet converged — a fresh clone is + exactly this, since git carries no ``.venv`` — and there ``lc init`` + is the whole answer. """ directory = (directory or Path.cwd()).resolve() - missing = [ - name for name in ("pyproject.toml", "uv.lock", ".venv") if not (directory / name).exists() - ] - if missing: + missing = [name for name in _ENVIRONMENT_FILES if not (directory / name).exists()] + if not missing: + return directory + declared = (directory / "pyproject.toml").exists() or (directory / SPEC_FILENAME).exists() + if not declared: raise ProjectError( - f"{directory} is missing {', '.join(missing)} — not a project " - "root. Run `lc init` to converge one." + f"{directory} is not a Lightcone project — `lc run` uses the " + "directory it is invoked from, and there is no project here. " + "cd to the root of one and try again." ) - return directory + raise ProjectError( + f"{directory} is a Lightcone project that has not been built yet " + f"— missing {', '.join(missing)}. Run `lc init` here first." + ) # ============================================================================= diff --git a/tests/test_cli.py b/tests/test_cli.py index 311cff86..2c9a9979 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -288,7 +288,7 @@ def test_outside_a_project_is_a_clean_error( monkeypatch.chdir(tmp_path) result = runner.invoke(main, ["run", "true"]) assert result.exit_code == 1 - assert "lc init" in result.output + assert "not a Lightcone project" in result.output def test_a_subdirectory_of_a_project_is_not_the_project( @@ -301,7 +301,7 @@ def test_a_subdirectory_of_a_project_is_not_the_project( monkeypatch.chdir(nested) result = runner.invoke(main, ["run", "true"]) assert result.exit_code == 1 - assert "not a project" in result.output + assert "not a Lightcone project" in result.output def test_run_needs_no_spec_file( diff --git a/tests/test_run.py b/tests/test_run.py index 96e1ab59..56428748 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -59,19 +59,29 @@ def test_a_subdirectory_is_not_the_project(project: Path) -> None: the directory that is used, or it is an error.""" nested = project / "a" / "b" nested.mkdir(parents=True) - with pytest.raises(ProjectError, match="not a project"): + with pytest.raises(ProjectError, match="is not a Lightcone project"): current_project(nested) -def test_the_missing_pieces_are_named(project: Path) -> None: - (project / "uv.lock").unlink() - with pytest.raises(ProjectError, match="missing uv.lock"): +def test_an_unbuilt_project_is_told_to_build_it(project: Path) -> None: + """A fresh clone is exactly this — git carries no `.venv` — so here + `lc init` is the whole answer, and the missing pieces are named.""" + (project / ".venv").rmdir() + with pytest.raises(ProjectError, match="not been built yet") as raised: current_project(project) + assert "missing .venv" in str(raised.value) + assert "lc init" in str(raised.value) -def test_a_directory_without_an_environment_says_so(tmp_path: Path) -> None: - with pytest.raises(ProjectError, match="lc init"): +def test_the_wrong_directory_is_told_to_move_not_to_scaffold(tmp_path: Path) -> None: + """The complaint that produced this split: standing in `$HOME`, being + told to run `lc init` is advice to scaffold a project in your home + directory. The wrong *place* deserves "go to the right one".""" + with pytest.raises(ProjectError, match="is not a Lightcone project") as raised: current_project(tmp_path) + message = str(raised.value) + assert "cd to the root of one" in message + assert "lc init" not in message def test_the_default_is_the_working_directory( From 0f0cf6196d7c53dd2d5953dfb371b511991a4972 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Tue, 18 Aug 2026 14:51:33 +0200 Subject: [PATCH 08/23] Make the project tree writable, and fix the outside-a-project message (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two small changes to `lc run`, on top of layer 5. ## The project tree is writable now The boundary is meant to be a no-install stand-in for running the project in a container. What it should catch is a command reaching **outside** the project for a tool, library, or data file the image would not have. Forbidding writes *inside* the project served none of that — a container bind-mounts the working tree read-write — so the project joins the write set. This cannot be narrowed to "everything but `.venv`": Landlock unions rights over ancestors and has no way to subtract, so a writable project is necessarily a writable `.venv`. What keeps the environment honest is `uv sync --exact` at the next convergence, not the sandbox. Two things existed *only* to serve the read-only tree and go with it: - the filter that dropped `/tmp` from the write scope when the project lived under it; - `PYTHONPYCACHEPREFIX`, which redirected bytecode into a `$HOME` deleted after every run, so every in-tree module recompiled every time. A writable tree caches in place — what a container does, and what the scaffolded `.gitignore` already expects. The denial for a readable-but-not-writable path now means *a write outside the project* (a system path, or a declared input) and says so. Verified by hand against a real project: writing `results/out.csv` works; reading an undeclared file under `$HOME`, running `git`, and writing `/etc/...` are all still denied with their remedies. ### One test note worth reading The enforcement suite's `outside` fixture moves from `tmp_path` to `$HOME`. `/tmp` is in the write baseline, so anything `tmp_path` hands us is **granted** — those denial tests had been passing only because a project under `/tmp` used to drop `/tmp` from the policy. Left alone they would have gone green while testing nothing. ## `lc run` outside a project `/home/francois is missing pyproject.toml, uv.lock, .venv — not a project root. Run \`lc init\` to converge one.` listed files nobody expected to be there and advised scaffolding a project in your home directory. The two failures are different mistakes and now get different advice. No project markers at all is the wrong *place*: it says so and says to `cd` to the root of one, with no mention of `lc init`. A `pyproject.toml` or `astra.yaml` present but no built environment is the right place, unconverged — a fresh clone is exactly this, since git carries no `.venv` — and there `lc init` really is the answer. --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DqTza2GqRLZ4yY5ywAusQH --------- Co-authored-by: Claude Opus 5 (1M context) --- CLAUDE.md | 60 ++++++++++++----- src/lightcone/cli/commands.py | 8 ++- src/lightcone/engine/project.py | 8 +-- src/lightcone/engine/run.py | 8 ++- src/lightcone/engine/sandbox/denial.py | 21 +++--- src/lightcone/engine/sandbox/model.py | 2 +- src/lightcone/engine/sandbox/policy.py | 66 ++++++++++++------ src/lightcone/engine/sandbox/seatbelt.py | 48 +++++++------ tests/test_sandbox_denial.py | 73 +++++++++++++++----- tests/test_sandbox_enforcement.py | 62 ++++++++++++++--- tests/test_sandbox_policy.py | 85 +++++++++++++++++++++--- tests/test_sandbox_wrap.py | 8 ++- 12 files changed, 334 insertions(+), 115 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3164833b..f2205c78 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -304,8 +304,8 @@ user owns: the site registry supplies `UV_CACHE_DIR` on Perlmutter (spec §4). - `--compile-bytecode` is the one genuinely per-project cost: bytecode is generated into the venv, never linked (~55 MB of 216 MB here). It is a - deliberate trade — the environment is read-only at execution time, so - without it every recipe run re-compiles. + deliberate trade: paying compilation once here beats paying it on the + first import of every run. - **Every external tool goes through one seam**, `project._run`, which tests monkeypatch — so the suite never shells out, and every call is inspectable. `_check_call` turns a nonzero exit into a `ProjectError`: @@ -469,10 +469,10 @@ the guarantee for the people who do. **SBPL is last-match-wins; Landlock unions.** This asymmetry decides where a rule can live, and it cuts both ways. A later `(deny …)` can take -back an earlier allow, which is how `_read_only_guard` claws back the -write the vendored defaults grant on `/tmp`; Landlock has no equivalent — -a narrower rule only ever *adds* — so the Linux side solves that by -leaving the root out of the policy (`policy._write_roots`). +back an earlier allow, which is how `_read_only_guard` takes back write +on the read roots the vendored defaults would otherwise hand out; +Landlock has no equivalent — a narrower rule only ever *adds* — so on +Linux the policy simply never names them. But the same asymmetry means SBPL does **not** give nesting for free. Landlock unions, so a writable output directory inside a readable project @@ -585,17 +585,43 @@ only checks that the guard is present. Verified empirically, not assumed. persisted. An earlier draft carried a `to_manifest()` with no caller — deleted, because "no dead code" applies to this layer's own conveniences too. It lands with layer 4, which is what needs it. -- **`/tmp` is dropped from the write scope when the project lives under - it**, which §7 does not contemplate. Granting `/tmp` unconditionally - would make the tree of a `/tmp`-hosted project writable and silently - void the read-only-tree guarantee. `TMPDIR` points into the private - scope regardless, so `tempfile` works either way. The drop is - Linux-only in effect: on macOS the vendored defaults grant - `/private/tmp` write unconditionally and the guard only claws back our - *read* roots, so the surrounding `/tmp` stays writable there. The - project itself is protected on both — it is a read root, so the guard - names it — and `/tmp` writability is not a channel undeclared inputs - arrive through, so the asymmetry is recorded rather than closed. +- **`results/` is writable; the rest of the tree is not**, where §4 gives + a probe no output and therefore no in-tree write scope at all. A probe + gets the same write scope a recipe does, so a probe that works means a + recipe will — and the environment a run starts with is the one it + finishes with. + - **The shape was chosen because all three mechanisms express it + natively.** A writable directory *nested inside* a read-only tree is + the widening direction: Landlock unions rights over ancestors, SBPL + restates the write tier after the guard, and podman mounts the + project `:ro` with `results` `:rw` over it — all verified by running + them. The reverse — a writable tree with `.venv` carved out — needs + rights *subtraction*, which podman and SBPL can do and **Landlock + cannot at all**. That asymmetry is the whole argument: the read-only + shape is the only one direct mode and containerized mode can both + deliver, so it is the only one that gives a single UX. + - This was briefly reversed (PR #174) on the premise that "a container + bind-mounts the working tree read-write". True of the default, not of + the mount table we will write — and the experiment that settled it is + two `podman -v` flags. Don't re-derive it from the default again. + - **`_exec_set` grants no directory anywhere**, including `.venv/bin`. + A directory grant is a grant on whatever the directory holds *later*, + which was a live hole for as long as the tree was writable: `cp + /usr/bin/git .venv/bin/` ran a tool the allowlist denies by name. + Belt-and-braces under a read-only tree, one scandir, keep it. + - **A write-denial test must target a path the OS would let you + write.** `printf x > /etc/…` passes with no sandbox at all — the OS + refuses it for any non-root user — so it pins nothing. The sound + target is a user-owned path that the *policy* makes read-only: a + declared input. Mutation-check every denial test by running the same + command through `Unavailable()` and confirming it succeeds. + - **Enforcement fixtures must not live under `/tmp`.** It is in the + write baseline, so anything pytest's `tmp_path` hands you is + *granted*. The old denial tests passed only because a project under + `/tmp` used to drop `/tmp` from the policy; with that gone they went + green while testing nothing. `tests/test_sandbox_enforcement.py`'s + `outside` fixture is rooted at `$HOME`, which is outside every grant + by construction. - **`/run` is granted whole**, where codex names only `/run/current-system/sw`. It reaches `/run/user/$UID` — dconf, the gnupg and keyring sockets, portal state. Kept deliberately: the test diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 252d1d58..2efa63c8 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -164,9 +164,11 @@ def run(command: tuple[str, ...]) -> None: """Run COMMAND in the project environment, inside the sandbox. Byte-for-byte the environment recipes get — same lock, same - ``.venv``, same boundary — so a probe that works means a recipe - will. Reads the project tree and the inputs declared in - ``astra.yaml``; writes nowhere but its own temporary scope. + ``.venv``, same boundary — so a command that works here means a + recipe will. The project and the inputs declared in ``astra.yaml`` + are readable, ``results/`` and scratch are writable, and anything + outside that — a host tool, a system library, an undeclared file — + is refused. """ from lightcone.engine import run as engine_run from lightcone.engine.project import current_project diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 8f082ecf..ee1ec035 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -433,10 +433,10 @@ def _env_is_current(directory: Path) -> bool: # --locked so a drifted lock is an error rather than a silent relock; -# --exact because a plain sync is additive; --compile-bytecode because the -# environment is read-only at execution time, so a recipe cannot write .pyc -# as it imports — paying compilation once here instead of on every recipe -# run. +# --exact because a plain sync is additive, and because it is what puts the +# environment back in agreement with the lock after anything a run did to +# it; --compile-bytecode to pay compilation once here rather than on the +# first import of every run. _SYNC_ARGS = ["sync", "--locked", "--exact", "--compile-bytecode"] diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index 108c1d41..f44a88ee 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -5,9 +5,11 @@ point: if a probe works, the recipe will, and if a probe is denied, the recipe would have been. -A probe has no output, which is what makes it the strictest consumer of -the boundary: it may read the project and the declared inputs, and write -only its own tmp scope. Nothing it does can land in the project tree. +What the boundary catches is a reach *outside* the declared set — a +tool, a library, or a data file that is on this machine and would not be +in the image. The tree itself is read-only apart from ``results/``, +which is where output goes, so the environment a run started with is the +one it finishes with. """ from __future__ import annotations diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py index 6b43ff41..0936460f 100644 --- a/src/lightcone/engine/sandbox/denial.py +++ b/src/lightcone/engine/sandbox/denial.py @@ -60,8 +60,8 @@ def explain(stderr: str, policy: Policy, *, cwd: Path) -> list[str]: - a path granted for **neither** access — an undeclared tool or an undeclared input; - a path granted for **read but not write** — which, since reading - it was allowed, can only have been a write attempt into the - read-only project tree. + it was allowed, can only have been a write into the read-only + tree, a system directory, or a declared input. """ # Access-aware, and that distinction is what keeps the message # honest: every allowlisted binary lives under `/usr`, which the read @@ -161,19 +161,20 @@ def _render_data(path: Path) -> list[str]: def _render_write(path: Path) -> list[str]: - """The project tree is readable but not writable — say so plainly. + """Readable but not writable — say where writes are allowed to go. - Reading it was allowed, so this can only have been a write. There is - no declaration that makes the tree writable to a probe: a probe has - no output, and that is the answer rather than a limitation to work - around. + Reading it was allowed, so this can only have been a write. Every + such path is one a container would refuse too: the tree it mounts + read-only, a system path baked into the image, or an input that is + somebody else's file. """ return _message( f"cannot write {path}", [ - " the project tree is read-only in here — nothing a probe does can", - " land in it. Write somewhere scratch instead:", - " import tempfile; tempfile.mkdtemp() # or $TMPDIR, /tmp", + " output goes in results/ — the rest of the tree is read-only, so", + " the environment a run starts with is the one it ends with. For", + " anything that is not output, write somewhere scratch:", + " import tempfile; tempfile.mkdtemp() # or $TMPDIR", ], ) diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index 05ba1f67..69a91c63 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -51,7 +51,7 @@ class Policy: #: write scope; the caller owns removing it. tmp_home: Path #: Environment the boundary overlays: HOME, the XDG trio, - #: MPLCONFIGDIR, PYTHONPYCACHEPREFIX. + #: MPLCONFIGDIR, PYTHONPYCACHEPREFIX, TMPDIR, PATH. env: dict[str, str] = field(default_factory=dict) def grants(self, path: Path, roots: tuple[Path, ...]) -> bool: diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py index 08df5035..6f47a673 100644 --- a/src/lightcone/engine/sandbox/policy.py +++ b/src/lightcone/engine/sandbox/policy.py @@ -4,11 +4,21 @@ touch; :mod:`~lightcone.engine.sandbox.landlock` and :mod:`~lightcone.engine.sandbox.seatbelt` only decide how to say it. -The shape it encodes is the design's filesystem policy: read the project -and the declared inputs and the OS baseline; write only a private -scratch scope; execute only the environment plus a versioned utility -allowlist. Everything a recipe reaches for outside that set fails -loudly, which is the point. +The shape it encodes is what a container gives the command, minus the +container: the project and the declared inputs and the OS baseline +readable, ``results/`` and a private scratch scope writable, and only +the project's own environment plus a versioned utility allowlist +runnable. What it catches is a command reaching *outside* that set — a +tool, a library, or a data file that is on this machine and would not be +in the image. + +A read-only tree with one writable directory inside it is the shape all +three mechanisms express *natively*, which is why it is the shape: +Landlock unions rights over ancestors, so a nested grant only ever +widens; SBPL restates the write tier after the guard; and podman mounts +the project ``:ro`` with ``results`` ``:rw`` over it. The reverse — a +writable tree with a read-only hole in it — needs rights *subtraction*, +which Landlock cannot do at all. """ from __future__ import annotations @@ -155,12 +165,17 @@ def utility(name: str) -> Path | None: def probe_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: - """The policy for ``lc run`` — a probe. + """The policy for ``lc run``. - A probe has no output, so it gets **no in-tree write scope at all**: - it may read the project and the declared inputs, and write only to - its own tmp scope. That is stricter than a recipe's policy, and it is - what stops a probe from quietly materializing something. + The tree is read-only except for ``results/``, which is where output + goes. That keeps the environment itself — ``.venv``, ``uv.lock``, + the spec — exactly as the lock describes it for the whole run, and + it is the same scope a recipe gets, so a probe that works means a + recipe will. + + ``results/`` is granted only if it is already there. Convergence + creates it, and a policy that made directories would be a side + effect nobody asked a *probe* for. Creates the per-run HOME on disk as a side effect; the caller owns removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). @@ -173,7 +188,7 @@ def probe_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: # EXECUTE on the interpreter *file*; READ on the install root beside # it, for the stdlib. See :func:`_venv_python` and :func:`_stdlib_root`. stdlib = _stdlib_root(python) - write = _existing([tmp_home, *_write_roots(project)]) + write = _existing([tmp_home, project / "results", *_write_roots(project)]) read = _existing([project, *read_paths, *stdlib, *(Path(p) for p in _OS_READ_BASELINE)]) return Policy( @@ -188,13 +203,13 @@ def probe_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: def _write_roots(project: Path) -> list[Path]: """The write baseline, minus any root that would swallow the project. - ``/tmp`` is writable by design — but a project that - *lives* under it would then be writable too, which would silently - void the read-only-tree guarantee for exactly the people who keep - scratch analyses in ``/tmp``. Dropping the offending root is safe - because ``TMPDIR`` points at the private scope regardless, so - ``tempfile`` keeps working either way. The device entries can never - contain a project, so the filter is a no-op for them. + ``/tmp`` is writable by design — but a project that *lives* under it + would then be writable too, silently voiding the read-only tree for + exactly the people who keep scratch analyses in ``/tmp``. Dropping + the offending root is safe because ``TMPDIR`` points at the private + scope regardless, so ``tempfile`` keeps working either way. The + device entries can never contain a project, so the filter is a no-op + for them. """ resolved = project.resolve() roots = tuple(Path(root).resolve() for root in _WRITE_BASELINE) @@ -215,8 +230,8 @@ def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: what the command resolves has to be what the policy granted. ``PYTHONPYCACHEPREFIX`` is here for the same reason at the other end: - the project tree is read-only inside the boundary, so without it - every ``import`` of an in-tree module fails to write its + the tree is read-only apart from ``results/``, so without it every + ``import`` of an in-tree module fails trying to write its ``__pycache__``. ``TMPDIR`` points inside too, so ``tempfile`` works even where the shared ``/tmp`` had to leave the write set. """ @@ -290,7 +305,16 @@ def _exec_set(project: Path, python: Path | None) -> list[Path]: paths: list[Path] = [] bin_dir = project / ".venv" / "bin" if bin_dir.is_dir(): - paths.append(bin_dir) + # Per *file*, never the directory, for the same reason `/usr/bin` + # is: a directory grant is a grant on whatever the directory holds + # *later*. The read-only tree means nothing a run does can put a + # binary here, so this is belt-and-braces today — but it was a live + # hole for as long as the tree was writable (`cp /usr/bin/git + # .venv/bin/` ran a tool the allowlist denies by name), and + # enumerating costs one scandir. + paths.extend( + entry.resolve() for entry in bin_dir.iterdir() if os.access(entry, os.X_OK) + ) if python is not None: paths.append(python) # macOS framework builds `posix_spawn` themselves into diff --git a/src/lightcone/engine/sandbox/seatbelt.py b/src/lightcone/engine/sandbox/seatbelt.py index 13a654ec..09cd63ef 100644 --- a/src/lightcone/engine/sandbox/seatbelt.py +++ b/src/lightcone/engine/sandbox/seatbelt.py @@ -135,18 +135,18 @@ def profile_params(policy: Policy) -> list[tuple[str, str]]: def generate_profile(policy: Policy) -> str: """The SBPL for *policy*: upstream base, our tiers, upstream defaults. - Order is load-bearing, because **SBPL is last-match-wins**. That is - why ``(deny default)`` can lead the base and still be overridden, why - the read-only guard can take back the write the upstream defaults - hand out on ``/tmp`` — and why the write tier is restated *after* the - guard. - - That last one is not cosmetic. Landlock unions rights, so a narrower - grant only ever widens: a writable directory nested inside a readable - tree works there for free. Reproducing that here means the write set - must have the final word, or the guard's ``(deny file-write* …)`` - over the read roots would silently revoke any nested writable path — - Linux would allow and macOS would refuse the same policy. + Order is load-bearing, because **SBPL is last-match-wins**: it is + what lets ``(deny default)`` lead the base and still be overridden, + what lets :func:`_read_only_guard` take a write back, and why the + write tier is restated *after* that guard. + + That last one is not cosmetic — it is what makes ``results/`` work. + Landlock unions rights, so a writable directory nested inside a + readable tree needs no help there. Reproducing it here means the + write set must have the final word, or the guard's ``(deny + file-write* PROJECT)`` would revoke the grant on ``results/`` that + sits inside it: Linux would materialize and macOS would refuse the + very same policy. """ return "\n".join( [ @@ -160,7 +160,7 @@ def generate_profile(policy: Policy) -> str: # fails on macOS alone. Landlock does not gate mmap at # all, so read already implies it there: granting it here # is what makes the two platforms mean the same thing. - "read: project tree + declared inputs", + "read: the project, the declared inputs, and the OS baseline", "(allow file-read* file-test-existence file-map-executable", "READ", len(policy.read), @@ -187,7 +187,7 @@ def generate_profile(policy: Policy) -> str: _read_only_guard(policy), # Last, so a nested writable path beats the guard above it. _tier( - "write: the per-run private scope and scratch — never the project tree", + "write: the project, the per-run private scope, and shared scratch", "(allow file-read* file-write*", "WRITE", len(policy.write), @@ -207,13 +207,19 @@ def _tier(comment: str, opener: str, prefix: str, count: int) -> str: def _read_only_guard(policy: Policy) -> str: """Take back writes on everything readable that is not also writable. - The upstream defaults grant write to shared scratch (``/tmp``, - ``/var/tmp``) unconditionally, which would otherwise reopen exactly - the hole :func:`~lightcone.engine.sandbox.policy._write_roots` closes - on Linux — a project living under ``/tmp`` becoming writable. It is - only expressible because SBPL is last-match-wins; Landlock, which - unions rights, has to solve the same problem by leaving the root out - of the policy instead. + This is what keeps the profile's write set equal to the *policy's* + write set. The Linux side gets that for free — Landlock grants only + what the policy names — but here the upstream fragments hand out + writes of their own, on shared scratch (``/tmp``, ``/var/tmp``) and + on devices, and only a later ``deny`` can take one back. + +It emits a ``deny`` over every read root that is not also writable — + the project tree, ``/usr``, ``/etc``, the interpreter's stdlib root. + The project is the one that has to be a ``deny`` rather than an + absence: the upstream fragments grant write on ``/tmp``, so a project + living there would otherwise be writable through *their* rule. That + is the hole ``policy._write_roots`` closes on the Linux side, by + leaving the offending root out of the policy instead. Deliberately narrow: it names *our* read roots and nothing else, so the device and pty writes the upstream defaults grant survive. And it diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py index 3f06592a..3bb215ec 100644 --- a/tests/test_sandbox_denial.py +++ b/tests/test_sandbox_denial.py @@ -27,14 +27,29 @@ def project(tmp_path: Path) -> Path: @pytest.fixture -def policy(project: Path, tmp_path: Path) -> Policy: +def declared_input(tmp_path: Path) -> Path: + """A declared input: readable, and never writable. + + The readable-but-not-writable case used to be the project tree. Now + that the tree is writable, this is what is left of it — and it is + the honest example, since an input really is somebody else's file. + """ + source = tmp_path / "inputs" + source.mkdir() + (source / "catalog.csv").write_text("id\n") + return source + + +@pytest.fixture +def policy(project: Path, declared_input: Path, tmp_path: Path) -> Policy: scratch = tmp_path / "scratch" scratch.mkdir() venv_bin = project / ".venv" / "bin" venv_bin.mkdir(parents=True) return Policy( - read=(project, Path("/usr")), - write=(scratch,), + read=(project, declared_input, Path("/usr")), + # `results/` only — the rest of the tree is read-only. + write=(scratch, project / "results"), execute=(venv_bin,), tmp_home=scratch, env={}, @@ -71,16 +86,39 @@ def test_an_undeclared_data_file_gets_the_astra_snippet( def test_an_in_tree_write_is_its_own_kind_of_denial(policy: Policy, project: Path) -> None: - """Reading the project was allowed, so an EACCES on a project file - can only have been a write — and "declare it as an input" would be - exactly the wrong advice.""" + """Reading the project was allowed, so an EACCES on a project file can + only have been a write — and "declare it as an input" would be exactly + the wrong advice.""" stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" joined = "\n".join(denial.explain(stderr, policy, cwd=project)) assert "cannot write" in joined - assert "read-only in here" in joined + assert "output goes in results/" in joined + assert "inputs:" not in joined + + +def test_a_write_to_a_declared_input_says_the_same_thing( + policy: Policy, project: Path, declared_input: Path +) -> None: + """An input is somebody else's file: readable because it is declared, + never writable — and "declare it as an input" would be absurd advice + for a file that already is one.""" + target = declared_input / "catalog.csv" + stderr = f"PermissionError: [Errno 13] Permission denied: '{target}'\n" + joined = "\n".join(denial.explain(stderr, policy, cwd=project)) + assert "cannot write" in joined assert "inputs:" not in joined +def test_a_write_into_results_is_not_a_denial_at_all(policy: Policy, project: Path) -> None: + """`results/` is granted, so an EACCES there is the OS's problem, not + the sandbox's.""" + (project / "results").mkdir(exist_ok=True) + target = project / "results" / "out.csv" + target.write_text("") + stderr = f"PermissionError: [Errno 13] Permission denied: '{target}'\n" + assert denial.explain(stderr, policy, cwd=project) == [] + + # ---- what must not be reported -------------------------------------------- @@ -139,21 +177,26 @@ def test_a_bare_command_name_is_resolved_against_the_host( def test_a_relative_path_is_resolved_against_the_working_directory( - policy: Policy, project: Path + policy: Policy, project: Path, tmp_path: Path ) -> None: - """Python reports the string the command passed, so an in-tree denial - arrives as `'astra.yaml'` with no directory at all.""" - stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" + """Python reports the string the command passed, so a denial can + arrive as a bare relative path with no directory at all.""" + (tmp_path / "elsewhere.fits").write_text("") + stderr = "PermissionError: [Errno 13] Permission denied: '../elsewhere.fits'\n" assert denial.explain(stderr, policy, cwd=project) != [] -def test_no_escape_hatch_is_ever_offered(policy: Policy, project: Path) -> None: +def test_no_escape_hatch_is_ever_offered( + policy: Policy, project: Path, declared_input: Path +) -> None: """There is no way to run outside the sandbox, so no message may suggest one.""" + target = declared_input / "catalog.csv" joined = "\n".join( - denial.explain("PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n", policy, - cwd=project) + denial.explain( + f"PermissionError: [Errno 13] Permission denied: '{target}'\n", policy, cwd=project + ) ) - assert "read-only in here" in joined + assert "cannot write" in joined assert "--no-sandbox" not in joined assert "sandbox-debug" not in joined diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py index fb34e337..e6614083 100644 --- a/tests/test_sandbox_enforcement.py +++ b/tests/test_sandbox_enforcement.py @@ -32,7 +32,8 @@ import subprocess import sys import sysconfig -from collections.abc import Sequence +import tempfile +from collections.abc import Iterator, Sequence from pathlib import Path import pytest @@ -81,13 +82,27 @@ def project(tmp_path: Path) -> Path: @pytest.fixture -def outside(tmp_path: Path) -> Path: - """A directory the project never declares — the leak's other end.""" - elsewhere = tmp_path / "elsewhere" - elsewhere.mkdir() +def outside() -> Iterator[Path]: + """A directory the project never declares — the leak's other end. + + Under `$HOME`, not `tmp_path`. `/tmp` is in the write baseline, so + anything pytest hands us there is *granted*, and a denial test + written against it would pass only for as long as something else + happened to keep `/tmp` out of the policy. The real home is outside + every grant by construction — the boundary hands the command a + private one — which is the property these tests actually need. + + `mkdtemp`, not a fixed name: two suites sharing a home (xdist, two + checkouts, two CI jobs) would otherwise race on the same directory, + and a fixed name is also somebody's real path to delete. + """ + elsewhere = Path(tempfile.mkdtemp(prefix="lc-enforcement-", dir=Path.home())) (elsewhere / "secret.txt").write_text("undeclared\n") (elsewhere / "sneaky.py").write_text("VALUE = 'undeclared import'\n") - return elsewhere + try: + yield elsewhere + finally: + shutil.rmtree(elsewhere, ignore_errors=True) def run( @@ -346,17 +361,46 @@ def test_the_real_home_is_not_readable(backend: sandbox.Backend, project: Path) # ---- the write scope ------------------------------------------------------- -def test_the_project_tree_cannot_be_written( +def test_the_tree_outside_results_cannot_be_written( backend: sandbox.Backend, project: Path ) -> None: - """A probe has no output, so nothing it does may land in - the tree — and the file has to be *unchanged*, not merely reported.""" + """The environment a run starts with is the one it ends with — and the + file has to be *unchanged*, not merely reported.""" with sandbox.scope(project) as policy: result = shell(backend, policy, "printf clobbered > data.txt", cwd=project) assert result.returncode != 0 assert (project / "data.txt").read_text() == "in-tree\n", "the file changed anyway" +def test_results_can_be_written(backend: sandbox.Backend, project: Path) -> None: + """Writable inside a read-only tree — the nesting both mechanisms have + to agree on, and the one a container gets from a second bind mount.""" + (project / "results").mkdir() + with sandbox.scope(project) as policy: + result = shell(backend, policy, "printf out > results/out.csv", cwd=project) + assert result.returncode == 0, result.stderr + assert (project / "results" / "out.csv").read_text() == "out" + + +def test_a_declared_input_is_read_only( + backend: sandbox.Backend, project: Path, outside: Path +) -> None: + """Declaring an input makes it readable, never writable — it is + somebody else's file. + + Deliberately not a write to `/etc`: the OS denies that to any + non-root user on its own, so such a test passes with no sandbox at + all and pins nothing. This target is owned by the user running the + suite, so the *only* thing that can refuse the write is the + boundary. + """ + target = outside / "secret.txt" + with sandbox.scope(project, read_paths=[outside]) as policy: + result = shell(backend, policy, f"printf clobbered > {target}", cwd=project) + assert result.returncode != 0 + assert target.read_text() == "undeclared\n", "the file changed anyway" + + def test_the_private_scope_is_writable(backend: sandbox.Backend, project: Path) -> None: with sandbox.scope(project) as policy: target = policy.tmp_home / "result.txt" diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py index 2ce44ee8..b48732ee 100644 --- a/tests/test_sandbox_policy.py +++ b/tests/test_sandbox_policy.py @@ -36,11 +36,48 @@ def built(tmp_path: Path) -> Iterator[policy_module.Policy]: # ---- the write scope ------------------------------------------------------ -def test_a_probe_never_writes_in_the_tree(built: policy_module.Policy, tmp_path: Path) -> None: - """The invariant that makes `lc run` safe to hand to an agent: a probe - has no output, so nothing it does can land in the project.""" +def test_the_tree_is_read_only_apart_from_results( + built: policy_module.Policy, tmp_path: Path +) -> None: + """The environment a run starts with is the one it ends with: nothing + it does can touch `.venv`, the lock, or the spec.""" + project = tmp_path / "proj" + assert not built.grants(project / "astra.yaml", built.write) + assert not built.grants(project / ".venv" / "bin" / "python", built.write) + + +def test_results_is_writable(tmp_path: Path) -> None: + """Output goes here, and it is the same scope a recipe gets — which + is what makes a probe a probe of the real thing. + + A writable directory nested inside a read-only tree is the shape all + three mechanisms express natively: Landlock unions rights so a nested + grant only widens, SBPL restates the write tier after the guard, and + podman mounts `results` `:rw` over a `:ro` project.""" project = tmp_path / "proj" - assert not any(root == project or project.is_relative_to(root) for root in built.write) + (project / "results").mkdir(parents=True) + with scope(project) as built: + assert built.grants(project / "results" / "out.csv", built.write) + + +def test_results_is_granted_only_if_it_exists(tmp_path: Path) -> None: + """Convergence makes it. A policy that made directories would be a + side effect nobody asked a probe for.""" + project = tmp_path / "bare" + project.mkdir() + with scope(project) as built: + assert not built.grants(project / "results", built.write) + assert not (project / "results").exists() + + +def test_the_project_is_also_readable(built: policy_module.Policy, tmp_path: Path) -> None: + """Not redundant with the write grant. Landlock's write bits include + the read ones, but SBPL's write tier grants `file-read* file-write*` + and *not* `file-map-executable` — which macOS gates `dlopen` on, and + every compiled extension module under site-packages needs. Dropping + the project from `read` as duplication breaks `import numpy` on macOS + alone, with Linux CI still green.""" + assert built.grants(tmp_path / "proj" / "astra.yaml", built.read) def test_the_private_home_is_writable(built: policy_module.Policy) -> None: @@ -57,8 +94,8 @@ def test_the_shared_tmp_is_writable_for_a_project_outside_it() -> None: def test_a_project_living_under_tmp_does_not_become_writable() -> None: """`/tmp` is writable by design, so a project that *lives* there would - otherwise be writable too — voiding the read-only-tree guarantee for - exactly the people who keep scratch analyses in /tmp.""" + otherwise be writable through that grant — voiding the read-only tree + for exactly the people who keep scratch analyses in /tmp.""" shared = Path("/tmp").resolve() with scope(shared / "lc-policy-under-tmp") as built: assert shared not in built.write @@ -261,7 +298,10 @@ def test_the_venv_and_the_interpreter_behind_it_are_granted(tmp_path: Path) -> N with scope(project) as built: install_root = store.parent.resolve() - assert bin_dir.resolve() in built.execute + # The *directory* is deliberately not granted: the tree is + # writable, so that would be an exec grant on whatever is written + # into `.venv/bin` next. See `test_the_venv_bin_is_granted_per_file`. + assert bin_dir.resolve() not in built.execute assert real.resolve() in built.execute assert install_root in built.read # Its own tree, so EXECUTE too: a framework build re-execs itself @@ -270,6 +310,31 @@ def test_the_venv_and_the_interpreter_behind_it_are_granted(tmp_path: Path) -> N assert install_root in built.execute +def test_the_venv_bin_is_granted_per_file_not_as_a_directory(tmp_path: Path) -> None: + """The exec allowlist has to survive a writable project. + + `.venv/bin` is exec-granted and the tree is writable, so granting the + *directory* would grant whatever is written there next — copy a host + tool in and it runs, making the by-name denial of that same tool + meaningless. The grants are taken per file when the policy is built. + """ + project = tmp_path / "proj" + bin_dir = project / ".venv" / "bin" + bin_dir.mkdir(parents=True) + script = bin_dir / "pytest" + script.write_text("#!/bin/sh\n") + script.chmod(0o755) + + with scope(project) as built: + assert script.resolve() in built.execute, "a console script that exists is granted" + assert bin_dir.resolve() not in built.execute, "but never the directory" + # What a run writes there afterwards is therefore not runnable. + later = bin_dir / "smuggled" + later.write_text("#!/bin/sh\n") + later.chmod(0o755) + assert not built.grants(later, built.execute) + + def test_a_system_interpreter_does_not_make_the_whole_prefix_executable( tmp_path: Path, ) -> None: @@ -339,7 +404,11 @@ def test_home_and_friends_point_into_the_write_scope(built: policy_module.Policy assert Path(built.env[key]).is_relative_to(built.tmp_home), key -def test_bytecode_is_redirected_out_of_the_read_only_tree(built: policy_module.Policy) -> None: +def test_bytecode_is_redirected_out_of_the_read_only_tree( + built: policy_module.Policy, +) -> None: + """Without it every `import` of an in-tree module fails trying to + write its `__pycache__` into a tree it may not write.""" assert Path(built.env["PYTHONPYCACHEPREFIX"]).is_relative_to(built.tmp_home) diff --git a/tests/test_sandbox_wrap.py b/tests/test_sandbox_wrap.py index d5de9a2c..c5245c1e 100644 --- a/tests/test_sandbox_wrap.py +++ b/tests/test_sandbox_wrap.py @@ -92,8 +92,8 @@ def test_the_landlock_policy_travels_as_json_the_shim_understands(policy: Policy def test_the_overlay_is_the_seam_s_job_not_each_backend_s(policy: Policy) -> None: """Composed once, inside the wrap, for *every* backend — including the null one, which must run in the same environment as a sandboxed - run (same private `$HOME`, same PYTHONPYCACHEPREFIX) or the two stop - being comparable.""" + run (same private `$HOME`, same `TMPDIR`, same `PATH`) or the two + stop being comparable.""" prefixed = [*env_argv(policy), "true"] # Deliberately not a literal path: the exec set grants whatever the # utility search path resolved, and a second answer to that question @@ -220,7 +220,9 @@ def test_the_read_tier_can_map_executables(policy: Policy) -> None: on mapping. Without this `import numpy` fails on macOS alone, while Landlock (which does not gate mmap) is fine either way.""" profile = seatbelt.generate_profile(policy) - read_form = profile[profile.index(";; read: project tree") :].splitlines()[1] + # Anchored on the tier marker, not its prose: the comment is a label, + # not the contract. + read_form = profile[profile.index(";; read:") :].splitlines()[1] assert "file-map-executable" in read_form, read_form From 9395dcf6295cb37700b6a10f5b5206e734c025a8 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Wed, 19 Aug 2026 15:20:00 +0200 Subject: [PATCH 09/23] Layers 2 and 4: identity, provenance, and `lc materialize` (#175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lc materialize` executes an ASTRA analysis: it plans the graph, runs each recipe under the layer-5 boundary, and commits every output together with the manifest that describes it. Layer 2 folds in, because the fabric is what needs identity and nothing else did. Layer table: rows **2** and **4** → ✅. ## Results are versioned in the repository On the DataLad model — **git carries the pointers and the history, git-annex carries the bytes** — without taking the `datalad` package. Every operation the layer needs is one git or git-annex command, and `uv.lock` is `env_version`'s first term, so 43 transitive packages would stale every output in every project on each boto3 bump. Compatibility is a commitment rather than a coincidence: `lc init` writes a `datalad.dataset.id`, so a project is a DataLad dataset **from birth** (verified: `datalad status` needs no `--force` adoption step), and the run record is the format `datalad rerun` reads. lc never imports datalad, never requires it, and never parses `.datalad/`. git stops being optional — a project without version control had nowhere to put a result. ## Identity ``` env_version = sha256(uv.lock ‖ .python-version ‖ canonical install settings) code_version = sha256(recipe ‖ canonical decisions ‖ env_version) ``` Both length-framed, so a boundary shift between fields cannot produce one digest from two inputs. The git commit is **recorded with every output and hashed into neither** — a commit must not stale the world; per-output code invalidation is available by declaring source files as ASTRA inputs. The lock scan **refuses** path/directory/editable dependencies (the lock records where they were, not what was in them, so two syncs can install different code while every hash agrees), **reports** registry packages built from sdist, and is **advisory** on non-default dependency groups. ## Execution | Module | Role | |---|---| | `plan.py` | spec × universes → tasks; sub-analyses flatten to `.`, a second nesting level is refused rather than ignored | | `assets.py` | the output directory, the manifest, and `staleness()` — one predicate, two callers | | `worker.py` | one output: gates, reset, recipe, hash, manifest. Never raises, never writes git. Also the `python -m` entry point the run record names | | `materialize.py` | the driver: dirty-tree refusal, Dask, and the serialized save/restore loop | | `dataset.py` | the one git + git-annex seam, routed through the existing `project._run` | | `identity.py` | the two hashes and the lock scan | - **Dask owns the ordering.** Tasks are submitted with their upstream futures as arguments; there is no ready-set loop and no hand-rolled topological sort in the execution path. - **The driver owns git alone.** Concurrent git operations on one repository race on the index lock — this is `datalad-slurm`'s schedule/finish split. - **A run leaves the tree exactly as clean as it found it.** `ok` → save, `failed`/`blocked`/never-reported → restore, in a `try/finally`. That is what makes the dirty-tree refusal survivable rather than a trap. - **`--check` differs from a worker by one input value**, not by logic: `None` for anything already classified as would-run, meaning "this is going to change". `sandbox/policy.py` gains `recipe_policy()` beside `probe_policy()`, sharing one baseline: a recipe may write its own output directory and nothing else under `results/`. Reads are unchanged — a carve-out there is something Landlock cannot express at all. ## Three things running it turned up that the design did not have - **`.gitattributes` needs a leading `* annex.largefiles=nothing`.** `git annex add` annexes whatever it is handed, so without it the documented save turns `src/fit.py` into a read-only symlink into the object store and the next edit fails with `EACCES`. - **`git annex add -A` is not a thing.** `git add` needs `-A`; git-annex rejects it. A test extracts the line from `data/README.md` and runs it, because that spelling was wrong once. - **`git check-ignore` must be asked with a trailing slash and `--no-index`**, or the `results/*` an older scaffold wrote — which ignores the directory's *contents*, not its name — goes unseen, and a materialize reports success while committing nothing. ## Dependencies `git-annex` (0 transitive; wheel-only, so it sets the CLI's install floor — see the risk note in the plan) and `distributed`. **`datalad` is a dev-group dependency only**: the run record's format fails *silently* (its parser returns nothing on a mismatch and `rerun` then exits 0), so the suite asserts through datalad's own parser **and** runs a real `datalad rerun`, rather than golden-testing our own JSON. ## Verification ``` 348 passed ruff: clean mypy: clean ``` End-to-end from a clean `uv pip install .` into a throwaway venv (not the global tool env): `lc init` → commit → `--check` → `materialize` → idempotent re-run → decision change cascades to dependents → old bytes recovered from an old commit. Landlock enforced and recorded in `hermeticity`, results annexed, manifests in plain git, `datalad status` clean, one `[DATALAD RUNCMD]` commit per output. Also pinned by tests, against a real annex and the real boundary: a recipe cannot write into a sibling output (mutation-checked against `Unavailable()`), a recipe *can* read an annexed input through its symlink, a failing recipe commits nothing and leaves `git status` empty, and `python -m lightcone.engine.worker` imports neither click nor rich. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01GaahDTd6djz9PCm1Rh6SP4 --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/tests.yml | 4 +- CLAUDE.md | 815 +++++++++++++++++- pyproject.toml | 11 +- src/lightcone/_sandbox_exec.py | 103 ++- src/lightcone/cli/__init__.py | 5 + src/lightcone/cli/commands.py | 215 ++++- src/lightcone/engine/assets.py | 439 ++++++++++ src/lightcone/engine/dataset.py | 303 +++++++ src/lightcone/engine/identity.py | 284 ++++++ src/lightcone/engine/materialize.py | 676 +++++++++++++++ src/lightcone/engine/plan.py | 324 +++++++ src/lightcone/engine/project.py | 412 ++++++--- src/lightcone/engine/run.py | 87 +- src/lightcone/engine/sandbox/__init__.py | 2 + src/lightcone/engine/sandbox/boundary.py | 106 ++- src/lightcone/engine/sandbox/denial.py | 50 +- src/lightcone/engine/sandbox/landlock.py | 30 +- src/lightcone/engine/sandbox/model.py | 43 +- src/lightcone/engine/sandbox/policy.py | 102 ++- src/lightcone/engine/sandbox/seatbelt.py | 80 +- src/lightcone/engine/templates/__init__.py | 319 +++++-- .../templates/files/data-README.md.tmpl | 10 + .../templates/files/datalad-config.tmpl | 2 + .../engine/templates/files/gitattributes.tmpl | 14 + .../engine/templates/files/gitignore.tmpl | 2 - src/lightcone/engine/worker.py | 368 ++++++++ tests/conftest.py | 135 ++- tests/test_assets.py | 391 +++++++++ tests/test_cli.py | 270 +++++- tests/test_dataset.py | 420 +++++++++ tests/test_identity.py | 287 ++++++ tests/test_materialize.py | 737 ++++++++++++++++ tests/test_plan.py | 359 ++++++++ tests/test_project.py | 176 +++- tests/test_run.py | 26 +- tests/test_sandbox_enforcement.py | 48 +- tests/test_sandbox_policy.py | 24 +- tests/test_templates.py | 115 ++- tests/test_worker.py | 466 ++++++++++ 39 files changed, 7688 insertions(+), 572 deletions(-) create mode 100644 src/lightcone/engine/assets.py create mode 100644 src/lightcone/engine/dataset.py create mode 100644 src/lightcone/engine/identity.py create mode 100644 src/lightcone/engine/materialize.py create mode 100644 src/lightcone/engine/plan.py create mode 100644 src/lightcone/engine/templates/files/data-README.md.tmpl create mode 100644 src/lightcone/engine/templates/files/datalad-config.tmpl create mode 100644 src/lightcone/engine/templates/files/gitattributes.tmpl create mode 100644 src/lightcone/engine/worker.py create mode 100644 tests/test_assets.py create mode 100644 tests/test_dataset.py create mode 100644 tests/test_identity.py create mode 100644 tests/test_materialize.py create mode 100644 tests/test_plan.py create mode 100644 tests/test_worker.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8903ee7..b45d8ee4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,12 +59,12 @@ jobs: from pathlib import Path from lightcone.engine.sandbox import detect from lightcone.engine.sandbox.policy import ( - EXEC_ALLOWLIST, _UTILITY_PATH, probe_policy, + EXEC_ALLOWLIST, _UTILITY_PATH, exec_policy, ) c = detect().capability print(f'mechanism: {c.kind} abi: {c.landlock_abi} {c.detail}') print(f'utility search path: {_UTILITY_PATH}') - policy = probe_policy(Path(tempfile.mkdtemp()) / 'proj') + policy = exec_policy(Path(tempfile.mkdtemp()) / 'proj') granted = set(policy.execute) for name in EXEC_ALLOWLIST: found = shutil.which(name, path=_UTILITY_PATH) diff --git a/CLAUDE.md b/CLAUDE.md index f2205c78..4ada5042 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,6 +15,12 @@ hooks, or plugins. lightcone-cli depends on ASTRA. The `astra` CLI handles spec operations; the `lc` CLI handles execution. +That split is enforced in code, not just described: everything about what +a spec *means* — scoping, `from:` references, conditional outputs, +universe resolution, the recipe placeholder grammar — is answered by +`astra.resolve` and validated by `astra.validation` before lc acts on it. +lc's whole ASTRA surface is ten functions; see Key Invariants (layer 4). + ## ⚠️ This repository is a clean rebuild in progress The codebase is being **re-added layer by layer** on top of the normative @@ -39,13 +45,17 @@ speculatively. | # | Layer | State | |---|-------|-------| | 1 | **Project scaffolding** — `lc init` | ✅ **done** | -| 2 | Environment layer — `env_version`, lock scan, manifest schema | ⬜ | +| 2 | **Environment layer** — `env_version`, lock scan, manifest schema | ✅ **done** | | 3 | The `lc` entrypoint — launcher: discover → mode-detect → scrub `UV_*` → converge → delegate | ⬜ | -| 4 | Fabric — `lc materialize`, worker sequence, mid-run relock gate | ⬜ | +| 4 | **Fabric** — `lc materialize`, worker sequence, mid-run relock gate | ✅ **done** | | 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | | 6 | Container hatch — `[tool.lightcone.image]`, generated Containerfile, `lc build` | ⬜ | | 7 | Venues — Perlmutter, hub/GKE, Cloud Build | ⬜ | -| 8 | `lc status`, `lc verify`, WRROC export | ⬜ | +| 8 | `lc verify`, WRROC export | ⬜ | + +`lc status` landed with the invalidation model rather than at layer 8: +once an output can be *behind*, something has to say which ones are, and +the verb is the same classification walk `--check` already does. Layer 5 landed **out of order**, ahead of 2–4: `lc run` is the spec's *probe* verb, and a probe has no output, so it needs neither manifests @@ -115,7 +125,8 @@ recipes run under Landlock/ execution world: driver, workers, Identity: `env_version = sha256(uv.lock ‖ .python-version ‖ canonical install-settings ‖ canonical [tool.lightcone.image] ‖ Containerfile.extra -hash)`, sitting inside `code_version`. Every output records what +hash)`, recorded beside `definition_version` rather than folded into it. +Every output records what enforcement it actually ran under (`hermeticity`). See spec §1–§3, §7. ### Namespace contract @@ -139,15 +150,21 @@ src/lightcone/ # namespace — NO __init__.py ├── _sandbox_exec.py # the Landlock shim — stdlib only, zero lightcone imports ├── cli/ # the CLI only: flags, rendering, exit codes │ ├── __init__.py # exposes main(); the launcher hooks in here (layer 3) -│ └── commands.py # lc init, lc run +│ └── commands.py # lc init, lc run, lc materialize └── engine/ ├── __init__.py # docstring only ├── project.py # what a project is: convergence + discovery + ├── dataset.py # the git + git-annex seam: how a project stores + ├── identity.py # env_version, definition_version, the lock scan + ├── assets.py # an output: its directory, its manifest, its state + ├── plan.py # the spec, read as a graph of tasks + ├── worker.py # making one output; also the `python -m` entry point + ├── materialize.py # the driver: dirty gate, Dask, the save/restore loop ├── run.py # what `lc run` is: the probe + the uv hop ├── sandbox/ # the exec boundary │ ├── __init__.py # the public surface (detect, run, scope, the types) │ ├── model.py # Policy · Capability · Attestation · Backend protocol - │ ├── policy.py # what a probe may touch + │ ├── policy.py # what a probe and a recipe may touch │ ├── boundary.py # detect() + run(): the mechanism-blind half │ ├── landlock.py # Linux backend │ ├── seatbelt.py # macOS backend @@ -213,13 +230,31 @@ command callbacks and builds the rich console lazily, so `lc --help` and shell completion pay for neither. Keep this up as verbs land: a module-scope engine import would make every invocation pay for the heaviest layer. +**The scaffold comes from `astra.scaffold`, not `astra.cli`.** Both export +`create_boilerplate`, and the second drags Click, Rich and the validation +stack — measured, 37 ms against 4 ms. `astra.scaffold` is stdlib-only +(checked: it pulls no linkml_runtime, click, rich, pydantic or +jsonschema), which is why this one astra import sits at module scope in +`project.py` where `astra.validation` and `astra.resolve` must not. + **Templates are files, not string literals.** `engine/templates/files/*.tmpl` -are package data, loaded through `importlib.resources`; `templates/__init__.py` -exposes one function per scaffolded file. Placeholders are +are package data, loaded through `importlib.resources`. Placeholders are `string.Template` (`${name}`) — **not** `str.format` — because several templates legitimately contain braces (TOML tables, MyST `{astra}` roles). Substitution is strict, so a missing key raises. +**A template gets a function only when there is something to decide** — a +value the caller supplies (`pyproject`, `datalad_config`, `index_md`) or a +merge policy for a file the user already owns (`gitignore_repair`, +`gitattributes_repair`). Everything else is read by name: +`templates.read("myst.yml.tmpl")`, or `partial(templates.read, …)` where +convergence wants a thunk. This reverses the earlier "one function per +scaffolded file" rule, which had five of the module's twenty functions +doing nothing but rename a file — a second place for the name to be +wrong, and one the type checker cannot catch. The two `*_repair` +functions stay named because `_Converger.file` hands them the text alone, +so the template name has to be bound before the call site. + **No engine constants for the environment.** The scaffolded `.python-version` is the interpreter `lc` itself is running on, and `requires-python` is lightcone-cli's own `Requires-Python`, verbatim. These @@ -247,6 +282,10 @@ user owns: | `.python-version` | The exact patch of the interpreter `lc` is running on | | `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | | `.gitignore` | One managed block of patterns; convergence ensures each is present | +| `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository | +| `.gitattributes` | The storage policy: what git-annex holds and what git carries. Line-managed, like `.gitignore` | +| `.datalad/config` | A `datalad.dataset.id` UUID, generated once. lc never reads it back | +| `data/` + `README.md` | Where declared inputs live; annexed, and committed before anything computes on them | | `results/` + `README.md` | Where outputs land; the README states the materialize-don't-hand-write contract | | `myst.yml`, `index.md` | Template MyST report referencing `astra.yaml` *by path* | @@ -257,7 +296,9 @@ user owns: layout, and the boilerplate's `python src/main.py` is a placeholder. Universes are discovered by `glob("*.yaml")`, which is empty-not-error on a missing directory. `tests/test_project.py::test_a_clone_of_a_converged_project_is_converged` - pins this: a clone must need nothing but `.venv`. + pins this: a clone must need nothing but `.venv` and `git annex init`. + Those two are the exemptions, and for one reason — they are local state + git does not clone. - **Convergence, not scaffolding.** Each item is created if missing, offered to a conservative `repair(text) -> str | None` hook otherwise, and left alone when the hook returns `None`. `--check` computes the @@ -312,19 +353,694 @@ user owns: nothing convergence invokes is allowed to fail silently. Every uv invocation carries an explicit `--project` — uv's own walk-up discovery is never trusted (spec §4). -- **uv is required; git is optional.** An absent uv is a refusal (it is - the environment substrate); an absent git is simply nothing to converge, - since a project without version control is still a valid project. +- **uv, git and git-annex are all required.** Each is a refusal, not a + warning: uv is the environment substrate and git + git-annex are the + storage substrate, and results are versioned in the repository, so there + is no useful project without any of them. git is the one tool uv cannot + install and the single admitted exception to a uv-installable stack; + git-annex ships as a wheel and is therefore a locked dependency, which + makes its wheel platforms the CLI's install floor. + (This reverses layer 1's original "git is optional" — a project without + version control had nowhere to put a result.) +- **Two questions about git, not one.** `_in_repository` is a pure + filesystem walk-up and answers for a directory that does not exist yet, + which is what check mode needs. `_can_ask_git` adds `is_dir()`, because + every git *invocation* needs an existing working directory — inside an + enclosing repository the walk-up says "in a repository" for a + directory that is not there, and running git in it raises + `FileNotFoundError` out of `Popen` rather than answering anything. - **`git init` checks for an *enclosing* work tree**, not just a `.git` in the directory, so `lc init subdir/` inside a repository can't create a nested one. (`.git` may be a file — linked worktree or submodule — so the - test is `exists`, not `is_dir`.) + test is `exists`, not `is_dir`.) The annex asks git-annex's own question, + `git config --get annex.uuid`, so an enclosing repository that already + has one is adopted rather than re-initialized. - **`lc init` has exactly two flags**, `--check` and `--json`. `--no-git` and `--no-sync` were deleted: neither had a caller outside the test suite, `--no-git` was a workaround for the missing enclosing-repo check, and `--no-sync`'s real home is containerized mode (layer 6), where the host `.venv` is inert. Don't add a flag whose only user is a test — stub `project._run` instead. +## Key Invariants (storage) + +**Results are versioned in the project's own repository**, on the DataLad +model: **git carries the pointers and the history, git-annex carries the +bytes.** `engine/dataset.py` is the whole seam, and every command in it +goes through `project._run` — the same one convergence uses, so there is +one monkeypatch point and the `tools` fixture already covers git. + +**git-annex is a wheel, and that sets the CLI's install floor.** +`manylinux_2_34` on x86_64/aarch64, macOS 14+ arm64 or 15+ x86_64, +win_amd64 — and **no sdist**, so a host below the floor fails to install +rather than building from source. Because git-annex is a *hard* runtime +dependency and `lc init` pins `lightcone-cli` into every project, that +floor gates installing lc at all, including `lc run`, which never touches +the annex. If it ever bites a real user, that coupling is the thing to +revisit — an extra, or a probed requirement like git — not the floor. + +**Perlmutter clears it** (checked 2026-08-19, login node): the wheel +installs and `git annex version` runs. That was the open question this +stack was most likely to fail on, so it is written down rather than +re-derived. Two things it does *not* settle, both layer 7's: + +- **Lustre/GPFS behaviour is unmeasured.** arXiv:2505.06558 documents + symlink, many-small-files and inode pressure on parallel filesystems. + Correctness is not the worry; cost is, and one output is one file plus + one hard link to its object. Results are already thin (above); adjusted + (unlocked) branches remain the untried lever, as does whether Lustre + makes the hard link behave differently. Time `git annex add` and + `git annex get` on `$SCRATCH` and `$CFS` before designing around + either. +- **`git-annex-shell` is not on the default remote PATH.** `git annex get` + from a laptop dispatches to it over a non-login ssh session, and the + wheel installs it into `.venv/bin`. Configuration, not design: + `git config remote..annex-shell `. + +**`filter=annex` is what makes an ordinary `git add` do the right thing.** +`git annex init` configures the smudge/clean filter itself; the template +adds `* filter=annex`, and `annex.largefiles` then decides what counts as +content. So a researcher types `git add -A . && git commit` — the same +git they already know — and a 200 KB input lands in the annex as a +101-byte pointer while `src/main.py` and the manifests stay real. Nothing +lc scaffolds, prints or documents asks anyone to run a git-annex command, +and `dataset.save` does not run one either. + +**The `annex.largefiles=nothing` default is load-bearing.** It has to come +first, with outputs and inputs opting out; last matching line wins. +Without it `filter=annex` routes *everything* into the annex, analysis +code included. `tests/test_dataset.py::test_analysis_code_stays_in_git_and_stays_writable` +pins it against a real annex. + +**Manifests stay in git, deliberately.** `**/.lightcone-manifest.json` is +exempted back out of the annex so it is readable on a clone that has +fetched no annex content at all — which is what lets `lc materialize +--check` classify a whole project on a laptop that holds none of the +bytes. + +**An unfetched file exists, and that is the trap — in two shapes.** +`assets.data_version` refuses both with `ContentNotFetchedError`, naming +`git annex get`, and it checks for both regardless of which one lc's own +writes produce: `annex.thin` and `git annex lock` are the researcher's to +set on their clone, so the shape a file arrives in is not ours to assume. + +- **Unlocked** — what `filter=annex` writes, hard-linked or copied alike. + A clone without content holds a ~100-byte *pointer file* where the data + would be: it exists, it is readable, and hashing it yields a perfectly + well-formed digest of the wrong thing. The test is git-annex's own + `isPointerFile` — a `/annex/objects/` prefix within the first 32 KiB. + Measured before it was fixed: the same input hashed differently on a + clone, silently. +- **Locked** — a symlink into the object store, which without the content + dangles. This one is quieter and worse: `is_file()` answers False for a + dangling symlink, so a directory walk filtered on that alone drops the + absent file from the digest *without a word*, reporting a hash of + whatever subset happens to be present. Only dangling symlinks are added + back to the walk — one that resolves to a file already is one, and one + that resolves to a directory is not content. + +`--check` catches the error and reports "not in this clone" rather than +"changed", because the two are different facts and only one of them means +a rebuild. + +**PATH is part of the contract.** `git annex` is not a builtin — git +dispatches it by searching `PATH` for a `git-annex` executable, and +`uv tool install lightcone-cli` links only lc's own entry points into +`~/.local/bin`. So `dataset.put_our_bin_first()` **prepends** +`Path(sys.executable).parent`, idempotently, before any git call. Prepend, +never append: a system copy winning would make the version the project's +lock records a fiction. `project.require_git_annex()` probes after the +prepend and by the name git itself searches for. + +Measured, not assumed: with only the tool's `lc` on `PATH` — exactly what +`uv tool install` produces — `git annex version` fails, and with the +prepend disabled `lc init` refuses. Under `uv run` it is a no-op, because +uv already fronts the project's `.venv/bin`. + +**A project can sit inside a larger repository, so `dataset.status` is +scoped and relativised.** `lc init subdir/` adopts an enclosing work +tree rather than nesting a new one — that is a supported layout — and +`git status --porcelain` otherwise covers the *whole* work tree and names +paths from *its* root. Unscoped, an edit anywhere else in the repository +refuses every run in the project, and lc's own writes arrive as +`subdir/results/…`, which the refusal's path-class split cannot +recognise. So the call carries `-- .` and the `rev-parse --show-prefix` +is stripped off each path. A wholly untracked project collapses to `.`, +which is git's own summary of it and exactly what the refusal should tell +someone to add. + +**git needs an identity, and that is asked before a run, not at the first +commit.** A fresh container or CI image has none — the case an agent-run +CLI meets most — and discovering it at `dataset.save` would throw away +whatever the recipe had already computed. `require_committer` asks +`git var GIT_COMMITTER_IDENT`, which is the question a commit itself +asks: an identity resolves from `user.email`, `EMAIL`, the author and +committer variables, or three levels of config, and a probe that +reimplements that lookup is one that can disagree with the thing it +stands in for. Measured: `git annex init` does *not* need one (it +tolerates the missing identity and exits 0), so `lc init` is not gated on +it — only the verb that commits is. + +**The ignore probe asks about the directory as a directory.** +`check-ignore` is run with a trailing slash (`results/`), because the rule +that matters most — the `results/*` an older lc scaffold wrote — ignores +the directory's *contents* and does not match the bare name at all. And +with `--no-index`, which asks about the *rules* rather than the index: +without it git answers "not ignored" for anything already tracked, which +is exactly the project where someone committed one result by hand and left +the rule for the next. The item records **`blocked`**, not a warning — +`git add` skips ignored paths in silence, so a materialize would report +success and commit nothing — and it names `::`, +because `.gitignore` convergence only ever appends and cannot fix this. + +**`.gitattributes` can be *unrepairable*, and that is a second blocked +item.** The file is last-match-wins, and a repair only appends — so a +project that already carries `results/** annex.largefiles=anything` +without the `*` defaults gets `* annex.largefiles=nothing` appended +*below* it, and every result then lands in git as a plain blob while the +report says repaired and converged. `templates.gitattributes_disorder` +judges **the text a repair would produce**, not the text as it stands (a +file missing the defaults is in order until they are appended), and only +compares lines setting the **same attribute** — `* filter=annex` landing +under an `annex.largefiles` line is not disorder, and treating it as one +blocks a perfectly good file. Blocked rather than fixed, for the reason +the ignore rule is: reordering a file the user wrote is not something +append-only convergence gets to do, so it names the line and prints the +order the managed lines belong in. + +**An lc project is a DataLad dataset from birth.** `.datalad/config` +carries a `datalad.dataset.id` UUID, generated once by `lc init` and never +regenerated — it identifies the dataset across clones and siblings. +Verified, not assumed: `datalad status` recognises a freshly scaffolded +project with no `--force` adoption step, and `Dataset('.').id` is the UUID +we wrote. The reciprocal is a standing non-goal: **lc never requires +datalad, never imports it, and never parses `.datalad/`.** A researcher +who wants `datalad get`, siblings or RIA stores runs `uv add datalad` in +their own project. + +**Annexed files are ordinary writable files.** `filter=annex` keeps them +unlocked, so an output can be overwritten in place and nothing needs to +remove it first. (This reverses the earlier symlink model, where a +rebuild hit `PermissionError` on a path that looked perfectly ordinary.) + +**Results are committed thin; declared inputs are not.** `dataset.save` +passes `-c annex.thin=true` to its `git add`, so a result is hard-linked +to its annex object instead of copied — measured, 39 MB to 20 MB for one +20 MB file, since an unlocked file otherwise exists both in the tree and +in the object store. + +It is safe *there* and nowhere else, and the reason is specific: +**thin's hazard is an in-place write.** With the file hard-linked to the +object, one `open('r+b')` rewrites the object under the key that names +it — measured, the committed version becomes unrecoverable (`git annex +get` reports no known copies) and `fsck` only notices later, moving it to +`.git/annex/bad/`. lc never writes in place: a worker *removes* an output +directory before rebuilding it, and an unlink merely drops the link +count, leaving the object intact (measured — an old version still checks +out clean afterwards). So the flag is passed **per-add and never written +to the repository's config**, because repo-wide it would reach `data/`, +which researchers add with their own `git add` and whose tools — +`h5py.File(p, 'r+')`, astropy `mode='update'` — very much do open files +for update. + +Thin-ness is *not* recorded anywhere: it is local working-tree state, so +a clone re-decides at `git annex get` time. That is why detection has to +handle every shape rather than the one we write. + +**`restore` is scoped, and asymmetric.** `git clean -qfdx` always, plus +`git checkout HEAD --` **only when HEAD has the path** — a first +materialization has nothing to go back to, and the naive form exits +nonzero on the pathspec. Never `git checkout HEAD -- .`: a failed task +must not discard edits made elsewhere while the graph was running. + +## Key Invariants (layer 2) + +**Two hashes, and they answer different questions** (`identity.py`). +`definition_version = sha256(recipe ‖ canonical decisions)` is what the +spec says an output *is*. `env_version = sha256(uv.lock bytes ‖ +.python-version bytes ‖ canonical install-settings JSON)` is what it ran +under. + +**`env_version` is not part of `definition_version`, and that is the whole +shape of the model.** An environment moves for reasons that have nothing +to do with any particular output — one `uv add` for one plotting script +rewrites the lock for the entire project — while a research artifact costs +hours to remake and has usually already been looked at. So an environment +edit stales nothing; it makes an output **behind**, which is reported and +left alone. (This reverses the original design, where `env_version` sat +inside `code_version` and therefore staled every output in the project on +any dependency change, and every output in every project on an lc upgrade. +That was recorded as an accepted cost; it was the bug.) + +Nothing is lost by not rebuilding: the manifest records the environment +*and* the commit, and that commit's `uv.lock` reconstructs the +environment exactly. Over-sensitivity in `env_version` is affordable +precisely because it no longer spends compute. + +**Both are length-framed.** Concatenating fields raw lets a boundary shift +between them produce one digest from two different inputs. `_frame` +writes label, length, then bytes. The test lives on `env_version`, where a +shift is actually constructible — two raw file bodies, adjacent — rather +than on `definition_version`, whose second field is canonical JSON and +cannot be shifted into. Mutation-checked: breaking `_frame` fails it. + +**The lock's raw bytes, not a parse.** A comment reflow moves +`env_version`, deliberately: the alternative is a parse of our own that +can silently disagree with uv about what the lock means, and +over-invalidation is the failure that costs time rather than correctness. + +**The install-settings list is closed** (`_INSTALL_SETTINGS`), and every +key is hashed whether or not the project sets it. A setting outside the +list must not move the hash, or every uv config nicety stales the world; +a setting whose value merely *matches* today's default must, because that +default can change under a project that never said anything. + +**The settings are read where uv reads them, and `uv.toml` *replaces* +`[tool.uv]`** rather than merging with it (measured, uv 0.12.5 — uv warns +about the pair itself, and `tool_warnings()` already lifts that into the +report). Reading both would hash settings uv is ignoring, reporting two +environments where uv installs one; reading only `pyproject.toml` left a +`uv.toml` free to change what gets installed without moving `env_version` +at all. Only the *values* are hashed, never which file supplied them — +two projects that install the same artifacts are one environment however +they spell it. `scan_lock` reads `default-groups` through the same +function, because it is asking uv's question too. + +What this deliberately cannot reach is **user-level configuration** +(`~/.config/uv/uv.toml`), which uv merges in underneath the project's own +(measured). That is machine state, not project state: hashing it would +make one commit answer differently on two hosts, so a colleague's clone +would report every output as behind. The residue is real and unguarded — +a user-level `no-build` changes what a sync installs and nothing records +it. + +**The git commit is recorded, never hashed, and never a signal.** It goes +in the manifest so the code that produced a result stays recoverable. It +is out of `definition_version` because git has one sha for the whole +tree — hashing it would stale every output in the repository on a README +edit — and it is not a `behind` trigger for the same reason: it moves on +every commit, and a signal that is always on is not one. + +The honest consequence, which is a **decision and not an oversight**: +editing `src/fit.py` remakes nothing, because the recipe *string* is +unchanged. Per-output code invalidation is available by declaring the +source files as ASTRA inputs, and that declaration is deliberately the +researcher's to make rather than something lc infers — for an expensive +output, "the code moved and the result still stands" is a legitimate and +common position. Do not add a heuristic that scans a recipe's command line +for repo paths. + +**Names are compared in PEP 503 form.** uv writes the normalized name +into `uv.lock`; `pyproject.toml` carries whatever the author wrote, and +`project_name()` keeps `_` and `.`. Raw comparison makes a packaged +project called `my_project` fail to recognise *itself*, and the lock scan +then refuses the whole run over the project's own code. + +**The lock scan refuses only what cannot be audited.** A path, directory, +or editable dependency records *where* it was rather than *what was in +it*, so two syncs of one lock can install different code while every hash +agrees they are identical — that is a refusal. A registry package with no +wheel is a *report* (identity covers the sdist, not the build of it), and +a non-default dependency group is *advisory*. The project's own package +is exempt: it is the project, and the repository already records its +bytes. + +## Key Invariants (layer 4) + +**What the spec *means* is ASTRA's to say, and `plan.py` asks rather than +re-derives it.** `astra.resolve` settles each universe's decisions, +resolves every output's inputs to what supplies them, drops the outputs +whose `when:` does not hold, and renders the recipe grammar. Scoping, +`from:` aliases, sub-analysis nesting and the placeholder grammar are all +*read* here, never re-implemented — a second implementation of one +specification is how the two start disagreeing, and ours had: + +- it could not build `examples/iris_pipeline` at all, ASTRA's own + canonical nested example; +- it ignored `when:` on an output, so a universe ran a recipe the spec + excludes and committed a manifest for it; +- it invented a dotted input id (`inputs: [hod.mass_function]`) and an + implicit "same universe id" fallback for sub-analyses, neither of which + `astra validate` accepts — `UniverseNode.universe` names it explicitly. + +`lightcone.engine.plan` therefore holds only what execution adds: +`Task`, `Graph`, and the mapping of resolved outputs onto directories, +edges and `definition_version`. When something about the spec's meaning looks +wrong, the fix is in astra-tools, not here. + +**A spec ASTRA rejects never reaches a recipe.** `build` runs +`validate_analysis_schema`, `validate_analysis_file` and +`validate_universe_file` before resolving anything, and refuses with +ASTRA's own errors. This is a contract requirement, not a courtesy: +resolution answers what a *valid* spec means and does not re-check that +it is one, so without the gate an invalid spec surfaces later as a +missing decision or an unresolvable input — blaming the run for a fault +in the file, far from the line at fault. It caught three of lc's own +test fixtures the first time it ran. + +**The layout is flat and path-addressed.** `results///`, +`data/` for declared inputs, and the path in a rendered recipe *is* the +path on disk — no staging, no scratch, no relocation. The `output_id` is +ASTRA's **qualified** id, so a sub-analysis output lands at +`results//./` and one addressing scheme spans +however deep the spec nests. Nesting is not capped: the dot separator is +unambiguous because ASTRA ids match `^[a-z][a-z0-9_]*$`. + +**One rule names a path, and both the recipe and the run record use it** +(`plan.declared_path`). Project-relative inside the tree, absolute +outside it, never resolved. A declared input may name an absolute +`source:` — ASTRA allows it and an HPC project pointing at a shared +catalog is the obvious case — and there is no project-relative spelling +for one, so a bare `relative_to` was a `ValueError` traceback out of +`lc status`, `--check` and `materialize` alike. Two copies of this rule +existed; the second is what made the first easy to miss. + +An input outside the project is **reported, not refused**: its bytes are +hashed into the manifest like any other, so a change to it still +cascades, but it is not in the repository and the commit recording the +output cannot bring it back. That is a weaker promise than the rest of +the layer makes, and saying so is the whole obligation — the same +treatment `sdist_built` gets. + +**Two universes cannot share an id.** The id names a directory under +`results/`, and the graph is keyed on `(universe_id, output_id)` — so the +second file simply replaced the first and one universe's outputs went +missing with nothing said. The way in is the natural one: copy +`baseline.yaml`, edit the decisions, forget the id inside. `build` +refuses, naming both files. + +**Because the path is composed, `output_dir` refuses an id that is not one +path component.** An empty universe or output id collapses +`results//` onto a *parent* — `results/` itself, for two — and the +worker empties that directory before running a recipe in it, so the +consequence of an unchecked id is deleting every other universe's +outputs. A `/`, `.` or `..` is refused for the same reason. This is the +guard that lets the reset stay a whole-directory operation. + +**The reset takes the whole directory, and cannot take a named list.** A +recipe declares an output *id*, never filenames, so there is no set of +"expected files" to remove — and a previous run that crashed can have left +anything in there, which would otherwise survive into this run's +`data_version` as though the recipe had written it. What bounds the blast +radius is the guard above, not a narrower delete. + +**Dask owns the ordering.** Every task is submitted with its upstream +futures as arguments, so the dependency order, the parallelism, and the +scheduling all fall out of the argument graph. There is no ready-set loop +and no hand-rolled topological sort in the execution path. +`Graph.order()` exists for the read-only walk, which has to classify a +task after everything upstream of it — and for submitting in an order +where a task's upstream handles already exist. + +**The driver owns git, alone.** Workers execute and return a +`TaskResult`; the driver commits, in one thread, as results arrive. +Concurrent git operations on one repository race on the index lock — +this is the `datalad-slurm` schedule/finish split, and it is not a +preference. + +**A dependent does run while its upstream is being annexed, and that is +safe.** Dask releases a task the moment its upstream's *worker* returns, +milliseconds before the driver finishes committing that upstream's +directory — so a recipe reads an input directory while git's clean filter +is moving its content into the annex. Measured before relying on it: git-annex +hard-links the content into the object store and then renames the symlink +over the file, so the path never stops existing and never holds partial +bytes (448 concurrent full-content reads across 24 MB: no missing paths, +no short reads, no wrong bytes). Don't "fix" this by moving the save into +the task — that is what puts git back in the workers. + +**A declared input is hashed once per run** (`assets.Versions`). Without +it a multiverse spec re-reads the same bytes once per `(universe, +output)` that names it — eight universes times four outputs sharing one +catalog is thirty-two full hashes of one file, paid again on the +"nothing to do" path because classification needs the digest before it can +skip. Memoizing is sound for exactly as long as a run lasts: a run +refuses to start on a dirty tree, and the only in-tree path a recipe may +write is its own output directory. A class rather than a closure, so it +keeps one dict alive and not whatever scope built it; and deliberately +unlocked, because a lock would serialise every hash and would not survive +being handed to a worker in another process. + +**HEAD is read once per run, by the driver, and handed down.** The driver +commits each output as it lands, so HEAD *moves* during a run: a +per-task `dataset.head` would stamp later manifests with a commit this +same run created, and whether it did would depend on whether a recipe +finished before or after the previous save. Nondeterminism in a +provenance field is worse than either answer. + +**The worker never raises, and that is enforced at the unit boundary.** +It returns `ok`, `current`, `behind`, `failed`, or `blocked`. A task whose upstream +did not report — failed, or never finished at all — returns `blocked` +without running. Raising would make Dask re-raise in the driver and abort +every task in flight, and reporting all independent failures in one run +is most of what owning the loop buys. `worker.materialize` wraps the +whole unit, so the contract holds for failure modes nobody enumerated; +the one inner guard that remains exists because "your recipe failed" and +"your recipe worked and we could not record it" deserve different words. + +**`data_version` is computed in the worker, before anything is staged.** +The dependent's argument *is* the upstream worker's return value, so the +digest has to exist at return time — when the files are still untracked +and unannexed. Deriving it from `git annex find` instead was the first +instinct and is wrong twice: nothing is annexed yet, so every output would +record `sha256([])` — one constant, silently disabling the whole chain +with green tests — and it would make the digest a function of +`annex.backend`. The annex backend is therefore not load-bearing and is +not pinned. + +**A skip returns the *recorded* digest, never a recomputed one.** On a +clone that has fetched no annex content the files are dangling symlinks, +and rehashing them would quietly report a different output. `--check` +obeys the same rule for the same reason — it reads each unchanged +upstream's manifest rather than hashing the directory, or it would report +a rebuild for a project that is entirely up to date. The honest +consequence: a hand-edited-and-committed output does not cascade in this +layer. Catching that is `lc verify`'s job. + +**Three states, and the line between them is the layer's whole shape** +(`assets.classify`). + +| state | means | what happens | +|---|---|---| +| `stale` | the artifact **contradicts** the project: the spec defines it differently than it was made, or it records deriving from bytes the project no longer holds | remade | +| `behind` | it is still exactly what the spec asks for; only the **environment** moved | reported, left alone | +| `current` | neither | nothing | + +The distinction is *contradiction* versus *circumstance*. A stale artifact +is mislabelled — what is on disk is not an instance of what the spec +declares — so keeping it would be a lie. A behind artifact is not wrong in +any way; its environment is recorded and its commit reconstructs that +environment, so remaking it buys nothing and can cost a week of +allocation. + +**`Verdict.calls_for_a_remake(refresh=)` is the one place that turns a +state into an action**, and it has three callers — the worker, `check`, +and the walk that feeds the cascade. `stale` always; `behind` only when +asked. Do not re-spell it inline; the third copy is where they start to +disagree. + +**One classification rule, and the two callers differ by one value** +(`assets.classify`). It compares `definition_version` against the +manifest, the declared input *set* against the recorded one, each recorded +`input_versions[…]` against the version it is handed, and finally +`env_version`. The set comparison is separate on purpose: +`definition_version` hashes the recipe and the decisions, neither of which +an input the spec no longer declares moves — so without it a dropped +dependency leaves the output reporting current forever. The worker hands +live digests; the read-only walk hands `None` for anything it has already +decided will run, meaning "this is going to change". That single value is +the entire difference between them — one input, conservatively chosen, not +a second body of logic — the same discipline as layer 1's +`converge(write=False)`. It is the one place in the layer where a bug is +quiet rather than loud, which is exactly why it may not have two +implementations. + +**`stale` wins over `behind` when both apply.** The artifact is going to +be remade either way, and reporting "left alone" about something the run +is about to rebuild is the one wrong answer. + +**`behind` does not propagate, and a behind upstream still feeds its +dependents.** It says the environment moved, not that the bytes are wrong, +so `TaskResult.usable` includes it and `data_version` flows on unchanged. +Propagating it would mark one old artifact's entire downstream forever and +kill the signal; the mosaic — how many environments and commits an output's +ancestry spans — is a project-level question, not a per-output flag. + +**`--refresh` widens a run by exactly one state.** It is not an escape +hatch (it asks for *more* work, not less), and it must not become a +rebuild-the-world flag: a `current` output stays current under it. There +is deliberately no flag in the other direction — nothing suppresses the +rebuild of a stale output, because deleting the directory is the user's +own file operation and is stronger consent than a flag. + +**`up_to_date` does not count `behind`, and is not true of a run that +failed.** `lc materialize --check` is a gate, and a project of curated +results would otherwise never pass it again — that is the `behind` half. +The other half is that `made` stays empty when *every* recipe fails, so +`not made and not planned` reported "nothing to do" over a list of +failures. It is `ok and not made and not planned`; those two are the +first keys of the JSON report and are what an agent branches on. + +**`lc status` reports; `--check` gates.** Status always exits 0 — a state +is not a failure — and it is the only verb that shows the commit each +output was made at, for every state and not only the interesting ones. +It reads manifests and hashes inputs; it runs nothing, commits nothing, +and does not mind a dirty tree, because the moment you most need to know +what state a project is in is when it is not clean. Two verbs answering +the same question with different exit codes is how a script comes to +depend on the wrong one, so keep the split sharp. + +**A read-only verb never tracebacks, and an entry point never +misattributes.** `lc status` and `--check` read projects that are *in a +state* — that is what they are for — so anything `_predicted` cannot read +becomes the same `None` an absent input already produces: it will be +remade, and the recipe is where that failure belongs with a real error. +(The concrete way in: `data_version`'s directory walk keeps dangling +symlinks deliberately, so an unfetched annexed file cannot drop silently +out of a digest — one that is *not* an annex link then reaches `open()`.) +The same rule at the other end: `worker.main` is what every +`[DATALAD RUNCMD]` record names, and its "no output ``" message covers +the task lookup **only**. It once wrapped the whole body, so a `KeyError` +raised anywhere inside astra's validation or resolution was reported as a +bad target — a rerun misdiagnosing itself, at the one place nobody is +watching. + +**`git_sha` in a manifest is the commit the run *started* at**, not the +commit the run went on to create. It is the code that produced the output. +A test that reads `dataset.head()` after materializing and expects a match +is asserting the wrong thing. + +**One uv hop, one spelling** (`project.uv_prefix(root, *, sync)`). The +flags are also what `run_record` writes verbatim into every commit +message, so a second copy is a second thing that can drift out of the +provenance. The only thing callers disagree about is `sync`: a probe +converges the environment it is about to describe, a recipe must not, or +every concurrent worker writes the same `.venv`. + +**A run syncs the environment; it does not report on it.** `uv run +--locked` asserts only that `uv.lock` still matches `pyproject.toml`, and +workers pass `--no-sync` — so a lock edited without a sync would leave +recipes importing packages the lock does not describe while every manifest +recorded the *new* lock's `env_version`. Measured: the recipe imported +`packaging 26.3` under a lock saying `24.2`, and uv accepted it silently. +`materialize()` calls `project.sync()` before anything else, so the state +is made impossible rather than detected. `--check` needs neither: +`env_version` is the lock's bytes, so a drifted `.venv` cannot change what +it answers. + +**A run takes every core, and there is no flag to say otherwise.** How +much of a machine a run may use — and which machine — is one question, and +it belongs to a declared execution backend rather than to a `--jobs` knob +only a `LocalCluster` could honour. + +**A dirty tree is a refusal, and `--check` is exempt.** Every +materialization is committed with the code that produced it, so a run that +started dirty could not say what that code was. Check mode does not +refuse: reading the state of a project before deciding what to commit is +what it is for. The refusal splits by path class because the remedies are +opposite — work the researcher owns gets committed, and anything under +`results/` is lc's to write and is wreckage to discard. + +**A run leaves the tree exactly as clean as it found it.** The worker +resets the output directory *before* executing, so a failed recipe, a +crash, or a Ctrl-C would otherwise leave tracked files deleted or +half-written — and the next run's refusal would tell the user to commit +truncated, manifest-less garbage into `results/`, destroying the one +property the layer exists for. So `ok` → `dataset.save`, and `failed`, +`blocked` or never-reported → `dataset.restore`, with the consumption +loop in a `try/finally` so an interrupt restores whatever is still +outstanding. This is what makes the dirty-tree refusal survivable rather +than a trap. + +**The run record names declared paths, never resolved ones.** Every +declared input under `data/` is an annex symlink, so a `Path.resolve()` +in the record's `inputs` writes `.git/annex/objects/SHA256E-…` — the +storage rather than the input, and a path nothing can `datalad get`. This +shipped once; `_inside` is lexical now. + +**The run record is genuinely re-runnable, and its `cmd` is the worker +module.** `python -m lightcone.engine.worker /`, +behind `uv run --locked --project .` — which reconstructs the environment +from *that commit's* lock, and therefore the exact engine that produced +the output. The bare recipe would reconstruct nothing lc adds (no locked +environment, no boundary, no gates, no manifest) and would commit bytes +the identity model never produced; `lc materialize` cannot be it either, +because `datalad rerun` removes the declared outputs first and that +dirties the tree materialize refuses to start from. + +**The worker module is not an `lc` verb and not a console script.** It +makes the output unconditionally, commits nothing, leaves the tree dirty by +design. `lc --help` advertising it would hand people a footgun, and a +`[project.scripts]` entry would put it on `$PATH` through +`uv tool install`. `lightcone/_sandbox_exec.py` is the same shape for the +same reason. Keep it cheap to import — **no click, no rich** — it is on +the path of every task and every rerun; two tests pin that. + +**The record's format is datalad's, so it is tested through datalad.** +`get_run_info` matches with a regex and returns `(None, None)` on any +mismatch, after which `rerun` says "no command; skipping" and **exits 0** — +a golden test over our own JSON would stay green through a silent break. +So the suite asserts through datalad's parser *and* runs a real +`datalad rerun`, and `datalad` is a **dev** dependency only. Nothing in lc +imports it. + +**A policy is a description, and `scope()` owns every policy's lifetime.** +`exec_policy` creates no directory — the worker resets the output +directory, so the whole of an output's lifecycle stays in the module that +owns it, and a policy that could not be built without touching the +filesystem would be the impurity `wrap` is already pinned against. +`sandbox.scope` takes a *built* policy, so the `rmtree` of the private +`$HOME` has one owner. + +**There is one policy, `exec_policy`, and a recipe gets exactly what a +probe gets.** The tree is read-only apart from `results/`, for both. So +layer 5's promise is not "in the direction that matters" — it is simply +true: a command that works under `lc run` works as a recipe, with nothing +in between to reason about. + +A recipe is deliberately **not** narrowed to its own output directory. +That would be a second answer to "are these bytes what produced them", +and the manifest's `data_version` is the first — content-addressed, +checked by `lc verify`, and the only one that survives a rebuild on +another machine. Two mechanisms for one guarantee is one more than can be +kept honest, and the sandbox's is the one that cannot travel. + +The residue, recorded rather than argued away: a cross-write that lands +*before* the victim task hashes leaves a manifest that is self-consistent +and wrong, which no checksum can see. It needs concurrent tasks and a +hardcoded sibling path, and the threat model here is accidental leakage +rather than a hostile recipe — but `lc verify` will not catch that one, so +do not describe it as covered. + +**`cluster_for_run()` is the seam, and it is two methods wide.** +`submit(fn, *args, key=…)` and `completed(handles)`. That is all the +driver asks of a scheduler and all a venue has to supply — which is what +lets the suite run a graph inline in-process, and what will let something +larger than a laptop land behind it without the driver noticing. + +### Recorded deviations from the spec (layers 2 and 4) + +- **`git_dirty` is not written.** Spec §3 lists it; the start-of-run + refusal makes it constant. The limitation that comes with that, stated: + the check is at start of run while manifests are written per-output much + later, so a user who edits `src/fit.py` while a long graph runs gets a + manifest whose `git_sha` no longer describes the code that ran, and + nothing records it. +- **The manifest carries what this layer can honestly fill.** + `schema_version`, `output_id`, `universe_id`, `recipe`, + `definition_version`, `env_version`, `data_version`, `decisions`, + `input_versions`, `git_sha`, + `git_remote`, `lc_version`, `hermeticity`. Spec §3's longer list — + `uv_version`, `platform`, `python_build`, `worker_runtime`, `image`, + `dpkg_snapshot_sha256`, `sdist_built`, `env_snapshot`, `gpu_driver` — + is either layer 6's or attestation nothing here reads; it lands with the + verb that reads it. +- **`env_version` has three terms, not five.** The + `[tool.lightcone.image]` and `Containerfile.extra` terms are layer 6's, + and hashing an empty shape for them now would be foreshadowing. +- **A recipe is handed to `bash -c`.** ASTRA recipes are command lines, + not argv, and `bash` is in the exec allowlist — so it is granted by the + same rule that grants everything else a recipe may run. + ## Key Invariants (layer 5) **The seam is a pure argv rewrite.** Every mechanism is a @@ -467,6 +1183,16 @@ is the whole point of the environment overlay. Reading `pyvenv.cfg`'s guard is the fix; failing loudly on a layout nobody uses beats voiding the guarantee for the people who do. +**The two mechanisms disagree about the write root itself.** Landlock +follows POSIX — unlinking a directory needs write on its *parent* — so a +recipe granted write on its output directory cannot remove that +directory. Seatbelt's `(allow file-write* (subpath …))` covers the +directory node, and permits it. Found by CI, which is the point of +running one suite on both. Nothing depends on either answer (the worker +resets the directory anyway, and a recipe that removes it fails to record +its output), so the asymmetry is documented rather than papered over — +but a test that asserts one mechanism's answer will go red on the other. + **SBPL is last-match-wins; Landlock unions.** This asymmetry decides where a rule can live, and it cuts both ways. A later `(deny …)` can take back an earlier allow, which is how `_read_only_guard` takes back write @@ -635,9 +1361,15 @@ only checks that the guard is present. Verified empirically, not assumed. | To... | Read | Key patterns | |---|---|---| | Add the next layer | the spec (§11 = the layer ordering) | Land code + tests + deps together; update the layer table above. Docs are deliberately deferred | -| Change what a scaffolded file contains | `src/lightcone/engine/templates/files/` | Edit the `.tmpl`; add new ones to `TEMPLATE_NAMES` + a renderer | +| Change what a scaffolded file contains | `src/lightcone/engine/templates/files/` | Edit the `.tmpl`; add new ones to `TEMPLATE_NAMES`, and a renderer only if the file needs a substituted value or a merge policy | | Add a value to the scaffold | `src/lightcone/engine/templates/__init__.py` | Derive it from the environment or our own metadata before introducing a constant | | Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | +| Change how a project stores bytes | `src/lightcone/engine/dataset.py` + `templates/files/gitattributes.tmpl` | Every command through `project._run`; test it against a real annex (`real_tools`) | +| Change how an output is identified | `src/lightcone/engine/identity.py` + `tests/test_identity.py` | Sensitivity tests both ways: what must move the hash, and what must not | +| Change when an output is remade | `src/lightcone/engine/assets.py` + `tests/test_assets.py` | One `classify()`, two callers; `--check` differs by one input value, never by logic. Ask first whether the thing that moved *contradicts* the project or is a *circumstance* — the second is `behind`, not `stale` | +| Change how the spec becomes a graph | `src/lightcone/engine/plan.py` + `tests/test_plan.py` | Ask `astra.resolve`; if the answer is missing, the fix is a PR to astra-tools. Anything ambiguous is a `ProjectError`, never a guess | +| Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | +| Change what a run commits | `src/lightcone/engine/materialize.py` + `tests/test_materialize.py` | The driver owns git alone; the tree ends as clean as it started | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | | Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | | Change what a sandboxed command may touch | `sandbox/policy.py` + `tests/test_sandbox_policy.py` | Path sets only — no mechanism ever leaks in here | @@ -647,17 +1379,46 @@ only checks that the guard is present. Verified empirically, not assumed. - `tests/conftest.py` — the `tools` autouse fixture stubs `engine.project._run`, emulating each tool's observable effect (`uv lock` - writes `uv.lock`, `uv sync` makes `.venv`, `git init` makes `.git`) and - recording every argv. `uv_calls(tools)` narrows it to uv. Tests are - hermetic: no network, no resolution, no subprocesses. + writes `uv.lock`, `uv sync` makes `.venv`, `git init` makes `.git`, + `git annex init` marks the repository annexed) and recording every argv. + `uv_calls(tools)` narrows it to uv. Tests are hermetic: no network, no + resolution, no subprocesses. The `real_tools` fixture opts back out, + putting the real `_run` back. +- `tests/test_dataset.py` — the storage seam, and **the one place in the + non-sandbox suite that runs real tools**, via `real_tools`. That is + deliberate: the question is whether bytes land in the annex or as a blob + in git, and a fake answering it would only restate what the code already + believes. Every bug this file found (the missing `.gitattributes` + default, the pointer-file trap, `filter=annex`) was invisible to a stub. - `tests/test_project.py` — discovery and convergence semantics, called directly. This is where scaffold behavior is tested. +- `tests/test_plan.py` tests what lc adds — directories, edges, + `definition_version`, target resolution, the validation gate — and **not** + what a spec means. Scoping, `from:`, `when:` and the recipe grammar are + covered by `astra-tools`' own suite; asserting them here again would + re-create the second implementation this layer just deleted. Every + fixture must be a spec `astra validate` accepts, which the gate now + enforces for free. +- `tests/test_identity.py`, `test_assets.py`, `test_plan.py` — **pure**, + and the whole of layer 2 plus the graph. Nothing spawns, nothing on disk + beyond `tmp_path`. +- `tests/test_worker.py`, `tests/test_materialize.py` — real recipes, + through the real boundary, against a real repository, via the `analysis` + fixture in `conftest.py`. That is the price of testing execution: whether + the gates hold, whether bytes land in the annex, and whether the tree is + clean afterwards are not questions a stub can answer. It is cheap anyway + — the fixture's project declares no dependencies, so `uv lock` and + `uv sync` together cost milliseconds. + - **`cluster_for_run()` is the one new monkeypatch point.** Most tests + swap in an inline scheduler and never start Dask; exactly one starts a + real `LocalCluster`, because a seam is only worth having if the thing + it abstracts still fits through it. - `tests/test_cli.py` — the CLI surface only: flags reaching the engine, rendering, exit codes, error translation. Rich wraps output at terminal width, so assert on short unwrappable fragments. - `tests/test_templates.py` — template loading (guards the packaging of - package data), strict substitution, and the `.gitignore` entry/repair - logic. Content assertions live here; `test_project.py` asserts only that + package data), strict substitution, and the `.gitignore` / + `.gitattributes` entry/repair logic. Content assertions live here; `test_project.py` asserts only that the file written *is* the template, so a template edit touches one file. The sandbox suite splits along the seam, which is what makes it cheap: @@ -691,8 +1452,9 @@ layer works. Four properties, each of which it would be easy to lose: *a leak only Linux catches is a leak*. macOS is in the CI matrix for exactly this: it is the sole place the generated SBPL is ever executed. -2. **The real policy.** It runs against `probe_policy` — what an actual - `lc run` gets — never a policy hand-built to make the point. A test +2. **The real policy.** It runs against `exec_policy` — what an actual + `lc run` *and* an actual recipe get — never a policy hand-built to + make the point. A test that grants exactly what it is testing cannot discover that the *shipped* policy grants something else. (This is how `/usr` sat in the exec set through a full green suite.) @@ -715,5 +1477,12 @@ silent. Every leak case here was checked that way. - Ruff for linting (E, F, I, N, W, UP), line length 100, target Python 3.11 - mypy strict mode with `namespace_packages = true`, `explicit_package_bases = true` -- Docstrings and comments carry *why*, and cite the spec section - (`spec §7`) when they encode a design decision +- **Google-style docstrings** on every public function: an imperative + one-line summary, then `Args:` / `Returns:` / `Raises:` / `Yields:` + where they carry information. Concise — a design decision gets a + sentence or two, not its history; the long form belongs in this file. + Two deliberate exceptions: **click command callbacks**, whose docstring + *is* the `--help` text and must stay prose, and properties, whose value + the summary already describes. +- Comments and docstrings carry *why*, never a narrative of how the code + came to be diff --git a/pyproject.toml b/pyproject.toml index bc3763ed..43c61c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,16 +23,12 @@ classifiers = [ "Topic :: Scientific/Engineering", ] -# Clean rebuild: dependencies are re-added with the layer that needs -# them. Layer 1 (`lc init` scaffolding) needs the ASTRA spec tooling, the -# CLI framework, and console rendering — nothing else. dependencies = [ - # Pinned: the spec scaffold delegates to astra's boilerplate helper, - # whose surface has changed across releases (0.2.14 exposes the - # public `create_boilerplate`). Bump deliberately. - "astra-tools==0.2.14", + "astra-tools==0.2.16", "click>=8.0", "rich>=13.0", + "git-annex>=10.2026", + "distributed>=2026.7", ] [dependency-groups] @@ -41,6 +37,7 @@ dev = [ "pytest-cov", "ruff", "mypy", + "datalad", ] docs = [ "zensical>=0.0.33", diff --git a/src/lightcone/_sandbox_exec.py b/src/lightcone/_sandbox_exec.py index 233c1717..9ea94cea 100644 --- a/src/lightcone/_sandbox_exec.py +++ b/src/lightcone/_sandbox_exec.py @@ -107,13 +107,10 @@ def _libc() -> ctypes.CDLL: def abi() -> int: - """The Landlock ABI this kernel supports, or 0 if it supports none. + """Probe the Landlock ABI this kernel supports. - A version query, so it is cheap and side-effect free. Every failure - mode — kernel < 5.13, the syscall blocked by seccomp, an - architecture we do not vouch for — answers 0 rather than raising: - the caller's question is "can I sandbox here", and "no" is a valid - answer to it. Recording *which* ABI answered is the caller's job. + Returns: + The ABI level, or 0 if the kernel supports none. """ if os.uname().machine not in _SUPPORTED_ARCHES: return 0 @@ -129,23 +126,14 @@ def abi() -> int: def handled_access(abi_level: int) -> int: - """The rights the ruleset declares it governs, for *abi_level*. - - Unknown bits make ``landlock_create_ruleset`` fail EINVAL, so this - only ever widens with the ABI. Widening matters in both directions: - a right the ruleset does not *handle* is one the kernel does not - check at all, so ABI 1 silently permits ``ftruncate`` — while a - handled right must also be granted somewhere or nothing can use it. - - ``REFER`` is the case worth naming: when a ruleset does not handle - it, the kernel denies *every* cross-directory rename and link, which - is the ABI-1 EXDEV the denial classifier knows about. Handling it - (and granting it on the write roots) is what lets a recipe rename its - own temporary files. - - ``IOCTL_DEV`` (ABI 5) is deliberately left unhandled: ioctls on - device nodes are outside the accidental-leakage threat model, and - handling it would break anything opening a terminal afresh. + """Build the mask of rights the ruleset declares it governs. + + Args: + abi_level: The ABI reported by :func:`abi`. + + Returns: + Every right this ABI knows about. A right the kernel does not + know is a ruleset it rejects outright. """ handled = _ABI1_ALL if abi_level >= 2: @@ -156,16 +144,29 @@ def handled_access(abi_level: int) -> int: def write_bits(abi_level: int) -> int: - """The rights a writable root is granted: everything but EXECUTE. + """Build the mask a writable root is granted. + + Args: + abi_level: The ABI reported by :func:`abi`. - Write implies read — a directory you may create files in but not - list is not a useful grant — so the read bits are included. + Returns: + Everything but EXECUTE, which is granted per file. """ return (handled_access(abi_level) & ~ACCESS_FS_EXECUTE) | READ_BITS def create_ruleset(handled_fs: int) -> int: - """A new, empty Landlock ruleset FD governing *handled_fs*.""" + """Create an empty Landlock ruleset. + + Args: + handled_fs: The rights the ruleset governs. + + Returns: + A ruleset file descriptor. + + Raises: + OSError: If the kernel refuses the ruleset. + """ attr = _RulesetAttr(handled_fs, 0) fd = _libc().syscall( SYS_LANDLOCK_CREATE_RULESET, @@ -179,13 +180,15 @@ def create_ruleset(handled_fs: int) -> int: def add_path_rule(ruleset_fd: int, path: str, access: int) -> None: - """Grant *access* beneath *path*. + """Grant access beneath a path. + + Args: + ruleset_fd: The ruleset to add to. + path: The directory or file to grant on. + access: The rights to grant, masked to what the path can bear. - Rights are masked down for a regular-file rule, because the kernel - rejects directory-only rights on one. That is what makes a per-file - EXECUTE grant work — and per-file is the point: ``/usr/bin`` holds - both the utility allowlist and every undeclared tool on the host, so - granting the directory would defeat the layer. + Raises: + OSError: If the path cannot be opened or the rule is refused. """ parent_fd = os.open(path, os.O_PATH | os.O_CLOEXEC) try: @@ -214,10 +217,17 @@ def add_path_rule(ruleset_fd: int, path: str, access: int) -> None: def build_ruleset(policy: dict[str, object], abi_level: int) -> int: - """A ruleset FD realizing *policy*, or raise :class:`OSError`. + """Build a ruleset realizing a policy document. + + Args: + policy: The read, write and execute path lists from argv. + abi_level: The ABI reported by :func:`abi`. - A path that has vanished between policy construction and here grants - nothing, which is safe, so it is skipped rather than fatal. + Returns: + A ruleset file descriptor. + + Raises: + OSError: If the ruleset cannot be created or a rule refused. """ fd = create_ruleset(handled_access(abi_level)) try: @@ -238,11 +248,13 @@ def build_ruleset(policy: dict[str, object], abi_level: int) -> int: def restrict_self(ruleset_fd: int) -> None: - """Apply *ruleset_fd* to this process, irreversibly. + """Apply a ruleset to this process, irreversibly. + + Args: + ruleset_fd: The ruleset to enter. - ``PR_SET_NO_NEW_PRIVS`` first: the kernel refuses an unprivileged - ``landlock_restrict_self`` without it, and it is also what stops a - setuid binary from escaping the domain. + Raises: + OSError: If ``no_new_privs`` or the restriction is refused. """ libc = _libc() if libc.prctl(_PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0: @@ -267,11 +279,14 @@ def _fail(message: str) -> None: def main(argv: list[str] | None = None) -> None: - """``--policy -- ...`` — restrict, then become it. + """Restrict this process, then exec the command it was given. + + ``--policy -- ...``. Never falls through to running + the command unsandboxed: a setup failure exits with a reserved code + that nothing a command could return will collide with. - Hand-parsed rather than argparse'd: this runs before every sandboxed - exec, and everything after ``--`` belongs to the command, including - arguments that look like our own options. + Args: + argv: Defaults to the process arguments. """ args = list(sys.argv[1:] if argv is None else argv) if len(args) < 2 or args[0] != "--policy": diff --git a/src/lightcone/cli/__init__.py b/src/lightcone/cli/__init__.py index 1fc431b3..53c32342 100644 --- a/src/lightcone/cli/__init__.py +++ b/src/lightcone/cli/__init__.py @@ -4,6 +4,11 @@ def main() -> None: + """Run the CLI. + + The console-script entry point. Imports the command module lazily, so + the cost of click and the engine is paid only once a command runs. + """ from lightcone.cli.commands import main as _main _main() diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 2efa63c8..f67c60d5 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from rich.console import Console + from lightcone.engine.materialize import MaterializeReport from lightcone.engine.project import ConvergenceReport logger = logging.getLogger(__name__) @@ -22,8 +23,12 @@ @functools.cache def _console() -> Console: - """The rich console, built on first use to avoid startup cost at each - invocation of the cli even when the console is not needed.""" + """Build the rich console, once and on first use. + + Returns: + The console. Deferred so ``lc --help`` and shell completion never + pay for it. + """ from rich.console import Console return Console() @@ -39,6 +44,17 @@ class _EngineErrorGroup(click.Group): """ def invoke(self, ctx: click.Context) -> object: + """Run a command, rendering engine errors as clean CLI errors. + + Args: + ctx: The click context. + + Returns: + Whatever the command returned. + + Raises: + click.ClickException: In place of any ``ProjectError``. + """ from lightcone.engine.project import ProjectError try: @@ -86,13 +102,6 @@ def init(directory: Path, check_only: bool, as_json: bool) -> None: Safe to re-run at any time: creates whatever is missing, repairs the pieces lightcone manages, and never overwrites files you own. - - The spec scaffold (``astra.yaml``, ``universes/baseline.yaml``) - follows the ``astra init`` boilerplate; on top of it sit the uv - project (``pyproject.toml`` with lightcone-cli locked in, - ``.python-version``, ``uv.lock``, ``.venv``), ``.gitignore`` entries, - ``results/``, and a template MyST report (``myst.yml`` + - ``index.md``). """ from lightcone.engine.project import converge @@ -115,14 +124,6 @@ def init(directory: Path, check_only: bool, as_json: bool) -> None: def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: bool) -> None: """Print a convergence report: the items, then the verdict. - - ``--check`` and a real run print the *same* report — they disagree - only about tense — so ``dry_run`` selects the mood and nothing else, - mirroring the ``write`` flag convergence itself takes. Blocked items - are listed with created and repaired ones because they count the same - way: they are what lets a report say "not converged" (see - :class:`~lightcone.engine.project.ConvergenceReport`). Their reason - arrives separately, as a warning. """ mark, style = ("·", "yellow") if dry_run else ("✓", "green") @@ -161,26 +162,184 @@ def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: @main.command(context_settings={"ignore_unknown_options": True, "allow_interspersed_args": False}) @click.argument("command", nargs=-1, required=True, type=click.UNPROCESSED) def run(command: tuple[str, ...]) -> None: - """Run COMMAND in the project environment, inside the sandbox. - - Byte-for-byte the environment recipes get — same lock, same - ``.venv``, same boundary — so a command that works here means a - recipe will. The project and the inputs declared in ``astra.yaml`` - are readable, ``results/`` and scratch are writable, and anything - outside that — a host tool, a system library, an undeclared file — - is refused. + """Run COMMAND in the project environment, under isolation. """ from lightcone.engine import run as engine_run from lightcone.engine.project import current_project outcome = engine_run.probe(current_project(), command) if outcome.notes: - # click.echo, not rich: these lines are built to be pasted, and - # reflowing a `uv add` remedy would break the one thing the - # denial message is for. click.echo("\n".join(["", *outcome.notes]), err=True) # `Popen.returncode` is negative for a signal, and `sys.exit(-9)` # truncates to 247. `lc run` is a proxy for the command it runs, so # an OOM-killed probe comes back as the shell's conventional 128+N. code = outcome.returncode sys.exit(128 - code if code < 0 else code) + + +# ============================================================================= +# lc materialize +# ============================================================================= + + +@main.command() +@click.argument("targets", nargs=-1) +@click.option( + "--check", + "check_only", + is_flag=True, + help=( + "Report what would run and why, without executing or committing " + "anything; exit 1 if anything is out of date." + ), +) +@click.option( + "--refresh", + is_flag=True, + help=( + "Also remake outputs that are behind — still what the analysis " + "asks for, but made under an earlier environment." + ), +) +@click.option( + "--json", + "as_json", + is_flag=True, + help="Emit the report as JSON on stdout.", +) +def materialize( + targets: tuple[str, ...], check_only: bool, refresh: bool, as_json: bool +) -> None: + """Make the analysis's outputs, and commit each one as it lands. + + Each output is committed together with its manifest, in a commit that + records the command that produced it. The git tree needs to be clean + before the run can start. + + An output is remade when the analysis defines it differently than it + was made — a changed recipe or decision — or when one of its declared + inputs changed. Inputs are compared by content, so a rebuild that + comes out byte-identical stops there instead of cascading. + + An output made under an earlier environment is reported as behind and + left alone: it is still what the analysis asks for, and the manifest + records the environment and the commit that produced it. Pass + --refresh to remake those too. + """ + from lightcone.engine import materialize as engine + from lightcone.engine.project import current_project + + root = current_project() + if check_only: + report = engine.check(root, targets, refresh=refresh) + else: + report = engine.materialize(root, targets, refresh=refresh) + + if as_json: + click.echo(json.dumps(report.as_dict(), indent=2)) + else: + if report.notes: + click.echo("\n".join(["", *report.notes]), err=True) + _render_materialize_output(report, root, dry_run=check_only) + + if not report.ok or (check_only and not report.up_to_date): + sys.exit(1) + + +# ============================================================================= +# lc status +# ============================================================================= + + +@main.command() +@click.option( + "--json", + "as_json", + is_flag=True, + help="Emit the report as JSON on stdout.", +) +def status(as_json: bool) -> None: + """Report what state each of the analysis's outputs is in. + + For every output the analysis declares: whether it is current, behind + or stale, and — once it has been materialized — the commit it was made + at. An output that is behind is not wrong; that commit is where the + code and the environment which produced it can be read back. + + Reads only. It runs nothing, commits nothing, does not mind an unclean + tree, and always exits 0 — a state is not a failure. Use + `lc materialize --check` for a gate that exits nonzero. + """ + from lightcone.engine import materialize as engine + from lightcone.engine.project import current_project + + report = engine.status(current_project()) + if as_json: + click.echo(json.dumps(report.as_dict(), indent=2)) + return + + marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} + width = max((len(o.output) for o in report.outputs), default=0) + lines = [ + # The commit gets a column of its own, for every state and not only + # the interesting ones: "which code made this" is the question the + # verb exists to answer, and it has an answer for a current output + # too. + f" {marks[o.status]} {o.status:<8} {o.output:<{width}} " + f"{o.git_sha[:7] or '—':<7}" + (f" [dim]{o.why}[/dim]" if o.why else "") + for o in report.outputs + ] + lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + + counts = report.counts + if not report.outputs: + lines.append("[dim]The analysis declares no output with a recipe.[/dim]") + else: + lines.append("") + lines.append( + " · ".join(f"{count} {state}" for state, count in counts.items() if count) + ) + _console().print("\n".join(lines)) + + +# ============================================================================= +# Rendering +# ============================================================================= + + +def _render_materialize_output(report: MaterializeReport, root: Path, *, dry_run: bool) -> None: + """Print what ran, or what would. + """ + lines = [ + f" [yellow]·[/yellow] would run {name} — {why}" + for name, why in report.planned.items() + ] + lines += [f" [green]✓[/green] made {name}" for name in report.made] + # Behind is not a warning and not a problem: it is a fact about where + # an output came from, and the only line here that tells you something + # you could not have worked out from the exit code. + lines += [ + f" [cyan]·[/cyan] behind {name} — {why}" for name, why in report.behind.items() + ] + lines += [f" [dim]·[/dim] up to date {name}" for name in report.current] + lines += [f" [red]✗[/red] failed {name}" for name in report.failed] + lines += [f" [red]✗[/red] blocked {name}" for name in report.blocked] + lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + + if not report.ok: + verdict = f"[red]✗[/red] {root} did not finish" + elif report.up_to_date: + verdict = f"[green]✓[/green] {root} is up to date — nothing to do" + elif dry_run: + verdict = f"[yellow]![/yellow] {len(report.planned)} output(s) would be made" + else: + verdict = f"[green]✓[/green] Made {len(report.made)} output(s) in {root}" + # On the verdict line as well as in the listing: on a large analysis the + # listing scrolls away, and this is the one state that reports something + # rather than doing it. + if report.behind: + verdict += f" · {len(report.behind)} behind — `--refresh` remakes them" + + if lines: + lines.append("") + _console().print("\n".join([*lines, verdict])) diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py new file mode 100644 index 00000000..e56c7fb5 --- /dev/null +++ b/src/lightcone/engine/assets.py @@ -0,0 +1,439 @@ +"""What a materialized output *is*: where it lives, what it records, and +whether it is still current. + +An asset is a directory — ``results///`` — holding +whatever the recipe wrote, plus a manifest beside it. The manifest is the +only part lc writes itself, and it is kept out of the annex so it stays +readable on a clone that has fetched no content at all. + +The rule that classifies an output — ``current``, ``behind`` or +``stale`` — lives here too, next to the manifest it reads and the hashes +it compares. It is the one place in the layer where a bug is quiet rather +than loud: a rule that under-reports leaves an output silently describing +bytes that no longer follow from its inputs. It is a **content-hash** +rule, so a byte-identical rebuild stops the cascade and a restored file +with an old mtime cannot hide. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Literal + +from lightcone.engine.project import ProjectError + +MANIFEST_FILENAME = ".lightcone-manifest.json" +SCHEMA_VERSION = 1 + +#: Excluded from the content hash: the manifest is written *after* the +#: hash it contains, so hashing it would be circular. +_HASH_EXCLUDE = frozenset({MANIFEST_FILENAME}) + +#: The marker every annexed path carries, in both of the shapes an annexed +#: file takes on disk — the pointer file's first bytes, and the locked +#: symlink's target. Detecting the first is the same test git-annex's own +#: ``isPointerFile`` makes, and both have to be made: an unfetched file is +#: readable, or absent, but never obviously wrong. +_ANNEX_OBJECTS = "/annex/objects/" +_POINTER_PREFIX = _ANNEX_OBJECTS.encode() +_POINTER_MAX_BYTES = 32 * 1024 + + +class ContentNotFetchedError(ProjectError): + """An annexed file whose content is not in this clone.""" + + +def output_dir(root: Path, universe_id: str, output_id: str) -> Path: + """Locate a ``(universe, output)`` pair's directory. + + Path-addressed: the path in a rendered recipe is this path, with no + staging, scratch or relocation in between. + + Args: + root: The project root. + universe_id: The universe the output was made under. + output_id: The output's id, qualified for a sub-analysis. + + Returns: + ``/results//``. + + Raises: + ProjectError: If either id is not a single path component. The + path is *composed* from them, so an empty one collapses it + onto a parent — ``results/`` itself, for two — and a worker + empties this directory before running a recipe in it. + """ + for label, value in (("universe", universe_id), ("output", output_id)): + if not value or "/" in value or "\\" in value or value in {".", ".."}: + raise ProjectError( + f"{label} id {value!r} is not a single path component, so it " + f"cannot name a directory under results/." + ) + return root / "results" / universe_id / output_id + + +# ============================================================================= +# Content identity +# ============================================================================= + + +def data_version(path: Path) -> str: + """Hash *path*'s content — its bytes, and nothing else about it. + + A directory hashes each file in sorted relative-path order with the + relative path fed in beside the bytes, so a rename moves the digest. A + file hashes its own bytes. The two are framed apart, so a directory + holding one file cannot collide with that file alone. + + Never mtime or size: a content hash is what lets a byte-identical + rebuild stop cascading, and what stops a file restored with an old + timestamp passing as unchanged. + + Args: + path: A file or directory. The manifest is excluded from a + directory's digest, since it carries the result. + + Returns: + The digest, as ``sha256:``. + + Raises: + FileNotFoundError: If *path* does not exist. Never a constant + digest, which would silently disable the staleness chain. + ContentNotFetchedError: If any file is annexed without its content + being in this clone, in either of the shapes that takes. + """ + if path.is_symlink() and not path.exists(): + _refuse_unfetched(path) + if not path.exists(): + raise FileNotFoundError(path) + h = hashlib.sha256() + if path.is_file(): + _refuse_unfetched(path) + h.update(b"file:") + _feed(h, path) + return f"sha256:{h.hexdigest()}" + + h.update(b"dir:") + # A dangling symlink is an unfetched *locked* file, and ``is_file()`` + # answers False for one — so filtering on that alone would drop it from + # the digest without a word, reporting a hash of the subset that happens + # to be present. Only dangling ones are added back: a symlink that + # resolves to a file is already a file, and one that resolves to a + # directory is not content. + files = [ + p + for p in path.rglob("*") + if p.name not in _HASH_EXCLUDE and (p.is_file() or (p.is_symlink() and not p.exists())) + ] + for p in sorted(files, key=lambda x: x.relative_to(path).as_posix()): + _refuse_unfetched(p) + h.update(b"path:") + h.update(p.relative_to(path).as_posix().encode()) + h.update(b"\0data:") + _feed(h, p) + h.update(b"\0") + return f"sha256:{h.hexdigest()}" + + +class Versions: + """Content identities, computed once per run. + + A declared input is hashed once per ``(universe, output)`` that names + it, which for a multiverse spec is the same bytes over and over: eight + universes times four outputs sharing one catalog reads it thirty-two + times. Memoizing is sound for exactly as long as a run lasts — a run + refuses to start on a dirty tree, and the only in-tree path a recipe + may write is its own output directory, so a declared input's bytes + cannot change underneath it. + + A class rather than a closure, so what it keeps alive is one dict and + not whatever scope built it. Deliberately unlocked: concurrent workers + can race to compute the same digest, which wastes one hash rather than + serialising every hash behind a lock — and a lock would not survive + being handed to a worker in another process. + """ + + def __init__(self) -> None: + self._known: dict[Path, str] = {} + + def of(self, path: Path) -> str: + """Return *path*'s content identity, hashing it at most once. + + Args: + path: A declared input, file or directory. + + Returns: + The digest, as ``sha256:``. + """ + resolved = path.resolve() + if (known := self._known.get(resolved)) is None: + known = self._known[resolved] = data_version(path) + return known + + +def _refuse_unfetched(path: Path) -> None: + """Refuse a file whose content this clone does not hold. + + An annexed file takes one of two shapes, and a researcher can convert + between them whenever they like — so both are checked rather than + whichever one lc's own writes produce. An *unlocked* file is a small + regular file holding the object's path, hard-linked to the object or + copied from it depending on ``annex.thin``; both look identical without + the content. A *locked* file is a symlink into the object store, which + without the content simply dangles. + + Args: + path: A file about to be hashed. + + Raises: + ContentNotFetchedError: If *path* is either shape without its + content. Loud, because both alternatives are silent: hashing a + pointer yields a well-formed digest of the wrong bytes, and a + dangling symlink drops out of a directory's digest entirely. + """ + if path.is_symlink() and not path.exists(): + unfetched = _ANNEX_OBJECTS in path.readlink().as_posix() + elif path.stat().st_size > _POINTER_MAX_BYTES: + unfetched = False + else: + with path.open("rb") as f: + unfetched = f.read(len(_POINTER_PREFIX)) == _POINTER_PREFIX + if unfetched: + raise ContentNotFetchedError( + f"{path}: the content is not in this clone — git-annex holds a " + f"reference to it, not the data. Fetch it with `git annex get {path}`." + ) + + +def _feed(h: hashlib._Hash, path: Path) -> None: + """Stream *path* into *h* — outputs are not assumed to fit in memory.""" + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 16), b""): + h.update(chunk) + + +# ============================================================================= +# The manifest +# ============================================================================= + + +@dataclass(frozen=True) +class Manifest: + """What one materialization recorded about itself. + + Every field is filled by the run that wrote it; there are no optional + halves. ``input_versions`` is the chain — each declared input's content + identity at the moment this output was made — and it is what lets a + change to anything upstream reach here without a timestamp. + """ + + output_id: str + universe_id: str + recipe: str + #: What the spec says this output is: its recipe and decisions. The + #: rebuild trigger. + definition_version: str + #: The environment it was made under. Recorded, never a rebuild + #: trigger — a difference here makes the output *behind*, not stale. + env_version: str + data_version: str + decisions: dict[str, str] + input_versions: dict[str, str] + #: The commit the working tree was at when the run started. Recorded, + #: never hashed into `definition_version` — a commit must not stale + #: every output in the repository, and this is what makes the code that + #: produced a result recoverable anyway. + git_sha: str + #: The `origin` URL, or empty when the repository has no remote. + git_remote: str + lc_version: str + #: What the sandbox actually enforced, as the boundary attested it. + hermeticity: dict[str, Any] + schema_version: int = field(default=SCHEMA_VERSION) + + def as_dict(self) -> dict[str, Any]: + """Return the manifest as JSON-ready data, ``schema_version`` first. + + Returns: + Every field, in declaration order. + """ + data = asdict(self) + return {"schema_version": data.pop("schema_version"), **data} + + +def read(directory: Path) -> Manifest | None: + """Read the manifest in *directory*. + + Args: + directory: An output directory. + + Returns: + The manifest, or ``None`` when it is absent or unparseable — which + the staleness rule reads as "make it again", the safe direction. + + Raises: + OSError: Deliberately not caught. A permission problem is a real + fault and must not look like an output needing a rebuild. + """ + path = directory / MANIFEST_FILENAME + if not path.is_file(): + return None + try: + data = json.loads(path.read_text()) + return Manifest(**{k: v for k, v in data.items() if k in _FIELDS}) + except (json.JSONDecodeError, TypeError): + return None + + +def write(directory: Path, manifest: Manifest) -> Path: + """Write *manifest* into *directory*, atomically. + + The rename is the commit point: a reader sees the previous manifest or + this one, never half of either. + + Args: + directory: The output directory to write into. + manifest: The record to write. + + Returns: + The path written. + """ + path = directory / MANIFEST_FILENAME + temporary = directory / f"{MANIFEST_FILENAME}.tmp" + temporary.write_text(json.dumps(manifest.as_dict(), indent=2, sort_keys=False) + "\n") + temporary.replace(path) + return path + + +_FIELDS = frozenset(Manifest.__dataclass_fields__) + + +# ============================================================================= +# Classification — one rule, and it is the only place the rule lives +# ============================================================================= +# +# Three states, and the line between them is what the whole model turns on. +# +# An output is **stale** when it contradicts the project as it now stands: +# the spec defines it differently than the artifact was made, or it records +# deriving from bytes the project no longer holds. Either way what is on +# disk is mislabelled, so it is remade. +# +# An output is **behind** when it is still exactly what the spec asks for, +# but was made under an earlier environment. Nothing about it is wrong — +# the environment it ran under is in its manifest and the commit beside it +# reconstructs that environment — so it is reported and left alone. A +# caller that wants it remade asks for that. +# +# The git commit takes part in neither. It is tree-wide, so a README edit +# moves it for every output at once; using it as a signal would mean +# everything is always behind, and a signal that is always on is not one. +# It is recorded, and shown as the context of a `behind` line. + + +#: What an output is, relative to the project as it now stands. +Status = Literal["current", "behind", "stale"] + + +@dataclass(frozen=True) +class Reason: + """Why an output no longer describes what it was made from.""" + + kind: Literal["missing", "definition", "declaration", "input"] + #: Which declared input it was about, for the two input kinds. + input: str = "" + + def __str__(self) -> str: + if self.kind == "missing": + return "no manifest — it has never been materialized" + if self.kind == "definition": + return "the recipe or its decisions changed" + if self.kind == "declaration": + return f"the output no longer declares the same inputs (`{self.input}`)" + return f"the input `{self.input}` changed" + + +@dataclass(frozen=True) +class Verdict: + """One output's state, and the sentence explaining it.""" + + status: Status + #: Why, for ``stale`` and ``behind``. Empty for ``current``. + why: str = "" + + def calls_for_a_remake(self, *, refresh: bool) -> bool: + """Whether a run would make this output again. + + ``stale`` always, because the artifact contradicts the project. + ``behind`` only when asked, because it does not. + + Args: + refresh: Whether the caller asked for behind outputs too. + + Returns: + Whether to run the recipe. + """ + return self.status == "stale" or (refresh and self.status == "behind") + + +def classify( + *, + definition_version: str, + env_version: str, + manifest: Manifest | None, + inputs: Mapping[str, str | None], +) -> Verdict: + """Decide what an output is, relative to the project as it now stands. + + The only place this rule lives. Values rather than objects, because + the two callers arrive at them differently and must not diverge in + anything else. + + Args: + definition_version: What the spec currently says this output is. + env_version: The run's environment identity. + manifest: The output's recorded manifest, or ``None``. + inputs: Each declared input's current content identity. ``None`` + for an input the caller has already decided will be remade — + check mode's sentinel, meaning "this is going to change", + since it cannot know whether a rebuild is byte-identical. + + Returns: + ``stale``, ``behind`` or ``current``, with the reason for the + first two. + """ + if (reason := _stale(definition_version, manifest, inputs)) is not None: + return Verdict("stale", str(reason)) + assert manifest is not None # `_stale` returns a reason when it is None + if manifest.env_version != env_version: + # The sentence says what happened; *where* it happened is the + # manifest's `git_sha`, which a caller with a column for it reads + # from the record rather than from prose. + return Verdict("behind", "made under an earlier environment") + return Verdict("current") + + +def _stale( + definition_version: str, + manifest: Manifest | None, + inputs: Mapping[str, str | None], +) -> Reason | None: + """Why the artifact contradicts the project, or ``None`` if it does not.""" + if manifest is None: + return Reason("missing") + if manifest.definition_version != definition_version: + return Reason("definition") + # The *set* first, and separately: `definition_version` hashes the recipe + # and the decisions, neither of which an input the spec no longer declares + # moves — so without this the loop below would never look at it. Adding an + # input is caught either way; dropping one is only caught here. + if changed := set(inputs) ^ set(manifest.input_versions): + return Reason("declaration", sorted(changed)[0]) + for name, current in inputs.items(): + if current is None or manifest.input_versions[name] != current: + return Reason("input", name) + return None diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py new file mode 100644 index 00000000..30f72f1d --- /dev/null +++ b/src/lightcone/engine/dataset.py @@ -0,0 +1,303 @@ +"""The git + git-annex seam: how a project stores what it produced. + +Storage follows the DataLad model — git carries the pointers and the +history, git-annex carries the bytes — reached through ordinary ``git`` +and ``git annex`` commands. ``.gitattributes`` routes content: the default +is ``annex.largefiles=nothing``, and outputs and declared inputs opt out +of it, so manifests and analysis code stay in git while results and data +go to the annex. + +Every command goes through :func:`~lightcone.engine.project._run`, the +same seam convergence uses, so there is one place to monkeypatch and every +invocation is inspectable. + +``.gitattributes`` sets ``filter=annex``, so an ordinary ``git add`` +routes content to the annex. Nothing here — and nothing lc documents — +asks anyone to run a git-annex command by hand. +""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Iterable +from pathlib import Path + +from lightcone.engine import project + + +def put_our_bin_first() -> None: + """Prepend the directory holding our own interpreter to ``PATH``. + + ``git annex`` is dispatched by git searching ``PATH`` for a + ``git-annex`` executable, and ``uv tool install lightcone-cli`` links + only *our* entry points into ``~/.local/bin`` — the git-annex we + resolved as a dependency sits beside the interpreter instead. Verified: + with only the tool's ``lc`` on ``PATH``, ``git annex version`` fails and + ``lc init`` refuses. (Under ``uv run`` the prepend is a no-op, because + uv already fronts the project's ``.venv/bin``.) + + Prepend, never append: a system git-annex winning would make the + version recorded in the lock a fiction. Idempotent, and applied to our + own environment because that is what child processes inherit. + """ + ours = str(Path(sys.executable).parent) + path = os.environ.get("PATH", "") + if path.split(os.pathsep)[:1] == [ours]: + return + os.environ["PATH"] = ours + os.pathsep + path if path else ours + + +# ============================================================================= +# The repository +# ============================================================================= + + +def init_git(directory: Path) -> None: + """Create the repository the project's history lives in. + + Args: + directory: Where to run ``git init``. + """ + _git(["init", "-q"], cwd=directory) + + +def init_annex(directory: Path) -> None: + """Create the object store the bytes live in. + + Args: + directory: A directory inside the repository to annex. + """ + _git(["annex", "init", "-q"], cwd=directory) + + +def is_annexed(directory: Path) -> bool: + """Report whether the enclosing repository has an annex. + + Asks git-annex's own question — ``annex.uuid`` is the marker it writes + on ``init`` — rather than guessing from a directory listing. + + Args: + directory: A directory inside the repository. + + Returns: + True if the repository has been annexed. + """ + return _git_ok(["config", "--get", "annex.uuid"], cwd=directory) + + +def ignore_rule(directory: Path, path: str) -> str | None: + """Find the ignore rule covering *path*, if there is one. + + ``--no-index`` asks about the *rules* rather than the index: without + it, git answers "not ignored" for anything already tracked, which is + exactly the project where someone tracked a result by hand and left + the rule for the next one. + + Args: + directory: The repository to ask in. + path: The pathspec to ask about, with a trailing slash for a + directory — a rule like ``results/*`` ignores the contents and + does not match the bare name. + + Returns: + ``::``, or ``None`` if nothing ignores it. + Convergence cannot repair this, so the message must name the line. + """ + proc = project._run(["git", "check-ignore", "-v", "--no-index", "--", path], cwd=directory) + if proc.returncode != 0 or not proc.stdout.strip(): + return None + # `::\t`, one line per pathspec. + return proc.stdout.splitlines()[0].split("\t")[0] + + +# ============================================================================= +# What a run does to the repository +# ============================================================================= + + +def status(directory: Path) -> list[tuple[str, str]]: + """List *directory*'s uncommitted changes. + + Honours ``.gitignore``, so ``.venv/`` never counts; ``data/`` and + ``results/`` do, which is the point — inputs are committed before + anything computes on them. + + Scoped to *directory* and reported relative to it, both deliberately. + A project can sit inside a larger repository — ``lc init subdir/`` + adopts an enclosing work tree rather than nesting a new one — and + porcelain otherwise covers that whole tree and names paths from *its* + root: an edit somewhere else in the repository would refuse every run, + and lc's own writes would arrive as ``subdir/results/…``, which no + caller sorting by path class can recognise. + + Args: + directory: The project to inspect. + + Returns: + ``(status code, path)`` for each change, paths relative to + *directory*, empty when clean. A wholly untracked project collapses + to ``.``, which is git's own summary of it and what the caller + would tell the user to add. + """ + prefix = _git(["rev-parse", "--show-prefix"], cwd=directory).strip() + lines = _git(["status", "--porcelain", "--", "."], cwd=directory).splitlines() + return [ + (line[:2], line[3:].removeprefix(prefix) or ".") for line in lines if line.strip() + ] + + +def require_committer(directory: Path) -> None: + """Refuse a repository that cannot make a commit yet. + + git needs an identity to commit, and a fresh container or CI image has + none — the case this CLI is most often run in. Asked at the start of a + run rather than discovered at the first save, because by then a recipe + has already run and its work is about to be restored away over a + setting that takes one command to fix. + + Asked as ``git var``, which is the question a commit itself asks: + an identity can come from ``user.email``, ``EMAIL``, the author and + committer variables, or three levels of config, and reimplementing + that lookup here is how a probe comes to disagree with the thing it + is standing in for. + + Args: + directory: The repository that is about to be committed to. + + Raises: + ProjectError: If no committer identity resolves. + """ + put_our_bin_first() + proc = project._run(["git", "var", "GIT_COMMITTER_IDENT"], cwd=directory) + if proc.returncode != 0: + raise project.ProjectError( + "git has no identity to commit with, and every materialized output " + "is committed:\n" + ' git config --global user.name "Your Name"\n' + ' git config --global user.email "you@example.com"' + ) + + +def head(directory: Path) -> tuple[str, str]: + """Read the commit ``HEAD`` is at and the ``origin`` URL. + + Both are reads and neither takes the index lock. + + Args: + directory: The repository to read. + + Returns: + ``(commit sha, origin URL)``. The URL is empty rather than absent + when the repository has no remote — a manifest field that is + sometimes missing reads worse than one that is sometimes blank. + """ + remote = project._run(["git", "config", "--get", "remote.origin.url"], cwd=directory) + return ( + _git(["rev-parse", "HEAD"], cwd=directory).strip(), + remote.stdout.strip() if remote.returncode == 0 else "", + ) + + +def dataset_id(directory: Path) -> str: + """Read the dataset's UUID out of ``.datalad/config``. + + Through ``git config`` rather than by parsing: lc writes that file and + otherwise leaves ``.datalad/`` to datalad. + + Args: + directory: The project root. + + Returns: + The UUID, or empty for a project assembled without ``lc init``. + """ + found = project._run( + ["git", "config", "-f", ".datalad/config", "--get", "datalad.dataset.id"], + cwd=directory, + ) + return found.stdout.strip() if found.returncode == 0 else "" + + +def save(directory: Path, paths: Iterable[Path], message: str) -> bool: + """Commit *paths*. + + A plain ``git add``: ``.gitattributes`` sets ``filter=annex``, so git's + own add routes content to the annex and everything else into git. lc + runs no annex command here for the same reason it asks nobody else to. + + ``annex.thin`` is set for this add alone, so a result is hard-linked to + its annex object rather than copied — one copy on disk instead of two. + It is safe precisely here: thin's hazard is editing a file in place, + which rewrites the object under the key that names it, and lc never + does — the worker removes an output directory before rebuilding it, and + an unlink leaves the object untouched. Setting it repository-wide would + reach declared inputs instead, which a researcher adds with their own + `git add` and whose tools very much do open files for update, so it is + passed per-add and never written to the repository's config. + + Args: + directory: The repository root. + paths: What to stage, absolute or repository-relative. + message: The commit message. + + Returns: + False if there was nothing to commit. + """ + relative = [_rel(directory, p) for p in paths] + _git(["-c", "annex.thin=true", "add", "-A", "--", *relative], cwd=directory) + if _git_ok(["diff", "--cached", "--quiet"], cwd=directory): + return False + _git(["commit", "-q", "-m", message], cwd=directory) + return True + + +def restore(directory: Path, paths: Iterable[Path]) -> None: + """Put *paths* back the way the last commit had them. + + ``clean`` first for what a run wrote, then ``checkout`` for what it + deleted or truncated — and only when the path is in ``HEAD``, since a + first materialization has nothing to go back to. + + Args: + directory: The repository root. + paths: What to restore. Scoped to these and never the whole tree: + a failed run must not discard edits made while it ran. + """ + for path in paths: + rel = _rel(directory, path) + _git(["clean", "-qfdx", "--", rel], cwd=directory) + if _git_ok(["cat-file", "-e", f"HEAD:{rel}"], cwd=directory): + _git(["checkout", "-q", "HEAD", "--", rel], cwd=directory) + + +# ============================================================================= +# Running git +# ============================================================================= + + +def _git(argv: list[str], *, cwd: Path) -> str: + """Run git in *cwd*, returning its stdout; a nonzero exit raises.""" + put_our_bin_first() + proc = project._run(["git", *argv], cwd=cwd) + if proc.returncode != 0: + raise project.ProjectError(f"`git {' '.join(argv)}` failed:\n{proc.stderr.strip()}") + return str(proc.stdout or "") + + +def _git_ok(argv: list[str], *, cwd: Path) -> bool: + """Run git as a yes/no probe: exit status is the answer, not a failure.""" + put_our_bin_first() + return bool(project._run(["git", *argv], cwd=cwd).returncode == 0) + + +def _rel(directory: Path, path: Path) -> str: + """*path* as a repository-relative POSIX pathspec. + + git pathspecs are ``/``-separated whatever the platform, and an + absolute path would silently mean something else inside a repository + reached through a symlink. + """ + resolved = Path(path) + if resolved.is_absolute(): + resolved = resolved.relative_to(directory.resolve()) + return resolved.as_posix() diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py new file mode 100644 index 00000000..a0e0d1c8 --- /dev/null +++ b/src/lightcone/engine/identity.py @@ -0,0 +1,284 @@ +"""What a materialized output is identified by. + +Two hashes, and they answer different questions on purpose. + +``definition_version`` is what the spec says an output *is*: its rendered +recipe and the decisions it was made under. It is the rebuild trigger — +when it moves, the artifact on disk is no longer an instance of the thing +the spec describes, so keeping it would be mislabelling it. + +``env_version`` is the environment's identity: the lock's bytes, the +interpreter pin's bytes, and the settings that decide *which artifacts* +``uv sync`` materializes from that lock. It is deliberately +over-sensitive — raw lock bytes, so a comment reflow moves it — because +the alternative is a parse that silently disagrees with uv about what the +lock means. + +**``env_version`` is not part of ``definition_version``**, and that is the +whole shape of the model. An environment moves for reasons that have +nothing to do with a given output — one added dependency rewrites the lock +for the whole project — and a research artifact costs hours to remake and +is often already looked at. So an environment edit does not stale +anything; it makes an artifact *behind*, which is a fact the report states +and a rebuild the caller can ask for. Over-sensitivity is affordable +exactly because it no longer spends compute. + +What makes that safe is that nothing is lost: the environment an output +was made under is recorded in its manifest, and the commit alongside it +reconstructs that environment from the lock of the day. + +The git commit is not an input to either hash. It is tree-wide, so +hashing it would stale every output in the repository on a README edit. + +Both hashes are length-framed. Concatenating fields raw lets a boundary +shift between them produce the same digest from different inputs — a +recipe ending in a character the decisions begin with, say — and nothing +about a content hash is worth having if it can be shifted. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import tomllib +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from lightcone.engine.project import ProjectError + +#: The closed, audited list of uv settings that change *which artifacts* a +#: sync materializes from an unchanged lock. Closed on purpose: anything +#: outside it is either already covered by the lock's bytes or does not +#: affect what ends up installed, and a set that grew by guesswork would +#: report every output in every project as behind each time it did. +_INSTALL_SETTINGS = ( + "default-groups", + "no-binary", + "no-binary-package", + "no-build", + "no-build-package", + "config-settings", + "no-build-isolation", + "no-build-isolation-package", +) + + +# ============================================================================= +# The hashes +# ============================================================================= + + +def env_version(root: Path) -> str: + """Compute the environment identity of a project. + + ``sha256(uv.lock bytes ‖ .python-version bytes ‖ canonical + install-settings JSON)``, length-framed. Read fresh on every call: this + is also the mid-run gate's baseline, and a cached value would check + nothing. + + Args: + root: The project root. + + Returns: + The digest, as ``sha256:``. + + Raises: + ProjectError: If ``uv.lock`` or ``.python-version`` is missing. + """ + lock = _required(root, "uv.lock", "run `uv lock` (or `lc init`) to lock the environment") + pin = _required( + root, + ".python-version", + "the exact interpreter pin is part of the environment's identity; " + "run `lc init` to scaffold it", + ) + + h = hashlib.sha256() + _frame(h, "uv.lock", lock.read_bytes()) + _frame(h, "python-version", pin.read_bytes()) + _frame(h, "install-settings", _install_settings(root).encode()) + return f"sha256:{h.hexdigest()}" + + +def definition_version(*, recipe: str, decisions: Mapping[str, str]) -> str: + """Compute what the spec says one output is. + + ``sha256(recipe ‖ canonical decisions)``, length-framed. The + environment is deliberately absent: see this module's docstring. + + Args: + recipe: The rendered recipe command. + decisions: The decisions this output declares, as id → option. + + Returns: + The digest, as ``sha256:``. + """ + h = hashlib.sha256() + _frame(h, "recipe", recipe.encode()) + _frame(h, "decisions", _canonical(dict(decisions)).encode()) + return f"sha256:{h.hexdigest()}" + + +def _frame(h: hashlib._Hash, label: str, data: bytes) -> None: + """Feed one labelled, length-delimited field into *h*.""" + h.update(label.encode()) + h.update(b"\0") + h.update(str(len(data)).encode("ascii")) + h.update(b"\0") + h.update(data) + + +def _canonical(value: Any) -> str: + """JSON with a single spelling per value — sorted keys, no padding.""" + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _install_settings(root: Path) -> str: + """The audited uv settings, canonically, absent ones included. + + Every key is emitted whether or not the project sets it, so adding a + setting whose value happens to be uv's default still moves + ``env_version`` — a project that says what it means and a project that + relies on a default are the same environment only until uv's default + changes. + + Only the values matter, never which file supplied them: two projects + that install the same artifacts are one environment however they spell + it. + """ + return _canonical({key: _uv_config(root).get(key) for key in _INSTALL_SETTINGS}) + + +# ============================================================================= +# The lock scan +# ============================================================================= + + +@dataclass(frozen=True) +class LockScan: + """What the lock says about how far identity actually reaches.""" + + #: Dependencies whose bytes the lock does not pin. A refusal. + refusals: tuple[str, ...] + #: Registry packages shipping no wheel, so the sdist is built at sync + #: time. Reported: identity covers the sdist, not the build of it. + sdist_built: tuple[str, ...] + #: Groups outside uv's default set. Advisory: they are installable + #: states `env_version` does not distinguish. + non_default_groups: tuple[str, ...] + + +def scan_lock(root: Path) -> LockScan: + """Scan the lock for dependencies that weaken an output's identity. + + A path, directory or editable dependency records *where* it was rather + than what was in it, so two syncs of one lock can install different + code while every hash agrees they are identical. That is the refusal. + The project's own package is exempt: the repository records its bytes. + + Args: + root: The project root. + + Returns: + The refusals, the registry packages built from sdist, and the + dependency groups outside uv's default set. + + Raises: + ProjectError: If ``uv.lock`` is missing or is not valid TOML. + """ + lock_path = _required(root, "uv.lock", "run `uv lock` (or `lc init`)") + try: + lock = tomllib.loads(lock_path.read_text()) + except tomllib.TOMLDecodeError as e: + raise ProjectError(f"{lock_path}: invalid TOML: {e}") from e + + pyproject = _pyproject(root) + own = _canonical_name(pyproject.get("project", {}).get("name") or "") + + refusals: list[str] = [] + sdist_built: list[str] = [] + for package in lock.get("package", []): + name = package.get("name", "?") + source = package.get("source", {}) or {} + if kind := next((k for k in ("path", "directory", "editable") if k in source), None): + if _canonical_name(name) != own: + refusals.append( + f"{name}: {kind} dependency — the lock records where it was, " + "not what was in it, so two syncs can install different code" + ) + elif "registry" in source and "sdist" in package and not package.get("wheels"): + sdist_built.append(name) + + groups = set(pyproject.get("dependency-groups", {}) or {}) + default = _uv_config(root).get("default-groups", ["dev"]) + non_default = set() if default == "all" else groups - set(default) + + return LockScan( + refusals=tuple(sorted(refusals)), + sdist_built=tuple(sorted(sdist_built)), + non_default_groups=tuple(sorted(non_default)), + ) + + +# ============================================================================= +# Reading the project's files +# ============================================================================= + + +def _canonical_name(name: str) -> str: + """A distribution name in PEP 503 form. + + Both sides need it: uv writes the normalized name into ``uv.lock``, + while ``pyproject.toml`` carries whatever the author wrote — and + ``project_name()`` keeps ``_`` and ``.``. Comparing them raw makes a + packaged project called ``my_project`` fail to recognise *itself*, and + the lock scan then refuses the whole run over the project's own code. + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def _required(root: Path, name: str, remedy: str) -> Path: + """*root*/*name*, or a refusal naming what to do about its absence.""" + path = root / name + if not path.is_file(): + raise ProjectError(f"{root}: no {name} — {remedy}.") + return path + + +def _uv_config(root: Path) -> dict[str, Any]: + """uv's settings for this project, read the way uv itself reads them. + + A ``uv.toml`` beside ``pyproject.toml`` **replaces** ``[tool.uv]`` + wholesale rather than merging with it, so a project carrying both has + settings uv ignores — and hashing those would say two environments + differ when uv installs the same thing in each. uv warns about the + pair itself, and convergence already lifts its warnings. + + What this cannot reach is user-level configuration + (``~/.config/uv/uv.toml``), which uv *does* merge in underneath. That + is machine state rather than project state: hashing it would make one + commit answer differently on two hosts, reporting every output as + behind on a colleague's clone. + """ + config = root / "uv.toml" + if not config.is_file(): + return _pyproject(root).get("tool", {}).get("uv", {}) or {} + try: + return tomllib.loads(config.read_text()) + except tomllib.TOMLDecodeError as e: + raise ProjectError(f"{config}: invalid TOML: {e}") from e + + +def _pyproject(root: Path) -> dict[str, Any]: + path = _required( + root, + "pyproject.toml", + "the environment is pyproject.toml + uv.lock + .python-version; run `lc init`", + ) + try: + return tomllib.loads(path.read_text()) + except tomllib.TOMLDecodeError as e: + raise ProjectError(f"{path}: invalid TOML: {e}") from e diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py new file mode 100644 index 00000000..f5d26ee1 --- /dev/null +++ b/src/lightcone/engine/materialize.py @@ -0,0 +1,676 @@ +"""Making a whole analysis: what runs, in what order, and what gets committed. + +The driver's three jobs, and the order matters. + +**It refuses to start on a dirty tree.** Every materialization is +committed together with the code that produced it, so a run that began +with uncommitted changes could not honestly say which code that was. + +**It hands the graph to Dask and gets out of the way.** Every task is +submitted with its upstream futures as arguments, so the ordering, the +parallelism, and the scheduling are Dask's — there is no ready-set loop +here to get wrong. + +**It owns git, alone.** Workers execute and return; the driver commits, in +one thread, as results arrive. That is not a preference: concurrent git +operations on one repository race on the index lock. The same loop +restores what a failed or interrupted task left behind, so the tree ends +exactly as clean as it started — which is what makes the refusal above +survivable rather than a trap. + +One consequence, checked rather than assumed: a dependent starts as soon +as its upstream's *worker* returns, which is milliseconds before the +driver finishes annexing that upstream — so a recipe does read an input +directory while ``git annex add`` is replacing its files with symlinks. +That is safe, because git-annex hard-links the content into the object +store first and then renames the symlink over the file: the path never +stops existing and never holds partial bytes. Measured, on a run of +concurrent full-content reads across 24 MB: no missing paths, no short +reads, no wrong bytes. Do not "fix" this by moving the save into the +task — that is what puts git back in the workers. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Protocol + +from lightcone.engine import assets, dataset, identity, plan, project, worker +from lightcone.engine.plan import Graph, Key, Task +from lightcone.engine.project import ProjectError + + +@dataclass +class MaterializeReport: + """What a run did, or — in check mode — what it would do.""" + + #: ``universe/output`` for each output this run produced and committed. + made: list[str] = field(default_factory=list) + #: Nothing about them changed, and the environment is the one they were + #: made under. + current: list[str] = field(default_factory=list) + #: Still what the spec asks for, but made under an earlier environment + #: — ``universe/output`` → the context, naming the commit. Left alone; + #: ``refresh`` is what remakes them. + behind: dict[str, str] = field(default_factory=dict) + #: The recipe failed, or the environment moved under it. + failed: list[str] = field(default_factory=list) + #: Not attempted, because something upstream did not finish. + blocked: list[str] = field(default_factory=list) + #: Check mode only: ``universe/output`` → why it would run. + planned: dict[str, str] = field(default_factory=dict) + #: lc's own prose: what the lock scan found, why a task did not finish. + warnings: list[str] = field(default_factory=list) + #: Console lines from the boundary, verbatim — a downgrade notice, a + #: denial and its remedies. Kept apart from ``warnings`` because they + #: are built to be *pasted*: reflowing a `uv add numpy` to the terminal + #: width breaks the one thing a denial message is for. The caller + #: prints these unwrapped, exactly as ``lc run`` does. + notes: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + """Whether everything that was attempted finished.""" + return not self.failed and not self.blocked + + @property + def up_to_date(self) -> bool: + """Whether the analysis needed nothing done to it. + + ``behind`` does not count against it. An output made under an + earlier environment is not out of date — it is what the spec asks + for, and saying otherwise would put ``--check`` back in the + business of demanding compute. + + A run that failed is not up to date either, however little it + managed to produce: ``made`` stays empty when every recipe fails, + so without ``ok`` here the first two keys of the JSON report would + read "nothing to do" over a list of failures. + """ + return self.ok and not self.made and not self.planned + + def as_dict(self) -> dict[str, Any]: + """Return the report as JSON-ready data. + + Returns: + Every field, with ``ok`` and ``up_to_date`` first. + """ + return {"ok": self.ok, "up_to_date": self.up_to_date, **asdict(self)} + + +# ============================================================================= +# Check mode +# ============================================================================= + + +def check(root: Path, targets: Sequence[str], *, refresh: bool = False) -> MaterializeReport: + """Classify every task without executing or committing anything. + + Not subject to the dirty-tree refusal: reading the state of a project + before deciding what to commit is what this is for. The walk is + topological because a task can only be classified after everything + upstream of it, and an upstream already classified as would-run is + passed down as ``None`` — check mode cannot know whether a rebuild + comes out byte-identical, and assuming it will would under-report. + + Args: + root: The project root. + targets: What to classify; empty means everything. + refresh: Whether an output that is merely behind would be remade. + + Returns: + ``planned`` naming each output that would run and why, ``behind`` + those made under an earlier environment, and ``current`` the + rest. + + Raises: + ProjectError: If the spec, the universes or the lock cannot be + read, or a target matches nothing. + """ + report = MaterializeReport() + for key, verdict, _ in _classified(root, targets, report, refresh=refresh): + name = _name(key) + if verdict.calls_for_a_remake(refresh=refresh): + report.planned[name] = verdict.why + elif verdict.status == "behind": + report.behind[name] = verdict.why + else: + report.current.append(name) + return report + + +def _classified( + root: Path, targets: Sequence[str], report: MaterializeReport, *, refresh: bool +) -> list[tuple[Key, assets.Verdict, assets.Manifest | None]]: + """Classify every task in topological order, reading nothing but disk. + + The walk both read-only modes share, so there is one answer to "what + is this output" and not one per verb. Topological because a task can + only be classified after everything upstream of it, and an upstream + already decided to run is passed down as ``None`` — nothing here can + know whether a rebuild comes out byte-identical, and assuming it will + would under-report. + + Args: + root: The project root. + targets: What to classify; empty means everything. + report: Collects the lock scan's warnings, and the note about + inputs this clone has not fetched. + refresh: Whether behind outputs count as running, which is what + decides whether their dependents see the sentinel. + + Returns: + One ``(key, verdict, manifest)`` per task, upstream first. + """ + graph, env_version = _graph(root, targets, report) + versions = assets.Versions() + would_run: set[Key] = set() + unfetched: set[str] = set() + + classified = [] + for key in graph.order(): + task = graph.tasks[key] + manifest = assets.read(task.output_dir) + verdict = assets.classify( + definition_version=task.definition_version, + env_version=env_version, + manifest=manifest, + inputs=_predicted(root, task, would_run, versions, unfetched), + ) + if verdict.calls_for_a_remake(refresh=refresh): + would_run.add(key) + classified.append((key, verdict, manifest)) + + if unfetched: + report.warnings.append( + "reported as out of date because their content is not in this " + f"clone, not because they changed: {', '.join(sorted(unfetched))}. " + "Fetch with `git annex get `." + ) + return classified + + +def _predicted( + root: Path, + task: Task, + would_run: set[Key], + versions: assets.Versions, + unfetched: set[str], +) -> dict[str, str | None]: + """Each input's version as check mode can know it. + + Args: + root: The project root, for naming a path in the report. + task: The output being classified. + would_run: Task keys already decided to be remade. + versions: The run's content-hash memo. + unfetched: Collects declared inputs whose content is not local, so + the report can say why they read as out of date. + + Returns: + Each input's version as check mode can know it: ``None`` for + anything that will be rebuilt, is absent, or cannot be read. + """ + predicted: dict[str, str | None] = {} + for name, path in task.inputs.items(): + upstream = task.produced_by.get(name) + if upstream is not None: + # The upstream's *recorded* digest, for the same reason a worker + # returns it rather than rehashing: on a clone that has fetched + # no annex content the files are dangling symlinks, and hashing + # them would report a different output and cascade a rebuild + # over a project that is perfectly up to date. + manifest = None if upstream in would_run else assets.read(path) + predicted[name] = manifest.data_version if manifest else None + elif not path.exists(): + predicted[name] = None + else: + try: + predicted[name] = versions.of(path) + except assets.ContentNotFetchedError: + # Not "it changed" — "I cannot tell". Conservative, and + # said out loud, because reporting a rebuild for a clone + # that simply has not fetched its inputs is misleading on + # its own. + unfetched.add(plan.declared_path(root, path)) + predicted[name] = None + except OSError: + # Unreadable for any other reason — a broken symlink inside + # the directory, a permission wall. Same answer as an input + # that is not there at all: it will be remade, and the + # recipe is where that failure belongs, with a real error. + predicted[name] = None + return predicted + + +# ============================================================================= +# Status — what the project holds, and where each of it came from +# ============================================================================= + + +@dataclass(frozen=True) +class OutputStatus: + """One output: what it is now, and the commit it came from.""" + + #: ``universe/output_id``. + output: str + status: assets.Status + #: Why, for ``stale`` and ``behind``. Empty for ``current``. + why: str + #: The commit the output was materialized at, or empty if it never was. + #: This is the whole point of the verb: an artifact that is behind is + #: not wrong, and this is where the code and environment that produced + #: it can be read back. + git_sha: str + #: Its content identity, or empty if it was never materialized. + data_version: str + + def as_dict(self) -> dict[str, Any]: + """Return the record as JSON-ready data. + + Returns: + Every field, in declaration order. + """ + return asdict(self) + + +@dataclass +class StatusReport: + """Every output the spec declares, in dependency order.""" + + outputs: list[OutputStatus] = field(default_factory=list) + #: The lock scan's prose, and inputs this clone has not fetched. + warnings: list[str] = field(default_factory=list) + + @property + def counts(self) -> dict[str, int]: + """How many outputs are in each state, states with none included.""" + tally = {"current": 0, "behind": 0, "stale": 0} + for output in self.outputs: + tally[output.status] += 1 + return tally + + def as_dict(self) -> dict[str, Any]: + """Return the report as JSON-ready data. + + Returns: + The counts, then every output, then the warnings. + """ + return { + "counts": self.counts, + "outputs": [output.as_dict() for output in self.outputs], + "warnings": self.warnings, + } + + +def status(root: Path) -> StatusReport: + """Report what state every declared output is in. + + Reads manifests and hashes declared inputs; runs nothing, commits + nothing, and does not care whether the tree is clean. Classified with + ``refresh=False``, because this says what the project *is* rather than + what some run would do to it. + + Args: + root: The project root. + + Returns: + One record per declared output, upstream first. + + Raises: + ProjectError: If the spec, the universes or the lock cannot be + read. + """ + report = MaterializeReport() + result = StatusReport() + for key, verdict, manifest in _classified(root, [], report, refresh=False): + result.outputs.append( + OutputStatus( + output=_name(key), + status=verdict.status, + why=verdict.why, + git_sha=manifest.git_sha if manifest else "", + data_version=manifest.data_version if manifest else "", + ) + ) + result.warnings = report.warnings + return result + + +# ============================================================================= +# Executing +# ============================================================================= + + +def materialize( + root: Path, targets: Sequence[str], *, refresh: bool = False +) -> MaterializeReport: + """Make everything *targets* names, committing each output as it lands. + + Args: + root: The project root. + targets: What to make; empty means everything. Asking for an + output asks for what it is made of. + refresh: Also remake outputs that are merely behind — still what + the spec asks for, but made under an earlier environment. + + Returns: + What was made, what was current or behind, what failed or was + blocked, plus the boundary's notes and the lock scan's warnings. + + Raises: + ProjectError: If a required tool or git's committer identity is + missing, the tree has uncommitted changes, or the lock cannot + be audited. + """ + project.require_uv() + project.require_git() + project.require_git_annex() + dataset.require_committer(root) + # Before anything else: workers pass `--no-sync`, so this is the only + # place the environment is made to match the lock. + report = MaterializeReport() + report.warnings.extend(f"uv: {w}" for w in project.sync(root)) + if changes := dataset.status(root): + raise ProjectError(_dirty(root, changes)) + + graph, env_version = _graph(root, targets, report) + if not graph.tasks: + return report + + dsid = dataset.dataset_id(root) + # Read once, for every task: the driver commits each output as it lands, + # so HEAD moves during the run, and a per-task read would give later + # manifests a commit this run created — nondeterministically, depending + # on whether a recipe finished before or after the previous save. + head = dataset.head(root) + # One memo for the run, for the same reason as one HEAD read: a + # declared input shared by several outputs — or by one output across + # several universes — is the same bytes every time it is asked for. + versions = assets.Versions() + outstanding: dict[Key, Task] = dict(graph.tasks) + try: + with cluster_for_run() as scheduler: + pending: dict[Key, Any] = {} + # Submitted in dependency order so a task's upstream futures + # exist to be passed to it. Dask still derives the *execution* + # order — from those arguments, not from this loop. + for key in graph.order(): + task = graph.tasks[key] + pending[key] = scheduler.submit( + worker.materialize, + root, + task, + env_version, + head, + versions, + refresh, + *[pending[dep] for dep in task.depends_on], + key=_name(key), + ) + for result in scheduler.completed(list(pending.values())): + _consume(root, graph.tasks[result.key], result, dsid, report) + outstanding.pop(result.key, None) + finally: + # Whatever never reported — an interrupt, a dead cluster — left a + # reset output directory behind. Scoped to this run's outputs and + # never to the whole tree, so edits made while the graph ran + # survive. + for task in outstanding.values(): + dataset.restore(root, [task.output_dir]) + return report + + +def _consume( + root: Path, task: Task, result: worker.TaskResult, dsid: str, report: MaterializeReport +) -> None: + """Record one finished task, and commit or undo what it left on disk.""" + name = _name(task.key) + if lines := [note for note in result.notes if note]: + # Named on a line of their own rather than prefixed onto each: a + # prefix would land in the middle of a multi-line remedy and make + # it uncopyable, which is the whole reason these travel separately. + report.notes.extend([f"{name}:", *lines]) + + if result.status == "ok": + dataset.save(root, [task.output_dir], run_record(root, task, dsid)) + report.made.append(name) + return + + if result.status == "current": + report.current.append(name) + return + + if result.status == "behind": + report.behind[name] = result.reason + return + + dataset.restore(root, [task.output_dir]) + getattr(report, result.status).append(name) + report.warnings.append(f"{name}: {result.reason}") + + +class Scheduler(Protocol): + """How the driver talks to whatever is running the graph. + + Two methods, because that is all the driver needs and all a venue has + to supply: hand over a task with its upstream handles, and iterate the + results as they land. Keeping it this narrow is what lets the suite + run the graph inline — and what will let a venue larger than a laptop + land behind :func:`cluster_for_run` without the driver noticing. + """ + + def submit(self, fn: Any, *args: Any, key: str) -> Any: + """Schedule a call. + + Args: + fn: The function to run. + *args: Its arguments, upstream handles included. + key: A display name for the task. + + Returns: + A handle to pass to dependents. + """ + ... + + def completed(self, handles: list[Any]) -> Iterator[worker.TaskResult]: + """Yield results as they land. + + Args: + handles: Everything submitted. + + Yields: + Each task's result, in completion order. + """ + ... + + +@dataclass(frozen=True) +class _Dask: + """A Dask client, narrowed to what the driver asks of it.""" + + client: Any + + def submit(self, fn: Any, *args: Any, key: str) -> Any: + """Schedule a call on the Dask client. See :class:`Scheduler`.""" + return self.client.submit(fn, *args, key=key) + + def completed(self, handles: list[Any]) -> Iterator[worker.TaskResult]: + """Yield results as Dask completes them. See :class:`Scheduler`.""" + # distributed ships no type information, so this one call is + # annotated rather than the module exempted. + from distributed import as_completed + + for _, result in as_completed(handles, with_results=True): # type: ignore[no-untyped-call] + yield result + + +@contextmanager +def cluster_for_run() -> Iterator[Scheduler]: + """Open a scheduler for one run. The seam venues will land behind. + + Every core, with no knob to say otherwise: how much of a machine a run + may use, and which machine, is one question and it belongs to the + declaration of an execution backend. + + Threads rather than processes — every task's real work happens in a + subprocess behind the exec boundary, so a worker spends its time in + ``wait()`` with the GIL released, and a threaded cluster costs no + interpreter startup and no pickling of results. + + Yields: + A scheduler bound to a local Dask cluster, closed on exit. + """ + from distributed import Client, LocalCluster + + with LocalCluster( # type: ignore[no-untyped-call] + n_workers=1, + threads_per_worker=os.cpu_count() or 1, + processes=False, + dashboard_address=None, + ) as cluster: + with Client(cluster) as client: # type: ignore[no-untyped-call] + yield _Dask(client) + + +# ============================================================================= +# The commit +# ============================================================================= + + +def run_record(root: Path, task: Task, dsid: str) -> str: + """Build the commit message for one materialized output. + + A ``[DATALAD RUNCMD]`` record — datalad's format, not ours, so all of + it is written and none of it abbreviated. ``datalad rerun`` reads it + with a regex and reports "no command; skipping" on any mismatch. + + ``cmd`` is the worker module: the bare recipe would reconstruct nothing + lc adds, and ``lc materialize`` cannot be it because a rerun removes + the declared outputs first, dirtying the tree materialize refuses to + start from. + + Args: + root: The project root. + task: The output that was made. + dsid: The dataset's UUID, which ``rerun`` reads. + + Returns: + The full commit message, subject and record. + """ + info = { + "chain": [], + "cmd": ( + "uv run --locked --project . -- python -m lightcone.engine.worker " + f"{task.universe_id}/{task.output_id}" + ), + "dsid": dsid, + "exit": 0, + "inputs": sorted(plan.declared_path(root, path) for path in task.inputs.values()), + "outputs": [plan.declared_path(root, task.output_dir)], + "pwd": ".", + } + body = json.dumps(info, indent=1, sort_keys=True, ensure_ascii=False) + return ( + f"[DATALAD RUNCMD] {task.output_id} [{task.universe_id}]\n\n" + "=== Do not change lines below ===\n" + f"{body}\n" + "^^^ Do not change lines above ^^^" + ) + + +# ============================================================================= +# Reading the project +# ============================================================================= + + +def _graph(root: Path, targets: Sequence[str], report: MaterializeReport) -> tuple[Graph, str]: + """The tasks a run covers, and the environment they will be compared to. + + The lock scan runs here, once, for both modes: what it refuses is a + dependency whose bytes the lock does not pin, which makes every hash + below it a claim nobody can check. + """ + scan = identity.scan_lock(root) + if scan.refusals: + raise ProjectError( + "the lock has dependencies that cannot be audited, so an output's " + "identity would not mean anything:\n " + + "\n ".join(scan.refusals) + + "\nPublish them, or vendor them into the project." + ) + if scan.sdist_built: + report.warnings.append( + "built from source at sync time, so identity covers the sdist and " + f"not the build of it: {', '.join(scan.sdist_built)}" + ) + if scan.non_default_groups: + report.warnings.append( + "dependency groups outside uv's default set are installable states " + f"the environment's identity does not distinguish: {', '.join(scan.non_default_groups)}" + ) + + env_version = identity.env_version(root) + graph = plan.build(root) + if targets: + graph = graph.closure(graph.resolve(list(targets))) + + # A declared input outside the project is hashed into the manifest like + # any other, so a change to it still cascades — but it is not in the + # repository, so the commit that records the output cannot bring it + # back. That is a weaker promise than the rest of the layer makes, and + # the only honest thing to do about it is say so. + outside = { + plan.declared_path(root, path) + for task in graph.tasks.values() + for name, path in task.inputs.items() + if name not in task.produced_by and root not in path.parents + } + if outside: + report.warnings.append( + "declared inputs outside the project are recorded by content but " + "not stored in it, so a commit cannot restore them: " + + ", ".join(sorted(outside)) + ) + return graph, env_version + + +def _name(key: Key) -> str: + return f"{key[0]}/{key[1]}" + + +def _dirty(root: Path, changes: Sequence[tuple[str, str]]) -> str: + """The refusal, split by what the right remedy actually is. + + Two path classes, because they call for opposite actions: work the + researcher owns has to be committed, and anything under ``results/`` + is lc's to write, so a change there is wreckage to discard rather than + a contribution to keep. + """ + theirs = [c for c in changes if not c[1].startswith("results/")] + ours = [c for c in changes if c[1].startswith("results/")] + + lines = [ + f"uncommitted changes in {root} — every materialization is committed " + "with the code that produced it, so a run cannot start from a tree " + "that does not say what that code is.", + ] + if theirs: + lines += [ + "", + ' commit these: git add -A . && git commit -m "…"', + *(f" {code.strip() or '??'} {path}" for code, path in theirs), + ] + if ours: + lines += [ + "", + " discard these (lc writes results/):", + " git restore --staged --worktree results/ && git clean -fd results/", + *(f" {code.strip() or '??'} {path}" for code, path in ours), + ] + return "\n".join(lines) diff --git a/src/lightcone/engine/plan.py b/src/lightcone/engine/plan.py new file mode 100644 index 00000000..154af1f5 --- /dev/null +++ b/src/lightcone/engine/plan.py @@ -0,0 +1,324 @@ +"""The spec, read as a graph of tasks. + +``astra.yaml`` × ``universes/*.yaml`` gives one task per +``(universe, output)`` pair that has a recipe. A task carries everything +executing it needs and nothing about *how* it will be executed: the +rendered command, where its bytes go, what it reads, which decisions it +was made under, and its ``definition_version``. + +What the spec *means* is ASTRA's to say. ``astra.resolve`` settles each +universe's decisions, resolves every output's inputs to what supplies +them, drops the outputs whose ``when:`` does not hold, and renders the +recipe grammar — so scoping, ``from:`` references and sub-analysis +nesting are read here rather than re-derived. A qualified output id +(``classification.accuracy``) is used verbatim as the directory name, so +one addressing scheme spans however deep the spec nests. + +Nothing here schedules anything. Ordering is Dask's job at execution time +and a topological walk's job in ``--check``; this module only says which +task depends on which. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from graphlib import CycleError, TopologicalSorter +from pathlib import Path + +from lightcone.engine import assets, identity +from lightcone.engine.project import SPEC_FILENAME, ProjectError + +#: A task's identity within a run: which universe, which output. +Key = tuple[str, str] + + +@dataclass(frozen=True) +class Task: + """One output, in one universe: everything needed to make it.""" + + universe_id: str + output_id: str + output_dir: Path + #: The recipe with its placeholders substituted — a shell command. + recipe: str + #: Declared input name → the path it resolves to. Upstream outputs are + #: their directories; everything else is whatever ``source`` named. + inputs: dict[str, Path] + #: The subset of ``inputs`` another task produces, and which one. + produced_by: dict[str, Key] + decisions: dict[str, str] + definition_version: str + + @property + def key(self) -> Key: + """This task's identity within a run.""" + return (self.universe_id, self.output_id) + + @property + def depends_on(self) -> tuple[Key, ...]: + """The tasks that must be made first, deduplicated, in order.""" + return tuple(dict.fromkeys(self.produced_by.values())) + + +@dataclass(frozen=True) +class Graph: + """Every task a run could make, and how they relate.""" + + tasks: dict[Key, Task] + + def order(self) -> list[Key]: + """Return the tasks in dependency order. + + Only ``--check`` needs this, to classify a task after everything + upstream of it. Execution never calls it: Dask derives the same + order from the futures it is handed. + + Returns: + Every task key, dependencies first. + + Raises: + ProjectError: If the outputs depend on each other in a cycle. + """ + sorter = TopologicalSorter({k: set(t.depends_on) for k, t in self.tasks.items()}) + try: + return list(sorter.static_order()) + except CycleError as e: + raise ProjectError(f"the outputs depend on each other in a cycle: {e.args[1]}") from e + + def closure(self, targets: list[Key]) -> Graph: + """Narrow the graph to *targets* and what they depend on. + + Asking for an output asks for what it is made of; anything less + runs a recipe against inputs that were never brought up to date. + + Args: + targets: The task keys asked for. + + Returns: + A graph holding *targets* plus their transitive dependencies. + """ + wanted: set[Key] = set() + pending = list(targets) + while pending: + key = pending.pop() + if key in wanted: + continue + wanted.add(key) + pending.extend(self.tasks[key].depends_on) + return Graph(tasks={k: t for k, t in self.tasks.items() if k in wanted}) + + def resolve(self, targets: list[str]) -> list[Key]: + """Turn what a user typed into task keys. + + Args: + targets: Each an output id — matching every universe that has + it — or ``/`` for exactly one. + + Returns: + The matching task keys, in the order given. + + Raises: + ProjectError: If a target matches nothing. Quietly making + nothing is the least useful thing a build tool can do. + """ + keys: list[Key] = [] + for target in targets: + universe, _, output = target.rpartition("/") + matched = [ + key for key in self.tasks if key[1] == output and universe in ("", key[0]) + ] + if not matched: + known = ", ".join(sorted(f"{u}/{o}" for u, o in self.tasks)) or "none" + raise ProjectError(f"no output matches `{target}`. Available: {known}") + keys.extend(matched) + return keys + + +# ============================================================================= +# Building the graph +# ============================================================================= + + +def build(root: Path) -> Graph: + """Read a project's spec and universes into a graph of tasks. + + Args: + root: The project root. + + Returns: + One task per ``(universe, output)`` pair that has a recipe and is + active in that universe. + + Raises: + ProjectError: If the spec is missing, declares no universe, gives + two universes the same id, or names an input nothing provides. + """ + from astra.helpers import load_yaml, resolve_analysis_tree + + spec_path = root / SPEC_FILENAME + if not spec_path.is_file(): + raise ProjectError( + f"{root}: no {SPEC_FILENAME} — there is no analysis to materialize." + ) + universes = sorted((root / "universes").glob("*.yaml")) + if not universes: + raise ProjectError( + f"{root}/universes/ declares no universe — a run needs at least one " + "set of decisions to make outputs under." + ) + _validate(spec_path, universes) + spec = dict(resolve_analysis_tree(load_yaml(spec_path), root)) + + tasks: dict[Key, Task] = {} + declared_in: dict[str, Path] = {} + for path in universes: + universe = load_yaml(path) + universe_id = str(universe.get("id") or path.stem) + # A universe id names a directory under results/, so two files + # claiming one would write to the same place — and since the second + # simply replaces the first here, the outputs of one of them would + # go missing with nothing said. + if (first := declared_in.get(universe_id)) is not None: + raise ProjectError( + f"{first.name} and {path.name} both declare the universe " + f"`{universe_id}`, so both would materialize into " + f"results/{universe_id}/. Give each universe its own id." + ) + declared_in[universe_id] = path + for task in _tasks(root, universe_id, spec, universe): + tasks[task.key] = task + return Graph(tasks=tasks) + + +def declared_path(root: Path, path: Path) -> str: + """Name *path* the way the analysis declared it. + + Project-relative inside the tree, absolute outside it — the two forms a + recipe, a manifest and a ``[DATALAD RUNCMD]`` record all want, and the + reason a declared input with an absolute ``source:`` has somewhere to + be written down rather than being a crash. + + Deliberately **not** resolved. Every declared input under ``data/`` is + an annex symlink, so resolving would name + ``.git/annex/objects/SHA256E-…`` — the storage rather than the input, + and a path no one can fetch. + + Args: + root: The project root. + path: What to name. + + Returns: + A POSIX path, relative to *root* where it is under it. + """ + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.as_posix() + + +def _validate(spec_path: Path, universes: list[Path]) -> None: + """Refuse a spec ASTRA rejects, before anything is resolved. + + Resolution answers what a valid spec *means*; it does not re-check + that it is one. So an invalid spec reaches it as a missing decision or + an unresolvable input — blaming the run for a fault in the file, and + at a point far from the line that caused it. Asking ASTRA first costs + one pass over a spec file and moves the error to where it can be + fixed. + + Args: + spec_path: The project's ``astra.yaml``. + universes: Every universe file that will be resolved against it. + + Raises: + ProjectError: Listing every structural and semantic error found, + in ASTRA's own words. + """ + from astra.validation import ( + validate_analysis_file, + validate_analysis_schema, + validate_universe_file, + ) + + problems = [ + *validate_analysis_schema(spec_path), + *(str(e) for e in validate_analysis_file(spec_path)), + ] + for path in universes: + problems += [f"{path.name}: {e}" for e in validate_universe_file(path, spec_path)] + if problems: + listed = "\n".join(f" {problem}" for problem in problems) + raise ProjectError( + f"{spec_path} does not validate, so there is nothing to materialize " + f"from it:\n{listed}" + ) + + +def _tasks( + root: Path, + universe_id: str, + spec: dict[str, object], + universe: dict[str, object], +) -> list[Task]: + """Every task one universe contributes. + + ``resolve_outputs`` has already dropped what this universe does not + produce, so the only filter left is whether an output carries a + command: a re-export names bytes another output makes, and making it + twice under two ids is not a thing to do. + """ + from astra.resolve import render_command, resolve_outputs + + resolved = resolve_outputs(spec, universe, root) + executable = {out.id for out in resolved if out.command} + + tasks = [] + for out in resolved: + if not out.command: + continue + output_dir = assets.output_dir(root, universe_id, out.id) + values: dict[str, str] = {} + paths: dict[str, Path] = {} + produced_by: dict[str, Key] = {} + for declared in out.inputs: + if declared.produced_by in executable: + produced_by[declared.id] = (universe_id, declared.produced_by) + paths[declared.id] = assets.output_dir( + root, universe_id, declared.produced_by + ) + elif declared.source: + # An absolute `source:` wins over the join — pathlib's own + # rule, and the one anyone writing one expects. + paths[declared.id] = root / declared.source + else: + raise ProjectError( + f"output `{out.id}` declares the input `{declared.id}`, but no " + "output produces it and no declared input gives it a source." + ) + values[declared.id] = declared_path(root, paths[declared.id]) + + try: + recipe = render_command( + out.command, + inputs=values, + decisions=out.decisions, + output=declared_path(root, output_dir), + ) + except ValueError as e: + raise ProjectError(f"output `{out.id}`: {e}") from e + + tasks.append( + Task( + universe_id=universe_id, + output_id=out.id, + output_dir=output_dir, + recipe=recipe, + inputs=paths, + produced_by=produced_by, + decisions=out.decisions, + definition_version=identity.definition_version( + recipe=recipe, decisions=out.decisions + ), + ) + ) + return tasks diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index ee1ec035..ca89dcd6 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -7,11 +7,15 @@ import re import shutil import subprocess +import uuid from collections.abc import Callable from dataclasses import asdict, dataclass, field +from functools import partial from pathlib import Path -from lightcone.engine import templates +from astra.scaffold import create_boilerplate + +from lightcone.engine import dataset, templates SPEC_FILENAME = "astra.yaml" @@ -47,17 +51,19 @@ def converged(self) -> bool: """Whether the project needed nothing done to it. Note the tense: after a write run that created files this is - ``False``. It reports what convergence *found* — which is the - question check mode asks — not "the project is now good". + ``False``. It reports what convergence *found*, not whether the + project is now good. """ return not self.created and not self.repaired and not self.blocked def as_dict(self) -> dict[str, object]: - """The report as JSON-ready data, ``converged`` first. + """Return the report as JSON-ready data, ``converged`` first. + + Built from :func:`dataclasses.asdict`, so a field added to the + report cannot silently go missing from ``lc init --json``. - Built from :func:`dataclasses.asdict` so a field added to the - report cannot silently go missing from ``lc init --json``, which is - the agent-facing contract. + Returns: + Every field, with ``converged`` first. """ return {"converged": self.converged, **asdict(self)} @@ -86,12 +92,16 @@ def item( ) -> None: """Converge something whose *presence* is the question. - ``is_current`` makes it a *derived* artifact instead, one whose - agreement with its inputs is the question: a ``uv.lock`` that no - longer matches ``pyproject.toml``, or a ``.venv`` that no longer - matches the lock, is exactly as unconverged as a missing one, and - reports as ``repaired``. It is consulted only when the item is - present, so a fresh project pays no probe. + Args: + name: What to call it in the report. + present: Whether it is already there. + apply: Creates or repairs it. Called only when writing. + is_current: Makes it a *derived* artifact instead, one whose + agreement with its inputs is the question — a ``uv.lock`` + that no longer matches ``pyproject.toml`` is exactly as + unconverged as a missing one, and reports as ``repaired``. + Consulted only when present, so a fresh project pays no + probe. """ if not present: self.report.created.append(name) @@ -110,17 +120,18 @@ def file( template: Callable[[], str], repair: Callable[[str], str | None] | None = None, ) -> None: - """Create *path* from *template* if missing; else offer it to *repair*. - - *template* is a thunk rather than a string, so check mode renders - nothing at all and a steady-state run renders only the files it - actually writes. Parent directories are created, so a managed file - never depends on an earlier item having made its directory first. - - ``repair`` receives the current text and returns the fixed text, or - ``None`` when the file is already fine. Repairs must be - conservative by construction — see - :func:`~lightcone.engine.templates.gitignore_repair`. + """Create a file from a template, or offer it to a repair. + + Args: + name: What to call it in the report. + path: Where it goes. Parent directories are created, so a + managed file never depends on an earlier item. + template: Renders the file. A thunk rather than a string, so + check mode renders nothing and a steady-state run renders + only what it writes. + repair: Receives the current text and returns the fixed text, + or ``None`` when the file is already fine. Must be + conservative by construction. """ if not path.exists(): self.report.created.append(name) @@ -138,7 +149,11 @@ def blocked(self, name: str, reason: str) -> None: """Record an item convergence cannot complete, and why. Stronger than :meth:`warn`: the project is not converged, so - ``--check`` fails instead of reporting a file that isn't there. + ``--check`` fails rather than reporting a file that is not there. + + Args: + name: What to call it in the report. + reason: What the user must do, recorded as a warning. """ self.report.blocked.append(name) self.report.warnings.append(reason) @@ -154,39 +169,41 @@ def warn(self, message: str) -> None: def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: - """Converge *directory* into an ASTRA project. Idempotent. + """Converge a directory into an ASTRA project. Idempotent. Creates whatever is missing, repairs the pieces lightcone manages, and never overwrites a file the user owns. A directory that already holds an ``astra.yaml`` is adopted, not rejected. - With ``write=False`` nothing touches the filesystem — not even the - directory itself — and the returned report describes what a real run - would have done. + Args: + directory: The project root, created if absent. + write: False for check mode, which touches nothing — not even the + directory — and reports what a real run would have done. - Raises :class:`ProjectError` if uv is missing or fails — it is the - environment substrate, so there is no useful project without it. - """ - # Imported here rather than at module scope because `astra.cli` costs - # ~0.5 s to import — Click, Rich, and the validation stack, which pulls - # linkml_runtime. astra-tools#102 moves the scaffold to a stdlib-only - # `astra.scaffold`; once that lands this can go back up top. - from astra.cli import create_boilerplate + Returns: + What was created, repaired, left alone or blocked, plus warnings. + Raises: + ProjectError: If uv, git or git-annex is missing, or any of them + fails. + """ directory = directory.resolve() require_uv() + require_git() + require_git_annex() c = _Converger(write=write) if write: directory.mkdir(parents=True, exist_ok=True) - # `create_boilerplate` is astra's public scaffold API, the same one + # `astra.scaffold` is astra's public scaffold API, the same one # `astra init` delegates to: it writes the spec — astra.yaml plus # universes/baseline.yaml, which converge as one item because the # baseline references the boilerplate's example decision — and nothing - # else. + # else. It is stdlib-only and imports in milliseconds, which is why it + # sits at module scope where the validation stack cannot. c.item( "astra.yaml", (directory / SPEC_FILENAME).exists(), @@ -196,19 +213,22 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: c.file( ".gitignore", directory / ".gitignore", - templates.gitignore, + partial(templates.read, "gitignore.tmpl"), repair=templates.gitignore_repair, ) - _converge_results(c, directory) + _converge_dataset(c, directory) + _converge_tracked_dir(c, directory, "data", partial(templates.read, "data-README.md.tmpl")) + _converge_tracked_dir( + c, directory, "results", partial(templates.read, "results-README.md.tmpl") + ) # The MyST report is a recommended add-on on top of the spec, not part # of it — which is why it is scaffolded here and not by `astra init`. - c.file("myst.yml", directory / "myst.yml", templates.myst_yml) + c.file("myst.yml", directory / "myst.yml", partial(templates.read, "myst.yml.tmpl")) c.file( "index.md", directory / "index.md", lambda: templates.index_md(title=directory.name or "My Analysis"), ) - _converge_git(c, directory) # Lock and sync last: they are the expensive steps, and they are # meaningless until pyproject.toml and .python-version exist. @@ -229,7 +249,11 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: def require_uv() -> None: - """Refuse early when uv is absent — it is the environment substrate.""" + """Refuse early when uv is absent — it is the environment substrate. + + Raises: + ProjectError: If uv is not on ``PATH``. + """ if shutil.which("uv") is None: raise ProjectError( "uv is required (the environment substrate). Install it: " @@ -237,6 +261,82 @@ def require_uv() -> None: ) +def require_git() -> None: + """Refuse early when git is absent. + + The one tool uv cannot install, and the only admitted exception to an + otherwise uv-installable stack. + + Raises: + ProjectError: If git is not on ``PATH``. + """ + if shutil.which("git") is None: + raise ProjectError( + "git is required (results are versioned in the repository). " + "Install it: https://git-scm.com/downloads" + ) + + +def require_git_annex() -> None: + """Refuse early when git-annex is not reachable as git reaches it. + + Probed after :func:`~lightcone.engine.dataset.put_our_bin_first`, and + by the name git itself searches for: ``git annex`` is git finding a + ``git-annex`` executable on ``PATH``, not a builtin. + + Raises: + ProjectError: If ``git-annex`` is not on ``PATH``. + """ + dataset.put_our_bin_first() + if shutil.which("git-annex") is None: + raise ProjectError( + "git-annex is required (it stores the bytes results are made of) " + "and is not on PATH. It ships as a wheel and installs with the " + "engine: `uv sync` in the project, or reinstall lightcone-cli." + ) + + +def uv_prefix(directory: Path, *, sync: bool) -> list[str]: + """Build the ``uv run`` hop that pins a command to a project. + + ``--locked`` makes a stale lock uv's loud error rather than a silent + relock, and ``--project`` is explicit because uv's walk-up discovery + is never trusted. + + Args: + directory: The project to pin to. + sync: True for a probe, which converges the environment it is + about to describe. False for a recipe: the environment was + converged before the run, and syncing per task would have + every concurrent worker writing the same ``.venv``. + + Returns: + The argv prefix, ending in ``--``. + """ + selection = ["--exact"] if sync else ["--no-sync"] + return ["uv", "run", "--locked", *selection, "--project", str(directory), "--"] + + +def sync(directory: Path) -> list[str]: + """Make ``.venv`` match ``uv.lock``. + + A run calls this before it starts rather than checking and refusing: + workers pass ``--no-sync``, so nothing else would notice a lock edited + without a sync, and a manifest recording an environment the recipe did + not run under is the identity model saying something untrue. + + Args: + directory: The project root. + + Returns: + Whatever uv warned about. + + Raises: + ProjectError: If uv fails. + """ + return _check_call(["uv", *_SYNC_ARGS, "--project", str(directory)], cwd=directory) + + def _converge_uv_project(c: _Converger, directory: Path) -> None: """pyproject.toml + .python-version — the environment definition. @@ -262,38 +362,137 @@ def _converge_uv_project(c: _Converger, directory: Path) -> None: c.file(".python-version", directory / ".python-version", templates.python_version) -def _converge_results(c: _Converger, directory: Path) -> None: - results_dir = directory / "results" - if results_dir.exists() and not results_dir.is_dir(): +def _converge_tracked_dir( + c: _Converger, directory: Path, name: str, readme: Callable[[], str] +) -> None: + """A directory the repository tracks, plus the README that makes it exist. + + Git carries no empty directories, so a tracked directory that starts + empty needs a file to survive a clone. The README is that file, and it + is also where what belongs in the directory is written down. + """ + path = directory / name + if path.exists() and not path.is_dir(): + c.blocked(f"{name}/", f"{name} exists but is not a directory.") + return + c.item(f"{name}/", path.is_dir(), path.mkdir) + c.file(f"{name}/README.md", path / "README.md", readme) + + +def _converge_dataset(c: _Converger, directory: Path) -> None: + """The repository: git for the pointers, git-annex for the bytes. + + Whether the directory *itself* holds a ``.git`` is the wrong question + for the repository: ``lc init subdir/`` inside an existing one must not + create a nested repository, so the check walks up. (``.git`` can be a + file — a linked worktree or submodule — hence ``exists`` rather than + ``is_dir``.) The annex is asked about the same way git-annex asks + itself, so an enclosing repository that already has one is adopted. + + Then the two files that make the storage policy: ``.gitattributes`` + routes results and inputs into the annex and keeps manifests in git, + and ``.datalad/config`` carries a dataset id — the one thing a git + + git-annex repository lacks to *be* a DataLad dataset, so a project is + one from birth rather than by later adoption. Neither is read back by + lc; the id is generated once and never regenerated, because ``file`` + writes only what is missing. + """ + c.item(".git", _in_repository(directory), lambda: dataset.init_git(directory)) + # After the item above, so a fresh project has a repository to annex. + c.item( + "git-annex", + _can_ask_git(directory) and dataset.is_annexed(directory), + lambda: dataset.init_annex(directory), + ) + attributes = directory / ".gitattributes" + # Read before the repair, not after: the check is on the text a repair + # would leave behind, and check mode never writes one. + authored = attributes.read_text() if attributes.exists() else "" + c.file( + ".gitattributes", + attributes, + partial(templates.read, "gitattributes.tmpl"), + repair=templates.gitattributes_repair, + ) + if misplaced := templates.gitattributes_disorder(authored): c.blocked( - "results/", - "results exists but is not a directory; outputs cannot " - "materialize until it is one.", + ".gitattributes", + f".gitattributes would put `{misplaced}` after a line that has to " + "come before it, and git-annex takes the last match — so results " + "would be committed to git as plain blobs instead of reaching the " + "annex. Convergence only ever appends, so it cannot reorder a file " + "the user wrote. Put lightcone's lines in this order:\n " + + "\n ".join(templates.entries("gitattributes.tmpl")), ) - return - c.item("results/", results_dir.is_dir(), results_dir.mkdir) - c.file("results/README.md", results_dir / "README.md", templates.results_readme) + c.file( + ".datalad/config", + directory / ".datalad" / "config", + lambda: templates.datalad_config(dataset_id=str(uuid.uuid4())), + ) + _converge_committable(c, directory) -def _converge_git(c: _Converger, directory: Path) -> None: - """``git init``, unless the project is already under version control. +def _converge_committable(c: _Converger, directory: Path) -> None: + """Refuse to call a project converged while its outputs are unignorable. - Whether the directory *itself* holds a ``.git`` is the wrong question: - ``lc init subdir/`` inside an existing repository must not create a - nested one, so the check walks up. (``.git`` can be a file — a linked - worktree or submodule — hence ``exists`` rather than ``is_dir``.) + Results and declared inputs are committed, so an ignore rule covering + either is not a preference — it is a project where materializing + reports success and commits nothing, silently, because ``git add`` + skips ignored paths without a word. - git is optional, unlike uv: a project without it is perfectly valid, so - an absent git is silently nothing to converge rather than a warning. + A repair is not available: ``.gitignore`` convergence only ever + appends, deliberately, so a rule the user wrote — or one an older + scaffold wrote before results were tracked — stays until they delete + it. Blocked rather than warned, because it is convergence failing at + something it is responsible for. """ - if shutil.which("git") is None: + if not _can_ask_git(directory): return - already = any((p / ".git").exists() for p in [directory, *directory.parents]) - c.item(".git", already, lambda: _check_call(["git", "init", "-q"], cwd=directory)) + for name in ("results", "data"): + # Asked with the trailing slash, because the rule that matters most + # here — the `results/*` an older lc scaffold wrote — ignores the + # directory's *contents*, and does not match the bare name at all. + if rule := dataset.ignore_rule(directory, f"{name}/"): + c.blocked( + f"{name}/", + f"{name}/ is git-ignored by `{rule}`, so nothing in it can be " + f"committed — and {name}/ is versioned in the repository. " + "Delete that line and run `lc init` again.", + ) + + +def _in_repository(directory: Path) -> bool: + """Whether *directory* is inside a git work tree, its own or an ancestor's. + + A pure filesystem question, so it answers for a directory that does not + exist yet — which is the whole point in check mode: ``lc init --check`` + on a new subdirectory of a repository must report that the repository + is already there, not that one would be created. + """ + return any((p / ".git").exists() for p in [directory, *directory.parents]) + + +def _can_ask_git(directory: Path) -> bool: + """Whether git can be *run* here — a stricter question than the above. + + Every git invocation needs an existing working directory, and check + mode does not create one. Inside an enclosing repository the walk-up + says "in a repository" for a directory that is not there yet, and + running git in it raises ``FileNotFoundError`` out of ``Popen`` rather + than answering anything. + """ + return directory.is_dir() and _in_repository(directory) def project_name(directory: Path) -> str: - """A PEP 503-ish project name derived from the directory name.""" + """Derive a project name from a directory name. + + Args: + directory: The project root. + + Returns: + A PEP 503-ish name, or ``analysis`` if nothing usable remains. + """ name = re.sub(r"[^A-Za-z0-9._-]+", "-", directory.name).strip("-._").lower() return name or "analysis" @@ -305,22 +504,23 @@ def project_name(directory: Path) -> str: def current_project(directory: Path | None = None) -> Path: - """*directory* (default: the working directory), taken as the project root. - - ``lc run`` assumes it is invoked from the root, so the only check is - that the environment is actually there. There is no walk-up: the - directory you are in is the directory that is used, or it is an - error. - - The two ways that fails are different mistakes and get different - advice. A directory with no project markers at all is the wrong - *place* — the answer is to go to the right one, and telling someone - standing in ``$HOME`` to run ``lc init`` there would be telling them - to scaffold a project in their home directory. A directory that - holds a ``pyproject.toml`` or an ``astra.yaml`` but lacks the built - environment is the right place, not yet converged — a fresh clone is - exactly this, since git carries no ``.venv`` — and there ``lc init`` - is the whole answer. + """Take a directory as the project root, checking it is one. + + There is no walk-up: the directory you are in is the directory that is + used, or it is an error. + + Args: + directory: Defaults to the working directory. + + Returns: + The resolved project root. + + Raises: + ProjectError: If the environment is not there. The two ways that + fails get different advice — a directory with no project + markers is the wrong *place*, while one that declares a + project but lacks the built environment is the right place, + not yet converged, which is what a fresh clone is. """ directory = (directory or Path.cwd()).resolve() missing = [name for name in _ENVIRONMENT_FILES if not (directory / name).exists()] @@ -357,12 +557,15 @@ def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: def child_env() -> dict[str, str]: - """The environment external tools run in: ours, minus ``VIRTUAL_ENV``. + """Build the environment external tools run in. + + Ours, minus ``VIRTUAL_ENV``. Every uv invocation names its project + explicitly, so an activated environment elsewhere is never what we + mean — and uv warns once per invocation when it ignores one, which + would otherwise land in the report and in ``--json``. - Every uv invocation names its project explicitly, so an activated - environment elsewhere is never what we mean — uv agrees, ignoring it and - warning that it did, once per invocation, which would otherwise land in - the report and in ``--json``. Explicit flags beat ambient variables. + Returns: + The current environment without ``VIRTUAL_ENV``. """ return {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} @@ -381,20 +584,22 @@ def _check_call(argv: list[str], *, cwd: Path) -> list[str]: def tool_warnings(stderr: str) -> list[str]: - """A tool's warnings, lifted out of its progress output. - - uv interleaves warnings with progress on stderr, so relaying the whole - stream would bury them under a line per installed package. A warning is - a line starting ``warning:`` plus its continuations, which uv aligns - under the 9-character ``warning: `` prefix. The two-space floor is what - separates those from uv's own change list (`` + pkg==1.0``), which is - indented by exactly one — folding those in swallowed the whole install - list into the warning text. - - The one that has to reach the user: when the uv cache and the project - are on different filesystems, uv cannot link and silently falls back to - copying every package — most of an environment stops being shared, and - nothing else would say so. + """Lift a tool's warnings out of its progress output. + + uv interleaves warnings with progress, so relaying the whole stream + would bury them under a line per installed package. A warning is a + line starting ``warning:`` plus its continuations, which uv indents by + at least two — one space is uv's own change list (`` + pkg==1.0``). + + The one that must reach the user: when the uv cache and the project + are on different filesystems, uv silently falls back to copying every + package, and nothing else would say so. + + Args: + stderr: A tool's captured stderr. + + Returns: + One entry per warning, continuations folded in. """ found: list[str] = [] for line in stderr.splitlines(): @@ -424,10 +629,9 @@ def _lock_is_current(directory: Path) -> bool: def _env_is_current(directory: Path) -> bool: """Whether ``.venv`` still satisfies ``uv.lock``. - Set-level, not byte-level: uv catches packages the - lock requires and the environment lacks, but not extras installed by - hand, which leave ``--check`` reporting "would make no changes". What - bounds what a recipe can import is the sandbox, not this probe. + Set-level, not byte-level: uv catches packages the lock requires and + the environment lacks, but not extras installed by hand. What bounds + what a recipe can import is the sandbox, not this probe. """ return _is_current(["sync", "--locked", "--exact", "--check"], directory=directory) diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index f44a88ee..d152f2a8 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -19,27 +19,33 @@ from typing import Any from lightcone.engine import sandbox -from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv +from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv, uv_prefix def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: - """Run *command* in the project environment, inside the boundary. + """Run a command in the project environment, inside the boundary. - *command* is required, and there is deliberately no bare-``lc run`` - shell. A probe is run far more often by an agent than by a person, - and an agent that opens an interactive shell waits forever for input - nobody is going to type. + Args: + project: The project root. + command: The argv to run. Required — there is deliberately no bare + ``lc run`` shell, since an agent that opens an interactive + shell waits forever for input nobody will type. + + Returns: + The exit code, what the boundary enforced, and any lines the + caller should print verbatim. """ require_uv() spec = read_spec(project) - with sandbox.scope(project, read_paths=input_paths(project, spec)) as policy: + built = sandbox.exec_policy(project, read_paths=input_paths(project, spec)) + with sandbox.scope(built) as policy: return sandbox.run( sandbox.detect(), policy, list(command), cwd=project, - prefix=uv_prefix(project), + prefix=uv_prefix(project, sync=True), # Same reason as convergence: this uv invocation names its # project explicitly, so an environment activated elsewhere # is never what we mean — and uv says so, once per run, in @@ -48,25 +54,18 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: ) -def uv_prefix(project: Path) -> list[str]: - """``uv run``, pinned to *project* and refusing to drift. +def read_spec(project: Path) -> dict[str, Any]: + """Read the project's spec, best-effort. - ``--locked`` makes a stale lock uv's loud error rather than a silent - relock, and ``--exact`` keeps a previously-installed extra from - surviving into the environment being probed. Always an explicit - ``--project``: uv's own walk-up discovery is never trusted. - """ - return ["uv", "run", "--locked", "--exact", "--project", str(project), "--"] + A probe exists to debug a project, and a spec whose sub-analysis + references are stale is exactly when someone runs one. + Args: + project: The project root. -def read_spec(project: Path) -> dict[str, Any]: - """The project's ``astra.yaml``, with sub-analyses merged in if possible. - - Best-effort by design: a probe exists to debug a project, and a spec - whose sub-analysis references are stale is exactly when someone runs - one. A tree that will not resolve degrades to the top-level document - rather than making the verb unusable — and no spec at all is simply - an empty one, with no declared inputs. + Returns: + The spec with sub-analyses merged in; the top-level document alone + if the tree will not resolve; an empty spec if there is none. """ from astra.helpers import load_yaml, resolve_analysis_tree @@ -81,23 +80,33 @@ def read_spec(project: Path) -> dict[str, Any]: def input_paths(project: Path, spec: dict[str, Any]) -> list[Path]: - """The declared inputs that are filesystem paths, resolved. + """Collect the declared inputs that are filesystem paths. + + ASTRA's ``source`` is free-form — a URI, a dotted name, a path — so + the test for "is this a path" is whether it resolves to something that + exists. Anything else is somebody else's input kind. + + Args: + project: The project root, for resolving relative sources. + spec: The spec to read inputs from. - A probe's read allowlist is the union of *all* declared inputs — it - has no output, so it has no narrower set to use. ASTRA's ``source`` - is free-form (a URI, a dotted name, a path), so the test for "is - this a path" is whether it resolves to something that exists. - Anything else is somebody else's input kind. + Returns: + The resolved paths that exist. """ from astra.helpers import get_inputs + from astra.resolve import iter_analysis_nodes found: list[Path] = [] - for declared in get_inputs(spec): - source = declared.get("source") - if not isinstance(source, str) or not source: - continue - candidate = Path(source) - resolved = candidate if candidate.is_absolute() else project / candidate - if resolved.exists(): - found.append(resolved.resolve()) - return found + # Every node, not just the root: a sub-analysis declares its own + # inputs, and a probe denied one is a denial the researcher cannot act + # on — the file is declared, just not at the top of the tree. + for _scope, node in iter_analysis_nodes(spec): + for declared in get_inputs(node): + source = declared.get("source") + if not isinstance(source, str) or not source: + continue + candidate = Path(source) + resolved = candidate if candidate.is_absolute() else project / candidate + if resolved.exists(): + found.append(resolved.resolve()) + return list(dict.fromkeys(found)) diff --git a/src/lightcone/engine/sandbox/__init__.py b/src/lightcone/engine/sandbox/__init__.py index 07bb5ff0..b8ef71bd 100644 --- a/src/lightcone/engine/sandbox/__init__.py +++ b/src/lightcone/engine/sandbox/__init__.py @@ -22,6 +22,7 @@ scope, ) from lightcone.engine.sandbox.model import Attestation, Backend, Capability, Policy +from lightcone.engine.sandbox.policy import exec_policy __all__ = [ "Attestation", @@ -31,6 +32,7 @@ "Policy", "Unavailable", "detect", + "exec_policy", "run", "scope", ] diff --git a/src/lightcone/engine/sandbox/boundary.py b/src/lightcone/engine/sandbox/boundary.py index fdb44bfd..8ccf4660 100644 --- a/src/lightcone/engine/sandbox/boundary.py +++ b/src/lightcone/engine/sandbox/boundary.py @@ -47,9 +47,16 @@ class Unavailable: capability: Capability = field(default_factory=lambda: Capability(kind="none")) def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + """Return *argv* unchanged — there is no mechanism to wrap with.""" return list(argv) def attest(self, policy: Policy) -> Attestation: + """Attest that nothing was enforced. + + Returns: + ``fs: open``. Saying so is the caller's job; pretending is + nobody's. + """ return Attestation(mechanism="none", fs="open") @@ -65,11 +72,15 @@ class Outcome: def detect() -> Backend: - """The best mechanism this host can offer. + """Pick the best mechanism this host can offer. + + The single platform branch in the codebase. A backend that probes + unavailable falls through to :class:`Unavailable`, so adding another + mechanism is one import and one line. - The single platform branch. A backend that probes unavailable falls - through to the next candidate and finally to :class:`Unavailable`, - so adding another mechanism is one import and one line. + Returns: + A backend, never ``None`` — an unenforced host gets one that says + so rather than a special case for callers to branch on. """ if sys.platform == "linux": from lightcone.engine.sandbox.landlock import LandlockBackend, capability @@ -91,19 +102,23 @@ def detect() -> Backend: @contextmanager -def scope(project: Path, *, read_paths: Sequence[Path] = ()) -> Iterator[Policy]: - """A probe policy, with its per-run HOME cleaned up afterwards. +def scope(policy: Policy) -> Iterator[Policy]: + """Own a policy's lifetime, cleaning up the directory it allocated. + + Every policy owns a private ``$HOME`` on disk, so leaking one is a + real cost on a machine that runs many. Taking the policy rather than + building it keeps that lifetime in one place for every caller. + + Args: + policy: An already-built policy. - The policy owns a real directory on disk (the private ``$HOME``), so - building one is not free and leaking one is a real cost on a machine - that runs many probes. A context manager makes the lifetime the - caller's, visibly. + Yields: + The same policy, with its ``tmp_home`` removed on exit. """ - built = policy_module.probe_policy(project, read_paths=read_paths) try: - yield built + yield policy finally: - shutil.rmtree(built.tmp_home, ignore_errors=True) + shutil.rmtree(policy.tmp_home, ignore_errors=True) def run( @@ -115,18 +130,26 @@ def run( env: dict[str, str], prefix: Sequence[str] = (), ) -> Outcome: - """Run *argv* through *backend*, and explain it if it fails. - - *prefix* is spawned **outside** the rewrite — it is how the caller - says "wrap the command, not this". ``lc run`` uses it for the - ``uv run`` hop, because uv, its config, and its caches are trusted - plumbing that must stay outside the boundary. - - stdout is inherited untouched, so a probe stays a probe — output - arrives live. stderr is teed: written through as it arrives *and* - retained, because the denial classifier needs text and the user - needs immediacy. (A denial printed only to stdout is therefore - missed; the trailer still fires.) + """Run a command through a backend, and explain it if it fails. + + stdout is inherited untouched, so output arrives live. stderr is teed + — written through as it arrives and retained — because the denial + classifier needs text and the user needs immediacy. + + Args: + backend: The mechanism to wrap with. + policy: What the command may touch. + argv: The command. + cwd: Where to run it. + env: The environment for everything outside the rewrite. The + policy's own overlay is applied inside it, not merged here. + prefix: Spawned *outside* the rewrite — how a caller says "wrap + the command, not this". Used for the ``uv run`` hop, since + uv's config and caches are trusted plumbing. + + Returns: + The exit code, what was actually enforced, and any lines the + caller should print verbatim. """ wrapped = [*prefix, *backend.wrap(policy, [*env_argv(policy), *argv])] attestation = backend.attest(policy) @@ -183,21 +206,22 @@ def run( def env_argv(policy: Policy) -> list[str]: - """``env K=V …``, prefixed to the command *inside* the wrap. - - One place, every backend — including :class:`Unavailable`, which - would otherwise silently run in a different environment than a - sandboxed run. - Composed inside the wrap rather than around it so the ``prefix`` - (the ``uv run`` hop) keeps the real environment: uv resolves its - cache from ``XDG_CACHE_HOME`` and its interpreters from - ``XDG_DATA_HOME``. - - ``env`` is resolved the same way the exec set resolved it, not by a - literal path: the set grants whatever - :func:`~lightcone.engine.sandbox.policy.utility` found, so a hardcoded - ``/usr/bin/env`` is a denial on the first exec of every run on any - host that keeps its copy elsewhere. + """Build the ``env K=V …`` prefix applied *inside* the wrap. + + One place, every backend — including :class:`Unavailable`, which would + otherwise run in a different environment than a sandboxed run. Inside + the wrap rather than around it, so the ``uv run`` prefix keeps the + real environment: uv resolves its cache from ``XDG_CACHE_HOME`` and + its interpreters from ``XDG_DATA_HOME``. + + Args: + policy: The policy whose overlay to apply. + + Returns: + The ``env`` invocation, empty when the policy overlays nothing. + ``env`` is resolved the way the exec set resolved it, since a + hardcoded path would be denied on any host that keeps it + elsewhere. """ if not policy.env: return [] @@ -238,6 +262,7 @@ def __init__(self, stream: IO[str]) -> None: self._size = 0 def run(self) -> None: + """Pump the stream through to stderr, keeping a bounded tail.""" for line in self._stream: sys.stderr.write(line) self._chunks.append(line) @@ -247,4 +272,5 @@ def run(self) -> None: sys.stderr.flush() def text(self) -> str: + """Return the retained tail, for the denial classifier.""" return "".join(self._chunks) diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py index 0936460f..8a4f3bb6 100644 --- a/src/lightcone/engine/sandbox/denial.py +++ b/src/lightcone/engine/sandbox/denial.py @@ -45,23 +45,24 @@ def explain(stderr: str, policy: Policy, *, cwd: Path) -> list[str]: - """Lines explaining the first confirmed denial in *stderr*, or ``[]``. - - "Confirmed" is doing real work here. A candidate is dropped when the - path does not exist (an ordinary missing-file bug, not a denial) or - when the policy fully grants it (someone else's permission problem). - Both checks run in the unsandboxed parent, where ``stat`` sees - everything — which is the whole reason the parent does the - explaining rather than the child. - - The three outcomes are three different mistakes, and conflating them - is how a sandbox message becomes useless: - - - a path granted for **neither** access — an undeclared tool or an - undeclared input; - - a path granted for **read but not write** — which, since reading - it was allowed, can only have been a write into the read-only - tree, a system directory, or a declared input. + """Explain the first confirmed denial in a child's stderr. + + "Confirmed" is doing real work: a candidate is dropped when the path + does not exist (an ordinary missing-file bug) or when the policy fully + grants it (someone else's permission problem). Both checks run in the + unsandboxed parent, where ``stat`` sees everything — which is why the + parent explains rather than the child. + + Args: + stderr: The child's captured stderr. + policy: What the command was allowed to touch. + cwd: Where it ran, for resolving relative paths. + + Returns: + Lines to print verbatim, or ``[]`` when nothing can be confirmed. + A path granted for neither access is an undeclared tool or input; + one granted for read but not write can only have been a write + into the read-only tree, a system directory, or a declared input. """ # Access-aware, and that distinction is what keeps the message # honest: every allowlisted binary lives under `/usr`, which the read @@ -84,13 +85,18 @@ def explain(stderr: str, policy: Policy, *, cwd: Path) -> list[str]: def trailer(mechanism: str) -> str: - """The one line printed after *every* failed sandboxed run. + """Build the line printed after every failed sandboxed run. Unconditional by design: :func:`explain` fires only when it can - extract and confirm a path, and the cases where it cannot — a recipe - that catches the ``PermissionError`` and exits with something else — - are exactly the cases where someone would otherwise spend an hour - fighting an invisible wall. + confirm a path, and the cases where it cannot — a command that catches + the ``PermissionError`` and exits with something else — are exactly + the cases where someone would otherwise fight an invisible wall. + + Args: + mechanism: What enforced the run. + + Returns: + One line, naming the mechanism. """ return ( f"this ran under the lc sandbox ({mechanism}) — a permissions or " diff --git a/src/lightcone/engine/sandbox/landlock.py b/src/lightcone/engine/sandbox/landlock.py index 6b36d16c..70a5b189 100644 --- a/src/lightcone/engine/sandbox/landlock.py +++ b/src/lightcone/engine/sandbox/landlock.py @@ -26,12 +26,12 @@ @functools.cache def capability() -> Capability: - """Whether this kernel can enforce, and at which ABI. + """Probe whether this kernel can enforce, and at which ABI. - Cached: it is a syscall whose answer cannot change inside one - process. The ABI is *recorded*, not merely gated on — a "best - effort" that succeeded against a kernel with no Landlock at all is - exactly the silent degradation this layer exists to make impossible. + Returns: + A ``landlock`` capability with its ABI, or ``none`` with the + reason. Cached: a syscall whose answer cannot change inside one + process. """ abi = _sandbox_exec.abi() if abi > 0: @@ -75,6 +75,17 @@ class LandlockBackend: interpreter: str = sys.executable def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + """Rewrite *argv* to run under the Landlock shim. + + Pure: no temporary files, no file descriptors, no global state. + + Args: + policy: What the command may touch. + argv: The command. + + Returns: + The rewritten command. + """ document = json.dumps(_document(policy), separators=(",", ":"), sort_keys=True) return [ self.interpreter, @@ -87,6 +98,15 @@ def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: ] def attest(self, policy: Policy) -> Attestation: + """Report what the wrapped command will have enforced. + + Args: + policy: The policy being wrapped. + + Returns: + The record written with every output, derived from the flags + actually applied. + """ return Attestation( mechanism="landlock", fs="declared", diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index 69a91c63..abd3ce30 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -55,12 +55,17 @@ class Policy: env: dict[str, str] = field(default_factory=dict) def grants(self, path: Path, roots: tuple[Path, ...]) -> bool: - """Whether *path* lies under any of *roots*. + """Test whether a path lies under any of a set of roots. - The one containment predicate for the layer. ``is_relative_to`` is - reflexive, so a root grants itself — spelling it `p == r or - p.is_relative_to(r)` is not just redundant, it teaches the next - reader that the stdlib does not do the obvious thing. + The one containment predicate for the layer. + + Args: + path: The path to test, resolved before comparison. + roots: One of the policy's path tiers. + + Returns: + True if *path* is under a root. ``is_relative_to`` is + reflexive, so a root grants itself. """ resolved = path.resolve() return any(resolved.is_relative_to(root) for root in roots) @@ -109,14 +114,36 @@ def capability(self) -> Capability: A read-only property rather than a bare attribute, so a frozen dataclass satisfies the protocol — an immutable backend is the - point, since ``wrap`` must be pure. + point, since :meth:`wrap` must be pure. """ ... def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: - """*argv*, rewritten into a command that sandboxes itself.""" + """Rewrite a command into one that sandboxes itself. + + Must be pure: no temporary files, no file descriptors, no global + state. That is what makes a backend testable on a host that + cannot run it. + + Args: + policy: What the command may touch. + argv: The command. + + Returns: + The rewritten command. + """ ... def attest(self, policy: Policy) -> Attestation: - """What :meth:`wrap`'s command will actually have enforced.""" + """Report what :meth:`wrap`'s command will actually have enforced. + + Derived from the flags applied, never from what the mechanism + matrix says should have happened. + + Args: + policy: The policy being wrapped. + + Returns: + The record written with every output. + """ ... diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py index 6f47a673..6943b36f 100644 --- a/src/lightcone/engine/sandbox/policy.py +++ b/src/lightcone/engine/sandbox/policy.py @@ -56,13 +56,18 @@ def utility(name: str) -> Path | None: - """Where *name* resolves from the fixed search path, if it is there. + """Resolve an allowlisted tool from the fixed search path. - The **only** way anything works out where an allowlisted tool lives. - Anywhere that hardcodes a path instead is a second answer to the same - question, and the two disagree the moment a host keeps its copy - somewhere else: the exec set would grant one file and the caller - would run the other, which is a denial on every single run. + The only way anything works out where an allowlisted tool lives. + Hardcoding a path is a second answer to the same question, and the two + disagree the moment a host keeps its copy elsewhere — the exec set + would grant one file and the caller would run the other. + + Args: + name: A tool from :data:`EXEC_ALLOWLIST`. + + Returns: + Its path, or ``None`` if it is not on this host. """ found = shutil.which(name, path=_UTILITY_PATH) return Path(found) if found else None @@ -164,21 +169,29 @@ def utility(name: str) -> Path | None: } -def probe_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: - """The policy for ``lc run``. +def exec_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: + """Build what a sandboxed command may touch. - The tree is read-only except for ``results/``, which is where output - goes. That keeps the environment itself — ``.venv``, ``uv.lock``, - the spec — exactly as the lock describes it for the whole run, and - it is the same scope a recipe gets, so a probe that works means a - recipe will. + The tree is read-only apart from ``results/``, so ``lc run`` and a + recipe get the same scope: a command that works under one works under + the other. - ``results/`` is granted only if it is already there. Convergence - creates it, and a policy that made directories would be a side - effect nobody asked a *probe* for. + A recipe is not narrowed to its own output directory. Whether an + output's bytes are its own is what the manifest's ``data_version`` + answers, and that answer travels; a second mechanism for it would not. + The residue: a cross-write landing before the victim task hashes + leaves a manifest that is self-consistent and wrong, which no checksum + can see. - Creates the per-run HOME on disk as a side effect; the caller owns - removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). + Args: + project: The project root, granted read. + read_paths: Declared inputs outside the tree. + + Returns: + The policy. ``results/`` is granted only if it exists — a policy + describes, it does not prepare. Creates the per-run HOME on disk; + the caller owns removing it (see + :func:`~lightcone.engine.sandbox.boundary.scope`). """ tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-")).resolve() for sub in _HOME_LAYOUT.values(): @@ -217,23 +230,25 @@ def _write_roots(project: Path) -> list[Path]: def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: - """HOME and friends, pointed at a fresh directory. + """Point ``HOME`` and friends at a fresh private directory. The real ``$HOME`` is neither readable nor writable inside the - boundary. Left alone, that breaks matplotlib, astropy, and R on first - import — and the obvious patch, mounting ``$HOME`` read-only, would - reopen the dotfile-steering channel the whole layer exists to close. - Giving them a private HOME instead is the Bazel/nix move: they work, - and they cannot be steered. - - ``PATH`` is set for a different reason, but the same one at heart: - what the command resolves has to be what the policy granted. - - ``PYTHONPYCACHEPREFIX`` is here for the same reason at the other end: - the tree is read-only apart from ``results/``, so without it every - ``import`` of an in-tree module fails trying to write its - ``__pycache__``. ``TMPDIR`` points inside too, so ``tempfile`` works - even where the shared ``/tmp`` had to leave the write set. + boundary, which breaks matplotlib, astropy and R on first import — + and mounting it read-only would reopen the dotfile-steering channel + the layer exists to close. A private HOME is the Bazel/nix move: they + work, and they cannot be steered. + + ``PATH`` is set so what the command resolves is what the policy + granted. ``PYTHONPYCACHEPREFIX`` and ``TMPDIR`` point inside the + private HOME, so an in-tree import can write its ``__pycache__`` and + ``tempfile`` works even where the shared ``/tmp`` left the write set. + + Args: + tmp_home: The per-run directory to point at. + project: The project, whose ``.venv/bin`` fronts ``PATH``. + + Returns: + The environment overlay the boundary applies inside the wrap. """ return { "HOME": str(tmp_home), @@ -254,7 +269,7 @@ def _venv_python(project: Path) -> Path | None: Resolved, because ``.venv/bin/python`` is a symlink and Landlock evaluates the target. What gets granted on it is :func:`_exec_set`'s decision, and its install root is separately a - read root (:func:`probe_policy`) for the standard library beside it. + read root (:func:`exec_policy`) for the standard library beside it. """ python = project / ".venv" / "bin" / "python" return python.resolve() if python.exists() else None @@ -334,13 +349,18 @@ def _exec_set(project: Path, python: Path | None) -> list[Path]: @functools.cache def elf_loaders() -> tuple[Path, ...]: - """The dynamic loaders present on this host, realpath'd. - - Scans each distinct directory once rather than globbing five - patterns: on a merged-``/usr`` system all five resolve to the same - directory, and `glob` re-lists and re-matches its ~8000 entries per - pattern — which measured as 95% of the whole policy build. Cached - because the answer cannot change while the process runs. + """Find the dynamic loaders present on this host. + + Landlock checks EXECUTE on the loader's open, so without these every + dynamically linked binary fails ``EACCES``. Scans each distinct + directory once rather than globbing five patterns — on a merged- + ``/usr`` system all five resolve to the same directory, and globbing + re-lists ~8000 entries per pattern, which measured as 95% of the + policy build. + + Returns: + The realpath'd loaders. Cached: the answer cannot change while + the process runs. """ found: set[Path] = set() for directory, patterns in _loader_patterns().items(): diff --git a/src/lightcone/engine/sandbox/seatbelt.py b/src/lightcone/engine/sandbox/seatbelt.py index 09cd63ef..e4eb093b 100644 --- a/src/lightcone/engine/sandbox/seatbelt.py +++ b/src/lightcone/engine/sandbox/seatbelt.py @@ -59,8 +59,14 @@ @functools.cache def read_profile(name: str) -> str: - """The raw text of an SBPL fragment, cached — it is immutable - package data read on every ``wrap``.""" + """Read an SBPL fragment. + + Args: + name: A fragment's file name. + + Returns: + Its text. Cached: immutable package data read on every wrap. + """ if name not in (BASE_PROFILE, NETWORK, PLATFORM_DEFAULTS): raise KeyError(f"unknown profile fragment: {name!r}") return (resources.files(__package__) / "profiles" / name).read_text(encoding="utf-8") @@ -68,14 +74,14 @@ def read_profile(name: str) -> str: @functools.cache def capability() -> Capability: - """Whether ``sandbox-exec`` is present and usable on this host. + """Probe whether ``sandbox-exec`` is present and usable on this host. - Seatbelt has been "deprecated" since 2012 and has never been given a - removal date, but the profile dialect does drift across releases — - so this runs a live canary rather than trusting the file's presence. + Deprecated since 2012 and never given a replacement, which is why it + is probed rather than assumed. - Cached like its Landlock twin: the canary is a subprocess, and the - answer cannot change inside one process. + Returns: + A ``seatbelt`` capability, or ``none`` with the reason. Cached: + the answer cannot change inside one process. """ if os.name != "posix" or not os.path.isfile(SANDBOX_EXEC): return Capability(kind="none", detail=f"{SANDBOX_EXEC} not present") @@ -102,6 +108,17 @@ class SeatbeltBackend: capability: Capability = field(default_factory=lambda: Capability(kind="seatbelt")) def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + """Rewrite *argv* to run under ``sandbox-exec``. + + Pure: no temporary files, no file descriptors, no global state. + + Args: + policy: What the command may touch. + argv: The command. + + Returns: + The rewritten command. + """ return [ SANDBOX_EXEC, "-p", @@ -112,6 +129,15 @@ def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: ] def attest(self, policy: Policy) -> Attestation: + """Report what the wrapped command will have enforced. + + Args: + policy: The policy being wrapped. + + Returns: + The record written with every output, derived from the flags + actually applied. + """ return Attestation( mechanism="seatbelt", fs="declared", @@ -120,7 +146,14 @@ def attest(self, policy: Policy) -> Attestation: def profile_params(policy: Policy) -> list[tuple[str, str]]: - """The ``-D`` bindings the generated profile refers to, in order.""" + """Build the ``-D`` bindings the generated profile refers to. + + Args: + policy: The policy being wrapped. + + Returns: + ``(name, value)`` pairs, in the order the profile uses them. + """ return [ (f"{prefix}_{index}", str(path)) for prefix, paths in ( @@ -133,20 +166,21 @@ def profile_params(policy: Policy) -> list[tuple[str, str]]: def generate_profile(policy: Policy) -> str: - """The SBPL for *policy*: upstream base, our tiers, upstream defaults. - - Order is load-bearing, because **SBPL is last-match-wins**: it is - what lets ``(deny default)`` lead the base and still be overridden, - what lets :func:`_read_only_guard` take a write back, and why the - write tier is restated *after* that guard. - - That last one is not cosmetic — it is what makes ``results/`` work. - Landlock unions rights, so a writable directory nested inside a - readable tree needs no help there. Reproducing it here means the - write set must have the final word, or the guard's ``(deny - file-write* PROJECT)`` would revoke the grant on ``results/`` that - sits inside it: Linux would materialize and macOS would refuse the - very same policy. + """Generate the SBPL profile for a policy. + + Upstream base, our tiers, then upstream defaults. Order is + load-bearing, because SBPL is last-match-wins: it is what lets + ``(deny default)`` lead the base and still be overridden, what lets + :func:`_read_only_guard` take a write back, and why the write tier is + restated *after* that guard — otherwise a writable directory inside a + readable tree would be revoked. + + Args: + policy: What the command may touch. + + Returns: + The profile text, referring to the bindings + :func:`profile_params` supplies. """ return "\n".join( [ diff --git a/src/lightcone/engine/templates/__init__.py b/src/lightcone/engine/templates/__init__.py index f5f97218..33ef4eff 100644 --- a/src/lightcone/engine/templates/__init__.py +++ b/src/lightcone/engine/templates/__init__.py @@ -12,9 +12,15 @@ interpret. Substitution is strict: a missing key raises rather than silently emitting a placeholder. -This module owns file *content*, including how content merges into a file -the user already owns (:func:`gitignore_repair`). It knows nothing about -convergence bookkeeping or the console. +A template gets a function here only when there is something to decide: +a value the caller supplies (:func:`pyproject`, :func:`datalad_config`, +:func:`index_md`) or a policy for merging into a file the user already +owns (:func:`gitignore_repair`, :func:`gitattributes_repair`). Everything +else is its own content, and callers read it by name through +:func:`read` — a wrapper that only renames the file would be a second +place for the name to be wrong. + +This module knows nothing about convergence bookkeeping or the console. """ from __future__ import annotations @@ -29,6 +35,9 @@ { "pyproject.toml.tmpl", "gitignore.tmpl", + "gitattributes.tmpl", + "datalad-config.tmpl", + "data-README.md.tmpl", "results-README.md.tmpl", "myst.yml.tmpl", "index.md.tmpl", @@ -37,7 +46,17 @@ def read(name: str) -> str: - """Return the raw text of template *name*.""" + """Read a template's raw text. + + Args: + name: A file name from :data:`TEMPLATE_NAMES`. + + Returns: + The template's text. + + Raises: + KeyError: If *name* is not a shipped template. + """ if name not in TEMPLATE_NAMES: raise KeyError(f"unknown template: {name!r}") return (resources.files(__name__) / "files" / name).read_text(encoding="utf-8") @@ -53,7 +72,14 @@ def _render(name: str, /, **values: str) -> str: def pyproject(*, name: str) -> str: - """The scaffolded ``pyproject.toml`` for a project called *name*.""" + """Render the scaffolded ``pyproject.toml``. + + Args: + name: The project name. + + Returns: + A virtual uv project depending on lightcone-cli. + """ return _render( "pyproject.toml.tmpl", name=name, @@ -63,30 +89,32 @@ def pyproject(*, name: str) -> str: def python_version() -> str: - """``.python-version`` — the exact interpreter patch, taken from the - interpreter ``lc`` is running on. + """Render ``.python-version``. Deliberately not an engine constant: a new project pins the python the researcher actually has, rather than one lc would have to download to - honor a number baked into a release. + honour a number baked into a release. + + Returns: + The exact patch of the interpreter ``lc`` is running on. """ v = sys.version_info return f"{v.major}.{v.minor}.{v.micro}\n" def requires_python() -> str: - """The scaffolded ``requires-python``, taken verbatim from - lightcone-cli's own ``Requires-Python``. - - A scaffolded project depends on the engine, so uv enforces this bound - during resolution regardless; declaring the same specifier states it - rather than inventing a second, unrelated one. Verbatim, so a compound - specifier carries over intact. - - It cannot conflict with :func:`python_version`: lc is only *able* to - run on an interpreter satisfying this specifier, so the ambient pin - always satisfies it. The fallback for a metadata-less install is the - running interpreter's own minor version, which holds that property too. + """Render the scaffolded ``requires-python``. + + Taken verbatim from lightcone-cli's own ``Requires-Python``: a + scaffolded project depends on the engine, so uv enforces this bound + during resolution regardless, and declaring the same specifier states + it rather than inventing a second one. It cannot conflict with + :func:`python_version`, since lc only runs on an interpreter that + already satisfies it. + + Returns: + The specifier, or the running interpreter's minor version for a + metadata-less install. """ from importlib.metadata import PackageNotFoundError, metadata @@ -101,13 +129,15 @@ def requires_python() -> str: def lightcone_requirement() -> str: - """The ``lightcone-cli`` requirement pinned into the scaffold. + """Render the ``lightcone-cli`` requirement for the scaffold. + + The engine lives inside the experiment's lock, pinned to the version + that ran ``lc init``, so the engine that produced a result stays + recoverable. - The engine lives *inside the experiment's lock* — pinned to the - version that ran ``lc init``, so driver and project stay in lockstep - and the engine that produced a result stays recoverable. Dev builds - fall back to unpinned: their versions aren't published, so pinning one - would make the project's lock unsolvable. + Returns: + A pinned requirement, or an unpinned one for a dev build, whose + version is not published and would make the lock unsolvable. """ from importlib.metadata import PackageNotFoundError, version @@ -119,39 +149,42 @@ def lightcone_requirement() -> str: # ============================================================================= -# .gitignore — converged entry-wise, not by marker +# Line-managed files — converged entry-wise, not by marker # ============================================================================= - - -def gitignore() -> str: - """The whole ``.gitignore``, for a project that has none.""" - return read("gitignore.tmpl") - - -def gitignore_header() -> str: - """The template's own leading comment. - - Read back out of the template rather than duplicated as a constant, so - rewording it there can never leave :func:`gitignore_repair` appending a - second header to a file that already carries the first. +# +# ``.gitignore`` and ``.gitattributes`` are both "a list of lines the user +# owns, some of which are ours". Convergence works against *the set of +# managed lines* rather than against a marker comment, so a project ends up +# with the right entries however its file got there — and lines added in a +# later lc release reach projects that already have one, instead of being +# skipped because a marker was present. +# +# `entries`, `missing` and `header` take the template name. The two +# `*_repair` functions are the exception to reading templates by name: +# convergence hands them to `_Converger.file` as callbacks over the text +# alone, so the name has to be bound here rather than at the call site. + + +#: The one line-managed template whose lines have to be in a particular +#: order to mean the right thing; ``.gitignore``'s only do so among +#: themselves, which appending already preserves. +_GITATTRIBUTES = "gitattributes.tmpl" + + +def entries(name: str) -> tuple[str, ...]: + """List the lines a template manages. + + Args: + name: A line-managed template's file name. + + Returns: + Its non-comment, non-blank lines, in template order. """ - return read("gitignore.tmpl").splitlines()[0] - + return tuple(_lines(read(name))) -def gitignore_entries() -> tuple[str, ...]: - """The patterns lightcone manages, in template order. - - The template minus its comments and blank lines. Convergence works - against *this set* rather than against a marker, so a project ends up - with the right ignores however its ``.gitignore`` got there — and - entries added in a later lc release reach projects that already have - one, instead of being skipped because a marker was present. - """ - return tuple(_patterns(read("gitignore.tmpl"))) - -def _patterns(text: str) -> list[str]: - """The meaningful (non-comment, non-blank) lines of a gitignore.""" +def _lines(text: str) -> list[str]: + """The meaningful (non-comment, non-blank) lines of *text*.""" return [ stripped for line in text.splitlines() @@ -159,62 +192,168 @@ def _patterns(text: str) -> list[str]: ] -def missing_gitignore_entries(text: str) -> list[str]: - """Which managed patterns *text* does not already carry, in order.""" - present = set(_patterns(text)) - return [e for e in gitignore_entries() if e not in present] +def missing(name: str, text: str) -> list[str]: + """Find the managed lines a file does not already carry. + Args: + name: A line-managed template's file name. + text: The file's current contents. -def gitignore_repair(text: str) -> str | None: - """*text* with every managed pattern present, or ``None`` if it already - carries them all. + Returns: + The absent lines, in template order. + """ + present = set(_lines(text)) + return [e for e in entries(name) if e not in present] + + +def _repair(name: str, text: str) -> str | None: + """*text* with every line of template *name* present, or ``None``. Only ever appends, and only what is missing — so idempotency is - structural: a pattern already in the file is never added again, whoever - put it there. - - The append preserves template order, which is what keeps - ``!results/README.md`` after the ``results/*`` it negates. (A file - holding the negation *without* ``results/*`` would end up with them - inverted; that state can only be hand-written, and re-ordering - someone's ignores to fix it would be the more surprising behavior.) + structural: a line already in the file is never added again, whoever + put it there. The append preserves template order, which is what an + ignore file's negation patterns depend on. + + Append-only also means a line a template *dropped* is never removed + from a file an earlier lc wrote. That is the right default — the file + is the user's — but it is why convergence checks separately that + ``results/`` is not ignored: a ``results/*`` inherited from an older + scaffold would make every materialized output silently uncommittable. """ - missing = missing_gitignore_entries(text) - if not missing: + absent = missing(name, text) + if not absent: return None - block = "\n".join(missing) + "\n" + block = "\n".join(absent) + "\n" # The header is cosmetic, so add it only when it isn't already there; - # a later repair then appends bare patterns under the first one. - header = gitignore_header() - if header not in text: - block = header + "\n" + block + # a later repair then appends bare lines under the first one. + first = header(name) + if first not in text: + block = first + "\n" + block if not text.strip(): return block return text.rstrip("\n") + "\n\n" + block +def header(name: str) -> str: + """Read a template's own leading comment. + + Read back out of the template rather than duplicated as a constant, so + rewording it there cannot leave a repair appending a second header. + + Args: + name: A line-managed template's file name. + + Returns: + Its first line. + """ + return read(name).splitlines()[0] + + +def gitignore_repair(text: str) -> str | None: + """Append the managed ignore patterns a file is missing. + + Args: + text: The file's current contents. + + Returns: + The repaired text, or ``None`` if nothing was missing. + """ + return _repair("gitignore.tmpl", text) + + +def gitattributes_repair(text: str) -> str | None: + """Append the managed attribute lines a file is missing. + + More is at stake than in ``.gitignore``: a ``.gitattributes`` the user + wrote first would leave result bytes routed into git, not the annex. + What appending cannot always achieve is the *order* they need to be in + — see :func:`gitattributes_disorder`. + + Args: + text: The file's current contents. + + Returns: + The repaired text, or ``None`` if nothing was missing. + """ + return _repair(_GITATTRIBUTES, text) + + +def gitattributes_disorder(text: str) -> str: + """Name the managed line a repair would leave in the wrong place. + + ``.gitattributes`` is last-match-wins, so the two ``*`` defaults have + to come *before* the lines that opt out of them. A repair only + appends, so a file already carrying ``results/** annex.largefiles= + anything`` gets ``* annex.largefiles=nothing`` added after it — and + every result then lands in git as a plain blob while convergence + reports the file repaired and the project converged. + + Judged on the text a repair would produce, not the text as it stands: + a file missing the defaults entirely is in order today and out of it + the moment they are appended. And only lines setting the *same* + attribute can override one another, so ``* filter=annex`` landing + below ``results/** annex.largefiles=anything`` is not disorder — + neither says anything about the other. + + Args: + text: The file's current contents. + + Returns: + The first managed line that ends up below one it has to precede, + or empty when the order is right. + """ + rank = {line: i for i, line in enumerate(entries(_GITATTRIBUTES))} + lowest: dict[str, int] = {} + for line in _lines(_repair(_GITATTRIBUTES, text) or text): + if (place := rank.get(line)) is None: + continue + for attribute in _attributes(line): + if place < lowest.get(attribute, -1): + return line + lowest[attribute] = place + return "" + + +def _attributes(line: str) -> list[str]: + """The attribute names one ``.gitattributes`` line sets. + + A line is a pattern followed by specs — ``attr``, ``-attr``, ``!attr`` + or ``attr=value``. + """ + return [spec.split("=")[0].lstrip("-!") for spec in line.split()[1:]] + + # ============================================================================= -# results/ and the MyST report +# Files the caller supplies a value for # ============================================================================= -def results_readme() -> str: - """``results/README.md`` — where outputs land. +def datalad_config(*, dataset_id: str) -> str: + """Render ``.datalad/config``, the file that makes a project a dataset. - ``results/`` is git-ignored and starts empty, so without the README - the directory is invisible in a clone and nothing explains the - ``results///`` layout. - """ - return read("results-README.md.tmpl") + A dataset id is the one thing a git + git-annex repository lacks to + *be* a DataLad dataset. lc never reads this back; datalad does. + Args: + dataset_id: A UUID, generated once and never regenerated. -def myst_yml() -> str: - """``myst.yml`` — the report's MyST configuration.""" - return read("myst.yml.tmpl") + Returns: + A git-config file carrying ``datalad.dataset.id``. + """ + return _render("datalad-config.tmpl", dataset_id=dataset_id) def index_md(*, title: str) -> str: - """The template report. References ``astra.yaml`` elements *by path*, - so numbers and figures stay single-sourced in the analysis.""" + """Render ``index.md``, the template report. + + References ``astra.yaml`` elements by path, so numbers and figures + stay single-sourced in the analysis. + + Args: + title: The report title. + + Returns: + A MyST document. + """ return _render("index.md.tmpl", title=title) diff --git a/src/lightcone/engine/templates/files/data-README.md.tmpl b/src/lightcone/engine/templates/files/data-README.md.tmpl new file mode 100644 index 00000000..608a07ae --- /dev/null +++ b/src/lightcone/engine/templates/files/data-README.md.tmpl @@ -0,0 +1,10 @@ +# data/ + +Declared inputs live here, and `astra.yaml` references them by path: + + inputs: + catalog: + source: data/catalog.fits + +git-annex holds the bytes and git holds a pointer, so the repository +records exactly which data produced a result without carrying its size. diff --git a/src/lightcone/engine/templates/files/datalad-config.tmpl b/src/lightcone/engine/templates/files/datalad-config.tmpl new file mode 100644 index 00000000..36d84271 --- /dev/null +++ b/src/lightcone/engine/templates/files/datalad-config.tmpl @@ -0,0 +1,2 @@ +[datalad "dataset"] + id = ${dataset_id} diff --git a/src/lightcone/engine/templates/files/gitattributes.tmpl b/src/lightcone/engine/templates/files/gitattributes.tmpl new file mode 100644 index 00000000..ffc33bdc --- /dev/null +++ b/src/lightcone/engine/templates/files/gitattributes.tmpl @@ -0,0 +1,14 @@ +# lightcone-cli — what git-annex stores, and what git carries. +# +# `filter=annex` is what makes an ordinary `git add` route content to the +# annex, so nobody has to run git-annex by hand. `annex.largefiles` then +# decides what counts as content: the default is nothing, and only outputs +# and declared inputs opt out. Last matching line wins. +# +# Manifests are exempted back out because they must be readable on a clone +# that has fetched no annex content. +* annex.largefiles=nothing +* filter=annex +results/** annex.largefiles=anything +data/** annex.largefiles=anything +**/.lightcone-manifest.json annex.largefiles=nothing diff --git a/src/lightcone/engine/templates/files/gitignore.tmpl b/src/lightcone/engine/templates/files/gitignore.tmpl index 30c8ba2c..8b1c3d8d 100644 --- a/src/lightcone/engine/templates/files/gitignore.tmpl +++ b/src/lightcone/engine/templates/files/gitignore.tmpl @@ -5,5 +5,3 @@ __pycache__/ .DS_Store .venv/ _build/ -results/* -!results/README.md diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py new file mode 100644 index 00000000..bdbe52c9 --- /dev/null +++ b/src/lightcone/engine/worker.py @@ -0,0 +1,368 @@ +"""Making one output — the unit of work, and the only thing that runs a recipe. + +Also an entry point: + + python -m lightcone.engine.worker / + +which is what the ``[DATALAD RUNCMD]`` record in every materialization +commit names. That is why it is a module rather than an ``lc`` verb: it +makes the output unconditionally, commits nothing, and leaves the tree dirty by +design — precisely the state ``lc materialize`` refuses to start from — +so advertising it in ``lc --help`` would hand people a footgun. A +``uv run --locked --project .`` in front of it reconstructs the exact +engine that produced an output from the lock of the commit that recorded +it, module path included. + +Keep this module cheap to import: no click, no rich. It is on the +``python -m`` path of every rerun, and of every task in every run. + +Nothing here writes to git, and nothing here raises. A task that fails +returns a result saying so, because Dask propagates an exception to every +dependent and "who actually failed" would stop being answerable — +reporting every independent failure in one run is most of the point of +owning the loop. +""" + +from __future__ import annotations + +import shutil +import sys +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Literal + +from lightcone.engine import assets, dataset, identity, plan, sandbox +from lightcone.engine.plan import Key, Task +from lightcone.engine.project import ( + ProjectError, + child_env, + current_project, + uv_prefix, +) + +#: The commit a run is identified against: ``(sha, origin URL)``. Read once +#: by the driver and handed to every task, because the driver commits as +#: outputs land and HEAD therefore moves under the run. +Head = tuple[str, str] + +#: The shell a recipe's command is handed to. A recipe is a command line, +#: not an argv — redirects and pipes are part of what people write — and +#: bash is in the exec allowlist, so it is granted by the same rule that +#: grants everything else the boundary lets a recipe run. +_SHELL = "bash" + + +@dataclass(frozen=True) +class TaskResult: + """What one task did. Returned, never raised, and handed to dependents.""" + + key: Key + status: Literal["ok", "current", "behind", "failed", "blocked"] + #: The output's content identity. Present for the three states in + #: which the bytes on disk are what the spec asks for — this is what a + #: dependent compares against. + data_version: str = "" + #: Why it did not finish. Shown to the user verbatim. + reason: str = "" + #: Console lines from the boundary: a downgrade notice, a denial. + notes: tuple[str, ...] = () + + @property + def usable(self) -> bool: + """Whether a dependent may proceed on this result. + + Returns: + True for ``ok``, ``current`` and ``behind`` — the states in + which the bytes on disk are what the spec asks for. ``behind`` + is among them deliberately: it says the environment moved, not + that the artifact is wrong. + """ + return self.status in ("ok", "current", "behind") + + +# ============================================================================= +# The Dask unit: decide, then execute +# ============================================================================= + + +def materialize( + root: Path, + task: Task, + env_version: str, + head: Head, + versions: assets.Versions, + refresh: bool, + *upstream: TaskResult, +) -> TaskResult: + """Make *task* if it needs making. What Dask submits, once per task. + + Where "the worker never raises" is enforced. Dask re-raises a task's + exception in the driver, which would abort every other task in flight, + so the contract is absolute — and one assembled from individually + guarded call sites is only as true as the last person to add one. + + Args: + root: The project root. + task: The output to make. + env_version: The run's environment identity, checked either side + of the recipe. + head: The run's ``(commit sha, origin URL)``, read once by the + driver because it commits as outputs land and HEAD moves. + versions: The run's content-hash memo for declared inputs. + refresh: Whether to remake an output that is merely behind. + *upstream: The results of this task's dependencies, arriving as + the futures it was given — which is what makes Dask the + scheduler rather than a loop here. + + Returns: + What happened. Never raises. + """ + try: + return _materialize(root, task, env_version, head, versions, refresh, upstream) + except Exception as e: # the contract is that this function returns + return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") + + +def _materialize( + root: Path, + task: Task, + env_version: str, + head: Head, + versions: assets.Versions, + refresh: bool, + upstream: tuple[TaskResult, ...], +) -> TaskResult: + reported = {u.key: u for u in upstream if u.usable} + if absent := [dep for dep in task.depends_on if dep not in reported]: + names = ", ".join(f"{u}/{o}" for u, o in absent) + return TaskResult(task.key, "blocked", reason=f"upstream did not finish: {names}") + + live = {key: u.data_version for key, u in reported.items()} + inputs = { + name: live[key] if (key := task.produced_by.get(name)) else versions.of(path) + for name, path in task.inputs.items() + } + manifest = assets.read(task.output_dir) + verdict = assets.classify( + definition_version=task.definition_version, + env_version=env_version, + manifest=manifest, + inputs=inputs, + ) + if verdict.calls_for_a_remake(refresh=refresh): + return execute(root, task, env_version, inputs, head=head) + + # Left alone, so the bytes on disk stand. Their *recorded* digest, + # never a recomputed one: on a clone that has fetched no annex content + # the files are dangling symlinks, and rehashing them would quietly + # report a different output. + assert manifest is not None and verdict.status != "stale" # the branch above + return TaskResult( + task.key, verdict.status, data_version=manifest.data_version, reason=verdict.why + ) + + +# ============================================================================= +# Executing one task, unconditionally +# ============================================================================= + + +def execute( + root: Path, + task: Task, + env_version: str, + input_versions: Mapping[str, str], + *, + head: Head, +) -> TaskResult: + """Run *task*'s recipe and record what it produced. + + The output directory is reset first: the recipe owns it, and a file + left from a previous run would otherwise enter the content hash and be + committed as part of an output that never produced it. + + Args: + root: The project root. + task: The output to make. + env_version: The run's environment identity, checked either side + of the recipe so a mid-run lock edit cannot be recorded as if + it had been in force. + input_versions: Each declared input's content identity, recorded + in the manifest as the chain. + head: The run's ``(commit sha, origin URL)``. + + Returns: + ``ok`` with the output's ``data_version``, or ``failed``. Commits + nothing and never touches git beyond reading HEAD. + """ + if moved := _gate(root, env_version): + return TaskResult(task.key, "failed", reason=moved) + + # The whole directory, not a list of expected files: a recipe declares + # an output id rather than filenames, and a previous run that crashed + # can have left anything in here — which would otherwise survive into + # this run's `data_version` as though the recipe had written it. The + # path is `results//` and `output_dir` refuses an id + # that could widen it. + if task.output_dir.exists(): + shutil.rmtree(task.output_dir) + task.output_dir.mkdir(parents=True) + + read_paths = [p for p in task.inputs.values() if p.exists()] + policy = sandbox.exec_policy(root, read_paths=read_paths) + with sandbox.scope(policy): + outcome = sandbox.run( + sandbox.detect(), + policy, + [_SHELL, "-c", task.recipe], + cwd=root, + prefix=uv_prefix(root, sync=False), + env=child_env(), + ) + + if outcome.returncode != 0: + return TaskResult( + task.key, + "failed", + reason=f"the recipe exited {outcome.returncode}", + notes=outcome.notes, + ) + if moved := _gate(root, env_version): + return TaskResult(task.key, "failed", reason=moved, notes=outcome.notes) + + # Guarded separately from the boundary catch above it, because these + # two failures deserve different words: "your recipe failed" and "your + # recipe worked and we could not record it" are different problems. + try: + sha, remote = head + data_version = assets.data_version(task.output_dir) + assets.write( + task.output_dir, + assets.Manifest( + output_id=task.output_id, + universe_id=task.universe_id, + recipe=task.recipe, + definition_version=task.definition_version, + env_version=env_version, + data_version=data_version, + decisions=dict(task.decisions), + input_versions=dict(input_versions), + git_sha=sha, + git_remote=remote, + lc_version=_lc_version(), + hermeticity=asdict(outcome.attestation), + ), + ) + except (OSError, ProjectError) as e: + return TaskResult( + task.key, + "failed", + reason=f"the recipe finished but its output could not be recorded: {e}", + notes=outcome.notes, + ) + return TaskResult(task.key, "ok", data_version=data_version, notes=outcome.notes) + + +def _gate(root: Path, env_version: str) -> str: + """Empty while the environment is still the one the run started in.""" + if identity.env_version(root) == env_version: + return "" + return ( + "the environment changed while the run was in flight — uv.lock, " + ".python-version, or an install setting was edited. Nothing was " + "recorded; re-run `lc materialize`." + ) + + +def _lc_version() -> str: + from importlib.metadata import PackageNotFoundError, version + + try: + return version("lightcone-cli") + except PackageNotFoundError: # pragma: no cover - only in a source tree + return "" + + +# ============================================================================= +# The entry point the run record names +# ============================================================================= + + +def main(argv: list[str]) -> int: + """Run one task from the command line, unconditionally. + + ``python -m lightcone.engine.worker /`` — what the + ``[DATALAD RUNCMD]`` record in every materialization commit names. A + thin wrapper over :func:`execute`, so this is an entry point rather + than a second implementation. Nothing is classified: a rerun is a rerun. + + Args: + argv: One argument, ``/``. + + Returns: + 0 on success, 1 if the task failed, 2 on a bad argument or an + unreadable project. + """ + if len(argv) != 1 or "/" not in argv[0]: + print("usage: python -m lightcone.engine.worker /", file=sys.stderr) + return 2 + + universe_id, _, output_id = argv[0].partition("/") + try: + root = current_project() + env_version = identity.env_version(root) + graph = plan.build(root) + if (task := graph.tasks.get((universe_id, output_id))) is None: + print(f"no output `{argv[0]}` in this project", file=sys.stderr) + return 2 + # This one-task run reads HEAD for itself, because it *is* the + # driver here — the rule is that the run's commit is read once by + # whoever owns the run, not that a worker never reads it. + result = execute( + root, task, env_version, _from_disk(task), head=dataset.head(root) + ) + except ProjectError as e: + print(f"error: {e}", file=sys.stderr) + return 2 + + for note in result.notes: + print(note, file=sys.stderr) + if result.status != "ok": + print(f"error: {result.reason}", file=sys.stderr) + return 1 + return 0 + + +def _from_disk(task: Task) -> dict[str, str]: + """Upstream versions, read from the manifests already in the tree. + + There is no graph in flight to take them from, and a single-task run + has nothing to share a memo with — the digests are read once each here + by construction. + """ + versions: dict[str, str] = {} + for name, path in task.inputs.items(): + if task.produced_by.get(name) is not None: + if (manifest := assets.read(path)) is None: + raise ProjectError( + f"the input `{name}` has never been materialized — there is no " + f"manifest in {path}. Run `lc materialize` instead." + ) + versions[name] = manifest.data_version + else: + try: + versions[name] = assets.data_version(path) + except OSError as e: + # `data_version` reports an absent or unreadable path with + # the OS's own exception, and this is the entry point a + # `datalad rerun` lands on — so it names the declared input + # rather than unwinding a traceback at whoever reads that. + raise ProjectError( + f"the declared input `{name}` cannot be read: {e}" + ) from e + return versions + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/conftest.py b/tests/conftest.py index 3dc99f57..1a2edef6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,17 @@ from __future__ import annotations import shutil +import textwrap +from collections.abc import Callable from pathlib import Path from unittest.mock import MagicMock import pytest from click.testing import CliRunner +from lightcone.engine import dataset, project, templates +from lightcone.engine.project import _run as _real_run + @pytest.fixture def runner() -> CliRunner: @@ -21,15 +26,22 @@ def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: hermetic — no network, no real resolution, no subprocesses. Models each tool's observable effect: ``uv lock`` writes ``uv.lock``, - ``uv sync`` materializes ``.venv``, ``git init`` makes ``.git``. The - ``--check`` probes answer from existence — enough for the convergence - tests; drift is exercised by tests that stub ``_run`` themselves, since - only uv can really tell a stale lock from a current one. + ``uv sync`` materializes ``.venv``, ``git init`` makes ``.git``, + ``git annex init`` marks the repository annexed. The ``--check`` + probes and ``git config --get annex.uuid`` answer from what those + left behind — enough for the convergence tests; drift is exercised by + tests that stub ``_run`` themselves, since only uv can really tell a + stale lock from a current one, and only git can really answer an + ignore rule. Returns the recorded argv lists, so a test can assert on *which* tools ran with *which* flags. :func:`uv_calls` narrows that to uv. """ calls: list[list[str]] = [] + # Repositories `git annex init` has been run in. Kept in memory + # rather than on disk because `.git` is a *file* in a linked + # worktree, so there is nowhere inside it to leave a marker. + annexed: set[Path] = set() def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: calls.append(list(argv)) @@ -44,6 +56,12 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: (project / ".venv" / "bin").mkdir(parents=True, exist_ok=True) elif argv[:2] == ["git", "init"]: (cwd / ".git").mkdir(exist_ok=True) + elif argv[:3] == ["git", "annex", "init"]: + annexed.add(_repo(cwd)) + elif argv[:2] == ["git", "config"] and argv[-1] == "annex.uuid": + return MagicMock(returncode=0 if _repo(cwd) in annexed else 1) + elif argv[:2] == ["git", "check-ignore"]: + return _fake_check_ignore(cwd, argv[-1]) return MagicMock(returncode=0, stdout="", stderr="") from lightcone.engine import project @@ -55,14 +73,14 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: # set from it, every tool resolving to a path that exists on Linux # and not on macOS, so the enforcement suite tested a policy no user # would ever get. And the answer is never invented: convergence asks - # only *whether* uv and git exist, and `_run` is faked too, so - # nothing ever execs what comes back. Where the tool is really - # installed, that is what is returned; where it is not, the stub - # says so rather than naming a plausible path that isn't there. + # only *whether* uv, git and git-annex exist, and `_run` is faked + # too, so nothing ever execs what comes back. Where the tool is + # really installed, that is what is returned; where it is not, the + # stub says so rather than naming a plausible path that isn't there. real_which = shutil.which def fake_which(name: str, path: str | None = None) -> str | None: - if name in ("uv", "git"): + if name in ("uv", "git", "git-annex"): return real_which(name) or f"/stub/{name}" return real_which(name, path=path) @@ -70,6 +88,105 @@ def fake_which(name: str, path: str | None = None) -> str | None: return calls +@pytest.fixture +def real_tools(monkeypatch: pytest.MonkeyPatch) -> None: + """Opt out of :func:`tools`, and run the real git and git-annex. + + The hermetic default is right for convergence, which needs only each + tool's observable effect. Storage is not: what ``git annex add`` does + to a working tree *is* the thing under test, and no fake can tell you + whether a file ended up as an annex symlink or as a blob in git. + """ + from lightcone.engine import project + + monkeypatch.setattr(project, "_run", _real_run) + + +@pytest.fixture +def analysis(tmp_path: Path, real_tools: None) -> Callable[..., Path]: + """Build a real, committed lc project — the one fixture that runs a graph. + + Everything is genuine: a uv environment, a git repository with an + annex, an ``astra.yaml``, and a first commit. That is the price of + testing execution at all — the questions are whether a recipe runs + under the boundary, whether bytes land in the annex, and whether the + tree is clean afterwards, and no stub answers any of them. + + It is cheap anyway: the project declares no dependencies, so `uv lock` + and `uv sync` together cost milliseconds. + """ + + def build( + spec: str, + files: dict[str, str] | None = None, + universes: dict[str, str] | None = None, + ) -> Path: + root = tmp_path / "analysis" + for name in ("universes", "results", "data"): + (root / name).mkdir(parents=True, exist_ok=True) + + (root / "pyproject.toml").write_text( + '[project]\nname = "analysis"\nversion = "0.1.0"\n' + 'requires-python = ">=3.11"\ndependencies = []\n' + ) + (root / ".python-version").write_text(templates.python_version()) + (root / ".gitattributes").write_text(templates.read("gitattributes.tmpl")) + (root / ".gitignore").write_text(templates.read("gitignore.tmpl")) + (root / "astra.yaml").write_text(textwrap.dedent(spec)) + for name, text in (universes or {"baseline": "id: baseline\ndecisions: {}\n"}).items(): + (root / "universes" / f"{name}.yaml").write_text(textwrap.dedent(text)) + for name, text in (files or {}).items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(text)) + + for argv in (["uv", "lock", "-q"], ["uv", "sync", "-q", "--locked", "--exact"]): + _must(project._run([*argv, "--project", str(root)], cwd=root), argv) + dataset.init_git(root) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=root) + dataset.init_annex(root) + (root / ".datalad").mkdir(exist_ok=True) + (root / ".datalad" / "config").write_text( + templates.datalad_config(dataset_id="4b7b5c1e-0000-4000-8000-000000000000") + ) + dataset.save(root, [root], "scaffold") + return root + + return build + + +def _must(proc: object, argv: list[str]) -> None: + if getattr(proc, "returncode", 1) != 0: + raise AssertionError(f"{' '.join(argv)} failed:\n{getattr(proc, 'stderr', '')}") + + +def _repo(cwd: Path) -> Path: + """The work tree *cwd* is in — an annex belongs to the repository, not + to the directory the command happened to run from.""" + return next((p for p in [cwd, *cwd.parents] if (p / ".git").exists()), cwd) + + +def _fake_check_ignore(cwd: Path, path: str) -> MagicMock: + """A deliberately literal stand-in for ``git check-ignore -v``. + + Not gitignore semantics — those belong to git, and the tests that need + them use a real repository. This recognises exactly the shapes that put + ``results/`` out of reach: the ``results/*`` an older lc scaffold wrote, + and the ``results/`` or ``results`` someone writes by hand. + """ + ignore = cwd / ".gitignore" + if not ignore.exists(): + return MagicMock(returncode=1, stdout="", stderr="") + wanted = path.rstrip("/") + for number, line in enumerate(ignore.read_text().splitlines(), start=1): + if (pattern := line.strip()) and pattern.rstrip("*").rstrip("/") == wanted: + return MagicMock( + returncode=0, stdout=f".gitignore:{number}:{pattern}\t{path}\n", stderr="" + ) + return MagicMock(returncode=1, stdout="", stderr="") + + def _project(argv: list[str]) -> Path: """The project root a uv invocation was pointed at.""" return Path(argv[argv.index("--project") + 1]) diff --git a/tests/test_assets.py b/tests/test_assets.py new file mode 100644 index 00000000..ed0345e7 --- /dev/null +++ b/tests/test_assets.py @@ -0,0 +1,391 @@ +"""Tests for `lightcone.engine.assets` — an output's bytes, record, and +whether it is still current. + +The classification table is the important part of this file. It is driven +the way both callers drive it — the worker with live content identities, +and `--check` with `None` for anything it has already decided will be +remade — because the entire justification for one rule is that those two +cannot disagree. + +The `stale` / `behind` split is the other half. `stale` means the artifact +contradicts the project and must be remade; `behind` means it is still +exactly what the analysis asks for and only the environment moved. Getting +that line wrong in either direction is expensive: one way spends compute +nobody asked for, the other way leaves a result quietly describing an +environment that no longer exists. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from lightcone.engine import assets +from lightcone.engine.assets import Manifest, classify, data_version, output_dir +from lightcone.engine.project import ProjectError + + +def _manifest(**overrides: object) -> Manifest: + base: dict[str, object] = { + "output_id": "best_fit", + "universe_id": "baseline", + "recipe": "python src/fit.py {output}", + "definition_version": "sha256:code", + "env_version": "sha256:env", + "data_version": "sha256:data", + "decisions": {"method": "mcmc"}, + "input_versions": {"catalog": "sha256:cat"}, + "git_sha": "abc123", + "git_remote": "https://example/demo.git", + "lc_version": "0.4.2", + "hermeticity": {"mechanism": "landlock", "fs": "declared", "network": "allowed"}, + } + return Manifest(**{**base, **overrides}) # type: ignore[arg-type] + + +# ---- where an asset lives -------------------------------------------------- + + +def test_an_asset_is_addressed_by_its_path(tmp_path: Path) -> None: + """The path in a rendered recipe is the path on disk — no staging, no + scratch, no relocation.""" + assert output_dir(tmp_path, "baseline", "best_fit") == tmp_path / "results/baseline/best_fit" + + +# ---- content identity ------------------------------------------------------ + + +def test_the_same_bytes_hash_the_same(tmp_path: Path) -> None: + for name in ("a", "b"): + (tmp_path / name).mkdir() + (tmp_path / name / "fit.csv").write_text("x,y\n1,2\n") + assert data_version(tmp_path / "a") == data_version(tmp_path / "b") + + +def test_changed_bytes_move_it(tmp_path: Path) -> None: + (tmp_path / "fit.csv").write_text("x,y\n1,2\n") + before = data_version(tmp_path) + (tmp_path / "fit.csv").write_text("x,y\n1,3\n") + assert data_version(tmp_path) != before + + +def test_a_rename_moves_it(tmp_path: Path) -> None: + """The path goes into the hash beside the bytes: the same content under + a different name is a different output.""" + (tmp_path / "fit.csv").write_text("x,y\n") + before = data_version(tmp_path) + (tmp_path / "fit.csv").rename(tmp_path / "result.csv") + assert data_version(tmp_path) != before + + +def test_touching_a_file_does_not_move_it(tmp_path: Path) -> None: + """Content, never mtime. A file restored from history carries an old + timestamp and must not look changed because of it.""" + (tmp_path / "fit.csv").write_text("x,y\n") + before = data_version(tmp_path) + (tmp_path / "fit.csv").touch() + assert data_version(tmp_path) == before + + +def test_the_manifest_is_not_part_of_its_own_hash(tmp_path: Path) -> None: + """It carries the hash, so hashing it would be circular — and the + driver commits both together, so the two must agree.""" + (tmp_path / "fit.csv").write_text("x,y\n") + before = data_version(tmp_path) + assets.write(tmp_path, _manifest()) + assert data_version(tmp_path) == before + + +def test_a_file_and_a_directory_holding_it_are_different(tmp_path: Path) -> None: + """Framed apart deliberately: a declared input can be either, and the + two must never collide.""" + (tmp_path / "one").mkdir() + (tmp_path / "one" / "fit.csv").write_text("x,y\n") + (tmp_path / "fit.csv").write_text("x,y\n") + assert data_version(tmp_path / "one") != data_version(tmp_path / "fit.csv") + + +@pytest.mark.parametrize("bad", ["", "/", "..", ".", "a/b", "results/../.."]) +def test_output_dir_refuses_an_id_that_is_not_one_path_component( + tmp_path: Path, bad: str +) -> None: + """The path is composed from the two ids, and a worker empties it + before running a recipe — so an id that collapses it onto a parent + would take every other universe's outputs with it.""" + with pytest.raises(ProjectError): + assets.output_dir(tmp_path, bad, "best_fit") + with pytest.raises(ProjectError): + assets.output_dir(tmp_path, "baseline", bad) + + +def test_output_dir_is_two_components_below_results(tmp_path: Path) -> None: + """The shape everything else in the layer addresses by.""" + assert assets.output_dir(tmp_path, "baseline", "best_fit") == ( + tmp_path / "results" / "baseline" / "best_fit" + ) + + +def test_an_unfetched_annexed_file_is_refused_not_hashed(tmp_path: Path) -> None: + """`filter=annex` leaves a pointer file where the content would be, so + the path *exists* and is readable. Hashing it would be a well-formed + answer to the wrong question, and would land in a manifest as if it + described the data.""" + pointer = tmp_path / "catalog.fits" + pointer.write_text( + "/annex/objects/SHA256E-s300000--4367c4a63392fa9b887bbcf046033d89.fits\n" + ) + + with pytest.raises(assets.ContentNotFetchedError, match="git annex get"): + data_version(pointer) + + +def test_a_directory_holding_an_unfetched_file_is_refused_too(tmp_path: Path) -> None: + """An output directory is hashed as a whole, so one absent file must + not be quietly folded in as its pointer.""" + (tmp_path / "fit.csv").write_text("a,b\n") + (tmp_path / "big.bin").write_text("/annex/objects/SHA256E-s9--abc.bin\n") + + with pytest.raises(assets.ContentNotFetchedError): + data_version(tmp_path) + + +def test_an_unfetched_locked_file_is_refused_like_an_unfetched_pointer( + tmp_path: Path, +) -> None: + """The other shape an annexed file takes. A researcher may run `git + annex lock`, or set `annex.thin`, whenever they like — so detection + cannot depend on which one lc's own writes happen to produce.""" + locked = tmp_path / "catalog.fits" + locked.symlink_to("../.git/annex/objects/2K/9P/SHA256E-s300000--4367c4a6.fits") + + assert locked.is_symlink() and not locked.exists() + with pytest.raises(assets.ContentNotFetchedError, match="git annex get"): + data_version(locked) + + +def test_a_directory_holding_an_unfetched_locked_file_is_refused_too( + tmp_path: Path, +) -> None: + """The quiet one. A dangling symlink answers False to `is_file()`, so + filtering a directory walk on that alone drops the absent file from the + digest without a word — reporting a hash of the subset that happens to + be present, which is a worse lie than the pointer's.""" + (tmp_path / "fit.csv").write_text("a,b\n") + (tmp_path / "big.bin").symlink_to("../.git/annex/objects/xx/yy/SHA256E-s9--abc.bin") + + with pytest.raises(assets.ContentNotFetchedError): + data_version(tmp_path) + + +def test_a_broken_symlink_that_is_not_annexed_is_still_loud(tmp_path: Path) -> None: + """Not git-annex's doing, so not `git annex get`'s to fix — but it may + not vanish from the digest either.""" + (tmp_path / "fit.csv").write_text("a,b\n") + (tmp_path / "scratch.dat").symlink_to("/tmp/gone-with-the-scratch-dir") + + with pytest.raises(FileNotFoundError): + data_version(tmp_path) + + +def test_a_file_that_merely_mentions_the_prefix_is_still_hashed(tmp_path: Path) -> None: + """The test is the prefix at the very start, as git-annex's own is — + a script that talks about annex paths is not a pointer.""" + script = tmp_path / "fit.py" + script.write_text("# reads /annex/objects/ sometimes\nprint(1)\n") + + assert data_version(script).startswith("sha256:") + + +def test_a_missing_path_is_an_error_not_an_empty_hash(tmp_path: Path) -> None: + """The failure mode worth refusing loudly: a constant digest for + everything absent would silently disable the whole chain.""" + with pytest.raises(FileNotFoundError): + data_version(tmp_path / "nothing") + + +def test_one_input_is_hashed_once_per_run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A declared input is asked for once per `(universe, output)` that names + it — for a multiverse spec, the same bytes over and over. Eight + universes times four outputs sharing one catalog is thirty-two reads of + one file.""" + catalog = tmp_path / "catalog.txt" + catalog.write_text("measured\n") + hashed: list[Path] = [] + real = assets.data_version + monkeypatch.setattr(assets, "data_version", lambda p: (hashed.append(p), real(p))[1]) + + versions = assets.Versions() + digests = {versions.of(catalog) for _ in range(32)} + + assert len(hashed) == 1 + assert digests == {real(catalog)} + + +def test_the_memo_does_not_confuse_two_inputs(tmp_path: Path) -> None: + (tmp_path / "a.txt").write_text("one\n") + (tmp_path / "b.txt").write_text("two\n") + versions = assets.Versions() + + assert versions.of(tmp_path / "a.txt") != versions.of(tmp_path / "b.txt") + + +# ---- the manifest ---------------------------------------------------------- + + +def test_a_manifest_round_trips(tmp_path: Path) -> None: + written = _manifest() + assets.write(tmp_path, written) + assert assets.read(tmp_path) == written + + +def test_the_manifest_is_readable_json_with_the_schema_first(tmp_path: Path) -> None: + """It stays out of the annex precisely so a clone with no content + fetched can read it — including with a plain `grep`.""" + assets.write(tmp_path, _manifest()) + text = (tmp_path / assets.MANIFEST_FILENAME).read_text() + assert next(iter(json.loads(text))) == "schema_version" + assert "best_fit" in text + + +def test_no_manifest_reads_as_none(tmp_path: Path) -> None: + assert assets.read(tmp_path) is None + + +def test_an_unparseable_manifest_reads_as_none(tmp_path: Path) -> None: + """The safe direction: an unreadable record means make it again, not + trust it.""" + (tmp_path / assets.MANIFEST_FILENAME).write_text("{not json") + assert assets.read(tmp_path) is None + + +def test_writing_a_manifest_replaces_the_previous_one_whole(tmp_path: Path) -> None: + assets.write(tmp_path, _manifest()) + assets.write(tmp_path, _manifest(data_version="sha256:second")) + + manifest = assets.read(tmp_path) + assert manifest is not None and manifest.data_version == "sha256:second" + assert not list(tmp_path.glob("*.tmp")) + + +# ---- classification -------------------------------------------------------- +# +# Every case names the environment it is classified against, because that +# argument is what decides `behind`, and a default would hide it. + +_ENV = "sha256:env" + + +def _classify(**overrides: object) -> assets.Verdict: + """Classify against the manifest `_manifest()` builds, unchanged.""" + call: dict[str, object] = { + "definition_version": "sha256:code", + "env_version": _ENV, + "manifest": _manifest(), + "inputs": {"catalog": "sha256:cat"}, + } + return classify(**{**call, **overrides}) # type: ignore[arg-type] + + +def test_a_never_materialized_output_is_stale() -> None: + verdict = _classify(manifest=None, inputs={}) + assert verdict.status == "stale" + assert verdict.calls_for_a_remake(refresh=False) + assert "never been materialized" in verdict.why + + +def test_an_unchanged_output_is_current() -> None: + verdict = _classify() + assert verdict.status == "current" + assert not verdict.calls_for_a_remake(refresh=False) + assert verdict.why == "" + + +def test_a_drifted_definition_is_stale() -> None: + """One reason covers the recipe and the decisions, because + `definition_version` is what both feed.""" + verdict = _classify(definition_version="sha256:other") + assert verdict.status == "stale" + assert "recipe or its decisions" in verdict.why + + +def test_a_drifted_input_is_stale_and_says_which() -> None: + verdict = _classify(inputs={"catalog": "sha256:new"}) + assert verdict.status == "stale" + assert "`catalog`" in verdict.why + + +def test_a_newly_declared_input_is_stale() -> None: + """The manifest has no version recorded for it, so the sets differ.""" + verdict = _classify(inputs={"catalog": "sha256:cat", "mask": "sha256:mask"}) + assert verdict.status == "stale" + assert "`mask`" in verdict.why + + +def test_an_input_the_output_no_longer_declares_is_stale() -> None: + """`definition_version` hashes the recipe and the decisions — neither of + which a dropped input moves — so nothing else would catch it and the + output would stay current with a dependency set that changed.""" + verdict = _classify(inputs={}) + assert verdict.status == "stale" + assert "`catalog`" in verdict.why + + +def test_an_input_that_will_be_remade_is_stale() -> None: + """`--check`'s sentinel. It cannot know whether a rebuild comes out + byte-identical, so `None` means "this is going to change" — deliberate + over-approximation, and the only difference between the two callers.""" + assert _classify(inputs={"catalog": None}).status == "stale" + + +def test_the_sentinel_and_a_real_drift_give_the_same_verdict() -> None: + """The property that justifies one rule with two callers: for the same + state they agree, and `--check` differs from the worker only in what it + is able to know.""" + assert _classify(inputs={"catalog": None}) == _classify( + inputs={"catalog": "sha256:rebuilt"} + ) + + +def test_a_byte_identical_rebuild_stops_the_cascade() -> None: + """What content hashing buys, and what the sentinel deliberately gives + up: the worker sees the upstream came out the same and does not rerun, + where `--check` had to assume it would.""" + assert _classify(inputs={"catalog": "sha256:cat"}).status == "current" + + +# ---- behind, which is the whole point of the split ------------------------- + + +def test_a_moved_environment_is_behind_and_not_stale() -> None: + """The headline. One added dependency rewrites `uv.lock` for the whole + project, and folding that into the rebuild trigger is what made a + week-old result disappear over a plotting library.""" + verdict = _classify(env_version="sha256:moved") + assert verdict.status == "behind" + assert not verdict.calls_for_a_remake(refresh=False) + + +def test_the_reason_says_what_happened_and_not_where() -> None: + """The commit an output came from is a field of its manifest, not a + phrase in a sentence — a caller with a column for it reads the record. + Interpolating it here would also have to handle a repository with no + commit yet, which records an empty sha.""" + verdict = _classify(env_version="sha256:moved") + assert verdict.why == "made under an earlier environment" + + +def test_stale_wins_over_behind() -> None: + """Both moved. The artifact has to be remade either way, so the reason + reported is the one that calls for the work — a `behind` verdict here + would say "left alone" about something about to run.""" + verdict = _classify(definition_version="sha256:other", env_version="sha256:moved") + assert verdict.status == "stale" + assert verdict.calls_for_a_remake(refresh=False) + + +def test_an_environment_that_did_not_move_is_not_behind() -> None: + """The negative half of the sensitivity pair: `behind` has to be off by + default, or every output reports it forever and the signal is dead.""" + assert _classify(env_version=_ENV).status == "current" diff --git a/tests/test_cli.py b/tests/test_cli.py index 2c9a9979..02f8a279 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,14 +23,23 @@ def test_help_lists_the_implemented_verb(runner: CliRunner) -> None: assert "init" in result.output -def test_help_does_not_advertise_unbuilt_verbs(runner: CliRunner) -> None: +def test_help_advertises_exactly_the_verbs_that_work(runner: CliRunner) -> None: """`lc --help` advertises only verbs that work — advertising others before they do would be a lie.""" result = runner.invoke(main, ["--help"]) - for verb in ("materialize", "status", "verify", "build", "export"): + for verb in ("init", "materialize", "run", "status"): + assert f" {verb}" in result.output + for verb in ("verify", "build", "export"): assert f" {verb}" not in result.output +def test_help_does_not_advertise_the_worker(runner: CliRunner) -> None: + """The unit a run record names is machinery, not a verb: it makes the + output unconditionally, commits nothing, and leaves the tree dirty by + design — the state `lc materialize` refuses to start from.""" + assert "worker" not in runner.invoke(main, ["--help"]).output + + def test_engine_errors_render_cleanly( runner: CliRunner, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -313,3 +322,260 @@ def test_run_needs_no_spec_file( result = runner.invoke(main, ["run", "true"]) assert result.exit_code == 0 assert spawned[0]["project"] == project.resolve() + + +# ---- lc materialize ------------------------------------------------------- + + +def _stub(monkeypatch: pytest.MonkeyPatch, **outcomes: object) -> list[tuple[str, object]]: + """Record which engine entry point the flags reached, and with what.""" + from lightcone.engine import materialize as engine + + seen: list[tuple[str, object]] = [] + + def record(name: str) -> object: + def call(root: Path, targets: object, **kwargs: object) -> object: + seen.append((name, (list(targets), kwargs))) + return outcomes.get(name, engine.MaterializeReport()) + + return call + + monkeypatch.setattr(engine, "check", record("check")) + monkeypatch.setattr(engine, "materialize", record("materialize")) + return seen + + +def test_check_reaches_check_mode_and_nothing_else( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen = _stub(monkeypatch) + + runner.invoke(main, ["materialize", "--check"]) + + assert [name for name, _ in seen] == ["check"] + + +def test_targets_reach_the_engine( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen = _stub(monkeypatch) + + runner.invoke(main, ["materialize", "baseline/fit", "report"]) + + assert seen == [("materialize", (["baseline/fit", "report"], {"refresh": False}))] + + +def test_refresh_reaches_both_modes( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`--check` has to answer the question the run would ask, or the gate + reports on a run nobody is going to make.""" + seen = _stub(monkeypatch) + + runner.invoke(main, ["materialize", "--refresh"]) + runner.invoke(main, ["materialize", "--check", "--refresh"]) + + assert seen == [ + ("materialize", ([], {"refresh": True})), + ("check", ([], {"refresh": True})), + ] + + +def test_there_is_no_flag_to_stop_a_stale_output_being_remade(runner: CliRunner) -> None: + """`--refresh` widens what a run does; nothing narrows it. An artifact + that contradicts the analysis is remade, and deleting the directory is + the user's own file operation if they want it gone instead.""" + output = runner.invoke(main, ["materialize", "--help"]).output + for flag in ("--force", "--keep-going", "--no-refresh", "--skip"): + assert flag not in output + + +def test_there_is_no_knob_for_how_much_of_the_machine_to_use(runner: CliRunner) -> None: + """A run takes every core. How much of a machine, and which machine, + is one question and it belongs to a declared execution backend.""" + assert "--jobs" not in runner.invoke(main, ["materialize", "--help"]).output + + +def test_a_run_with_nothing_to_do_exits_zero( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _stub(monkeypatch) + + result = runner.invoke(main, ["materialize"]) + + assert result.exit_code == 0 + assert "nothing to do" in result.output + + +def test_check_exits_nonzero_when_something_would_run( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An agent has to be able to ask "is this current?" and read the + answer off the exit status.""" + from lightcone.engine.materialize import MaterializeReport + + _stub(monkeypatch, check=MaterializeReport(planned={"baseline/fit": "no manifest"})) + + result = runner.invoke(main, ["materialize", "--check"]) + + assert result.exit_code == 1 + assert "would run baseline/fit" in result.output + + +def test_a_failure_exits_nonzero( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from lightcone.engine.materialize import MaterializeReport + + _stub( + monkeypatch, + materialize=MaterializeReport(failed=["baseline/fit"], blocked=["baseline/report"]), + ) + + result = runner.invoke(main, ["materialize"]) + + assert result.exit_code == 1 + assert "failed baseline/fit" in result.output + assert "blocked baseline/report" in result.output + + +def test_the_json_report_is_machine_readable( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from lightcone.engine.materialize import MaterializeReport + + _stub(monkeypatch, materialize=MaterializeReport(made=["baseline/fit"])) + + result = runner.invoke(main, ["materialize", "--json"]) + + assert json.loads(result.output) == { + "ok": True, + "up_to_date": False, + "made": ["baseline/fit"], + "current": [], + "behind": {}, + "failed": [], + "blocked": [], + "planned": {}, + "warnings": [], + "notes": [], + } + + +def test_an_engine_refusal_is_a_clean_error( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A dirty tree, a lock that cannot be audited — the user sees the + message, never a traceback.""" + from lightcone.engine import materialize as engine + from lightcone.engine.project import ProjectError + + def refuse(root: Path, targets: object, **kwargs: object) -> object: + raise ProjectError("uncommitted changes in the project") + + monkeypatch.setattr(engine, "materialize", refuse) + + result = runner.invoke(main, ["materialize"]) + + assert result.exit_code == 1 + assert "uncommitted changes" in result.output + assert "Traceback" not in result.output + + +# ============================================================================= +# lc status +# ============================================================================= + + +def _status_stub(monkeypatch: pytest.MonkeyPatch, report: object) -> None: + from lightcone.engine import materialize as engine + + monkeypatch.setattr(engine, "status", lambda root: report) + + +def _report() -> object: + from lightcone.engine.materialize import OutputStatus, StatusReport + + return StatusReport( + outputs=[ + OutputStatus("baseline/first", "current", "", "3f2a1c8ffff", "sha256:one"), + OutputStatus( + "baseline/second", + "behind", + "made under an earlier environment", + "3f2a1c8ffff", + "sha256:two", + ), + OutputStatus("baseline/third", "stale", "the input `first` changed", "", ""), + ] + ) + + +def test_status_shows_each_output_its_state_and_its_commit( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _status_stub(monkeypatch, _report()) + + result = runner.invoke(main, ["status"]) + + assert result.exit_code == 0 + for fragment in ("baseline/first", "current", "behind", "stale", "3f2a1c8"): + assert fragment in result.output + + +def test_status_exits_zero_even_with_stale_outputs( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It reports; it does not gate. `lc materialize --check` is the gate, + and two verbs answering the same question with different exit codes is + how a script comes to depend on the wrong one.""" + _status_stub(monkeypatch, _report()) + + assert runner.invoke(main, ["status"]).exit_code == 0 + + +def test_status_json_is_machine_readable( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _status_stub(monkeypatch, _report()) + + result = runner.invoke(main, ["status", "--json"]) + + assert json.loads(result.output) == { + "counts": {"current": 1, "behind": 1, "stale": 1}, + "outputs": [ + { + "output": "baseline/first", + "status": "current", + "why": "", + "git_sha": "3f2a1c8ffff", + "data_version": "sha256:one", + }, + { + "output": "baseline/second", + "status": "behind", + "why": "made under an earlier environment", + "git_sha": "3f2a1c8ffff", + "data_version": "sha256:two", + }, + { + "output": "baseline/third", + "status": "stale", + "why": "the input `first` changed", + "git_sha": "", + "data_version": "", + }, + ], + "warnings": [], + } + + +def test_status_has_exactly_one_flag(runner: CliRunner) -> None: + """Minimal by decision: it answers one question, and every way of + narrowing it is a way of getting a partial answer to that question.""" + # The options block alone: the prose above it points at + # `lc materialize --check`, which is a different verb's flag. + options = runner.invoke(main, ["status", "--help"]).output.partition("Options:")[2] + assert "--json" in options + for flag in ("--check", "--refresh", "--verbose", "--all"): + assert flag not in options diff --git a/tests/test_dataset.py b/tests/test_dataset.py new file mode 100644 index 00000000..ee1094ed --- /dev/null +++ b/tests/test_dataset.py @@ -0,0 +1,420 @@ +"""Tests for `lightcone.engine.dataset` — how a project stores what it made. + +Most of this file runs against a **real** git + git-annex repository, via +the `real_tools` fixture. That is deliberate and it is the exception in +this suite: the question these tests ask is whether bytes land in the +annex or as a blob in git, and a fake that answered it would only be +restating what the code already believes. +""" + +from __future__ import annotations + +import os +import shutil +import sys +from pathlib import Path + +import pytest + +from lightcone.engine import assets, dataset, project, templates + + +@pytest.fixture +def repo(tmp_path: Path, real_tools: None) -> Path: + """A converged-enough dataset: a repository, an annex, a storage policy. + + Identity is set locally rather than relied on from the host, because + `git annex init` and every `save` make a commit. + """ + root = tmp_path / "demo" + (root / "results").mkdir(parents=True) + (root / "data").mkdir() + (root / ".gitattributes").write_text(templates.read("gitattributes.tmpl")) + dataset.init_git(root) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=root) + dataset.init_annex(root) + dataset.save(root, [root], "scaffold") + return root + + +def _rebuild(output: Path) -> None: + """What a worker does before it runs a recipe: the recipe owns the + directory, and a stale file must not survive into the next hash. + + With `filter=annex` an annexed file is writable, so this is about + what a rebuild *means* rather than about permissions. + """ + shutil.rmtree(output, ignore_errors=True) + output.mkdir(parents=True) + + +def _annexed(repo: Path, path: Path) -> bool: + """Whether git-annex holds the bytes rather than git. + + Asked of git-annex, because `filter=annex` means an annexed file is an + ordinary writable file in the tree with no symlink to look at. + `lookupkey` names a key for content git-annex holds and says nothing + for content git carries itself. + """ + rel = str(path.relative_to(repo)) + # Not `_git`: lookupkey exits nonzero for a file git carries itself, + # which is an answer rather than a failure. + found = project._run(["git", "annex", "lookupkey", rel], cwd=repo) + return found.returncode == 0 and bool(found.stdout.strip()) + + +# ---- what git-annex stores, and what git carries --------------------------- + + +def test_save_puts_result_bytes_in_the_annex_and_the_manifest_in_git(repo: Path) -> None: + """The whole storage policy, exercised end to end. The manifest has to + stay a plain git blob: `lc` reads it on clones that have fetched no + annex content at all.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("a,b\n1,2\n") + (output / ".lightcone-manifest.json").write_text('{"data_version": "abc"}\n') + + assert dataset.save(repo, [output], "materialize best_fit") + + assert _annexed(repo, output / "fit.csv") + assert not _annexed(repo, output / ".lightcone-manifest.json") + assert not dataset.status(repo) + + +def test_a_plain_git_add_annexes_content_by_itself(repo: Path) -> None: + """`filter=annex` is what makes git's own add route content, which is + what lets lc — and everyone else — never run a git-annex command.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("a,b\n1,2\n") + + dataset._git(["add", "-A", "--", "results"], cwd=repo) + dataset._git(["commit", "-q", "-m", "plain git"], cwd=repo) + + assert _annexed(repo, output / "fit.csv") + + +def test_analysis_code_stays_in_git_and_stays_writable(repo: Path) -> None: + """The default `annex.largefiles=nothing` is what keeps `filter=annex` + from routing source files into the annex along with the data.""" + (repo / "src").mkdir() + (repo / "src" / "fit.py").write_text("print('hi')\n") + + dataset.save(repo, [repo], "the analysis") + + assert not _annexed(repo, repo / "src" / "fit.py") + (repo / "src" / "fit.py").write_text("print('edited')\n") + + +def test_data_is_annexed_too(repo: Path) -> None: + """Declared inputs are content as much as outputs are; the repository + is the complete record of what produced what.""" + (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 64) + dataset.save(repo, [repo / "data"], "input data") + + assert _annexed(repo, repo / "data" / "catalog.fits") + + +# ---- one copy on disk, and finding what is not there ----------------------- + + +def _content_object(repo: Path, path: Path) -> Path: + """Where git-annex keeps the bytes for *path*.""" + rel = str(path.relative_to(repo)) + key = dataset._git(["annex", "lookupkey", rel], cwd=repo).strip() + return (repo / dataset._git(["annex", "contentlocation", key], cwd=repo).strip()).resolve() + + +def test_a_saved_result_is_hard_linked_to_its_annex_object(repo: Path) -> None: + """`annex.thin` for lc's own add: a result exists once on disk, not + twice. Safe here and only here — thin's hazard is an in-place write, + and a worker removes an output directory before rebuilding it.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("a,b\n1,2\n") + + dataset.save(repo, [output], "materialize best_fit") + + result = output / "fit.csv" + assert result.stat().st_ino == _content_object(repo, result).stat().st_ino + assert result.stat().st_nlink == 2 + + +def test_a_researchers_own_add_is_left_as_a_copy(repo: Path) -> None: + """Thin is set for lc's add alone, never in the repository's config — + a declared input is added by the researcher, whose tools do open files + for update, and an in-place write to a thin file rewrites the annex + object under the key that names it.""" + (repo / "data" / "catalog.fits").write_bytes(b"\x00" * 64) + + dataset._git(["add", "-A", "--", "data"], cwd=repo) + dataset._git(["commit", "-q", "-m", "input data"], cwd=repo) + + catalog = repo / "data" / "catalog.fits" + assert _annexed(repo, catalog) + assert catalog.stat().st_nlink == 1 + + +def test_content_that_is_not_here_is_refused_thin_or_not(repo: Path) -> None: + """The requirement a researcher's own choices create: `annex.thin` and + `git annex lock` are theirs to set on their clone, so lc must recognise + an absent file in every shape it can arrive in — never hash it.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "thin.bin").write_bytes(b"\x01" * 4096) + (output / "fat.bin").write_bytes(b"\x02" * 4096) + dataset.save(repo, [output], "materialize best_fit") + # `save` made both thin; put one back to a copy, so the two shapes an + # *unlocked* file takes are both represented. + dataset._git(["-c", "annex.thin=false", "annex", "fix", "fat.bin"], cwd=output) + assert (output / "thin.bin").stat().st_nlink == 2 + assert (output / "fat.bin").stat().st_nlink == 1 + + for name in ("thin.bin", "fat.bin"): + dataset._git(["annex", "drop", "--force", "--", name], cwd=output) + with pytest.raises(assets.ContentNotFetchedError, match="git annex get"): + assets.data_version(output / name) + + with pytest.raises(assets.ContentNotFetchedError): + assets.data_version(output) + + +def test_a_locked_file_without_its_content_is_refused_too(repo: Path) -> None: + """`git annex lock` turns the tree back into symlinks, and an unfetched + one dangles rather than reading as a pointer.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("a,b\n1,2\n") + dataset.save(repo, [output], "materialize best_fit") + + dataset._git(["annex", "lock", "--", "fit.csv"], cwd=output) + dataset._git(["annex", "drop", "--force", "--", "fit.csv"], cwd=output) + + assert (output / "fit.csv").is_symlink() + assert not (output / "fit.csv").exists() + with pytest.raises(assets.ContentNotFetchedError, match="git annex get"): + assets.data_version(output) + + +# ---- committing ------------------------------------------------------------ + + +def test_save_reports_when_there_was_nothing_to_commit(repo: Path) -> None: + """`lc materialize` may not leave an empty commit behind for an output + that produced nothing new.""" + assert dataset.save(repo, [repo / "results"], "again") is False + + +def test_save_stages_what_a_rebuild_deleted(repo: Path) -> None: + """A rebuild resets the output directory, so a file the previous run + produced and this one did not has to leave the commit as well.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("first\n") + (output / "extra.csv").write_text("gone next time\n") + dataset.save(repo, [output], "first") + + _rebuild(output) + (output / "fit.csv").write_text("second\n") + assert dataset.save(repo, [output], "second") + + assert not dataset.status(repo) + tracked = dataset._git(["ls-files", "--", "results"], cwd=repo) + assert "extra.csv" not in tracked + + +# ---- leaving the tree as clean as it was found ----------------------------- + + +def test_restore_undoes_a_half_written_rebuild(repo: Path) -> None: + """The invariant that makes the dirty-tree refusal survivable: a recipe + that truncates a committed output and then fails must not leave the + next run telling the user to commit the wreckage.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("good\n") + dataset.save(repo, [output], "good") + + _rebuild(output) + (output / "fit.csv").write_text("truncated\n") + (output / "junk.tmp").write_text("half a run\n") + + dataset.restore(repo, [output]) + + assert not dataset.status(repo) + assert not (output / "junk.tmp").exists() + assert (output / "fit.csv").read_text() == "good\n" + + +def test_restore_of_a_never_committed_output_is_not_an_error(repo: Path) -> None: + """A first materialization has nothing in HEAD to go back to, and the + naive `git checkout HEAD -- ` exits nonzero on the pathspec.""" + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("never committed\n") + + dataset.restore(repo, [output]) + + assert not dataset.status(repo) + assert not (output / "fit.csv").exists() + + +def test_restore_is_scoped_to_the_paths_it_is_given(repo: Path) -> None: + """Never `git checkout HEAD -- .`: a failed task must not discard edits + made elsewhere while the graph was running.""" + (repo / "notes.md").write_text("original\n") + output = repo / "results" / "baseline" / "best_fit" + output.mkdir(parents=True) + (output / "fit.csv").write_text("good\n") + dataset.save(repo, [repo], "notes and a result") + + (repo / "notes.md").write_text("edited mid-run\n") + _rebuild(output) + (output / "fit.csv").write_text("wreckage\n") + + dataset.restore(repo, [output]) + + assert (repo / "notes.md").read_text() == "edited mid-run\n" + assert dataset.status(repo) == [(" M", "notes.md")] + + +# ---- reading the state of the tree ----------------------------------------- + + +def test_status_reports_uncommitted_changes(repo: Path) -> None: + assert not dataset.status(repo) + + (repo / "src").mkdir() + (repo / "src" / "fit.py").write_text("print('hi')\n") + + assert dataset.status(repo) + assert ("??", "src/") in dataset.status(repo) + + +def test_status_honours_gitignore(repo: Path) -> None: + """`.venv/` must never dirty a tree — a materialize would refuse to run + in any project that had been synced.""" + (repo / ".gitignore").write_text(templates.read("gitignore.tmpl")) + dataset.save(repo, [repo], "ignores") + (repo / ".venv" / "bin").mkdir(parents=True) + (repo / ".venv" / "bin" / "python").write_text("") + + assert not dataset.status(repo) + + +def test_status_is_scoped_to_the_project_inside_a_larger_repository( + tmp_path: Path, real_tools: None +) -> None: + """`lc init subdir/` adopts an enclosing work tree rather than nesting a + new one, so a project can sit inside a bigger repository. Unscoped, + porcelain covers that whole tree — an unrelated edit anywhere in it + would refuse every run in the project.""" + outer = tmp_path / "outer" + project_root = outer / "project" + (project_root / "results").mkdir(parents=True) + (project_root / "src.py").write_text("print('hi')\n") + (outer / "unrelated.txt").write_text("someone else's work\n") + dataset.init_git(outer) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=outer) + dataset._git(["add", "-A", "."], cwd=outer) + dataset._git(["commit", "-q", "-m", "outer"], cwd=outer) + + (outer / "unrelated.txt").write_text("edited while the project is clean\n") + assert not dataset.status(project_root) + + (project_root / "results" / "fit.csv").write_text("1\n") + # Relative to the project, not to the repository — a caller sorting by + # path class cannot recognise `project/results/`. + assert dataset.status(project_root) == [("??", "results/")] + + +def test_a_project_that_has_never_been_committed_reads_as_itself( + tmp_path: Path, real_tools: None +) -> None: + """git collapses a wholly untracked directory to its own name, which + the prefix strip then empties. `.` is what the refusal has to say.""" + outer = tmp_path / "outer" + (outer / "project").mkdir(parents=True) + (outer / "project" / "astra.yaml").write_text("name: demo\n") + dataset.init_git(outer) + + assert dataset.status(outer / "project") == [("??", ".")] + + +def test_a_repository_that_cannot_commit_is_refused_before_anything_runs( + tmp_path: Path, real_tools: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh container or CI image has no git identity, which is the case + this CLI is most often run in. Discovered at the first save, it would + cost whatever the recipe had already computed. + + `user.useConfigOnly` is how the absence is staged, rather than an empty + HOME: with no config git *guesses* from the username and the hostname, + and whether the guess is one it will accept depends on the host — a + Linux runner's `user@box.(none)` is refused and a macOS runner's is + not. This turns the guess off, which is git's own switch for it. + """ + root = tmp_path / "demo" + root.mkdir() + dataset.init_git(root) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "absent")) + monkeypatch.setenv("GIT_CONFIG_SYSTEM", str(tmp_path / "absent")) + for name in ("EMAIL", "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"): + monkeypatch.delenv(name, raising=False) + dataset._git(["config", "user.useConfigOnly", "true"], cwd=root) + + with pytest.raises(project.ProjectError, match="no identity to commit with"): + dataset.require_committer(root) + + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=root) + dataset.require_committer(root) + + +def test_ignore_rule_names_the_line_to_delete(repo: Path) -> None: + """Convergence cannot repair this one, so the message has to point at + the line rather than merely report that one exists.""" + (repo / ".gitignore").write_text("# lightcone-cli\n.venv/\nresults/*\n") + + assert dataset.ignore_rule(repo, "results/") == ".gitignore:3:results/*" + assert dataset.ignore_rule(repo, "data/") is None + + +def test_ignore_rule_sees_through_a_tracked_path(repo: Path) -> None: + """`--no-index` is load-bearing: without it git answers "not ignored" + for anything already tracked, which is exactly the project where + someone committed one result by hand and left the rule for the next.""" + (repo / ".gitignore").write_text("results/*\n") + (repo / "results" / "kept.txt").write_text("added with -f\n") + dataset._git(["add", "-f", "--", "results/kept.txt"], cwd=repo) + dataset._git(["commit", "-q", "-m", "forced"], cwd=repo) + + assert dataset.ignore_rule(repo, "results/") == ".gitignore:1:results/*" + + +# ---- how git finds git-annex ----------------------------------------------- + + +def test_our_bin_goes_to_the_front_of_path(monkeypatch: pytest.MonkeyPatch) -> None: + """Prepend, never append. `git annex` dispatches by searching PATH for + a `git-annex` executable — a system copy winning would make the version + the project's lock records a fiction.""" + ours = str(Path(sys.executable).parent) + monkeypatch.setenv("PATH", "/somewhere/else") + + dataset.put_our_bin_first() + assert os.environ["PATH"] == f"{ours}{os.pathsep}/somewhere/else" + + dataset.put_our_bin_first() + assert os.environ["PATH"].count(ours) == 1 + + +def test_git_dispatches_annex_after_the_prepend(repo: Path) -> None: + """The claim the prepend makes, checked by the spelling git itself uses + rather than by resolving `git-annex` ourselves.""" + assert "git-annex version:" in dataset._git(["annex", "version"], cwd=repo) + + diff --git a/tests/test_identity.py b/tests/test_identity.py new file mode 100644 index 00000000..e263bd77 --- /dev/null +++ b/tests/test_identity.py @@ -0,0 +1,287 @@ +"""Tests for `lightcone.engine.identity` — what an output is identified by. + +These are sensitivity tests. A content hash is only worth having if it +moves when the thing it identifies moves and stays put when it does not, +so almost every case here is "change one thing, assert the hash did or did +not follow" — including the ones that assert it *did not*, which are the +ones a careless formula breaks. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from lightcone.engine.identity import LockScan, definition_version, env_version, scan_lock +from lightcone.engine.project import ProjectError + +_LOCK = """version = 1 +requires-python = ">=3.11" + +[[package]] +name = "demo" +version = "0.1.0" +source = { virtual = "." } +""" + +_PYPROJECT = """[project] +name = "demo" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [] +""" + + +_REGISTRY = 'source = { registry = "https://pypi.org/simple" }' + + +def _with_package(root: Path, *lines: str) -> None: + """Append one `[[package]]` entry to the project's lock.""" + (root / "uv.lock").write_text(_LOCK + "\n[[package]]\n" + "\n".join(lines) + "\n") + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + """A project with just the three files identity is made of.""" + (tmp_path / "uv.lock").write_text(_LOCK) + (tmp_path / ".python-version").write_text("3.13.14\n") + (tmp_path / "pyproject.toml").write_text(_PYPROJECT) + return tmp_path + + +# ---- env_version ----------------------------------------------------------- + + +def test_env_version_is_stable_and_shaped_like_a_digest(root: Path) -> None: + assert env_version(root) == env_version(root) + assert env_version(root).startswith("sha256:") + + +def test_the_lock_moves_it(root: Path) -> None: + before = env_version(root) + (root / "uv.lock").write_text(_LOCK + '\n[[package]]\nname = "numpy"\n') + assert env_version(root) != before + + +def test_even_a_comment_in_the_lock_moves_it(root: Path) -> None: + """Raw bytes, deliberately. Over-invalidating beats a parse that + silently disagrees with uv about what the lock means.""" + before = env_version(root) + (root / "uv.lock").write_text("# regenerated\n" + _LOCK) + assert env_version(root) != before + + +def test_the_interpreter_pin_moves_it(root: Path) -> None: + before = env_version(root) + (root / ".python-version").write_text("3.12.9\n") + assert env_version(root) != before + + +def test_an_install_setting_moves_it(root: Path) -> None: + """The lock says what *could* be installed; these say which of it is.""" + before = env_version(root) + (root / "pyproject.toml").write_text(_PYPROJECT + "\n[tool.uv]\nno-binary = true\n") + assert env_version(root) != before + + +def test_a_setting_outside_the_audited_list_does_not_move_it(root: Path) -> None: + """The list is closed. A setting that changes nothing about which + artifacts a sync materializes must not stale every output in the + project.""" + before = env_version(root) + (root / "pyproject.toml").write_text(_PYPROJECT + "\n[tool.uv]\npackage = false\n") + assert env_version(root) == before + + +def test_a_setting_written_in_uv_toml_moves_it(root: Path) -> None: + """`uv.toml` is the other place uv reads these from, and it decides the + same thing: which artifacts a sync materializes from an unchanged lock.""" + before = env_version(root) + (root / "uv.toml").write_text("no-binary = true\n") + assert env_version(root) != before + + +def test_uv_toml_replaces_tool_uv_rather_than_merging_with_it(root: Path) -> None: + """uv's own precedence (measured, uv 0.12.5): a `uv.toml` beside + `pyproject.toml` makes `[tool.uv]` ignored wholesale. Hashing a setting + uv is ignoring would report two environments where uv installs one.""" + bare = env_version(root) + (root / "pyproject.toml").write_text(_PYPROJECT + "\n[tool.uv]\nno-binary = true\n") + assert env_version(root) != bare + + (root / "uv.toml").write_text("") + assert env_version(root) == bare + + +def test_where_a_setting_is_written_is_not_part_of_the_environment( + root: Path, tmp_path: Path +) -> None: + """Two projects that install the same artifacts are one environment, + whichever file each of them says so in.""" + (root / "pyproject.toml").write_text(_PYPROJECT + "\n[tool.uv]\nno-binary = true\n") + + other = tmp_path / "other" + other.mkdir() + (other / "uv.lock").write_text(_LOCK) + (other / ".python-version").write_text("3.13.14\n") + (other / "pyproject.toml").write_text(_PYPROJECT) + (other / "uv.toml").write_text("no-binary = true\n") + + assert env_version(other) == env_version(root) + + +def test_project_code_does_not_move_it(root: Path) -> None: + """The environment is not the analysis. Editing a recipe's source has + to stale that output, through the git record and through declared + inputs — never every output in the repository.""" + before = env_version(root) + (root / "src").mkdir() + (root / "src" / "fit.py").write_text("print('hi')\n") + assert env_version(root) == before + + +def test_a_missing_lock_names_what_to_do(root: Path) -> None: + (root / "uv.lock").unlink() + with pytest.raises(ProjectError, match="no uv.lock"): + env_version(root) + + +def test_a_missing_interpreter_pin_is_a_refusal(root: Path) -> None: + (root / ".python-version").unlink() + with pytest.raises(ProjectError, match=r"no \.python-version"): + env_version(root) + + +def test_fields_cannot_shift_into_one_another(root: Path) -> None: + """Length framing, exercised where it is load-bearing. `env_version` + concatenates two files' *raw* bytes, so without framing a byte moved + from the end of one to the start of the other feeds the hash the same + input and claims two different environments are one. + """ + (root / "uv.lock").write_bytes(b"lock\n3.13") + (root / ".python-version").write_bytes(b".1\n") + shifted_left = env_version(root) + + (root / "uv.lock").write_bytes(b"lock\n") + (root / ".python-version").write_bytes(b"3.13.1\n") + assert env_version(root) != shifted_left + + +# ---- definition_version ---------------------------------------------------- + + +def test_definition_version_follows_both_its_terms() -> None: + recipe, decisions = "python fit.py {output}", {"method": "mcmc"} + original = definition_version(recipe=recipe, decisions=decisions) + + assert definition_version(recipe=recipe + " -v", decisions=decisions) != original + assert definition_version(recipe=recipe, decisions={"method": "nested"}) != original + + +def test_decision_order_does_not_matter() -> None: + """Decisions are a mapping, not a sequence — two spellings of the same + choices define the same output.""" + a = definition_version(recipe="r", decisions={"x": "1", "y": "2"}) + b = definition_version(recipe="r", decisions={"y": "2", "x": "1"}) + assert a == b + + +def test_the_environment_is_not_part_of_what_an_output_is(root: Path) -> None: + """The load-bearing separation. `env_version` is recorded beside an + output and compared to say it is *behind*; folding it in here would + make one added dependency remake a project's every result.""" + before = definition_version(recipe="r", decisions={}) + (root / ".python-version").write_text("3.12.9\n") + assert env_version(root) # the environment did move + assert definition_version(recipe="r", decisions={}) == before + + +# ---- the lock scan --------------------------------------------------------- + + +def test_a_clean_lock_scans_clean(root: Path) -> None: + assert scan_lock(root) == LockScan(refusals=(), sdist_built=(), non_default_groups=()) + + +def test_a_path_dependency_is_refused(root: Path) -> None: + """The lock records where it was, not what was in it: two syncs of one + lock can install different code, and every hash here would agree they + were identical.""" + _with_package(root, 'name = "sibling"', 'source = { path = "../sibling" }') + scan = scan_lock(root) + assert len(scan.refusals) == 1 + assert "sibling" in scan.refusals[0] and "path" in scan.refusals[0] + + +def test_an_editable_dependency_is_refused(root: Path) -> None: + _with_package(root, 'name = "tool"', 'source = { editable = "../tool" }') + assert "tool" in scan_lock(root).refusals[0] + + +def test_the_projects_own_package_is_not_refused(root: Path) -> None: + """It *is* the project, and the repository already records its bytes.""" + _with_package(root, 'name = "demo"', 'source = { editable = "." }') + assert scan_lock(root).refusals == () + + +def test_the_own_package_exemption_survives_name_normalization(root: Path) -> None: + """uv writes the PEP 503 name into the lock; `pyproject.toml` carries + whatever the author wrote. Comparing them raw makes a packaged project + fail to recognise itself, and the scan then refuses its own code.""" + (root / "pyproject.toml").write_text(_PYPROJECT.replace('name = "demo"', 'name = "My_Demo"')) + _with_package(root, 'name = "my-demo"', 'source = { editable = "." }') + + assert scan_lock(root).refusals == () + + +def test_a_registry_package_with_no_wheel_is_reported(root: Path) -> None: + """Identity covers the sdist, not the build of it — reported, not + refused, because building from source is legitimate.""" + _with_package( + root, + 'name = "oldlib"', + _REGISTRY, + "", + "[package.sdist]", + 'url = "https://example/oldlib.tar.gz"', + ) + assert scan_lock(root).sdist_built == ("oldlib",) + + +def test_a_registry_package_with_wheels_is_not_reported(root: Path) -> None: + _with_package( + root, + 'name = "numpy"', + _REGISTRY, + "", + "[package.sdist]", + 'url = "https://example/numpy.tar.gz"', + "", + "[[package.wheels]]", + 'url = "https://example/numpy.whl"', + ) + assert scan_lock(root).sdist_built == () + + +def test_groups_outside_the_default_set_are_advisory(root: Path) -> None: + """They are installable states `env_version` does not distinguish.""" + (root / "pyproject.toml").write_text( + _PYPROJECT + "\n[dependency-groups]\ndev = []\nplots = []\n" + ) + assert scan_lock(root).non_default_groups == ("plots",) + + +def test_a_group_uv_installs_by_default_is_not_advisory(root: Path) -> None: + (root / "pyproject.toml").write_text(_PYPROJECT + "\n[dependency-groups]\ndev = []\n") + assert scan_lock(root).non_default_groups == () + + +def test_the_default_group_set_is_read_from_uv_toml_too(root: Path) -> None: + """The scan asks the same question `env_version` does, so it has to read + the answer from the same place uv would.""" + (root / "pyproject.toml").write_text( + _PYPROJECT + "\n[dependency-groups]\ndev = []\nplots = []\n" + ) + (root / "uv.toml").write_text('default-groups = ["dev", "plots"]\n') + assert scan_lock(root).non_default_groups == () diff --git a/tests/test_materialize.py b/tests/test_materialize.py new file mode 100644 index 00000000..af47e14f --- /dev/null +++ b/tests/test_materialize.py @@ -0,0 +1,737 @@ +"""Tests for `lightcone.engine.materialize` — running a whole analysis. + +Everything here runs against a real project with a real git repository and +a real annex, because what is being pinned is what ends up committed: one +commit per output, a run record datalad can read, and a tree exactly as +clean afterwards as it was before. + +Most tests replace the Dask cluster with an inline scheduler — the one new +monkeypatch point — so they cost nothing to start. One does not, because +the seam is only worth having if the real thing still fits through it. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +from lightcone.engine import assets, dataset, identity +from lightcone.engine import materialize as engine +from lightcone.engine.project import ProjectError, child_env +from lightcone.engine.worker import TaskResult + +_SPEC = """ +version: "0.0.13" +name: analysis + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: first + type: metric + decisions: [method] + recipe: + command: echo {decisions.method} > {output}/value.txt + + - id: second + type: report + inputs: [first] + recipe: + command: cat {inputs.first}/value.txt > {output}/copy.txt + +decisions: + method: + label: Method + default: alpha + options: + alpha: {label: alpha} + beta: {label: beta} +""" + +_UNIVERSE = "id: baseline\ndecisions:\n method: alpha\n" + + +@pytest.fixture +def root(analysis: Callable[..., Path]) -> Path: + return analysis(_SPEC, universes={"baseline": _UNIVERSE}) + + +class _Inline: + """Run the graph in this thread, in the order it was submitted. + + Submission is topological, so a dependent is submitted only after its + upstreams have already run — which means the "handles" it is passed + are the upstream results themselves, exactly what the worker expects. + """ + + def submit(self, fn: Any, *args: Any, key: str) -> Any: + return fn(*args) + + def completed(self, handles: list[Any]) -> Iterator[TaskResult]: + yield from handles + + +@pytest.fixture +def inline(monkeypatch: pytest.MonkeyPatch) -> None: + @contextmanager + def fake() -> Iterator[_Inline]: + yield _Inline() + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + +def _commits(root: Path) -> int: + return len(dataset._git(["log", "--oneline"], cwd=root).splitlines()) + + +# ---- a run, end to end ----------------------------------------------------- + + +def test_every_output_is_made_and_committed(root: Path, inline: None) -> None: + before = _commits(root) + + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert report.ok and not report.up_to_date + assert (root / "results/baseline/second/copy.txt").read_text() == "alpha\n" + assert _commits(root) == before + 2 + assert not dataset.status(root) + + +def test_an_output_and_its_manifest_land_in_one_commit(root: Path, inline: None) -> None: + """One commit is one complete, self-describing materialization — the + manifest can never come to describe different bytes.""" + engine.materialize(root, ["first"]) + + committed = dataset._git( + ["show", "--name-only", "--format=", "HEAD"], cwd=root + ).split() + assert sorted(committed) == [ + "results/baseline/first/.lightcone-manifest.json", + "results/baseline/first/value.txt", + ] + + +def test_the_bytes_go_to_the_annex_and_the_manifest_to_git(root: Path, inline: None) -> None: + """What git records is the test: a pointer for content, the real thing + for a manifest. The working tree looks the same either way now.""" + engine.materialize(root, ["first"]) + + def blob(rel: str) -> str: + return dataset._git(["cat-file", "-p", f"HEAD:{rel}"], cwd=root) + + assert blob("results/baseline/first/value.txt").startswith("/annex/objects/") + assert blob("results/baseline/first/.lightcone-manifest.json").startswith("{") + + +def test_a_second_run_does_nothing_and_commits_nothing(root: Path, inline: None) -> None: + engine.materialize(root, []) + after_first = _commits(root) + + report = engine.materialize(root, []) + + assert report.made == [] + assert report.current == ["baseline/first", "baseline/second"] + assert report.up_to_date + assert _commits(root) == after_first + + +# ---- behind: the environment moved, the analysis did not ------------------- + + +def _move_the_environment(root: Path) -> None: + """Change `env_version` for real, and commit it. + + An install setting is hashed into the environment's identity but is not + in the lock, so `uv.lock` still matches `pyproject.toml` and both the + driver's `uv sync --locked` and the workers' `uv run --locked` go + through — which is what lets this test the classification rather than + an incidental uv refusal. + """ + pyproject = root / "pyproject.toml" + pyproject.write_text(pyproject.read_text() + "\n[tool.uv]\nno-binary = true\n") + dataset.save(root, [root], "an environment edit") + + +def test_a_moved_environment_is_reported_and_nothing_is_remade( + root: Path, inline: None +) -> None: + """The change the layer turns on. A rewritten environment says nothing + about whether a result is still right, and remaking one can cost hours, + so it is reported and left where it is.""" + engine.materialize(root, []) + after_first = _commits(root) + first = (root / "results/baseline/first/value.txt").read_text() + _move_the_environment(root) + + report = engine.materialize(root, []) + + assert report.made == [] + assert set(report.behind) == {"baseline/first", "baseline/second"} + assert "earlier environment" in report.behind["baseline/first"] + assert report.up_to_date, "behind is not out of date" + assert _commits(root) == after_first + 1, "only the environment edit" + assert (root / "results/baseline/first/value.txt").read_text() == first + + +def test_refresh_remakes_what_is_behind_and_commits_it(root: Path, inline: None) -> None: + """The other half: the report is not the only thing on offer, and asking + is one flag.""" + engine.materialize(root, []) + _move_the_environment(root) + before = _commits(root) + + report = engine.materialize(root, [], refresh=True) + + assert set(report.made) == {"baseline/first", "baseline/second"} + assert report.behind == {} + assert _commits(root) == before + 2 + manifest = assets.read(root / "results/baseline/first") + assert manifest is not None + assert manifest.env_version == identity.env_version(root) + + +def test_check_reports_behind_without_planning_it(root: Path, inline: None) -> None: + """`--check` is a gate, and `behind` must not close it — a project of + curated results would never pass again.""" + engine.materialize(root, []) + _move_the_environment(root) + + report = engine.check(root, []) + + assert report.planned == {} + assert set(report.behind) == {"baseline/first", "baseline/second"} + assert report.up_to_date + + +def test_check_with_refresh_plans_what_is_behind(root: Path, inline: None) -> None: + engine.materialize(root, []) + _move_the_environment(root) + + report = engine.check(root, [], refresh=True) + + assert set(report.planned) == {"baseline/first", "baseline/second"} + assert report.behind == {} + assert not report.up_to_date + + +def test_a_stale_output_is_stale_even_when_the_environment_also_moved( + root: Path, inline: None +) -> None: + """Both moved, and only one of them calls for work. Reporting `behind` + here would say "left alone" about something the next run will remake.""" + engine.materialize(root, []) + _move_the_environment(root) + (root / "universes" / "baseline.yaml").write_text( + "id: baseline\ndecisions:\n method: beta\n" + ) + dataset.save(root, [root], "switch method") + + report = engine.check(root, []) + + assert "the recipe or its decisions" in report.planned["baseline/first"] + assert "baseline/first" not in report.behind + + +# ---- lc status ------------------------------------------------------------- + + +def test_status_names_the_commit_each_output_came_from(root: Path, inline: None) -> None: + """The verb's whole reason to exist: an output that is behind is not + wrong, and this is where the code that produced it can be read back.""" + ran_against = dataset.head(root)[0] + engine.materialize(root, []) + + report = engine.status(root) + + assert [o.output for o in report.outputs] == ["baseline/first", "baseline/second"] + assert all(o.status == "current" for o in report.outputs) + # The commit the tree was at when the run *started* — the code that + # produced the output, not the commit the run itself went on to make. + assert all(o.git_sha == ran_against for o in report.outputs) + assert report.counts == {"current": 2, "behind": 0, "stale": 0} + + +def test_status_reports_behind_after_the_environment_moves( + root: Path, inline: None +) -> None: + made_at = dataset.head(root)[0] + engine.materialize(root, []) + _move_the_environment(root) + + report = engine.status(root) + + assert report.counts == {"current": 0, "behind": 2, "stale": 0} + assert all(o.git_sha == made_at for o in report.outputs), "the commit it was made at" + assert "earlier environment" in report.outputs[0].why + + +def test_status_leaves_a_never_materialized_output_without_a_commit(root: Path) -> None: + """There is nothing to name — and an empty string rather than HEAD, + which would claim the output came from a commit that never made it.""" + report = engine.status(root) + + assert report.counts == {"current": 0, "behind": 0, "stale": 2} + assert all(o.git_sha == "" and o.data_version == "" for o in report.outputs) + assert "never been materialized" in report.outputs[0].why + + +def test_status_does_not_mind_a_dirty_tree(root: Path, inline: None) -> None: + """It reads. Refusing here would make the one verb that tells you what + state you are in unavailable exactly when you need it.""" + engine.materialize(root, []) + (root / "results/baseline/first/value.txt").write_text("edited by hand\n") + + assert engine.status(root).counts["current"] == 2 + + +def test_asking_for_an_output_makes_what_it_is_made_of(root: Path, inline: None) -> None: + report = engine.materialize(root, ["second"]) + + assert report.made == ["baseline/first", "baseline/second"] + + +def test_a_changed_decision_remakes_the_output_and_its_dependents( + root: Path, inline: None +) -> None: + engine.materialize(root, []) + (root / "universes" / "baseline.yaml").write_text("id: baseline\ndecisions:\n method: beta\n") + dataset.save(root, [root], "switch method") + + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert (root / "results/baseline/second/copy.txt").read_text() == "beta\n" + + +def test_the_previous_bytes_are_still_there_at_the_previous_commit( + root: Path, inline: None +) -> None: + """The property the whole layer exists for: given a commit, recover the + exact bytes it produced.""" + engine.materialize(root, ["first"]) + original = dataset._git(["rev-parse", "HEAD"], cwd=root).strip() + (root / "universes" / "baseline.yaml").write_text("id: baseline\ndecisions:\n method: beta\n") + dataset.save(root, [root], "switch method") + engine.materialize(root, ["first"]) + assert (root / "results/baseline/first/value.txt").read_text() == "beta\n" + + dataset._git(["checkout", original, "--", "results/baseline/first"], cwd=root) + + assert (root / "results/baseline/first/value.txt").read_text() == "alpha\n" + + +# ---- check mode ------------------------------------------------------------ + + +def test_check_says_what_would_run_and_why(root: Path) -> None: + report = engine.check(root, []) + + assert set(report.planned) == {"baseline/first", "baseline/second"} + assert "never been materialized" in report.planned["baseline/first"] + assert not report.up_to_date + + +def test_check_writes_nothing_and_commits_nothing(root: Path) -> None: + before = _commits(root) + + engine.check(root, []) + + assert not (root / "results/baseline/first").exists() + assert _commits(root) == before + + +def test_check_cascades_through_an_output_it_already_decided_to_rebuild( + root: Path, inline: None +) -> None: + """The `None` sentinel. Check mode cannot know whether a rebuild comes + out byte-identical, so it assumes it will not — the one place it is + deliberately more pessimistic than a worker.""" + engine.materialize(root, []) + (root / "universes" / "baseline.yaml").write_text("id: baseline\ndecisions:\n method: beta\n") + dataset.save(root, [root], "switch method") + + report = engine.check(root, []) + + assert "the recipe or its decisions" in report.planned["baseline/first"] + assert report.planned["baseline/second"] == "the input `first` changed" + + +def test_check_does_not_refuse_a_dirty_tree(root: Path) -> None: + """Reading the state of a project before deciding what to commit is + exactly what check mode is for.""" + (root / "notes.md").write_text("in progress\n") + + assert engine.check(root, []).planned + + +# ---- the refusals ---------------------------------------------------------- + + +def test_a_dirty_tree_refuses_and_says_what_to_do_about_each_path( + root: Path, inline: None +) -> None: + """Two path classes, two opposite remedies: work the researcher owns is + committed, and anything under `results/` is lc's to write.""" + engine.materialize(root, ["first"]) + (root / "notes.md").write_text("in progress\n") + (root / "results/baseline/first/stray.txt").write_text("by hand\n") + + with pytest.raises(ProjectError) as raised: + engine.materialize(root, []) + + message = str(raised.value) + assert "commit these" in message and "notes.md" in message + assert "discard these" in message and "results/baseline/first/stray.txt" in message + + +def _consuming(source: str) -> str: + """`_SPEC` with `first` actually reading the declared input, from *source*.""" + return _SPEC.replace("source: data/catalog.fits", f"source: {source}").replace( + " decisions: [method]", " inputs: [catalog]\n decisions: [method]" + ) + + +def test_an_unreadable_declared_input_does_not_traceback_out_of_a_read_only_verb( + analysis: Callable[..., Path], +) -> None: + """A declared input directory can hold a symlink pointing nowhere — + the directory walk keeps dangling links deliberately, so that an + unfetched annexed file cannot silently drop out of the digest. One that + is not an annex link then reaches `open()`. `status` and `--check` read + projects that are in a state, so neither may raise.""" + root = analysis(_consuming("data/inputs"), universes={"baseline": _UNIVERSE}) + (root / "data" / "inputs").mkdir() + (root / "data" / "inputs" / "broken").symlink_to("nowhere.fits") + + assert engine.status(root).outputs + assert engine.check(root, []).planned + + +def test_a_declared_input_outside_the_project_is_reported_as_unrecoverable( + analysis: Callable[..., Path], tmp_path: Path +) -> None: + """Its bytes are hashed into the manifest like any other input, so a + change to it still cascades — but it is not in the repository, so the + commit that records the output cannot bring it back.""" + outside = tmp_path / "shared" / "catalog.fits" + outside.parent.mkdir() + outside.write_text("elsewhere\n") + root = analysis(_consuming(str(outside)), universes={"baseline": _UNIVERSE}) + + report = engine.check(root, []) + + assert any(str(outside) in w and "cannot restore them" in w for w in report.warnings) + + +def test_a_declared_input_inside_the_project_draws_no_such_warning( + analysis: Callable[..., Path], +) -> None: + """The mutation check on the test above: the same spec with the source + back under `data/` says nothing.""" + root = analysis(_consuming("data/catalog.fits"), universes={"baseline": _UNIVERSE}) + + assert not any("cannot restore" in w for w in engine.check(root, []).warnings) + + +def test_a_dependency_the_lock_does_not_pin_is_refused(root: Path) -> None: + """Its bytes are not recorded anywhere, so every hash below it would be + a claim nobody can check.""" + (root / "uv.lock").write_text( + (root / "uv.lock").read_text() + + '\n[[package]]\nname = "sibling"\nsource = { path = "../sibling" }\n' + ) + + with pytest.raises(ProjectError, match="cannot be audited"): + engine.check(root, []) + + +def test_a_lock_that_builds_from_source_is_a_warning_not_a_refusal(root: Path) -> None: + """Building from source is legitimate; identity just covers the sdist + rather than the build of it, and saying so is the whole obligation.""" + (root / "uv.lock").write_text( + (root / "uv.lock").read_text() + + '\n[[package]]\nname = "oldlib"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + '\n[package.sdist]\nurl = "https://example/oldlib.tar.gz"\n' + ) + + report = engine.check(root, []) + + assert report.ok + assert any("oldlib" in w for w in report.warnings) + + +# ---- leaving the tree as clean as it was found ----------------------------- + + +def test_a_failing_recipe_commits_nothing_and_leaves_the_tree_clean( + analysis: Callable[..., Path], inline: None +) -> None: + """The invariant that makes the dirty-tree refusal survivable: the next + run must not tell the user to commit truncated, manifest-less + garbage.""" + spec = _SPEC.replace("echo {decisions.method} > {output}/value.txt", "exit 1") + root = analysis(spec, universes={"baseline": _UNIVERSE}) + before = _commits(root) + + report = engine.materialize(root, []) + + assert report.failed == ["baseline/first"] + assert report.blocked == ["baseline/second"] + assert not report.ok + assert _commits(root) == before + assert not dataset.status(root) + + +def test_a_run_in_which_everything_failed_is_not_up_to_date( + analysis: Callable[..., Path], inline: None +) -> None: + """`made` stays empty when every recipe fails, so `up_to_date` alone + read "nothing to do" over a list of failures — and it is the second key + of the JSON report, which is what an agent branches on.""" + spec = _SPEC.replace("echo {decisions.method} > {output}/value.txt", "exit 1") + root = analysis(spec, universes={"baseline": _UNIVERSE}) + + report = engine.materialize(root, []) + + assert report.made == [] + assert not report.up_to_date + assert json.loads(json.dumps(report.as_dict()))["up_to_date"] is False + # The positive control is `test_a_second_run_does_nothing_and_commits_ + # nothing`, where the same empty `made` does mean up to date. + + +def test_a_rebuild_that_fails_puts_the_previous_output_back( + root: Path, inline: None +) -> None: + engine.materialize(root, ["first"]) + (root / "astra.yaml").write_text( + _SPEC.replace("echo {decisions.method} > {output}/value.txt", "exit 1") + ) + dataset.save(root, [root], "break the recipe") + at_break = _commits(root) + + report = engine.materialize(root, ["first"]) + + assert report.failed == ["baseline/first"] + assert (root / "results/baseline/first/value.txt").read_text() == "alpha\n" + assert _commits(root) == at_break + assert not dataset.status(root) + + +def test_an_interrupted_run_restores_what_never_reported( + root: Path, inline: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A sibling that already saved keeps its commit; the output still in + flight is put back, so the tree ends clean either way.""" + engine.materialize(root, []) + (root / "astra.yaml").write_text(_SPEC.replace("echo {decisions.method}", "echo changed")) + dataset.save(root, [root], "edit both recipes") + + class _Interrupted(_Inline): + def completed(self, handles: list[Any]) -> Iterator[TaskResult]: + yield handles[0] + raise KeyboardInterrupt + + @contextmanager + def fake() -> Iterator[_Interrupted]: + yield _Interrupted() + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + with pytest.raises(KeyboardInterrupt): + engine.materialize(root, []) + + assert not dataset.status(root) + + +# ---- the commit message ---------------------------------------------------- + + +def test_the_run_record_is_what_datalad_reads(root: Path, inline: None) -> None: + """Asserted through datalad's own parser rather than against our JSON: + it matches with a regex and returns nothing on any mismatch, after + which `rerun` reports "no command; skipping" and exits 0 — so a golden + test on the text would stay green through a silent break.""" + from datalad.api import Dataset + from datalad.local.rerun import get_run_info + + engine.materialize(root, ["second"]) + message = dataset._git(["log", "-1", "--format=%B"], cwd=root) + + subject, info = get_run_info(Dataset(str(root)), message) + + assert subject == "second [baseline]" + assert info is not None + assert info["cmd"].endswith("python -m lightcone.engine.worker baseline/second") + assert info["inputs"] == ["results/baseline/first"] + assert info["outputs"] == ["results/baseline/second"] + assert info["dsid"] == "4b7b5c1e-0000-4000-8000-000000000000" + assert info["chain"] == [] and info["pwd"] == "." + + +def test_the_record_names_the_declared_input_not_the_annex_object( + analysis: Callable[..., Path], inline: None +) -> None: + """Declared inputs are annex symlinks, so a resolved path records + `.git/annex/objects/SHA256E-…` — the storage rather than the input, and + something no one can `datalad get`.""" + spec = """ + version: "0.0.13" + name: analysis + inputs: + - id: catalog + type: data + source: data/catalog.txt + outputs: + - id: fit + type: metric + inputs: [catalog] + recipe: + command: cat {inputs.catalog} > {output}/seen.txt + """ + root = analysis(spec, files={"data/catalog.txt": "measured\n"}) + dataset.save(root, [root / "data"], "the catalog") + + engine.materialize(root, []) + + from datalad.api import Dataset + from datalad.local.rerun import get_run_info + + _, info = get_run_info(Dataset(str(root)), dataset._git(["log", "-1", "--format=%B"], cwd=root)) + assert info is not None + assert info["inputs"] == ["data/catalog.txt"] + + +def test_every_manifest_of_one_run_names_the_same_commit(root: Path, inline: None) -> None: + """The driver commits each output as it lands, so HEAD moves during the + run — and reading it per task would stamp later manifests with a commit + this same run created, nondeterministically.""" + engine.materialize(root, []) + + shas = { + assets.read(root / "results/baseline" / name).git_sha # type: ignore[union-attr] + for name in ("first", "second") + } + assert len(shas) == 1 + + +def test_check_agrees_with_a_run_on_a_clone_with_no_annex_content( + root: Path, inline: None, tmp_path: Path +) -> None: + """Manifests are in git, so an output whose bytes were never fetched + is still classifiable — check mode reads the recorded digest rather + than the pointer file sitting in its place.""" + engine.materialize(root, []) + clone = tmp_path / "clone" + dataset._git(["clone", "-q", str(root), str(clone)], cwd=tmp_path) + # `clone` does not copy identity, and `annex init` makes a commit — so + # this fails on any host without a usable global one, CI included. + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=clone) + dataset._git(["annex", "init", "-q", "clone"], cwd=clone) + pointer = (clone / "results/baseline/first/value.txt").read_text() + assert pointer.startswith("/annex/objects/") # content really is absent + + assert engine.check(clone, []).planned == {} + + +def test_a_drifted_environment_is_made_to_match_before_anything_runs( + root: Path, inline: None +) -> None: + """Workers pass `--no-sync`, so this is the only place the environment + is made to match the lock. Reported and refused, a lock edited without + a sync would leave recipes importing packages the lock does not + describe while every manifest recorded the new lock's `env_version`; + doing it instead is shorter and impossible to ignore.""" + pyproject = root / "pyproject.toml" + pyproject.write_text( + pyproject.read_text().replace("dependencies = []", 'dependencies = ["idna"]') + ) + from lightcone.engine import project as project_mod + + project_mod._run(["uv", "lock", "-q", "--project", str(root)], cwd=root) + dataset.save(root, [root], "add a dependency without syncing") + assert not project_mod._env_is_current(root) + + report = engine.materialize(root, ["first"]) + + assert report.ok + assert project_mod._env_is_current(root) + + +def test_the_recorded_command_reproduces_the_output(root: Path, inline: None) -> None: + """The claim the record makes, run literally. `datalad rerun` removes + the output, executes the recorded command, and commits what came + back — so the manifest is regenerated inside the same commit.""" + pytest.importorskip("datalad") + engine.materialize(root, ["first"]) + original = assets.read(root / "results/baseline/first") + assert original is not None + + proc = subprocess.run( + [sys.executable, "-c", "from datalad.api import rerun; rerun('HEAD')"], + cwd=root, + capture_output=True, + text=True, + env={**child_env(), "PYTHONPATH": _engine_path()}, + ) + + assert proc.returncode == 0, proc.stderr + rerun = assets.read(root / "results/baseline/first") + assert rerun is not None + assert rerun.data_version == original.data_version + assert rerun.data_version == assets.data_version(root / "results/baseline/first") + assert not dataset.status(root) + + +def _engine_path() -> str: + """Stand in for the ``lightcone-cli`` pin a real project's lock carries. + + The recorded command resolves the worker out of the project's own + environment, which is the whole reason the record reproduces the + *recorded* engine rather than today's. The fixture's project declares + no dependencies so it stays cheap to build, so the engine under test is + put on the path explicitly instead. + """ + import sysconfig + + return os.pathsep.join( + [str(Path(__file__).parent.parent / "src"), sysconfig.get_paths()["purelib"]] + ) + + +# ---- the scheduler seam ---------------------------------------------------- + + +def test_a_real_cluster_still_fits_through_the_seam(root: Path) -> None: + """The one test that starts Dask. The seam is only worth having if the + thing it abstracts still goes through it.""" + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert not dataset.status(root) + + +# ---- the report ------------------------------------------------------------ + + +def test_the_report_is_json_ready(root: Path, inline: None) -> None: + report = engine.materialize(root, ["first"]) + data = json.loads(json.dumps(report.as_dict())) + + assert data["ok"] is True + assert data["made"] == ["baseline/first"] diff --git a/tests/test_plan.py b/tests/test_plan.py new file mode 100644 index 00000000..0eb58be8 --- /dev/null +++ b/tests/test_plan.py @@ -0,0 +1,359 @@ +"""Tests for `lightcone.engine.plan` — the spec, read as a graph. + +Two things are being pinned here. That the graph says what the spec meant +— which outputs exist, what each depends on, and what its recipe came out +as — and that everything ambiguous is an error rather than a guess: a +placeholder nothing declares, a target nothing matches, a nesting depth +the addressing scheme cannot express. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from lightcone.engine import plan +from lightcone.engine.plan import Graph +from lightcone.engine.project import ProjectError + +_SPEC = """ +version: "0.0.13" +name: demo + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: fit + type: metric + inputs: [catalog] + decisions: [method] + recipe: + command: python src/fit.py {inputs.catalog} --method {decisions.method} {output} + + - id: report + type: report + inputs: [fit] + recipe: + command: python src/report.py {inputs.fit} {output} + + - id: reexport + type: metric + +decisions: + method: + label: Method + default: mcmc + options: + mcmc: {label: MCMC} + nested: {label: Nested} +""" + + +def _project(root: Path, spec: str = _SPEC, **universes: str) -> Path: + (root / "astra.yaml").write_text(textwrap.dedent(spec)) + (root / "universes").mkdir(exist_ok=True) + declared = universes or {"baseline": "id: baseline\ndecisions:\n method: mcmc\n"} + for name, text in declared.items(): + (root / "universes" / f"{name}.yaml").write_text(textwrap.dedent(text)) + return root + + +def _build(root: Path) -> Graph: + return plan.build(root) + + +# ---- what the graph contains ----------------------------------------------- + + +def test_one_task_per_universe_and_output_with_a_recipe(tmp_path: Path) -> None: + """A re-export declares no recipe, so it produces no bytes and there is + nothing to schedule for it.""" + graph = _build(_project(tmp_path)) + assert sorted(graph.tasks) == [("baseline", "fit"), ("baseline", "report")] + + +def test_every_universe_gets_its_own_task(tmp_path: Path) -> None: + graph = _build( + _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + alternative="id: alternative\ndecisions:\n method: nested\n", + ) + ) + assert {u for u, _ in graph.tasks} == {"baseline", "alternative"} + + +def test_a_decision_that_differs_gives_a_different_definition_version(tmp_path: Path) -> None: + """The whole point of a universe: the same output, made differently, is + a different thing.""" + graph = _build( + _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + alternative="id: alternative\ndecisions:\n method: nested\n", + ) + ) + assert ( + graph.tasks[("baseline", "fit")].definition_version + != graph.tasks[("alternative", "fit")].definition_version + ) + + +def test_an_output_that_ignores_a_decision_is_not_moved_by_it(tmp_path: Path) -> None: + """`definition_version` hashes the decisions the output *declares*, so an + unrelated choice does not stale it.""" + a = _build(_project(tmp_path, baseline="id: baseline\ndecisions:\n method: mcmc\n")) + b = _build(_project(tmp_path, baseline="id: baseline\ndecisions:\n method: nested\n")) + key = ("baseline", "report") + assert a.tasks[key].definition_version == b.tasks[key].definition_version + + +def test_an_output_addresses_its_own_directory(tmp_path: Path) -> None: + task = _build(_project(tmp_path)).tasks[("baseline", "fit")] + assert task.output_dir == tmp_path / "results" / "baseline" / "fit" + assert "results/baseline/fit" in task.recipe + + +def test_a_declared_input_resolves_to_its_source(tmp_path: Path) -> None: + task = _build(_project(tmp_path)).tasks[("baseline", "fit")] + assert task.inputs == {"catalog": tmp_path / "data" / "catalog.fits"} + assert task.produced_by == {} + assert "data/catalog.fits" in task.recipe + + +def test_a_declared_input_outside_the_project_keeps_its_absolute_path( + tmp_path: Path, +) -> None: + """A path in a recipe *is* the path on disk, and one outside the tree + has no project-relative spelling to fall back on. It used to be + rendered by `relative_to`, which raised a bare ValueError.""" + outside = tmp_path.parent / "shared" / "catalog.fits" + root = _project(tmp_path, _SPEC.replace("source: data/catalog.fits", f"source: {outside}")) + + task = _build(root).tasks[("baseline", "fit")] + assert task.inputs == {"catalog": outside} + assert str(outside) in task.recipe + + +def test_an_input_another_output_produces_becomes_an_edge(tmp_path: Path) -> None: + task = _build(_project(tmp_path)).tasks[("baseline", "report")] + assert task.produced_by == {"fit": ("baseline", "fit")} + assert task.depends_on == (("baseline", "fit"),) + assert "results/baseline/fit" in task.recipe + + +def test_an_input_nothing_provides_is_an_error(tmp_path: Path) -> None: + """Caught by ASTRA rather than by us: resolution answers what a valid + spec means, so `build` asks whether it is one first — and the error + names the line at fault instead of the run that tripped over it.""" + spec = _SPEC.replace("inputs: [catalog]", "inputs: [missing]").replace( + "{inputs.catalog}", "{inputs.missing}" + ) + with pytest.raises(ProjectError, match="does not validate"): + _build(_project(tmp_path, spec)) + + +def test_a_spec_astra_rejects_never_reaches_a_recipe(tmp_path: Path) -> None: + """The gate exists because the resolver assumes validity: without it an + invalid spec surfaces as a missing decision or an unresolvable input, + blaming the run for a fault in the file.""" + spec = _SPEC.replace('name: demo', "") + with pytest.raises(ProjectError, match="MISSING_ROOT_FIELD"): + _build(_project(tmp_path, spec)) + + +# ---- ordering, closure, targets -------------------------------------------- + + +def test_order_puts_dependencies_first(tmp_path: Path) -> None: + order = _build(_project(tmp_path)).order() + assert order.index(("baseline", "fit")) < order.index(("baseline", "report")) + + +def test_a_cycle_is_a_clean_error(tmp_path: Path) -> None: + spec = _SPEC.replace(" - id: reexport\n type: metric\n", "") + spec = spec.replace(" inputs: [catalog]", " inputs: [report]") + spec = spec.replace("{inputs.catalog}", "{inputs.report}") + with pytest.raises(ProjectError, match="cycle"): + _build(_project(tmp_path, spec)).order() + + +def test_asking_for_an_output_asks_for_what_it_is_made_of(tmp_path: Path) -> None: + graph = _build(_project(tmp_path)) + closure = graph.closure(graph.resolve(["report"])) + assert sorted(closure.tasks) == [("baseline", "fit"), ("baseline", "report")] + + +def test_a_bare_target_means_every_universe(tmp_path: Path) -> None: + graph = _build( + _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + alternative="id: alternative\ndecisions:\n method: nested\n", + ) + ) + assert sorted(graph.resolve(["fit"])) == [("alternative", "fit"), ("baseline", "fit")] + + +def test_a_qualified_target_means_exactly_one(tmp_path: Path) -> None: + graph = _build( + _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + alternative="id: alternative\ndecisions:\n method: nested\n", + ) + ) + assert graph.resolve(["baseline/fit"]) == [("baseline", "fit")] + + +def test_an_unknown_target_is_an_error_not_an_empty_run(tmp_path: Path) -> None: + """Quietly making nothing is the least useful thing a build tool can + do, so the message lists what there was.""" + graph = _build(_project(tmp_path)) + with pytest.raises(ProjectError, match="no output matches `typo`.*baseline/fit"): + graph.resolve(["typo"]) + + +# ---- refusals -------------------------------------------------------------- + + +def test_two_universes_cannot_claim_one_id(tmp_path: Path) -> None: + """They would materialize into one directory, and the second simply + replaced the first — so a universe went missing with nothing said. The + natural way to reach it is copying a universe file and forgetting to + change the id inside.""" + root = _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + copy="id: baseline\ndecisions:\n method: nested\n", + ) + with pytest.raises(ProjectError, match="both declare the universe `baseline`"): + _build(root) + + +def test_universes_are_told_apart_by_their_id_not_their_filename(tmp_path: Path) -> None: + root = _project( + tmp_path, + baseline="id: baseline\ndecisions:\n method: mcmc\n", + alternative="id: alternative\ndecisions:\n method: nested\n", + ) + assert {u for u, _ in _build(root).tasks} == {"baseline", "alternative"} + + +def test_no_spec_is_a_clean_error(tmp_path: Path) -> None: + (tmp_path / "universes").mkdir() + with pytest.raises(ProjectError, match="no astra.yaml"): + _build(tmp_path) + + +def test_no_universe_is_a_clean_error(tmp_path: Path) -> None: + (tmp_path / "astra.yaml").write_text(textwrap.dedent(_SPEC)) + with pytest.raises(ProjectError, match="declares no universe"): + _build(tmp_path) + + +# ---- sub-analyses ---------------------------------------------------------- + +_PARENT = """ +version: "0.0.13" +name: parent + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: mass_function + from: hod.mass_function + + - id: summary + type: report + inputs: [mass_function] + recipe: + command: python summarize.py {inputs.mass_function} {output} + +analyses: + hod: + path: ./analyses/hod +""" + +_SUB = """ +version: "0.0.13" +name: hod + +inputs: + - id: catalog + type: data + from: ../catalog + +outputs: + - id: mass_function + type: metric + inputs: [catalog] + decisions: [binning] + recipe: + command: python hod.py {inputs.catalog} --bins {decisions.binning} {output} + +decisions: + binning: + label: Binning + default: log + options: + log: {label: log} + linear: {label: linear} +""" + + +def _tree(root: Path) -> Path: + (root / "astra.yaml").write_text(textwrap.dedent(_PARENT)) + (root / "universes").mkdir() + # The sub-analysis's universe is named explicitly: ASTRA has no + # implicit "same id" fallback, and lc no longer invents one. + (root / "universes" / "baseline.yaml").write_text( + "id: baseline\ndecisions: {}\nanalyses:\n hod:\n universe: baseline\n" + ) + sub = root / "analyses" / "hod" + (sub / "universes").mkdir(parents=True) + (sub / "astra.yaml").write_text(textwrap.dedent(_SUB)) + (sub / "universes" / "baseline.yaml").write_text("id: baseline\ndecisions:\n binning: log\n") + return root + + +def test_a_sub_analysis_output_is_addressed_flat_and_qualified(tmp_path: Path) -> None: + """One addressing scheme and one place to look, whatever shape the + spec has.""" + graph = _build(_tree(tmp_path)) + assert sorted(graph.tasks) == [("baseline", "hod.mass_function"), ("baseline", "summary")] + task = graph.tasks[("baseline", "hod.mass_function")] + assert task.output_dir == tmp_path / "results" / "baseline" / "hod.mass_function" + + + + + + + + + + + + + + + + +# ---- rendering a recipe ---------------------------------------------------- + + + + + + + + diff --git a/tests/test_project.py b/tests/test_project.py index ff14baed..e9645e73 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -4,6 +4,7 @@ from __future__ import annotations import shutil +import uuid from pathlib import Path import pytest @@ -103,9 +104,9 @@ def test_converge_writes_the_templates_verbatim(tmp_path: Path) -> None: assert (project / "pyproject.toml").read_text() == templates.pyproject(name="proj") assert (project / ".python-version").read_text() == templates.python_version() - assert (project / ".gitignore").read_text() == templates.gitignore() - assert (project / "results/README.md").read_text() == templates.results_readme() - assert (project / "myst.yml").read_text() == templates.myst_yml() + assert (project / ".gitignore").read_text() == templates.read("gitignore.tmpl") + assert (project / "results/README.md").read_text() == templates.read("results-README.md.tmpl") + assert (project / "myst.yml").read_text() == templates.read("myst.yml.tmpl") assert (project / "index.md").read_text() == templates.index_md(title="proj") @@ -183,7 +184,12 @@ def test_adoption_adds_no_layout_of_its_own(tmp_path: Path) -> None: def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: """The property the removed `src/` item broke: everything convergence tracks has to be something git can carry, or a fresh clone reports - drift forever. `.venv` is exempt — it is git-ignored and rebuilt.""" + drift forever. + + Two items are exempt, and both for the same reason — they are local + state git does not clone. `.venv` is git-ignored and rebuilt from the + lock, and `git clone` of an annexed repository leaves the annex + uninitialized until someone runs `git annex init`.""" project = tmp_path / "proj" converge(project) @@ -198,7 +204,7 @@ def test_a_clone_of_a_converged_project_is_converged(tmp_path: Path) -> None: (clone / ".git").mkdir() report = converge(clone, write=False) - assert report.created == [".venv"], report.created + assert report.created == ["git-annex", ".venv"], report.created def test_converge_repairs_a_missing_piece(tmp_path: Path) -> None: @@ -236,7 +242,7 @@ def test_gitignore_repair_preserves_user_content(tmp_path: Path) -> None: text = (project / ".gitignore").read_text() assert text.startswith("mine.txt\nbuild/\n") - assert templates.missing_gitignore_entries(text) == [] + assert templates.missing("gitignore.tmpl", text) == [] def test_gitignore_repair_is_idempotent(tmp_path: Path) -> None: @@ -316,15 +322,82 @@ def test_results_that_is_not_a_directory_blocks_convergence(tmp_path: Path) -> N assert not converge(project).converged -# ---- git ------------------------------------------------------------------ +def test_a_gitattributes_the_repair_cannot_order_blocks_convergence( + tmp_path: Path, +) -> None: + """The repair only appends, so a file already opting `results/` into + the annex gets the `*` default added *below* it — and git-annex takes + the last match, so every result would be committed to git as a plain + blob while the report said the file was repaired.""" + project = tmp_path / "proj" + project.mkdir() + (project / ".gitattributes").write_text("results/** annex.largefiles=anything\n") + + report = converge(project) + assert ".gitattributes" in report.blocked + assert not report.converged + assert any("git-annex takes the last match" in w for w in report.warnings) + + # And it stays blocked, rather than the repair settling it into a lie. + assert not converge(project).converged + + +def test_a_gitattributes_in_the_right_order_is_repaired_and_converges( + tmp_path: Path, +) -> None: + """The mutation check on the test above: the same two lines the other + way round take the ordinary append and leave the project converged.""" + project = tmp_path / "proj" + project.mkdir() + (project / ".gitattributes").write_text( + "* annex.largefiles=nothing\nresults/** annex.largefiles=anything\n" + ) + + report = converge(project) + assert ".gitattributes" not in report.blocked + assert ".gitattributes" in report.repaired + assert converge(project).converged + + +# ---- the repository ------------------------------------------------------- -def test_initializes_a_repository(tmp_path: Path, tools: list[list[str]]) -> None: +def test_initializes_a_repository_with_an_annex( + tmp_path: Path, tools: list[list[str]] +) -> None: + """git for the pointers, git-annex for the bytes — a project is both + from birth, because results are versioned in it.""" project = tmp_path / "proj" report = converge(project) assert ["git", "init", "-q"] in tools + assert ["git", "annex", "init", "-q"] in tools assert ".git" in report.created + assert "git-annex" in report.created + + +def test_writes_the_storage_policy_and_the_dataset_id(tmp_path: Path) -> None: + """`.gitattributes` is what routes bytes to the annex; `.datalad/config` + is the one thing that makes the repository a DataLad dataset.""" + project = tmp_path / "proj" + report = converge(project) + + assert (project / ".gitattributes").read_text() == templates.read("gitattributes.tmpl") + assert ".datalad/config" in report.created + config = (project / ".datalad" / "config").read_text() + assert '[datalad "dataset"]' in config + uuid.UUID(config.split("id =")[1].strip()) + + +def test_the_dataset_id_is_generated_once(tmp_path: Path) -> None: + """It identifies the dataset across clones and siblings — regenerating + it on a re-`init` would make a project a different dataset every time.""" + project = tmp_path / "proj" + converge(project) + first = (project / ".datalad" / "config").read_text() + + converge(project) + assert (project / ".datalad" / "config").read_text() == first def test_does_not_nest_a_repository_inside_one(tmp_path: Path, tools: list[list[str]]) -> None: @@ -333,7 +406,7 @@ def test_does_not_nest_a_repository_inside_one(tmp_path: Path, tools: list[list[ (tmp_path / ".git").mkdir() report = converge(tmp_path / "nested" / "proj") - assert not any(c[0] == "git" for c in tools) + assert not any(c[:2] == ["git", "init"] for c in tools) assert ".git" in report.unchanged assert not (tmp_path / "nested" / "proj" / ".git").exists() @@ -345,23 +418,48 @@ def test_a_linked_worktree_counts_as_version_controlled(tmp_path: Path) -> None: assert ".git" in converge(tmp_path / "proj").unchanged -def test_git_absent_is_not_a_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, tools: list[list[str]] +def test_check_mode_can_ask_about_a_directory_that_does_not_exist_yet( + tmp_path: Path, ) -> None: - """git is optional, unlike uv: a project without it is valid, so an - absent git is nothing to converge rather than a warning.""" - from lightcone.engine import project as project_mod + """Check mode creates nothing, so inside an enclosing repository the + walk-up says "in a repository" for a directory that is not there — and + running git with a cwd that does not exist raises out of `Popen` + rather than answering anything.""" + (tmp_path / ".git").mkdir() - monkeypatch.setattr( - project_mod.shutil, - "which", - lambda name, path=None: None if name == "git" else f"/usr/bin/{name}", - ) - report = converge(tmp_path / "proj") + report = converge(tmp_path / "newproj", write=False) - assert not any(c[0] == "git" for c in tools) - assert ".git" not in report.created + report.unchanged - assert report.warnings == [] + assert ".git" in report.unchanged + assert "git-annex" in report.created + assert not (tmp_path / "newproj").exists() + + +def test_an_existing_annex_is_adopted(tmp_path: Path, tools: list[list[str]]) -> None: + """The annex is asked about the way git-annex asks itself, so an + enclosing repository that already has one is not re-initialized.""" + project = tmp_path / "proj" + converge(project) + tools.clear() + + report = converge(project) + assert "git-annex" in report.unchanged + assert not any(c[:3] == ["git", "annex", "init"] for c in tools) + + +def test_results_ignored_by_an_older_scaffold_blocks_convergence(tmp_path: Path) -> None: + """A `.gitignore` written before results were versioned keeps them + uncommittable, and `.gitignore` convergence only ever appends — so + convergence cannot fix this and must not call the project converged. + `git add` skips ignored paths in silence; a materialize would report + success and commit nothing.""" + project = tmp_path / "proj" + project.mkdir() + (project / ".gitignore").write_text("results/*\n") + + report = converge(project) + assert "results/" in report.blocked + assert not report.converged + assert any("results/*" in w and ".gitignore:1" in w for w in report.warnings) def test_surfaces_a_git_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -388,13 +486,39 @@ def fake_run(argv: list[str], *, cwd: Path) -> MagicMock: def test_requires_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - from lightcone.engine import project as project_mod - - monkeypatch.setattr(project_mod.shutil, "which", lambda name, path=None: None) + _absent(monkeypatch, "uv") with pytest.raises(ProjectError, match="uv is required"): converge(tmp_path / "proj") +def test_requires_git(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """git stops being optional the moment results are versioned in the + repository: there is no useful project without it, so an absent git is + a refusal rather than nothing to converge.""" + _absent(monkeypatch, "git") + with pytest.raises(ProjectError, match="git is required"): + converge(tmp_path / "proj") + + +def test_requires_git_annex(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """And by the name git itself searches for: `git annex` is not a + builtin, it is git finding a `git-annex` executable on PATH.""" + _absent(monkeypatch, "git-annex") + with pytest.raises(ProjectError, match="git-annex is required"): + converge(tmp_path / "proj") + + +def _absent(monkeypatch: pytest.MonkeyPatch, missing: str) -> None: + """Make exactly one tool unfindable, leaving the others resolvable.""" + from lightcone.engine import project as project_mod + + monkeypatch.setattr( + project_mod.shutil, + "which", + lambda name, path=None: None if name == missing else f"/usr/bin/{name}", + ) + + # ---- the uv seam ---------------------------------------------------------- diff --git a/tests/test_run.py b/tests/test_run.py index 56428748..98693565 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -11,7 +11,7 @@ import pytest from lightcone.engine import run as engine_run -from lightcone.engine.project import ProjectError, current_project +from lightcone.engine.project import ProjectError, current_project, uv_prefix SPEC = """\ title: Test @@ -105,6 +105,20 @@ def test_declared_file_inputs_become_read_paths(project: Path) -> None: assert engine_run.input_paths(project, spec) == [(project / "data" / "local.csv").resolve()] +def test_an_input_declared_inside_a_sub_analysis_is_a_read_path_too(project: Path) -> None: + """A denial the researcher cannot act on is worse than the access it + withheld: the file *is* declared, just not at the top of the tree.""" + (project / "data" / "stage.csv").write_text("x\n") + spec = { + "inputs": [{"id": "local", "source": "data/local.csv"}], + "analyses": {"stage": {"inputs": [{"id": "sub", "source": "data/stage.csv"}]}}, + } + assert engine_run.input_paths(project, spec) == [ + (project / "data" / "local.csv").resolve(), + (project / "data" / "stage.csv").resolve(), + ] + + def test_a_source_that_is_not_a_path_is_left_alone(project: Path) -> None: """ASTRA's `source` is free-form — a URI, a dotted name, a path — so "is this a path" is answered by whether it resolves to something that @@ -133,9 +147,17 @@ def test_an_unresolvable_tree_degrades_to_the_top_level_document(project: Path) def test_uv_is_pinned_to_the_project_and_refuses_to_drift(project: Path) -> None: """uv's own walk-up discovery is never trusted, and a stale lock must be uv's loud error rather than a silent relock.""" - prefix = engine_run.uv_prefix(project) + prefix = uv_prefix(project, sync=True) assert prefix[:2] == ["uv", "run"] assert "--locked" in prefix assert "--exact" in prefix assert prefix[prefix.index("--project") + 1] == str(project) assert prefix[-1] == "--" + + +def test_a_recipe_does_not_sync_where_a_probe_does(project: Path) -> None: + """The one thing the two hops disagree about: a probe converges the + environment it is about to describe, and a recipe must not, or every + concurrent worker writes the same `.venv`.""" + assert "--no-sync" in uv_prefix(project, sync=False) + assert "--exact" not in uv_prefix(project, sync=False) diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py index e6614083..f673ab25 100644 --- a/tests/test_sandbox_enforcement.py +++ b/tests/test_sandbox_enforcement.py @@ -10,7 +10,7 @@ Two deliberate choices: - **The real policy.** These run against - :func:`~lightcone.engine.sandbox.policy.probe_policy` — what an actual + :func:`~lightcone.engine.sandbox.policy.exec_policy` — what an actual ``lc run`` gets — not a policy hand-built to make a point. A test that grants exactly what it is testing cannot discover that the shipped policy grants something else. @@ -152,7 +152,7 @@ def test_the_expected_mechanism_is_in_use(backend: sandbox.Backend) -> None: def test_a_sandboxed_run_attests_a_scoped_filesystem( backend: sandbox.Backend, project: Path ) -> None: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: attestation = backend.attest(policy) assert attestation.fs == "declared" assert attestation.mechanism == backend.capability.kind @@ -167,7 +167,7 @@ def test_an_undeclared_host_tool_cannot_be_executed( """The #1 leakage channel: a recipe that works only because the author happens to have some tool installed.""" tool = undeclared_tool() - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, f"{tool} --version", cwd=project) assert result.returncode != 0, f"{tool} ran inside the sandbox" @@ -179,7 +179,7 @@ def test_an_undeclared_tool_is_readable_but_still_not_executable( readable so the dynamic linker works, which means an undeclared tool can be *seen* — running it is the leak, and only that is denied.""" tool = undeclared_tool() - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: seen = shell(backend, policy, f"test -r {tool} && echo READABLE", cwd=project) ran = shell(backend, policy, f"{tool} --version", cwd=project) assert "READABLE" in seen.stdout @@ -191,7 +191,7 @@ def test_an_allowlisted_utility_is_executable( ) -> None: """The other half: the allowlist has to actually work, or every recipe that pipes through sed breaks.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "printf 'b\\na\\n' | sort | head -1", cwd=project) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "a" @@ -203,7 +203,7 @@ def test_a_dynamically_linked_binary_runs_at_all( """Proves the loader tier — the ELF interpreter on Linux, dyld on macOS. Without it *nothing* dynamically linked starts, bash included, and every other test here would fail for the wrong reason.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "echo LOADER-OK", cwd=project) assert result.returncode == 0, result.stderr assert "LOADER-OK" in result.stdout @@ -215,7 +215,7 @@ def test_a_binary_dropped_into_the_writable_scope_cannot_be_run( """Write does not imply execute. Otherwise the allowlist is two lines from being defeated: copy a tool into scratch, run it from there.""" tool = undeclared_tool() - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: smuggled = policy.tmp_home / "smuggled" result = shell( backend, @@ -229,7 +229,7 @@ def test_a_binary_dropped_into_the_writable_scope_cannot_be_run( def test_the_projects_own_interpreter_runs_and_finds_its_stdlib( backend: sandbox.Backend, project: Path ) -> None: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, f"{project}/.venv/bin/python -c 'import json; print(json.dumps(1))'", cwd=project, @@ -247,7 +247,7 @@ def test_an_undeclared_python_module_cannot_be_imported( """Python-level leakage: a module reachable on the host but not part of the declared environment. Denied at *read*, so the import fails however sys.path was arranged.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -267,7 +267,7 @@ def test_a_compiled_extension_module_can_be_imported( read grant is enough, while macOS gates `dlopen` on `file-map-executable`. If the read tier lost that right, this is the test that fails — on macOS only.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -288,7 +288,7 @@ def test_an_undeclared_shared_library_cannot_be_loaded( smuggled = outside / library.name shutil.copy(library, smuggled) - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -319,7 +319,7 @@ def _a_compiled_extension() -> Path: def test_an_undeclared_data_file_cannot_be_read( backend: sandbox.Backend, project: Path, outside: Path ) -> None: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, f"cat {outside / 'secret.txt'}", cwd=project) assert result.returncode != 0 assert "undeclared" not in result.stdout @@ -331,14 +331,14 @@ def test_a_declared_input_outside_the_project_can_be_read( """The same file, declared. This is what makes the denial actionable rather than a wall: the remedy the message prints has to work.""" declared = outside / "secret.txt" - with sandbox.scope(project, read_paths=[declared]) as policy: + with sandbox.scope(sandbox.exec_policy(project, read_paths=[declared])) as policy: result = shell(backend, policy, f"cat {declared}", cwd=project) assert result.returncode == 0, result.stderr assert "undeclared" in result.stdout def test_the_project_tree_is_readable(backend: sandbox.Backend, project: Path) -> None: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "cat data.txt", cwd=project) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "in-tree" @@ -350,7 +350,7 @@ def test_the_real_home_is_not_readable(backend: sandbox.Backend, project: Path) canary = Path.home() / ".lc-enforcement-canary" canary.write_text("host home\n") try: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, f"cat {canary}", cwd=project) finally: canary.unlink(missing_ok=True) @@ -366,7 +366,7 @@ def test_the_tree_outside_results_cannot_be_written( ) -> None: """The environment a run starts with is the one it ends with — and the file has to be *unchanged*, not merely reported.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "printf clobbered > data.txt", cwd=project) assert result.returncode != 0 assert (project / "data.txt").read_text() == "in-tree\n", "the file changed anyway" @@ -376,7 +376,7 @@ def test_results_can_be_written(backend: sandbox.Backend, project: Path) -> None """Writable inside a read-only tree — the nesting both mechanisms have to agree on, and the one a container gets from a second bind mount.""" (project / "results").mkdir() - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "printf out > results/out.csv", cwd=project) assert result.returncode == 0, result.stderr assert (project / "results" / "out.csv").read_text() == "out" @@ -395,14 +395,14 @@ def test_a_declared_input_is_read_only( boundary. """ target = outside / "secret.txt" - with sandbox.scope(project, read_paths=[outside]) as policy: + with sandbox.scope(sandbox.exec_policy(project, read_paths=[outside])) as policy: result = shell(backend, policy, f"printf clobbered > {target}", cwd=project) assert result.returncode != 0 assert target.read_text() == "undeclared\n", "the file changed anyway" def test_the_private_scope_is_writable(backend: sandbox.Backend, project: Path) -> None: - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: target = policy.tmp_home / "result.txt" result = shell(backend, policy, f"printf ok > {target}", cwd=project) wrote = target.read_text() if target.exists() else "" @@ -415,7 +415,7 @@ def test_tempfile_works_inside_the_boundary( ) -> None: """TMPDIR points into the private scope, so the stdlib's own scratch keeps working even where the shared /tmp left the write set.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -432,7 +432,7 @@ def test_a_command_can_allocate_a_pty(backend: sandbox.Backend, project: Path) - """devpts and friends are granted, so pexpect and pytest's own capture work. Without them `pty.openpty()` fails as "out of pty devices" — a message naming neither a path nor the sandbox.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -451,7 +451,7 @@ def test_the_restriction_is_inherited_by_grandchildren( ) -> None: """Both mechanisms confine the whole descendant tree and neither can be shed — which is why wrapping the outermost command is enough.""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell( backend, policy, @@ -472,7 +472,7 @@ def test_a_denial_reaches_the_user_through_the_boundary( a real denial must produce the explanation *and* the trailer, or the sandbox is an invisible wall.""" tool = undeclared_tool() - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: outcome = sandbox.run( backend, policy, @@ -528,7 +528,7 @@ def test_an_allowlisted_tool_resolves_to_the_copy_that_was_granted( fronts homebrew, so `env bash` resolved `/opt/homebrew/bin/bash` while the policy had granted `/bin/bash` — and the sandbox denied bash itself with "Operation not permitted".""" - with sandbox.scope(project) as policy: + with sandbox.scope(sandbox.exec_policy(project)) as policy: result = shell(backend, policy, "command -v bash && echo RESOLVED", cwd=project) assert result.returncode == 0, result.stderr resolved = Path(result.stdout.splitlines()[0].strip()) diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py index b48732ee..bdb3b08d 100644 --- a/tests/test_sandbox_policy.py +++ b/tests/test_sandbox_policy.py @@ -29,7 +29,7 @@ def built(tmp_path: Path) -> Iterator[policy_module.Policy]: """ project = tmp_path / "proj" project.mkdir() - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: yield built @@ -56,7 +56,7 @@ def test_results_is_writable(tmp_path: Path) -> None: podman mounts `results` `:rw` over a `:ro` project.""" project = tmp_path / "proj" (project / "results").mkdir(parents=True) - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: assert built.grants(project / "results" / "out.csv", built.write) @@ -65,7 +65,7 @@ def test_results_is_granted_only_if_it_exists(tmp_path: Path) -> None: side effect nobody asked a probe for.""" project = tmp_path / "bare" project.mkdir() - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: assert not built.grants(project / "results", built.write) assert not (project / "results").exists() @@ -88,7 +88,7 @@ def test_the_shared_tmp_is_writable_for_a_project_outside_it() -> None: """`/tmp` specifically, not `gettempdir()`: on macOS the latter is the per-user `$TMPDIR` under /var/folders, a different directory that is not in the baseline at all.""" - with scope(Path.home() / ".lc-policy-test-project") as built: + with scope(policy_module.exec_policy(Path.home() / ".lc-policy-test-project")) as built: assert Path("/tmp").resolve() in built.write @@ -97,7 +97,7 @@ def test_a_project_living_under_tmp_does_not_become_writable() -> None: otherwise be writable through that grant — voiding the read-only tree for exactly the people who keep scratch analyses in /tmp.""" shared = Path("/tmp").resolve() - with scope(shared / "lc-policy-under-tmp") as built: + with scope(policy_module.exec_policy(shared / "lc-policy-under-tmp")) as built: assert shared not in built.write @@ -122,7 +122,7 @@ def test_declared_inputs_join_the_read_scope(tmp_path: Path) -> None: external.parent.mkdir() external.touch() - with scope(project, read_paths=[external]) as built: + with scope(policy_module.exec_policy(project, read_paths=[external])) as built: assert external.resolve() in built.read @@ -238,7 +238,7 @@ def test_the_allowlist_is_resolved_off_the_ambient_path( project = tmp_path / "proj2" project.mkdir() - with scope(project) as rebuilt: + with scope(policy_module.exec_policy(project)) as rebuilt: assert impostor.resolve() not in rebuilt.execute @@ -266,7 +266,7 @@ def test_the_env_the_seam_execs_is_one_the_policy_granted( project = tmp_path / "proj" project.mkdir() - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: spawned = Path(env_argv(built)[0]) assert spawned == elsewhere / "env" assert built.grants(spawned, built.execute) @@ -296,7 +296,7 @@ def test_the_venv_and_the_interpreter_behind_it_are_granted(tmp_path: Path) -> N real.chmod(0o755) (bin_dir / "python").symlink_to(real) - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: install_root = store.parent.resolve() # The *directory* is deliberately not granted: the tree is # writable, so that would be an exec grant on whatever is written @@ -325,7 +325,7 @@ def test_the_venv_bin_is_granted_per_file_not_as_a_directory(tmp_path: Path) -> script.write_text("#!/bin/sh\n") script.chmod(0o755) - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: assert script.resolve() in built.execute, "a console script that exists is granted" assert bin_dir.resolve() not in built.execute, "but never the directory" # What a run writes there afterwards is therefore not runnable. @@ -352,7 +352,7 @@ def test_a_system_interpreter_does_not_make_the_whole_prefix_executable( pytest.skip("no system python3") (bin_dir / "python").symlink_to(system) - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: assert Path("/usr") not in built.execute assert Path("/usr") in built.read assert system.resolve() in built.execute @@ -378,7 +378,7 @@ def test_an_interpreter_in_home_does_not_make_home_readable( (project / ".venv" / "bin").mkdir(parents=True) (project / ".venv" / "bin" / "python").symlink_to(interpreter) - with scope(project) as built: + with scope(policy_module.exec_policy(project)) as built: assert not built.grants(home, built.read), built.read # The interpreter file itself is still runnable. assert built.grants(interpreter, built.execute) diff --git a/tests/test_templates.py b/tests/test_templates.py index 9c7e5cb1..ca4e071e 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -99,32 +99,32 @@ def test_gitignore_entries_are_the_patterns_only() -> None: """Convergence compares patterns, so comments and blanks must not leak into the set — a comment treated as an entry would be re-appended forever.""" - entries = templates.gitignore_entries() + entries = templates.entries("gitignore.tmpl") assert not any(e.startswith("#") or not e.strip() for e in entries) assert ".venv/" in entries - assert "results/*" in entries - assert "!results/README.md" in entries -def test_gitignore_entries_keep_template_order() -> None: - """`!results/README.md` only works after the `results/*` it negates.""" - entries = templates.gitignore_entries() - assert entries.index("results/*") < entries.index("!results/README.md") +def test_the_template_does_not_ignore_what_the_repository_versions() -> None: + """`results/` and `data/` are committed, so an ignore rule covering + either would make every materialized output silently uncommittable — + `git add` skips ignored paths without a word.""" + entries = templates.entries("gitignore.tmpl") + assert not any(e.lstrip("!").startswith(("results", "data")) for e in entries) def test_gitignore_header_is_the_templates_own_first_line() -> None: """Derived, not duplicated: rewording the template's comment can't leave the repair appending a second header.""" - assert templates.gitignore().startswith(templates.gitignore_header() + "\n") + assert templates.read("gitignore.tmpl").startswith(templates.header("gitignore.tmpl") + "\n") def test_repair_is_none_when_nothing_is_missing() -> None: - assert templates.missing_gitignore_entries(templates.gitignore()) == [] - assert templates.gitignore_repair(templates.gitignore()) is None + assert templates.missing("gitignore.tmpl", templates.read("gitignore.tmpl")) == [] + assert templates.gitignore_repair(templates.read("gitignore.tmpl")) is None def test_repair_of_an_empty_file_is_just_the_template() -> None: - assert templates.gitignore_repair("") == templates.gitignore() + assert templates.gitignore_repair("") == templates.read("gitignore.tmpl") def test_repair_appends_only_what_is_missing_behind_the_header() -> None: @@ -132,22 +132,91 @@ def test_repair_appends_only_what_is_missing_behind_the_header() -> None: assert repaired is not None assert repaired.startswith("mine.txt\n.venv/\n\n") assert repaired.count(".venv/") == 1 - assert templates.gitignore_header() in repaired - assert templates.missing_gitignore_entries(repaired) == [] + assert templates.header("gitignore.tmpl") in repaired + assert templates.missing("gitignore.tmpl", repaired) == [] def test_repair_does_not_add_a_second_header() -> None: """The case a marker check would have skipped: header present, entries missing.""" - header = templates.gitignore_header() + header = templates.header("gitignore.tmpl") repaired = templates.gitignore_repair(f"{header}\n.venv/\n") assert repaired is not None assert repaired.count(header) == 1 - assert templates.missing_gitignore_entries(repaired) == [] + assert templates.missing("gitignore.tmpl", repaired) == [] def test_a_pattern_inside_a_comment_does_not_count_as_present() -> None: - assert "results/*" in templates.missing_gitignore_entries("# results/*\n") + assert ".venv/" in templates.missing("gitignore.tmpl", "# .venv/\n") + + +# ---- .gitattributes ------------------------------------------------------- + + +def test_gitattributes_routes_content_to_the_annex_and_everything_else_to_git() -> None: + """The whole storage policy. The default line is the load-bearing one: + `git annex add` annexes whatever it is handed, so without it the + documented save turns analysis code into read-only symlinks.""" + entries = templates.entries("gitattributes.tmpl") + assert entries[0] == "* annex.largefiles=nothing" + assert "results/** annex.largefiles=anything" in entries + assert "data/** annex.largefiles=anything" in entries + assert "**/.lightcone-manifest.json annex.largefiles=nothing" in entries + + +def test_gitattributes_exceptions_come_after_the_default() -> None: + """Last matching line wins, so a default written below the exceptions + would silently take the annex back out of the picture.""" + entries = templates.entries("gitattributes.tmpl") + assert entries.index("* annex.largefiles=nothing") < entries.index( + "results/** annex.largefiles=anything" + ) + + +def test_gitattributes_repair_appends_what_a_users_own_file_lacks() -> None: + """More is at stake here than in `.gitignore`: a `.gitattributes` the + user wrote first would leave result bytes routed into git.""" + repaired = templates.gitattributes_repair("*.fits filter=lfs\n") + assert repaired is not None + assert repaired.startswith("*.fits filter=lfs\n\n") + assert templates.missing("gitattributes.tmpl", repaired) == [] + assert templates.gitattributes_repair(templates.read("gitattributes.tmpl")) is None + + +def test_a_file_the_repair_can_fix_reports_no_disorder() -> None: + """The ordinary case: whatever the user already had, the managed lines + are appended in template order and the result means what it should.""" + assert templates.gitattributes_disorder("") == "" + assert templates.gitattributes_disorder("*.fits filter=lfs\n") == "" + assert templates.gitattributes_disorder(templates.read("gitattributes.tmpl")) == "" + + +def test_an_opt_out_the_defaults_would_land_below_is_named() -> None: + """The trap append-only cannot escape. A file that already opts + `results/` into the annex gets `* annex.largefiles=nothing` appended + *after* it, and last-match-wins then routes every result into git as a + plain blob — while convergence reports the file repaired.""" + misplaced = templates.gitattributes_disorder("results/** annex.largefiles=anything\n") + assert misplaced == "* annex.largefiles=nothing" + + +def test_a_hand_written_file_already_in_the_right_order_needs_nothing() -> None: + """Judged on meaning, not on who wrote it — and only lines setting the + *same* attribute can be out of order with each other, so the + `* filter=annex` the repair appends below these two is not disorder.""" + ordered = "* annex.largefiles=nothing\nresults/** annex.largefiles=anything\n" + assert templates.gitattributes_disorder(ordered) == "" + + +# ---- .datalad/config ------------------------------------------------------ + + +def test_datalad_config_carries_the_dataset_id() -> None: + """The one thing a git + git-annex repository lacks to *be* a DataLad + dataset, in the git-config syntax datalad reads it from.""" + text = templates.datalad_config(dataset_id="4b7b5c1e-0000-4000-8000-000000000000") + assert '[datalad "dataset"]' in text + assert "id = 4b7b5c1e-0000-4000-8000-000000000000" in text # ---- the rest ------------------------------------------------------------- @@ -164,6 +233,14 @@ def test_index_md_renders_the_title_and_keeps_myst_roles() -> None: def test_results_readme_explains_the_output_layout() -> None: - """`results/` is git-ignored and starts empty, so the README is the - only thing a clone shows for it.""" - assert "results///" in templates.results_readme() + """`results/` starts empty and git carries no empty directories, so the + README is the only thing a clone shows for it.""" + assert "results///" in templates.read("results-README.md.tmpl") + + +def test_data_readme_explains_where_declared_inputs_go() -> None: + """It says what the directory is for and nothing about how to fill it: + manipulating git-annex by hand is not something lc asks of anyone.""" + text = templates.read("data-README.md.tmpl") + assert "data/catalog.fits" in text + assert "git annex" not in text diff --git a/tests/test_worker.py b/tests/test_worker.py new file mode 100644 index 00000000..6accf882 --- /dev/null +++ b/tests/test_worker.py @@ -0,0 +1,466 @@ +"""Tests for `lightcone.engine.worker` — making one output. + +This file runs real recipes through the real boundary, against a real +project, because that is the only way to answer what it asks: whether the +environment gates hold, whether the output directory is really reset, +whether the manifest agrees with the bytes beside it, and whether a +recipe can reach something it was not given. + +Following `test_sandbox_enforcement.py`'s rule, every denial assertion is +mutation-checked — the same command is run through `Unavailable()` and +must *succeed*, or the test would pass without the sandbox doing anything. +""" + +from __future__ import annotations + +import subprocess +import sys +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path + +import pytest + +from lightcone.engine import assets, identity, plan, worker +from lightcone.engine.sandbox import Unavailable +from lightcone.engine.worker import TaskResult + +_SPEC = """ +version: "0.0.13" +name: analysis + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: first + type: metric + recipe: + command: echo one > {output}/value.txt + + - id: second + type: report + inputs: [first] + recipe: + command: cat {inputs.first}/value.txt > {output}/copy.txt +""" + + +@pytest.fixture +def root(analysis: Callable[..., Path]) -> Path: + return analysis(_SPEC) + + +def _task(root: Path, output_id: str) -> plan.Task: + graph = plan.build(root) + return graph.tasks[("baseline", output_id)] + + +#: What the driver reads once and hands to every task. +_HEAD = ("0123456789abcdef", "https://example/analysis.git") + + +def _make( + root: Path, output_id: str, *upstream: TaskResult, refresh: bool = False +) -> TaskResult: + """Run one task the way Dask would, handed its upstream results.""" + task = _task(root, output_id) + return worker.materialize( + root, task, identity.env_version(root), _HEAD, assets.Versions(), refresh, *upstream + ) + + +def _age(root: Path, output_id: str) -> None: + """Rewrite an output's manifest to name an environment that is not this + one — the shape a project takes when `uv add` rewrites the lock after a + result was made. + + The *recorded* value is what moves, never the run's: the environment + the recipe actually runs under has to stay the real one, or the mid-run + gate refuses the execution before any of this is exercised. + """ + directory = root / "results/baseline" / output_id + manifest = assets.read(directory) + assert manifest is not None + assets.write(directory, replace(manifest, env_version="sha256:an-earlier-environment")) + + +# ---- executing a recipe ---------------------------------------------------- + + +def test_a_recipe_runs_and_its_output_is_recorded(root: Path) -> None: + result = _make(root, "first") + + assert result.status == "ok" + assert (root / "results/baseline/first/value.txt").read_text() == "one\n" + assert result.data_version == assets.data_version(root / "results/baseline/first") + + +def test_the_manifest_is_complete_before_anything_is_saved(root: Path) -> None: + """The driver commits the directory and its manifest as one commit, so + a manifest that described the bytes only after the save could never be + in that commit.""" + _make(root, "first") + + manifest = assets.read(root / "results/baseline/first") + assert manifest is not None + assert manifest.data_version == assets.data_version(root / "results/baseline/first") + assert manifest.definition_version == _task(root, "first").definition_version + assert manifest.env_version == identity.env_version(root) + assert manifest.git_sha == _HEAD[0] and manifest.git_remote == _HEAD[1] + assert manifest.hermeticity["mechanism"] + + +def test_the_recipe_runs_under_the_boundary(root: Path) -> None: + """Whatever the host can enforce, the manifest records what it was — + never what the mechanism matrix says it should have been.""" + from lightcone.engine import sandbox + + _make(root, "first") + + manifest = assets.read(root / "results/baseline/first") + assert manifest is not None + assert manifest.hermeticity["mechanism"] == sandbox.detect().capability.kind + + +# ---- deciding whether to run at all ---------------------------------------- + + +def test_an_unchanged_output_is_current(root: Path) -> None: + made = _make(root, "first") + again = _make(root, "first") + + assert again.status == "current" + assert again.data_version == made.data_version + + +def test_a_skip_returns_the_recorded_digest_rather_than_rehashing(root: Path) -> None: + """On a clone that has fetched no annex content the files are dangling + symlinks, so a recompute would quietly report a different output.""" + _make(root, "first") + output = root / "results/baseline/first" + manifest = assets.read(output) + assert manifest is not None + (output / "value.txt").unlink() + + assert _make(root, "first").data_version == manifest.data_version + + +def test_a_moved_environment_leaves_the_output_alone(root: Path) -> None: + """The change this layer exists for. The recipe and the decisions still + define exactly this output, so it is reported and kept — remaking it + would spend the compute that a rewritten `uv.lock` never justified.""" + made = _make(root, "first") + (root / "results/baseline/first/value.txt").write_text("untouched\n") + _age(root, "first") + + again = _make(root, "first") + + assert again.status == "behind" + assert "earlier environment" in again.reason + assert again.data_version == made.data_version + assert (root / "results/baseline/first/value.txt").read_text() == "untouched\n" + + +def test_refresh_remakes_what_is_only_behind(root: Path) -> None: + """And the recipe really runs: the file the previous assertion left in + place is overwritten, so this cannot pass by skipping too.""" + _make(root, "first") + (root / "results/baseline/first/value.txt").write_text("untouched\n") + _age(root, "first") + + again = _make(root, "first", refresh=True) + + assert again.status == "ok" + assert (root / "results/baseline/first/value.txt").read_text() == "one\n" + + +def test_refresh_does_not_remake_what_is_current(root: Path) -> None: + """`--refresh` widens the run by one state, not to everything. Without + this it would be a rebuild-the-world flag wearing another name.""" + _make(root, "first") + + assert _make(root, "first", refresh=True).status == "current" + + +def test_a_behind_upstream_still_feeds_its_dependents(root: Path) -> None: + """`behind` says the environment moved, not that the bytes are wrong — + so a dependent proceeds on them rather than reporting blocked.""" + first = _make(root, "first") + _age(root, "first") + behind = _make(root, "first") + assert behind.status == "behind" + + second = _make(root, "second", behind) + + assert second.status == "ok" + manifest = assets.read(root / "results/baseline/second") + assert manifest is not None + assert manifest.input_versions == {"first": first.data_version} + + +def test_a_task_whose_upstream_did_not_finish_is_blocked(root: Path) -> None: + """Blocked without running: an exception would make Dask propagate to + every dependent and stop 'who actually failed' being answerable.""" + upstream = TaskResult(("baseline", "first"), "failed", reason="boom") + + result = _make(root, "second", upstream) + + assert result.status == "blocked" + assert "baseline/first" in result.reason + assert not (root / "results/baseline/second").exists() + + +def test_a_downstream_task_takes_its_upstreams_answer(root: Path) -> None: + first = _make(root, "first") + second = _make(root, "second", first) + + assert second.status == "ok" + manifest = assets.read(root / "results/baseline/second") + assert manifest is not None + assert manifest.input_versions == {"first": first.data_version} + + +# ---- the output directory the recipe owns ---------------------------------- + + +def test_a_stale_file_does_not_survive_a_rebuild(root: Path) -> None: + """It would otherwise land in the content hash and be committed as part + of an output that never produced it.""" + _make(root, "first") + output = root / "results/baseline/first" + (output / "leftover.txt").write_text("from a previous run\n") + + worker.execute(root, _task(root, "first"), identity.env_version(root), {}, head=_HEAD) + + assert not (output / "leftover.txt").exists() + assert (output / "value.txt").exists() + + +def test_a_failing_recipe_records_no_manifest(root: Path) -> None: + spec = _SPEC.replace("echo one > {output}/value.txt", "echo one > {output}/value.txt && false") + (root / "astra.yaml").write_text(spec) + + result = _make(root, "first") + + assert result.status == "failed" + assert "exited 1" in result.reason + assert assets.read(root / "results/baseline/first") is None + + +def test_a_recipe_that_removes_its_output_directory_fails_on_any_host(root: Path) -> None: + """The two mechanisms disagree about whether the removal is even + allowed — Landlock follows POSIX and refuses it, because unlinking the + directory needs write on `results/`, which is not granted; Seatbelt's + subpath grant covers the directory node itself and permits it. The + *contract* is the same either way, so that is what this asserts: a + `failed` result, never a raise into the driver.""" + (root / "astra.yaml").write_text( + _SPEC.replace("echo one > {output}/value.txt", "rm -rf {output}") + ) + + assert _make(root, "first").status == "failed" + + +def test_an_output_that_cannot_be_recorded_fails_rather_than_raises( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Recording is fallible too, and a raise here reaches Dask, which + re-raises in the driver and takes down every other task in flight — + where reporting one failure and letting the rest finish is the whole + point of owning the loop. Exercised without a mechanism, because that + is the host where a recipe really can delete what it was given.""" + from lightcone.engine import sandbox + + monkeypatch.setattr(sandbox, "detect", Unavailable) + (root / "astra.yaml").write_text( + _SPEC.replace("echo one > {output}/value.txt", "rm -rf {output}") + ) + + result = _make(root, "first") + + assert result.status == "failed" + assert "could not be recorded" in result.reason + + +# ---- the environment gates ------------------------------------------------- + + +def test_an_environment_that_moved_under_the_run_is_refused(root: Path) -> None: + """A manifest may not claim an environment that had already been edited + by the time the recipe ran.""" + result = worker.execute( + root, _task(root, "first"), "sha256:from-another-run", {}, head=_HEAD + ) + + assert result.status == "failed" + assert "environment changed" in result.reason + assert assets.read(root / "results/baseline/first") is None + + +# ---- what a recipe may touch ----------------------------------------------- + + +def test_a_recipe_cannot_write_outside_the_results_tree(root: Path) -> None: + """One policy for probes and recipes, so `results/` is the whole of a + recipe's in-tree write scope. Sibling outputs are *not* carved out — + the manifest's content hash is what says whether an output's bytes are + its own, and a second mechanism for one guarantee is one more than can + be kept honest.""" + from lightcone.engine import sandbox + + if sandbox.detect().capability.kind == "none": + pytest.skip("no sandbox mechanism on this host") + (root / "astra.yaml").write_text( + _SPEC.replace("echo one > {output}/value.txt", "echo tampered > src/injected.py") + ) + + assert _make(root, "first").status == "failed" + assert not (root / "src" / "injected.py").exists() + + +def test_that_write_would_have_succeeded_unsandboxed( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The mutation check. Without it the test above would pass on a host + that enforces nothing, and pin nothing at all.""" + from lightcone.engine import sandbox + + monkeypatch.setattr(sandbox, "detect", Unavailable) + (root / "src").mkdir(exist_ok=True) + (root / "astra.yaml").write_text( + _SPEC.replace("echo one > {output}/value.txt", "echo tampered > src/injected.py") + ) + + assert _make(root, "first").status == "ok" + assert (root / "src" / "injected.py").read_text() == "tampered\n" + + +def test_a_recipe_can_read_an_annexed_input(analysis: Callable[..., Path]) -> None: + """Declared inputs live in the annex. With `filter=annex` the working + tree holds the real bytes, so the project read root covers them — but + that is worth pinning, since it is what every recipe with an input + depends on.""" + from lightcone.engine import dataset + + spec = """ + version: "0.0.13" + name: analysis + + inputs: + - id: catalog + type: data + source: data/catalog.txt + + outputs: + - id: fit + type: metric + inputs: [catalog] + recipe: + command: cat {inputs.catalog} > {output}/seen.txt + """ + root = analysis(spec, files={"data/catalog.txt": "measured\n"}) + dataset.save(root, [root / "data"], "the catalog") + + result = _make(root, "fit") + + assert result.status == "ok", result.reason + assert (root / "results/baseline/fit/seen.txt").read_text() == "measured\n" + + +# ---- the entry point the run record names ---------------------------------- + + +def test_the_worker_module_imports_neither_click_nor_rich() -> None: + """It is on the `python -m` path of every rerun and every task, so a + CLI import here would be paid on all of them.""" + proc = subprocess.run( + [sys.executable, "-c", "import lightcone.engine.worker, sys; print(sorted(sys.modules))"], + capture_output=True, + text=True, + check=True, + ) + assert "'click'" not in proc.stdout + assert "'rich'" not in proc.stdout + + +def test_the_module_runs_one_task_and_commits_nothing(root: Path) -> None: + """What `datalad rerun` invokes. It leaves the tree dirty by design — + which is exactly why it is not an `lc` verb.""" + from lightcone.engine import dataset + + proc = subprocess.run( + [sys.executable, "-m", "lightcone.engine.worker", "baseline/first"], + cwd=root, + capture_output=True, + text=True, + ) + + assert proc.returncode == 0, proc.stderr + assert (root / "results/baseline/first/value.txt").read_text() == "one\n" + assert dataset.status(root) + + +def test_the_module_reruns_unconditionally( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A rerun is a rerun: the caller has already said what they want, and + a staleness check would answer a question nobody asked.""" + monkeypatch.chdir(root) + _make(root, "first") + (root / "results/baseline/first/value.txt").unlink() + + assert worker.main(["baseline/first"]) == 0 + assert (root / "results/baseline/first/value.txt").exists() + + +def test_the_module_refuses_an_argument_it_cannot_use(root: Path) -> None: + assert worker.main([]) == 2 + assert worker.main(["first"]) == 2 + + +def test_the_module_refuses_an_output_that_does_not_exist( + root: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.chdir(root) + assert worker.main(["baseline/nothing"]) == 2 + assert "no output `baseline/nothing`" in capsys.readouterr().err + + +def test_only_the_lookup_can_blame_the_target( + root: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The whole body used to sit under one `except KeyError`, so anything + raising one inside astra's validation or resolution was reported as + "no output ". This is the entry point every `[DATALAD RUNCMD]` + record names, so a rerun failing for an unrelated reason misdiagnosed + itself.""" + monkeypatch.chdir(root) + + def explode(_: Path) -> None: + raise KeyError("something inside the resolver") + + monkeypatch.setattr(worker.plan, "build", explode) + with pytest.raises(KeyError): + worker.main(["baseline/first"]) + assert "no output" not in capsys.readouterr().err + + +def test_a_declared_input_that_is_not_there_names_itself( + analysis: Callable[..., Path], monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """`data_version` reports an absent path with the OS's own exception, + which would unwind as a traceback at whoever is reading a rerun.""" + spec = _SPEC.replace( + " recipe:\n command: echo one > {output}/value.txt", + " inputs: [catalog]\n recipe:\n command: echo one > {output}/value.txt", + 1, + ) + root = analysis(spec) + monkeypatch.chdir(root) + + assert worker.main(["baseline/first"]) == 2 + assert "the declared input `catalog` cannot be read" in capsys.readouterr().err From 19986bb85f75fc8b2211757b4c52f6f6e65634a2 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Wed, 19 Aug 2026 15:32:34 +0200 Subject: [PATCH 10/23] Point the uv.toml limitation at the issue that owns it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The note said the residue was unguarded but not that it is unguardable with the tools uv provides — so the next reader's first instinct is the one that was already measured and rejected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GaahDTd6djz9PCm1Rh6SP4 --- CLAUDE.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4ada5042..38244824 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -631,7 +631,12 @@ What this deliberately cannot reach is **user-level configuration** make one commit answer differently on two hosts, so a colleague's clone would report every output as behind. The residue is real and unguarded — a user-level `no-build` changes what a sync installs and nothing records -it. +it. **There is no flag that closes it**: `--config-file` refuses a +`pyproject.toml` outright, and both `--config-file ` and +`--no-config` drop the project's own `[tool.uv]` along with everything +else (measured; `--no-config` also drops `[tool.uv.index]`, which is the +GPU mechanism, so adopting it would break working projects). Tracked as +issue #176 with the options and the measurements; don't re-derive them. **The git commit is recorded, never hashed, and never a signal.** It goes in the manifest so the code that produced a result stays recoverable. It From e7cea626c343020e6a0bfec565bce65eda6708b4 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 09:37:27 +0200 Subject: [PATCH 11/23] The engine is the host's uv tool, not a project dependency (#178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lc init` pinned `lightcone-cli` into every project's own lock, which is what forced **layer 3** — a launcher that discovers, mode-detects, scrubs `UV_*`, converges, and delegates to the lc inside the project's venv. Remove the pin and there is nothing to delegate to: the `lc` that was invoked is the engine that runs. Layer 3 is now marked removed by decision rather than pending. This reverses spec §2's engine-in-lock rule. The spec is a reference the rebuild drifts from deliberately; the reversal lands as a Recorded decision in `CLAUDE.md` rather than waiting on a spec rewrite. ## What changes A scaffolded `pyproject.toml` declares **no dependencies**. A project's resolution is no longer constrained by the engine's own pins (no astra-tools/click/rich/distributed/git-annex in any project lock), and `requires-python` now comes from the interpreter that wrote `.python-version` instead of from lightcone-cli's metadata — one source, so the bound and the pin cannot disagree. The git-annex wheel floor gates installing the tool and nothing else. ## What the pin bought, and where each guarantee went - **The engine inside `env_version`** — given up deliberately. An lc upgrade now moves nothing; no output ever reads `behind` over an engine release. This completes the reversal that already took `env_version` out of `definition_version`. The engine becomes attestation (`lc_version` in every manifest), not identity. - **Reruns reconstruct the exact engine** — re-provided in the record itself. `cmd` pins the engine by version through an ephemeral `uv run --no-project --with lightcone-cli==`, keeping the **module spelling** so the worker stays out of `lc --help` and off `$PATH`. A dev build's version is unpublished, so its record is the bare module — which only exists in a checkout whose own interpreter imports it. - **Driver/worker skew "structurally impossible"** — moot while workers are in-process threads; noted as a layer-7 connect-probe obligation. - **Layer 6** — the generated image can no longer get its in-image engine from the lock; noted that the Containerfile must install it as an explicit layer and that the version becomes a tag input. ## The worker now owns the project environment It has to. A rerun checks out the lock but never the `.venv`, and **`uv run --no-sync` against a missing `.venv` silently creates an empty one** (measured, uv 0.12.5) — recipes would have run in it while the manifest recorded the lock's `env_version`. `worker.main()` therefore syncs before anything executes, which also inherits the loud stale-lock refusal the outer `uv run --locked` used to provide. `current_project(synced=False)` is the affordance: the lock is repository content, the `.venv` is not. A new test reruns on a **fresh clone**, because the in-place rerun passes with or without the sync. `_engine_path()` is deleted — the suite faked the pin by injecting `src/` onto `PYTHONPATH`, and there is no pin left to fake. `test_the_run_record_is_what_datalad_reads` tightens its `.endswith` to a full-string match so a prefix appearing or vanishing cannot slip past. ## Verification - `uv run pytest` — 413 passed; `ruff` and `mypy --strict` clean. - End-to-end by hand: scaffolded a project (no `lightcone-cli` in `uv.lock`), materialized, `lc status`, then `datalad rerun` on a fresh clone with no `.venv` — the worker created it and the output came back byte-identical. - The pinned shape run literally, from a wheel built off this branch: an ephemeral engine env reproduced the output with a matching `data_version`, and git-annex is present beside that env's interpreter so `put_our_bin_first()` still resolves. Incidentally fixes a latent bug in `eval.yml`: the eval project's pin resolved a *released* engine from PyPI rather than the branch under test. No workflow change needed. ## Follow-up, not in this PR The `UV_*` ambient scrub (spec §4 step 3) is worth having and is independent of the pin — it protects `env_version`'s install-settings term, not the delegation that is gone. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01PGfhkvU3Ee2hhHcwAkeidg --------- Co-authored-by: Claude Fable 5 --- CLAUDE.md | 388 ++++++++++++------ pyproject.toml | 13 + src/lightcone/engine/assets.py | 2 + src/lightcone/engine/dataset.py | 28 -- src/lightcone/engine/materialize.py | 62 ++- src/lightcone/engine/project.py | 109 +++-- src/lightcone/engine/templates/__init__.py | 50 +-- .../templates/files/pyproject.toml.tmpl | 4 +- src/lightcone/engine/worker.py | 45 +- tests/test_cli.py | 12 +- tests/test_dataset.py | 40 +- tests/test_materialize.py | 153 +++++-- tests/test_project.py | 16 +- tests/test_run.py | 26 +- tests/test_templates.py | 56 +-- tests/test_worker.py | 21 + 16 files changed, 664 insertions(+), 361 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 38244824..320d2cdc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,13 +23,16 @@ lc's whole ASTRA surface is ten functions; see Key Invariants (layer 4). ## ⚠️ This repository is a clean rebuild in progress -The codebase is being **re-added layer by layer** on top of the normative -design spec: +The codebase is being **re-added layer by layer** on top of the design +spec: > **`../lightcone-cli/docs/design/execution-environment.md`** — *"the -> locked environment is the execution environment"*, v6.1. Normative. -> Read it before adding anything. It lives in the sibling checkout for -> now (branch `redesign_prototype`), alongside its decision records +> locked environment is the execution environment"*, v6.1. Read it before +> adding anything — but read it as a **reference, not gospel**: the +> rebuild deliberately drifts from it as implementation teaches better +> answers, and where this file's Recorded decisions disagree with the +> spec, the decisions win. It lives in the sibling checkout for now +> (branch `redesign_prototype`), alongside its decision records > (rationale, substrate tradeoffs, hermeticity enforcement, the v6 > review); it moves into this repo's `docs/design/` when the docs are > rewritten at the end of the rebuild. @@ -46,7 +49,7 @@ speculatively. |---|-------|-------| | 1 | **Project scaffolding** — `lc init` | ✅ **done** | | 2 | **Environment layer** — `env_version`, lock scan, manifest schema | ✅ **done** | -| 3 | The `lc` entrypoint — launcher: discover → mode-detect → scrub `UV_*` → converge → delegate | ⬜ | +| 3 | ~~The `lc` entrypoint — launcher~~ | ❌ **removed by decision** (2026-08) — see Recorded decisions: the host `lc` is the engine, so there is nothing to delegate to | | 4 | **Fabric** — `lc materialize`, worker sequence, mid-run relock gate | ✅ **done** | | 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | | 6 | Container hatch — `[tool.lightcone.image]`, generated Containerfile, `lc build` | ⬜ | @@ -63,10 +66,12 @@ Layer 5 landed **out of order**, ahead of 2–4: `lc run` is the spec's with it. That makes it the smallest honest consumer of the exec boundary, and the boundary is what layer 4 will then plug recipes into. -The spec's §11 (Migration) is the authoritative ordering; the table above -is the working map. **Layer boundaries are also dependency boundaries** — -a dependency enters `pyproject.toml` with the layer that needs it, not -before. +The spec's §11 (Migration) is the reference ordering; the table above is +the working map, and the spec is a reference the rebuild deliberately +drifts from — reversals land as Recorded decisions here rather than +waiting on a spec rewrite. **Layer boundaries are also dependency +boundaries** — a dependency enters `pyproject.toml` with the layer that +needs it, not before. ### Rules while rebuilding @@ -105,7 +110,9 @@ Each of these has been asked for in review at least once; none is optional. rebuild is complete. Same for `README.md` and `zensical.toml`. - **Port with intent.** Prior implementations (this repo's git history, and the `redesign_prototype` branch of the sibling `lightcone-cli` - checkout) are references, not sources of truth. The spec is. + checkout) are references, not sources of truth. Neither is the spec by + itself: the spec plus this file's Recorded decisions is the current + design, and the decisions override the spec where they disagree. - **Every layer ships tests.** See the per-layer test list in spec §11. ## Architecture (target) @@ -149,8 +156,8 @@ Any new `lightcone-*` package must: src/lightcone/ # namespace — NO __init__.py ├── _sandbox_exec.py # the Landlock shim — stdlib only, zero lightcone imports ├── cli/ # the CLI only: flags, rendering, exit codes -│ ├── __init__.py # exposes main(); the launcher hooks in here (layer 3) -│ └── commands.py # lc init, lc run, lc materialize +│ ├── __init__.py # exposes main(), lazily +│ └── commands.py # lc init, lc run, lc materialize, lc status └── engine/ ├── __init__.py # docstring only ├── project.py # what a project is: convergence + discovery @@ -170,7 +177,7 @@ src/lightcone/ # namespace — NO __init__.py │ ├── seatbelt.py # macOS backend │ └── denial.py # the denial UX └── templates/ # the scaffold's file content - ├── __init__.py # loader + one renderer per scaffolded file + ├── __init__.py # loader; a renderer only where there is a value to decide └── files/*.tmpl # the templates themselves, as real files evals/ # agentic eval seed: prompt.md + tasks// @@ -180,17 +187,19 @@ tests/ # pytest — mirrors src/ ## Development Commands ```bash -uv sync --group dev # installs pytest, ruff, mypy +uv sync --group dev # pytest (+ pytest-cov), ruff, mypy, datalad uv run pytest uv run ruff check src/ tests/ # --fix to apply uv run mypy src/ -uv build # wheel + sdist +uv build # wheel + sdist (CI runs this only to publish) ``` -These four are the whole loop, and they are exactly what CI runs -(`.github/workflows/{tests,lint}.yml`). There is deliberately no task +Test, lint and type-check are the whole loop, and they are what +`.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers -around them plus recipes for the frozen docs and the dormant eval. +around them plus recipes for the frozen docs and the dormant eval. The +other workflows are `eval.yml` (the agentic eval, on dispatch or PR +label), `pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. ## Key Invariants (layer 1) @@ -225,6 +234,17 @@ environment is there (`pyproject.toml`, `uv.lock`, `.venv`) and does not require an `astra.yaml`, so any uv project can be probed. No walk-up: the directory you invoke from is the project, or it is a clean error. +**Two questions about a project root, not one** — the same shape as +`_in_repository` / `_can_ask_git` below. `declared_project()` wants only +what the repository carries (`pyproject.toml`, `uv.lock`); +`current_project()` adds `.venv`. The split is named rather than a +`synced=` flag on one function, because "what makes a directory a +project" should not be negotiable per call site, and because a slice of +a constant would make the answer depend on the order its entries happen +to be in. The weaker question has exactly one caller — the worker entry +point, which builds the `.venv` a moment later — and that is the whole +reason it exists. + **CLI startup stays cheap.** `commands.py` imports the engine *inside* the command callbacks and builds the rich console lazily, so `lc --help` and shell completion pay for neither. Keep this up as verbs land: a module-scope @@ -257,20 +277,20 @@ so the template name has to be bound before the call site. **No engine constants for the environment.** The scaffolded `.python-version` is the interpreter `lc` itself is running on, and -`requires-python` is lightcone-cli's own `Requires-Python`, verbatim. These -can't conflict — lc only runs on an interpreter satisfying its own bound — -and neither is a number to maintain. Identity follows the project's files -from there: `env_version` hashes `.python-version`'s bytes, not anything in -the engine (spec §3). - -**`.gitignore` converges entry-wise, not by marker.** -`templates.gitignore_entries()` is the template minus comments and blanks; -`missing_gitignore_entries(text)` is what a repair appends, in template +`requires-python` is that interpreter's minor as a floor. Both come from +one place, so they can't conflict, and neither is a number to maintain. +Identity follows the project's files from there: `env_version` hashes +`.python-version`'s bytes, not anything in the engine (spec §3). + +**`.gitignore` and `.gitattributes` converge entry-wise, not by marker.** +`templates.entries(name)` is a template minus comments and blanks; +`templates.missing(name, text)` is what a repair appends, in template order. Idempotency is therefore structural — a pattern already in the file is never re-added, whoever wrote it — and a pattern introduced by a later lc release still reaches projects that already have a `.gitignore`, which -a "marker present ⇒ done" check would have skipped. `GITIGNORE_HEADER` is -cosmetic only; never make correctness depend on it. +a "marker present ⇒ done" check would have skipped. The header comment +(`templates.header`) is cosmetic only; never make correctness depend on +it. **What `lc init` converges** — idempotently, never overwriting a file the user owns: @@ -278,13 +298,13 @@ user owns: | Path | Role | |---|---| | `astra.yaml` + `universes/baseline.yaml` | astra's boilerplate spec, verbatim, as **one item keyed on `astra.yaml`** — the baseline references the boilerplate's example decision, so it must never land beside a user-authored spec. Its `container:` key is ignored outright — see Recorded decisions | -| `pyproject.toml` | The uv project: **virtual** (no `[build-system]`), `lightcone-cli` pinned in its own dependencies — the engine lives inside the experiment's lock | +| `pyproject.toml` | The uv project: **virtual** (no `[build-system]`), no dependencies — the engine is the host's uv tool, never a project dependency (see Recorded decisions), so the lock carries only what the analysis imports | | `.python-version` | The exact patch of the interpreter `lc` is running on | -| `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | +| `uv.lock`, `.venv` | **Derived** — converged by correctness, not existence: `uv lock --check` / `uv sync --locked --exact --check` decide, then `uv lock` / `uv sync --locked --exact --compile-bytecode` repair | | `.gitignore` | One managed block of patterns; convergence ensures each is present | | `.git` + the annex | `git init` then `git annex init` — results are versioned in the project's own repository | | `.gitattributes` | The storage policy: what git-annex holds and what git carries. Line-managed, like `.gitignore` | -| `.datalad/config` | A `datalad.dataset.id` UUID, generated once. lc never reads it back | +| `.datalad/config` | A `datalad.dataset.id` UUID, generated once. Read back only by `dataset.dataset_id`, through `git config -f`, for the run record's `dsid` | | `data/` + `README.md` | Where declared inputs live; annexed, and committed before anything computes on them | | `results/` + `README.md` | Where outputs land; the README states the materialize-don't-hand-write contract | | `myst.yml`, `index.md` | Template MyST report referencing `astra.yaml` *by path* | @@ -311,10 +331,9 @@ user owns: 0.12.3). A lock that no longer matches `pyproject.toml`, or an environment that no longer matches the lock, is exactly as unconverged as a missing one and reports as - `repaired`. Existence alone made `converge()` a no-op on drift, which - layer 3's launcher — which converges on every invocation precisely to - guarantee the environment matches the lock — would have silently - inherited. + `repaired`. Existence alone made `converge()` a no-op on drift, and + everything that converges before acting — `lc materialize`'s sync, the + worker entry point's — would have silently inherited it. - The probe is skipped when the artifact is absent (nothing to ask), so a fresh project costs none and the created/repaired split falls out of the same check. @@ -358,8 +377,8 @@ user owns: storage substrate, and results are versioned in the repository, so there is no useful project without any of them. git is the one tool uv cannot install and the single admitted exception to a uv-installable stack; - git-annex ships as a wheel and is therefore a locked dependency, which - makes its wheel platforms the CLI's install floor. + git-annex ships as a wheel and is therefore a dependency of the lc tool + itself, which makes its wheel platforms the CLI's install floor. (This reverses layer 1's original "git is optional" — a project without version control had nowhere to put a result.) - **Two questions about git, not one.** `_in_repository` is a pure @@ -393,10 +412,12 @@ one monkeypatch point and the `tools` fixture already covers git. `manylinux_2_34` on x86_64/aarch64, macOS 14+ arm64 or 15+ x86_64, win_amd64 — and **no sdist**, so a host below the floor fails to install rather than building from source. Because git-annex is a *hard* runtime -dependency and `lc init` pins `lightcone-cli` into every project, that -floor gates installing lc at all, including `lc run`, which never touches -the annex. If it ever bites a real user, that coupling is the thing to -revisit — an extra, or a probed requirement like git — not the floor. +dependency, that floor gates installing the lc tool at all, including +`lc run`, which never touches the annex — but only the tool: projects no +longer depend on lightcone-cli, so the floor never constrains a project's +own resolution. If it ever bites a real user, the hard dependency is the +thing to revisit — an extra, or a probed requirement like git — not the +floor. **Perlmutter clears it** (checked 2026-08-19, login node): the wheel installs and `git annex version` runs. That was the open question this @@ -413,8 +434,8 @@ re-derived. Two things it does *not* settle, both layer 7's: either. - **`git-annex-shell` is not on the default remote PATH.** `git annex get` from a laptop dispatches to it over a non-login ssh session, and the - wheel installs it into `.venv/bin`. Configuration, not design: - `git config remote..annex-shell `. + wheel installs it beside the tool's interpreter. Configuration, not + design: `git config remote..annex-shell `. **`filter=annex` is what makes an ordinary `git add` do the right thing.** `git annex init` configures the smudge/clean filter itself; the template @@ -446,10 +467,10 @@ set on their clone, so the shape a file arrives in is not ours to assume. - **Unlocked** — what `filter=annex` writes, hard-linked or copied alike. A clone without content holds a ~100-byte *pointer file* where the data would be: it exists, it is readable, and hashing it yields a perfectly - well-formed digest of the wrong thing. The test is git-annex's own - `isPointerFile` — a `/annex/objects/` prefix within the first 32 KiB. - Measured before it was fixed: the same input hashed differently on a - clone, silently. + well-formed digest of the wrong thing. The test follows git-annex's own + `isPointerFile`: a file no larger than 32 KiB whose bytes begin + `/annex/objects/`. Measured before it was fixed: the same input hashed + differently on a clone, silently. - **Locked** — a symlink into the object store, which without the content dangles. This one is quieter and worse: `is_file()` answers False for a dangling symlink, so a directory walk filtered on that alone drops the @@ -462,19 +483,42 @@ set on their clone, so the shape a file arrives in is not ours to assume. "changed", because the two are different facts and only one of them means a rebuild. -**PATH is part of the contract.** `git annex` is not a builtin — git -dispatches it by searching `PATH` for a `git-annex` executable, and -`uv tool install lightcone-cli` links only lc's own entry points into -`~/.local/bin`. So `dataset.put_our_bin_first()` **prepends** -`Path(sys.executable).parent`, idempotently, before any git call. Prepend, -never append: a system copy winning would make the version the project's -lock records a fiction. `project.require_git_annex()` probes after the -prepend and by the name git itself searches for. - -Measured, not assumed: with only the tool's `lc` on `PATH` — exactly what -`uv tool install` produces — `git annex version` fails, and with the -prepend disabled `lc init` refuses. Under `uv run` it is a no-op, because -uv already fronts the project's `.venv/bin`. +**git-annex is on `PATH` by construction, not by runtime repair.** +`git annex` is not a builtin — git dispatches it by searching `PATH` for +a `git-annex` executable — and an installer links only the requested +package's own executables, so a plain `uv tool install lightcone-cli` +would have left the one git command we tell people to type unable to run. +The fix is metadata: lightcone-cli re-declares the git-annex wheel's four +entry points verbatim in its own `[project.scripts]` (`git-annex`, +`git-annex-shell`, `git-remote-annex`, `git-remote-tor-annex`, all +`git_annex:cli`). The wheel ships no raw binaries in `bin/` — its +executables *are* that dispatcher, which `execv`s the real binary out of +package data with `argv[0]` preserved, and git-annex dispatches on +`argv[0]` busybox-style. So every install channel carries the +executables wherever it carries `lc`: the tool install links all five +names (verified), `uv run` and `uvx` front the venv's bin, and the +rerun's ephemeral engine environment gets them the same way. That covers +lc's own subprocesses *and* the researcher's bare `git add` with one +mechanism, which is why the old `put_our_bin_first()` PATH-prepend was +deleted rather than kept as a belt: a second answer to "which annex runs" +that could disagree with the shell's. + +The declaration is mirrored, never invented: +`test_the_annex_executables_are_ours_to_install` asserts ours match the +wheel's exactly, so an executable upstream adds, drops or renames fails +the suite rather than every user's install. (Considered and rejected: +there is no pyproject metadata to link a *dependency's* executables; +`--with-executables-from` is an install-time flag users won't type; a +post-install link repair contradicts "the install command is enough"; +vendoring the binaries into platform-specific lightcone-cli wheels is +five 14–36 MB wheels and a repack pipeline for what four lines of +metadata provide.) + +The accepted residue of dropping the prepend: `PATH` order is the +user's, so a system git-annex fronting the install's wins the dispatch — +for lc's subprocesses exactly as for their own git, and no different +from which `git` itself runs. Nothing records the annex version anyway +(see the Recorded decision on the engine's dependency closure). **A project can sit inside a larger repository, so `dataset.status` is scoped and relativised.** `lc init subdir/` adopts an enclosing work @@ -592,9 +636,11 @@ any dependency change, and every output in every project on an lc upgrade. That was recorded as an accepted cost; it was the bug.) Nothing is lost by not rebuilding: the manifest records the environment -*and* the commit, and that commit's `uv.lock` reconstructs the -environment exactly. Over-sensitivity in `env_version` is affordable -precisely because it no longer spends compute. +*and* the commit, and that commit's `uv.lock` reconstructs the project +environment exactly. (The engine is not in that lock — its reconstruction +is the run record's job, and its version the manifest's `lc_version`.) +Over-sensitivity in `env_version` is affordable precisely because it no +longer spends compute. **Both are length-framed.** Concatenating fields raw lets a boundary shift between them produce one digest from two different inputs. `_frame` @@ -738,8 +784,8 @@ path component.** An empty universe or output id collapses `results//` onto a *parent* — `results/` itself, for two — and the worker empties that directory before running a recipe in it, so the consequence of an unchecked id is deleting every other universe's -outputs. A `/`, `.` or `..` is refused for the same reason. This is the -guard that lets the reset stay a whole-directory operation. +outputs. A `/`, `\`, `.` or `..` is refused for the same reason. This is +the guard that lets the reset stay a whole-directory operation. **The reset takes the whole directory, and cannot take a named list.** A recipe declares an output *id*, never filenames, so there is no set of @@ -914,21 +960,31 @@ commit the run went on to create. It is the code that produced the output. A test that reads `dataset.head()` after materializing and expects a match is asserting the wrong thing. -**One uv hop, one spelling** (`project.uv_prefix(root, *, sync)`). The -flags are also what `run_record` writes verbatim into every commit -message, so a second copy is a second thing that can drift out of the -provenance. The only thing callers disagree about is `sync`: a probe +**One *project* uv hop, one spelling** (`project.uv_prefix(root, *, +sync)`). The only thing its callers disagree about is `sync`: a probe converges the environment it is about to describe, a recipe must not, or every concurrent worker writes the same `.venv`. +The run record's `cmd` is the deliberate second shape, and it is not the +drift the rule guards against: it is *project-less* by construction +(`uv run --no-project --with lightcone-cli==`), so it shares no flag +with `uv_prefix` — no `--project`, no `--locked`, no sync selection, +because there is no project environment involved. It builds an engine to +run, where `uv_prefix` enters an environment already built. Routing one +through the other would mean a helper with two disjoint output shapes. +What keeps *that* hop from drifting is that the worker it invokes +converges the project environment itself, so the record never has to +spell how. + **A run syncs the environment; it does not report on it.** `uv run --locked` asserts only that `uv.lock` still matches `pyproject.toml`, and workers pass `--no-sync` — so a lock edited without a sync would leave recipes importing packages the lock does not describe while every manifest recorded the *new* lock's `env_version`. Measured: the recipe imported `packaging 26.3` under a lock saying `24.2`, and uv accepted it silently. -`materialize()` calls `project.sync()` before anything else, so the state -is made impossible rather than detected. `--check` needs neither: +`materialize()` calls `project.sync()` right after the tool and committer +preflight — before the dirty check and the graph — so the state is made +impossible rather than detected. `--check` needs neither: `env_version` is the lock's bytes, so a drifted `.venv` cannot change what it answers. @@ -960,17 +1016,35 @@ than a trap. declared input under `data/` is an annex symlink, so a `Path.resolve()` in the record's `inputs` writes `.git/annex/objects/SHA256E-…` — the storage rather than the input, and a path nothing can `datalad get`. This -shipped once; `_inside` is lexical now. - -**The run record is genuinely re-runnable, and its `cmd` is the worker -module.** `python -m lightcone.engine.worker /`, -behind `uv run --locked --project .` — which reconstructs the environment -from *that commit's* lock, and therefore the exact engine that produced -the output. The bare recipe would reconstruct nothing lc adds (no locked -environment, no boundary, no gates, no manifest) and would commit bytes -the identity model never produced; `lc materialize` cannot be it either, -because `datalad rerun` removes the declared outputs first and that -dirties the tree materialize refuses to start from. +shipped once; `plan.declared_path` is lexical now, never resolved. + +**The run record is genuinely re-runnable, and every record pins its +engine.** `uv run --no-project --with '' -- python -m +lightcone.engine.worker /` — an ephemeral +environment that reconstructs the *engine*, while the worker itself +reconstructs the *project* environment from the rerun commit's own lock +(`main()` syncs before anything executes; `uv run --no-sync` against a +missing `.venv` silently creates an empty one, so a clone's rerun would +otherwise run recipes in a bare environment under a manifest recording +the lock's `env_version`). The bare recipe would reconstruct nothing lc +adds (no locked environment, no boundary, no gates, no manifest) and +would commit bytes the identity model never produced; `lc materialize` +cannot be it either, because `datalad rerun` removes the declared outputs +first and that dirties the tree materialize refuses to start from. + +The requirement (`materialize._engine_requirement`) pins a release by +version and a dev build by its source commit — hatch-vcs embeds the +commit in the version, and the repository URL comes from +`[project.urls]`, the engine's own metadata rather than a constant. So a +rerun works during development too, against the commit that ran +(verified against GitHub: uv resolves the short sha and hatch-vcs builds +the matching version from the clone). An unpushed commit fails a rerun +loudly at resolution, which beats silently finding another engine; a +dirty tree pins the last commit, and the version's `.dYYYYMMDD` marker in +the manifest is what says the bytes had drifted from it. The e2e rerun +tests monkeypatch the requirement seam to a wheel built from the working +tree (`UV_FIND_LINKS`), because a git pin can only ever build *committed* +code and the suite must execute the code under test. **The worker module is not an `lc` verb and not a console script.** It makes the output unconditionally, commits nothing, leaves the tree dirty by @@ -978,7 +1052,11 @@ design. `lc --help` advertising it would hand people a footgun, and a `[project.scripts]` entry would put it on `$PATH` through `uv tool install`. `lightcone/_sandbox_exec.py` is the same shape for the same reason. Keep it cheap to import — **no click, no rich** — it is on -the path of every task and every rerun; two tests pin that. +the path of every task and every rerun. +`test_the_worker_module_imports_neither_click_nor_rich` pins the imports +and `test_help_does_not_advertise_the_worker` the absence from `--help`; +nothing pins the absence of a `[project.scripts]` entry, so treat that +as a review item. **The record's format is datalad's, so it is tested through datalad.** `get_run_info` matches with a regex and returns `(None, None)` on any @@ -988,13 +1066,14 @@ So the suite asserts through datalad's parser *and* runs a real `datalad rerun`, and `datalad` is a **dev** dependency only. Nothing in lc imports it. -**A policy is a description, and `scope()` owns every policy's lifetime.** -`exec_policy` creates no directory — the worker resets the output -directory, so the whole of an output's lifecycle stays in the module that -owns it, and a policy that could not be built without touching the -filesystem would be the impurity `wrap` is already pinned against. -`sandbox.scope` takes a *built* policy, so the `rmtree` of the private -`$HOME` has one owner. +**A policy describes the project; it never prepares it.** `exec_policy` +creates nothing in the project tree — the worker resets the output +directory, so the whole of an output's lifecycle stays in the module +that owns it, and `results/` is granted only if it already exists. The +one thing it does put on disk is the per-run private `$HOME` +(`mkdtemp`), and that is why `sandbox.scope` takes a *built* policy: the +`rmtree` of that directory has one owner. (`wrap` stays pure; the +impurity lives in policy construction, once.) **There is one policy, `exec_policy`, and a recipe gets exactly what a probe gets.** The tree is read-only apart from `results/`, for both. So @@ -1020,7 +1099,11 @@ do not describe it as covered. `submit(fn, *args, key=…)` and `completed(handles)`. That is all the driver asks of a scheduler and all a venue has to supply — which is what lets the suite run a graph inline in-process, and what will let something -larger than a laptop land behind it without the driver noticing. +larger than a laptop land behind it without the driver noticing. A venue +owes one thing beyond the two methods: its worker processes must run an +interpreter that imports `lightcone.engine` at the driver's version — +see the engine-is-the-host's-uv-tool decision for how each venue +provides that, and for what workers do *not* need (git, git-annex). ### Recorded deviations from the spec (layers 2 and 4) @@ -1104,10 +1187,13 @@ Landlock unions rights over ancestors, so a single EXECUTE grant on layer enforcing nothing — while every test still passes, because the allowlisted binaries are exactly the ones that were going to work. This shipped once: the venv interpreter's *install root* was granted EXECUTE -so the resolved symlink target would be runnable, which is `/usr` for -any venv built on a system python (`uv venv --python-preference -only-system`, most CI images, HPC site pythons). The grant is on the -interpreter **file**; only READ goes to the install root, for the stdlib. +unconditionally, which is `/usr` for any venv built on a system python +(`uv venv --python-preference only-system`, most CI images, HPC site +pythons). The rule now: the interpreter **file** always; its install +root gets EXECUTE only when it is the interpreter's *own tree* (a +uv-managed store, a framework version directory — macOS framework +builds re-exec into `Resources/Python.app`), never a `_SHARED_PREFIXES` +member; and READ goes to the root regardless, for the stdlib. `test_a_system_interpreter_does_not_make_the_whole_prefix_executable` pins it. @@ -1215,6 +1301,64 @@ only checks that the guard is present. Verified empirically, not assumed. ### Recorded decisions +- **The engine is the host's uv tool, never a project dependency** + (2026-08, reversing spec §2's engine-in-lock rule and deleting layer 3). + `lc init` scaffolds no `lightcone-cli` dependency, and there is no + launcher and no delegation: the `lc` that was invoked is the engine that + runs. What the pin bought, and where each guarantee went: + - *The engine inside `env_version`.* Gone, and deliberately: an lc + upgrade now moves nothing — no output ever reads `behind` over an + engine release, completing the reversal that already took + `env_version` out of `definition_version`. The engine is attestation + (`lc_version` in every manifest, `worker.lc_version()`), not identity. + - *Reruns reconstruct the exact engine from the commit's lock.* + Re-provided in the run record itself: `cmd` pins the engine through + `uv run --no-project --with ''` — by version for a + release, by source commit for a dev build (see the layer-4 run-record + invariant). Needs PyPI or the git remote reachable at rerun time + where the lock needed neither; recorded, not hidden. + - *Driver/worker version skew "structurally impossible".* Re-provided + by sharing the **installation** instead of the lock. What a Dask + worker process actually needs is `lightcone.engine` importable at a + matching version — pickle serializes `worker.materialize` by + reference and `Task`/`Versions`/`TaskResult` by class reference. + Verified by hand (2026-08-20), not pinned by the suite — a full + materialization through a `LocalCluster(processes=True)`: the + current code works unchanged across process boundaries, and workers + need **no git and no git-annex** (the driver owns git alone; + `data_version` is pure file hashing). The shipped cluster is + threads; re-verify when layer 7 makes processes real. So per venue: on HPC the tool env lives on the shared + filesystem and the venue launches workers on the driver's own + interpreter (`sys.executable` — dask-jobqueue's `python=`), which + makes driver and workers the identical installation; in containerized + mode driver and workers share the image, same result. Both also + satisfy `distributed`'s own client/scheduler/worker coherence + requirement for free. A connect-time engine-version probe is needed + only for a cluster lc did not launch (a pre-existing gateway with its + own image). One venue cost to remember: the `assets.Versions` memo + degrades to once **per worker process**, so a declared input shared + by many tasks is re-hashed per process — efficiency, not correctness. + - *The engine's dependency closure left the record entirely, and is not + replaced.* The project lock used to pin what the engine resolved — + most concretely the git-annex build that wrote the bytes. Now + `lc_version` names the engine and nothing pins what it was made of. + Accepted rather than re-provided: the alternative is hashing an + environment lc does not own into artifacts it does, which is the + over-sensitivity the `behind` model exists to avoid. + - *Layer 6 note:* the generated image can no longer get its in-image + engine from the lock (`/opt/venv/bin/lc` existed because the pin did). + The Containerfile must install the engine as an explicit layer, and + that version becomes a tag input. + What it buys: a project's resolution is freed from the engine's own + pins (no astra-tools/click/rich/distributed/git-annex in any project + lock); the git-annex wheel floor gates only the tool install; the eval + workflow exercises the branch under test rather than a released engine + resolved from PyPI; and recipes no longer find `lc` or `git-annex` on + the sandbox PATH — the project `.venv/bin` no longer carries them, + which is the boundary telling the truth. The `UV_*` ambient scrub the + launcher would have done is still worth having and is tracked + separately; it protects `env_version`'s install-settings term, not the + delegation that is gone. - **Reads stay restricted, and the OS baseline is ours to maintain.** Codex restricts reads too now, but its Linux read baseline is a *mount table*, not a path list — there is nothing to adopt there. Keeping the @@ -1309,13 +1453,17 @@ only checks that the guard is present. Verified empirically, not assumed. future-facing surface; a system-layer declaration is currently inert, like `astra.yaml`'s `container:` key. - **The denial's remedies are only what works today** — `uv add` for a - Python package, plus the ASTRA input declaration. Nothing in a denial - message names a verb, flag, or declaration that does not exist. -- **No manifest is written, and no serializer for one.** A probe has no - output (§4), so the `Attestation` is returned and printed rather than - persisted. An earlier draft carried a `to_manifest()` with no caller — - deleted, because "no dead code" applies to this layer's own - conveniences too. It lands with layer 4, which is what needs it. + Python package, the ASTRA input declaration for data, and "output goes + in `results/`" plus `tempfile.mkdtemp()` for a write denial. Nothing in + a denial message names a verb, flag, or declaration that does not + exist. +- **`Attestation` has no serializer of its own.** An earlier draft + carried a `to_manifest()` with no caller — deleted, because "no dead + code" applies to this layer's own conveniences too. It is persisted by + the layer that needs it: the worker writes `asdict(outcome.attestation)` + into the manifest's `hermeticity` field. `lc run` is still a probe with + no output (§4), so there the attestation is returned and printed, never + persisted. - **`results/` is writable; the rest of the tree is not**, where §4 gives a probe no output and therefore no in-tree write scope at all. A probe gets the same write scope a recipe does, so a probe that works means a @@ -1335,11 +1483,13 @@ only checks that the guard is present. Verified empirically, not assumed. bind-mounts the working tree read-write". True of the default, not of the mount table we will write — and the experiment that settled it is two `podman -v` flags. Don't re-derive it from the default again. - - **`_exec_set` grants no directory anywhere**, including `.venv/bin`. - A directory grant is a grant on whatever the directory holds *later*, - which was a live hole for as long as the tree was writable: `cp - /usr/bin/git .venv/bin/` ran a tool the allowlist denies by name. - Belt-and-braces under a read-only tree, one scandir, keep it. + - **`_exec_set` grants no directory except the interpreter's own + install tree** (see the layer-5 EXECUTE invariant) — and never + `.venv/bin`. A directory grant is a grant on whatever the directory + holds *later*, which was a live hole for as long as the tree was + writable: `cp /usr/bin/git .venv/bin/` ran a tool the allowlist + denies by name. Belt-and-braces under a read-only tree, one scandir, + keep it. - **A write-denial test must target a path the OS would let you write.** `printf x > /etc/…` passes with no sandbox at all — the OS refuses it for any non-root user — so it pins nothing. The sound @@ -1386,15 +1536,19 @@ only checks that the guard is present. Verified empirically, not assumed. `engine.project._run`, emulating each tool's observable effect (`uv lock` writes `uv.lock`, `uv sync` makes `.venv`, `git init` makes `.git`, `git annex init` marks the repository annexed) and recording every argv. - `uv_calls(tools)` narrows it to uv. Tests are hermetic: no network, no + `uv_calls(tools)` narrows it to uv; `probes(calls)` to the read-only + `--check` probes. Under the stub, tests are hermetic: no network, no resolution, no subprocesses. The `real_tools` fixture opts back out, - putting the real `_run` back. -- `tests/test_dataset.py` — the storage seam, and **the one place in the - non-sandbox suite that runs real tools**, via `real_tools`. That is - deliberate: the question is whether bytes land in the annex or as a blob - in git, and a fake answering it would only restate what the code already - believes. Every bug this file found (the missing `.gitattributes` - default, the pointer-file trap, `filter=annex`) was invisible to a stub. + putting the real `_run` back — and everything built on it (`analysis`, + and `test_materialize.py`'s `engine_dist` wheel build plus the rerun + tests' ephemeral `uv run --with` resolve) does spawn, resolve, and may + touch the network. +- `tests/test_dataset.py` — the storage seam, tested against real tools + via `real_tools`, deliberately: the question is whether bytes land in + the annex or as a blob in git, and a fake answering it would only + restate what the code already believes. Every bug this file found (the + missing `.gitattributes` default, the pointer-file trap, `filter=annex`) + was invisible to a stub. - `tests/test_project.py` — discovery and convergence semantics, called directly. This is where scaffold behavior is tested. - `tests/test_plan.py` tests what lc adds — directories, edges, diff --git a/pyproject.toml b/pyproject.toml index 43c61c27..07c41c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,8 +44,21 @@ docs = [ ] +[project.urls] +Repository = "https://github.com/LightconeResearch/lightcone-cli" + [project.scripts] lc = "lightcone.cli:main" +# The git-annex wheel's own entry points, re-declared verbatim: an +# installer links only the requested package's executables, and a plain +# `uv tool install lightcone-cli` must put git-annex on the shell's PATH +# — `filter=annex` makes the researcher's own `git add` need it. The +# suite pins these against the wheel's, so an upstream change fails +# loudly here rather than silently at install time. +git-annex = "git_annex:cli" +git-annex-shell = "git_annex:cli" +git-remote-annex = "git_annex:cli" +git-remote-tor-annex = "git_annex:cli" [build-system] requires = ["hatchling", "hatch-vcs"] diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index e56c7fb5..ef808996 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -250,6 +250,8 @@ class Manifest: git_sha: str #: The `origin` URL, or empty when the repository has no remote. git_remote: str + #: The engine that made it. Attestation, not identity: outside both + #: hashes, so an lc upgrade neither stales an output nor puts it behind. lc_version: str #: What the sandbox actually enforced, as the boundary attested it. hermeticity: dict[str, Any] diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index 30f72f1d..cab0cc96 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -18,36 +18,11 @@ from __future__ import annotations -import os -import sys from collections.abc import Iterable from pathlib import Path from lightcone.engine import project - -def put_our_bin_first() -> None: - """Prepend the directory holding our own interpreter to ``PATH``. - - ``git annex`` is dispatched by git searching ``PATH`` for a - ``git-annex`` executable, and ``uv tool install lightcone-cli`` links - only *our* entry points into ``~/.local/bin`` — the git-annex we - resolved as a dependency sits beside the interpreter instead. Verified: - with only the tool's ``lc`` on ``PATH``, ``git annex version`` fails and - ``lc init`` refuses. (Under ``uv run`` the prepend is a no-op, because - uv already fronts the project's ``.venv/bin``.) - - Prepend, never append: a system git-annex winning would make the - version recorded in the lock a fiction. Idempotent, and applied to our - own environment because that is what child processes inherit. - """ - ours = str(Path(sys.executable).parent) - path = os.environ.get("PATH", "") - if path.split(os.pathsep)[:1] == [ours]: - return - os.environ["PATH"] = ours + os.pathsep + path if path else ours - - # ============================================================================= # The repository # ============================================================================= @@ -168,7 +143,6 @@ def require_committer(directory: Path) -> None: Raises: ProjectError: If no committer identity resolves. """ - put_our_bin_first() proc = project._run(["git", "var", "GIT_COMMITTER_IDENT"], cwd=directory) if proc.returncode != 0: raise project.ProjectError( @@ -277,7 +251,6 @@ def restore(directory: Path, paths: Iterable[Path]) -> None: def _git(argv: list[str], *, cwd: Path) -> str: """Run git in *cwd*, returning its stdout; a nonzero exit raises.""" - put_our_bin_first() proc = project._run(["git", *argv], cwd=cwd) if proc.returncode != 0: raise project.ProjectError(f"`git {' '.join(argv)}` failed:\n{proc.stderr.strip()}") @@ -286,7 +259,6 @@ def _git(argv: list[str], *, cwd: Path) -> str: def _git_ok(argv: list[str], *, cwd: Path) -> bool: """Run git as a yes/no probe: exit status is the answer, not a failure.""" - put_our_bin_first() return bool(project._run(["git", *argv], cwd=cwd).returncode == 0) diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index f5d26ee1..90087a65 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -34,6 +34,7 @@ import json import os +import re from collections.abc import Iterator, Sequence from contextlib import contextmanager from dataclasses import asdict, dataclass, field @@ -373,7 +374,8 @@ def materialize( project.require_git_annex() dataset.require_committer(root) # Before anything else: workers pass `--no-sync`, so this is the only - # place the environment is made to match the lock. + # place on a run's path where the environment is made to match the + # lock. (A rerun does not come through here; its entry point syncs.) report = MaterializeReport() report.warnings.extend(f"uv: {w}" for w in project.sync(root)) if changes := dataset.status(root): @@ -550,10 +552,14 @@ def run_record(root: Path, task: Task, dsid: str) -> str: it is written and none of it abbreviated. ``datalad rerun`` reads it with a regex and reports "no command; skipping" on any mismatch. - ``cmd`` is the worker module: the bare recipe would reconstruct nothing - lc adds, and ``lc materialize`` cannot be it because a rerun removes - the declared outputs first, dirtying the tree materialize refuses to - start from. + ``cmd`` is the worker module, never a console script, behind an + ephemeral ``uv run`` pinning the engine that made the output + (:func:`_engine_requirement`) — so the rerun executes that engine + rather than whatever the host has grown into. The bare recipe would + reconstruct nothing lc adds, and ``lc materialize`` cannot be it + because a rerun removes the declared outputs first, dirtying the tree + materialize refuses to start from. The worker rebuilds the *project* + environment itself from the lock of the commit being rerun. Args: root: The project root. @@ -565,9 +571,11 @@ def run_record(root: Path, task: Task, dsid: str) -> str: """ info = { "chain": [], + # Single-quoted because datalad hands cmd to a shell and the git + # form of the requirement contains spaces. "cmd": ( - "uv run --locked --project . -- python -m lightcone.engine.worker " - f"{task.universe_id}/{task.output_id}" + f"uv run --no-project --with '{_engine_requirement()}' -- " + f"python -m lightcone.engine.worker {task.universe_id}/{task.output_id}" ), "dsid": dsid, "exit": 0, @@ -584,6 +592,46 @@ def run_record(root: Path, task: Task, dsid: str) -> str: ) +def _engine_requirement() -> str: + """Build the requirement that reconstructs the running engine. + + A release pins by version, resolvable from an index. A dev build's + version is not published, but hatch-vcs embeds its source commit — + so the pin becomes that commit at the engine's own repository, and a + rerun during development still reconstructs the engine that ran. An + unpushed commit fails a rerun loudly at resolution, which beats + silently finding another engine. A dirty tree is the one + approximation: the commit names the code as last committed, and the + version's own dirty marker is what keeps that visible. + + Returns: + A PEP 508 requirement for ``uv run --with``. + """ + v = worker.lc_version() + commit = re.search(r"\+g([0-9a-f]+)", v) + if "dev" not in v or commit is None: + return f"lightcone-cli=={v}" + return f"lightcone-cli @ git+{_repository_url()}@{commit.group(1)}" + + +def _repository_url() -> str: + """Read the engine's repository URL out of its own metadata. + + From ``[project.urls]`` rather than a constant here, so the one place + the URL lives is the package metadata every install carries. + """ + from importlib.metadata import metadata + + for entry in metadata("lightcone-cli").get_all("Project-URL") or []: + label, _, url = str(entry).partition(",") + if label.strip().lower() == "repository": + return url.strip() + raise ProjectError( + "lightcone-cli's own metadata names no Repository URL, so a dev " + "engine cannot be pinned by commit — reinstall the engine." + ) + + # ============================================================================= # Reading the project # ============================================================================= diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index ca89dcd6..b8304e0e 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -8,7 +8,7 @@ import shutil import subprocess import uuid -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import asdict, dataclass, field from functools import partial from pathlib import Path @@ -280,19 +280,21 @@ def require_git() -> None: def require_git_annex() -> None: """Refuse early when git-annex is not reachable as git reaches it. - Probed after :func:`~lightcone.engine.dataset.put_our_bin_first`, and - by the name git itself searches for: ``git annex`` is git finding a - ``git-annex`` executable on ``PATH``, not a builtin. + Probed by the name git itself searches for: ``git annex`` is git + finding a ``git-annex`` executable on ``PATH``, not a builtin. Every + install channel puts one there by construction — lightcone-cli + declares the git-annex wheel's entry points as its own, so installers + link them beside ``lc`` — which makes this a refusal for broken + installs only. Raises: ProjectError: If ``git-annex`` is not on ``PATH``. """ - dataset.put_our_bin_first() if shutil.which("git-annex") is None: raise ProjectError( "git-annex is required (it stores the bytes results are made of) " - "and is not on PATH. It ships as a wheel and installs with the " - "engine: `uv sync` in the project, or reinstall lightcone-cli." + "and is not on PATH. It installs with lc itself: " + "`uv tool install --force lightcone-cli` repairs the install." ) @@ -320,10 +322,11 @@ def uv_prefix(directory: Path, *, sync: bool) -> list[str]: def sync(directory: Path) -> list[str]: """Make ``.venv`` match ``uv.lock``. - A run calls this before it starts rather than checking and refusing: - workers pass ``--no-sync``, so nothing else would notice a lock edited - without a sync, and a manifest recording an environment the recipe did - not run under is the identity model saying something untrue. + Both entry points that execute recipes call this before they start, + rather than checking and refusing: workers pass ``--no-sync``, so + nothing else would notice a lock edited without a sync, and a manifest + recording an environment the recipe did not run under is the identity + model saying something untrue. Args: directory: The project root. @@ -340,25 +343,13 @@ def sync(directory: Path) -> list[str]: def _converge_uv_project(c: _Converger, directory: Path) -> None: """pyproject.toml + .python-version — the environment definition. - An existing ``pyproject.toml`` is the user's: read, possibly warned - about, never edited. The warning is decided against the *pre-existing* - file, so one we just wrote — which always names the engine — can never - trigger it. + An existing ``pyproject.toml`` is the user's: read, never edited. """ - pyproject_path = directory / "pyproject.toml" - adopted = pyproject_path.exists() c.file( "pyproject.toml", - pyproject_path, + directory / "pyproject.toml", lambda: templates.pyproject(name=project_name(directory)), ) - if adopted and "lightcone-cli" not in pyproject_path.read_text(): - c.warn( - "pyproject.toml does not depend on lightcone-cli — the " - "engine should live inside the experiment's lock: " - "`uv add lightcone-cli`." - ) - c.file(".python-version", directory / ".python-version", templates.python_version) @@ -393,9 +384,9 @@ def _converge_dataset(c: _Converger, directory: Path) -> None: routes results and inputs into the annex and keeps manifests in git, and ``.datalad/config`` carries a dataset id — the one thing a git + git-annex repository lacks to *be* a DataLad dataset, so a project is - one from birth rather than by later adoption. Neither is read back by - lc; the id is generated once and never regenerated, because ``file`` - writes only what is missing. + one from birth rather than by later adoption. The id is generated + once and never regenerated, because ``file`` writes only what is + missing; the run record reads it back through ``dataset.dataset_id``. """ c.item(".git", _in_repository(directory), lambda: dataset.init_git(directory)) # After the item above, so a fresh project has a repository to annex. @@ -497,17 +488,43 @@ def project_name(directory: Path) -> str: return name or "analysis" -#: What makes a directory a project root: the environment ``lc run`` -#: enters. ``astra.yaml`` is deliberately not among them — a command can -#: be probed in any uv project, spec or no spec. -_ENVIRONMENT_FILES = ("pyproject.toml", "uv.lock", ".venv") +#: What the repository itself carries of the environment. ``astra.yaml`` +#: is deliberately not among them — a command can be probed in any uv +#: project, spec or no spec. +_DECLARED_FILES = ("pyproject.toml", "uv.lock") + +#: …plus the built environment. ``.venv`` is the one piece that is local +#: state rather than repository content, which is the whole difference +#: between the two questions below. +_ENVIRONMENT_FILES = (*_DECLARED_FILES, ".venv") + + +def declared_project(directory: Path | None = None) -> Path: + """Take a directory as a project root, needing only what git carries. + + The weaker of the two questions: it asks whether the project is + *declared*, not whether it is built. That is what the worker entry + point needs — a clone holds the lock and no ``.venv``, and the worker + converges the environment for itself a moment later. + + Args: + directory: Defaults to the working directory. + + Returns: + The resolved project root. + + Raises: + ProjectError: If ``pyproject.toml`` or ``uv.lock`` is absent. + """ + return _project_root(directory, _DECLARED_FILES) def current_project(directory: Path | None = None) -> Path: - """Take a directory as the project root, checking it is one. + """Take a directory as the project root, built environment and all. - There is no walk-up: the directory you are in is the directory that is - used, or it is an error. + The question every verb that runs something asks, and the stronger of + the two: a ``.venv`` that is not there is not something these callers + are going to create. Args: directory: Defaults to the working directory. @@ -516,14 +533,26 @@ def current_project(directory: Path | None = None) -> Path: The resolved project root. Raises: - ProjectError: If the environment is not there. The two ways that - fails get different advice — a directory with no project + ProjectError: If any of the environment is absent. + """ + return _project_root(directory, _ENVIRONMENT_FILES) + + +def _project_root(directory: Path | None, required: Sequence[str]) -> Path: + """Check *directory* against *required* and resolve it. + + There is no walk-up: the directory you are in is the directory that is + used, or it is an error. + + Raises: + ProjectError: If anything in *required* is absent. The two ways + that fails get different advice — a directory with no project markers is the wrong *place*, while one that declares a - project but lacks the built environment is the right place, - not yet converged, which is what a fresh clone is. + project but lacks a piece of the environment is the right + place, not yet converged. """ directory = (directory or Path.cwd()).resolve() - missing = [name for name in _ENVIRONMENT_FILES if not (directory / name).exists()] + missing = [name for name in required if not (directory / name).exists()] if not missing: return directory declared = (directory / "pyproject.toml").exists() or (directory / SPEC_FILENAME).exists() diff --git a/src/lightcone/engine/templates/__init__.py b/src/lightcone/engine/templates/__init__.py index 33ef4eff..c80e070a 100644 --- a/src/lightcone/engine/templates/__init__.py +++ b/src/lightcone/engine/templates/__init__.py @@ -78,13 +78,14 @@ def pyproject(*, name: str) -> str: name: The project name. Returns: - A virtual uv project depending on lightcone-cli. + A virtual uv project with no dependencies. The engine is not among + them: lc runs from the host's tool install, and the project's lock + carries only what the analysis itself imports. """ return _render( "pyproject.toml.tmpl", name=name, requires_python=requires_python(), - lc_requirement=lightcone_requirement(), ) @@ -105,49 +106,17 @@ def python_version() -> str: def requires_python() -> str: """Render the scaffolded ``requires-python``. - Taken verbatim from lightcone-cli's own ``Requires-Python``: a - scaffolded project depends on the engine, so uv enforces this bound - during resolution regardless, and declaring the same specifier states - it rather than inventing a second one. It cannot conflict with - :func:`python_version`, since lc only runs on an interpreter that - already satisfies it. + The running interpreter's minor version, consistent with + :func:`python_version`, which pins the exact patch of the same + interpreter — the bound and the pin come from one place, so they + cannot disagree. Returns: - The specifier, or the running interpreter's minor version for a - metadata-less install. + The specifier. """ - from importlib.metadata import PackageNotFoundError, metadata - - try: - # Message-style lookup: absent keys come back as None, not KeyError. - declared = metadata("lightcone-cli")["Requires-Python"] - except PackageNotFoundError: - declared = None - if declared: - return str(declared) return f">={sys.version_info.major}.{sys.version_info.minor}" -def lightcone_requirement() -> str: - """Render the ``lightcone-cli`` requirement for the scaffold. - - The engine lives inside the experiment's lock, pinned to the version - that ran ``lc init``, so the engine that produced a result stays - recoverable. - - Returns: - A pinned requirement, or an unpinned one for a dev build, whose - version is not published and would make the lock unsolvable. - """ - from importlib.metadata import PackageNotFoundError, version - - try: - v = version("lightcone-cli") - except PackageNotFoundError: - v = "" - return f"lightcone-cli=={v}" if v and "dev" not in v else "lightcone-cli" - - # ============================================================================= # Line-managed files — converged entry-wise, not by marker # ============================================================================= @@ -333,7 +302,8 @@ def datalad_config(*, dataset_id: str) -> str: """Render ``.datalad/config``, the file that makes a project a dataset. A dataset id is the one thing a git + git-annex repository lacks to - *be* a DataLad dataset. lc never reads this back; datalad does. + *be* a DataLad dataset. Read back only through ``dataset.dataset_id``, + for the run record's ``dsid``. Args: dataset_id: A UUID, generated once and never regenerated. diff --git a/src/lightcone/engine/templates/files/pyproject.toml.tmpl b/src/lightcone/engine/templates/files/pyproject.toml.tmpl index f851d29e..e17980b8 100644 --- a/src/lightcone/engine/templates/files/pyproject.toml.tmpl +++ b/src/lightcone/engine/templates/files/pyproject.toml.tmpl @@ -2,9 +2,7 @@ name = "${name}" version = "0.0.1" requires-python = "${requires_python}" -dependencies = [ - "${lc_requirement}", -] +dependencies = [] [tool.uv] required-version = ">=0.12" diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index bdbe52c9..0eb80c93 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -5,13 +5,15 @@ python -m lightcone.engine.worker / which is what the ``[DATALAD RUNCMD]`` record in every materialization -commit names. That is why it is a module rather than an ``lc`` verb: it -makes the output unconditionally, commits nothing, and leaves the tree dirty by -design — precisely the state ``lc materialize`` refuses to start from — -so advertising it in ``lc --help`` would hand people a footgun. A -``uv run --locked --project .`` in front of it reconstructs the exact -engine that produced an output from the lock of the commit that recorded -it, module path included. +commit names, behind an engine-pinning ``uv run --no-project --with …`` — +by version for a released engine, by source commit for a dev build. That +is why it is a module rather than an ``lc`` verb: it makes the output +unconditionally, commits nothing, and leaves the tree dirty by design — +precisely the state ``lc materialize`` refuses to start from — so +advertising it in ``lc --help`` would hand people a footgun. +:func:`main` converges the project environment from the rerun commit's +own lock before anything executes, so the record holds on a clone that +has never built one. Keep this module cheap to import: no click, no rich. It is on the ``python -m`` path of every rerun, and of every task in every run. @@ -25,6 +27,7 @@ from __future__ import annotations +import functools import shutil import sys from collections.abc import Mapping @@ -37,7 +40,8 @@ from lightcone.engine.project import ( ProjectError, child_env, - current_project, + declared_project, + sync, uv_prefix, ) @@ -250,7 +254,7 @@ def execute( input_versions=dict(input_versions), git_sha=sha, git_remote=remote, - lc_version=_lc_version(), + lc_version=lc_version(), hermeticity=asdict(outcome.attestation), ), ) @@ -275,7 +279,18 @@ def _gate(root: Path, env_version: str) -> str: ) -def _lc_version() -> str: +@functools.cache +def lc_version() -> str: + """Report the running engine's version, empty for a bare source tree. + + The one lookup behind both places the engine attests itself: the + manifest's ``lc_version`` and the run record's version pin. One + function, so the two cannot disagree about which engine ran. + + Cached: the metadata scan walks ``sys.path``, both callers are once + per output, and an installed version cannot change under a running + process. + """ from importlib.metadata import PackageNotFoundError, version try: @@ -297,6 +312,13 @@ def main(argv: list[str]) -> int: thin wrapper over :func:`execute`, so this is an entry point rather than a second implementation. Nothing is classified: a rerun is a rerun. + The environment is converged here, not assumed: a rerun checks out the + lock but never the ``.venv``, and the recipe's own ``uv run --no-sync`` + would silently hand it an *empty* environment while the manifest + recorded the lock's ``env_version``. The sync also carries + ``--locked``, so a lock that no longer matches ``pyproject.toml`` is a + loud refusal rather than a quiet relock. + Args: argv: One argument, ``/``. @@ -310,7 +332,8 @@ def main(argv: list[str]) -> int: universe_id, _, output_id = argv[0].partition("/") try: - root = current_project() + root = declared_project() + sync(root) env_version = identity.env_version(root) graph = plan.build(root) if (task := graph.tasks.get((universe_id, output_id))) is None: diff --git a/tests/test_cli.py b/tests/test_cli.py index 02f8a279..349b647d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -110,16 +110,8 @@ def test_blocked_items_are_rendered(runner: CliRunner, tmp_path: Path) -> None: assert "blocked results/" in result.output # Rich wraps on the terminal width, so assert on an unwrappable fragment. assert "✗" in result.output - - -def test_init_surfaces_warnings(runner: CliRunner, tmp_path: Path) -> None: - project = tmp_path / "proj" - project.mkdir() - (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') - - result = runner.invoke(main, ["init", str(project)]) - assert result.exit_code == 0 - assert "does not depend on lightcone-cli" in result.output + # The reason travels as a warning, and the console renders those too. + assert "not a directory" in result.output # ---- --check / --json (the agent-facing surface) -------------------------- diff --git a/tests/test_dataset.py b/tests/test_dataset.py index ee1094ed..ccfafe0e 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -9,9 +9,7 @@ from __future__ import annotations -import os import shutil -import sys from pathlib import Path import pytest @@ -398,23 +396,29 @@ def test_ignore_rule_sees_through_a_tracked_path(repo: Path) -> None: # ---- how git finds git-annex ----------------------------------------------- -def test_our_bin_goes_to_the_front_of_path(monkeypatch: pytest.MonkeyPatch) -> None: - """Prepend, never append. `git annex` dispatches by searching PATH for - a `git-annex` executable — a system copy winning would make the version - the project's lock records a fiction.""" - ours = str(Path(sys.executable).parent) - monkeypatch.setenv("PATH", "/somewhere/else") - - dataset.put_our_bin_first() - assert os.environ["PATH"] == f"{ours}{os.pathsep}/somewhere/else" - - dataset.put_our_bin_first() - assert os.environ["PATH"].count(ours) == 1 +def test_git_dispatches_annex_from_the_ambient_path(repo: Path) -> None: + """`git annex` is git finding a `git-annex` executable on PATH, not a + builtin — and lc no longer arranges PATH for its subprocesses: every + install channel carries the entry points beside the interpreter, so + the environment lc inherits already resolves them. Checked by the + spelling git itself uses rather than by resolving `git-annex` + ourselves.""" + assert "git-annex version:" in dataset._git(["annex", "version"], cwd=repo) -def test_git_dispatches_annex_after_the_prepend(repo: Path) -> None: - """The claim the prepend makes, checked by the spelling git itself uses - rather than by resolving `git-annex` ourselves.""" - assert "git-annex version:" in dataset._git(["annex", "version"], cwd=repo) +def test_the_annex_executables_are_ours_to_install() -> None: + """An installer links only the requested package's executables, and the + researcher's own `git add` needs git-annex on the *shell's* PATH — so + lightcone-cli re-declares the git-annex wheel's entry points verbatim. + Mirrored, not invented: asserted against the wheel's own metadata, so + an executable upstream adds, drops, or renames fails this test instead + of failing `uv tool install lightcone-cli` for every user.""" + from importlib.metadata import distribution + + ours = {e.name: e.value for e in distribution("lightcone-cli").entry_points} + theirs = {e.name: e.value for e in distribution("git-annex").entry_points} + assert theirs # the wheel stopped declaring entry points ⇒ redesign + for name, value in theirs.items(): + assert ours.get(name) == value, f"{name} is not re-declared as {value}" diff --git a/tests/test_materialize.py b/tests/test_materialize.py index af47e14f..e5c01f9c 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -13,7 +13,6 @@ from __future__ import annotations import json -import os import subprocess import sys from collections.abc import Callable, Iterator @@ -561,7 +560,9 @@ def fake() -> Iterator[_Interrupted]: # ---- the commit message ---------------------------------------------------- -def test_the_run_record_is_what_datalad_reads(root: Path, inline: None) -> None: +def test_the_run_record_is_what_datalad_reads( + root: Path, inline: None, monkeypatch: pytest.MonkeyPatch +) -> None: """Asserted through datalad's own parser rather than against our JSON: it matches with a regex and returns nothing on any mismatch, after which `rerun` reports "no command; skipping" and exits 0 — so a golden @@ -569,6 +570,7 @@ def test_the_run_record_is_what_datalad_reads(root: Path, inline: None) -> None: from datalad.api import Dataset from datalad.local.rerun import get_run_info + monkeypatch.setattr(engine.worker, "lc_version", lambda: "1.2.3") engine.materialize(root, ["second"]) message = dataset._git(["log", "-1", "--format=%B"], cwd=root) @@ -576,13 +578,38 @@ def test_the_run_record_is_what_datalad_reads(root: Path, inline: None) -> None: assert subject == "second [baseline]" assert info is not None - assert info["cmd"].endswith("python -m lightcone.engine.worker baseline/second") + # Full-string, not endswith: a flag appearing or vanishing here must + # fail this test, not slip past a suffix match. + assert info["cmd"] == ( + "uv run --no-project --with 'lightcone-cli==1.2.3' -- " + "python -m lightcone.engine.worker baseline/second" + ) assert info["inputs"] == ["results/baseline/first"] assert info["outputs"] == ["results/baseline/second"] assert info["dsid"] == "4b7b5c1e-0000-4000-8000-000000000000" assert info["chain"] == [] and info["pwd"] == "." +def test_the_engine_pin_follows_the_build(monkeypatch: pytest.MonkeyPatch) -> None: + """A release resolves from an index by version; a dev build cannot, + but hatch-vcs embeds its source commit, so the pin becomes that commit + at the engine's own repository — read from the engine's metadata, not + a constant. Dirty or clean, the commit is the last one; the version's + dirty marker is what says which.""" + monkeypatch.setattr(engine.worker, "lc_version", lambda: "1.2.3") + assert engine._engine_requirement() == "lightcone-cli==1.2.3" + + url = engine._repository_url() + assert url.startswith("https://") + monkeypatch.setattr(engine.worker, "lc_version", lambda: "1.3.dev2+g19986bb8") + assert engine._engine_requirement() == f"lightcone-cli @ git+{url}@19986bb8" + + monkeypatch.setattr( + engine.worker, "lc_version", lambda: "1.3.dev2+g19986bb8.d20260820" + ) + assert engine._engine_requirement() == f"lightcone-cli @ git+{url}@19986bb8" + + def test_the_record_names_the_declared_input_not_the_annex_object( analysis: Callable[..., Path], inline: None ) -> None: @@ -636,27 +663,39 @@ def test_check_agrees_with_a_run_on_a_clone_with_no_annex_content( is still classifiable — check mode reads the recorded digest rather than the pointer file sitting in its place.""" engine.materialize(root, []) - clone = tmp_path / "clone" - dataset._git(["clone", "-q", str(root), str(clone)], cwd=tmp_path) - # `clone` does not copy identity, and `annex init` makes a commit — so - # this fails on any host without a usable global one, CI included. - for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): - dataset._git(["config", key, value], cwd=clone) - dataset._git(["annex", "init", "-q", "clone"], cwd=clone) + clone = _clone(root, tmp_path) pointer = (clone / "results/baseline/first/value.txt").read_text() assert pointer.startswith("/annex/objects/") # content really is absent assert engine.check(clone, []).planned == {} +def _clone(root: Path, into: Path) -> Path: + """Clone *root* into a usable annexed repository, as a colleague would. + + The identity is set because `clone` does not copy one and + `annex init` makes a commit — so without it this fails on any host + with no usable global identity, CI included. No annex content is + fetched: a fresh clone holds pointers, and that is the state these + tests are about. + """ + clone = into / "clone" + dataset._git(["clone", "-q", str(root), str(clone)], cwd=into) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=clone) + dataset._git(["annex", "init", "-q", "clone"], cwd=clone) + return clone + + def test_a_drifted_environment_is_made_to_match_before_anything_runs( root: Path, inline: None ) -> None: - """Workers pass `--no-sync`, so this is the only place the environment - is made to match the lock. Reported and refused, a lock edited without - a sync would leave recipes importing packages the lock does not - describe while every manifest recorded the new lock's `env_version`; - doing it instead is shorter and impossible to ignore.""" + """Workers pass `--no-sync`, so this is the only place on a run's path + where the environment is made to match the lock (a rerun's worker + entry point syncs for itself). Reported and refused, a lock edited + without a sync would leave recipes importing packages the lock does + not describe while every manifest recorded the new lock's + `env_version`; doing it instead is shorter and impossible to ignore.""" pyproject = root / "pyproject.toml" pyproject.write_text( pyproject.read_text().replace("dependencies = []", 'dependencies = ["idna"]') @@ -673,11 +712,46 @@ def test_a_drifted_environment_is_made_to_match_before_anything_runs( assert project_mod._env_is_current(root) -def test_the_recorded_command_reproduces_the_output(root: Path, inline: None) -> None: +@pytest.fixture(scope="session") +def engine_dist(tmp_path_factory: pytest.TempPathFactory) -> tuple[str, Path]: + """Build the engine under test into a wheel the pinned record can find. + + The record pins ``lightcone-cli==`` and the suite's build is not + published, so the rerun's ephemeral environment is pointed here via + ``UV_FIND_LINKS`` — which is what lets the suite execute the same + record shape every real commit carries, rather than a test-only one. + + Returns: + The wheel's exact version, and the directory serving it. + """ + dist = tmp_path_factory.mktemp("engine-dist") + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(dist)], + cwd=Path(__file__).parent.parent, + check=True, + capture_output=True, + text=True, + ) + wheel = next(dist.glob("*.whl")) + return wheel.name.split("-")[1], dist + + +def test_the_recorded_command_reproduces_the_output( + root: Path, + inline: None, + engine_dist: tuple[str, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: """The claim the record makes, run literally. `datalad rerun` removes the output, executes the recorded command, and commits what came - back — so the manifest is regenerated inside the same commit.""" + back — so the manifest is regenerated inside the same commit, by the + pinned engine resolved into an ephemeral environment.""" pytest.importorskip("datalad") + version, dist = engine_dist + # The requirement seam, not `lc_version`: a git pin can only ever + # build committed code, so the suite pins the wheel built from the + # working tree — the code actually under test. + monkeypatch.setattr(engine, "_engine_requirement", lambda: f"lightcone-cli=={version}") engine.materialize(root, ["first"]) original = assets.read(root / "results/baseline/first") assert original is not None @@ -687,7 +761,7 @@ def test_the_recorded_command_reproduces_the_output(root: Path, inline: None) -> cwd=root, capture_output=True, text=True, - env={**child_env(), "PYTHONPATH": _engine_path()}, + env={**child_env(), "UV_FIND_LINKS": str(dist)}, ) assert proc.returncode == 0, proc.stderr @@ -698,21 +772,42 @@ def test_the_recorded_command_reproduces_the_output(root: Path, inline: None) -> assert not dataset.status(root) -def _engine_path() -> str: - """Stand in for the ``lightcone-cli`` pin a real project's lock carries. +def test_the_recorded_command_holds_on_a_fresh_clone( + root: Path, + inline: None, + tmp_path: Path, + engine_dist: tuple[str, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A clone checks out the lock but never `.venv`, and `uv run --no-sync` + against a missing environment silently creates an *empty* one — so the + record only reproduces the output because the worker converges the + environment for itself. The in-place rerun above cannot catch a missing + sync; this is the test that does.""" + pytest.importorskip("datalad") + version, dist = engine_dist + monkeypatch.setattr(engine, "_engine_requirement", lambda: f"lightcone-cli=={version}") + engine.materialize(root, ["first"]) + original = assets.read(root / "results/baseline/first") + assert original is not None - The recorded command resolves the worker out of the project's own - environment, which is the whole reason the record reproduces the - *recorded* engine rather than today's. The fixture's project declares - no dependencies so it stays cheap to build, so the engine under test is - put on the path explicitly instead. - """ - import sysconfig + clone = _clone(root, tmp_path) + assert not (clone / ".venv").exists() - return os.pathsep.join( - [str(Path(__file__).parent.parent / "src"), sysconfig.get_paths()["purelib"]] + proc = subprocess.run( + [sys.executable, "-c", "from datalad.api import rerun; rerun('HEAD')"], + cwd=clone, + capture_output=True, + text=True, + env={**child_env(), "UV_FIND_LINKS": str(dist)}, ) + assert proc.returncode == 0, proc.stderr + rerun = assets.read(clone / "results/baseline/first") + assert rerun is not None + assert rerun.data_version == original.data_version + assert (clone / ".venv").exists() + # ---- the scheduler seam ---------------------------------------------------- diff --git a/tests/test_project.py b/tests/test_project.py index e9645e73..459cc1ad 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -286,21 +286,18 @@ def test_check_mode_agrees_with_a_real_run(tmp_path: Path) -> None: # ---- warnings and blocked items ------------------------------------------ -def test_warns_when_pyproject_lacks_the_engine(tmp_path: Path) -> None: - """The engine belongs inside the experiment's lock; a pyproject we - didn't write is the user's, so warn rather than edit.""" +def test_an_adopted_pyproject_is_never_edited(tmp_path: Path) -> None: + """A pyproject we didn't write is the user's: read, never edited.""" project = tmp_path / "proj" project.mkdir() (project / "pyproject.toml").write_text('[project]\nname = "mine"\nversion = "0"\n') report = converge(project) - assert any("does not depend on lightcone-cli" in w for w in report.warnings) + assert report.warnings == [] assert 'name = "mine"' in (project / "pyproject.toml").read_text() -def test_no_engine_warning_for_a_pyproject_we_wrote(tmp_path: Path) -> None: - """The warning is decided against the pre-existing file, so a scaffolded - one — which always names the engine — can never trigger it.""" +def test_a_fresh_scaffold_warns_about_nothing(tmp_path: Path) -> None: report = converge(tmp_path / "proj") assert report.warnings == [] @@ -613,9 +610,8 @@ def drifted(argv: list[str], *, cwd: Path) -> MagicMock: def test_a_stale_environment_is_repaired_not_ignored( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The case layer 3's launcher depends on: an environment that no longer - satisfies the lock must be re-synced, not waved through because `.venv` - happens to be a directory.""" + """An environment that no longer satisfies the lock must be re-synced, + not waved through because `.venv` happens to be a directory.""" from unittest.mock import MagicMock from lightcone.engine import project as project_mod diff --git a/tests/test_run.py b/tests/test_run.py index 98693565..265e861d 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -11,7 +11,12 @@ import pytest from lightcone.engine import run as engine_run -from lightcone.engine.project import ProjectError, current_project, uv_prefix +from lightcone.engine.project import ( + ProjectError, + current_project, + declared_project, + uv_prefix, +) SPEC = """\ title: Test @@ -91,6 +96,25 @@ def test_the_default_is_the_working_directory( assert current_project() == project.resolve() +def test_a_declared_project_needs_only_what_git_carries(project: Path) -> None: + """The weaker question, and the difference between the two: a clone + holds the lock and no `.venv`, and the worker entry point builds one + rather than refusing.""" + (project / ".venv").rmdir() + assert declared_project(project) == project.resolve() + with pytest.raises(ProjectError, match="not been built yet"): + current_project(project) + + +def test_a_declared_project_still_needs_the_lock(project: Path) -> None: + """It is weaker, not absent: without `uv.lock` there is no environment + to converge and nothing to be exact about.""" + (project / "uv.lock").unlink() + with pytest.raises(ProjectError, match="not been built yet") as raised: + declared_project(project) + assert "missing uv.lock" in str(raised.value) + + def test_a_missing_spec_reads_as_an_empty_one(project: Path) -> None: (project / "astra.yaml").unlink() assert engine_run.read_spec(project) == {} diff --git a/tests/test_templates.py b/tests/test_templates.py index ca4e071e..f2f67ff3 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -29,12 +29,17 @@ def test_pyproject_renders_every_placeholder() -> None: assert "$" not in rendered assert 'name = "my-analysis"' in rendered assert f'requires-python = "{templates.requires_python()}"' in rendered - assert templates.lightcone_requirement() in rendered # Virtual by design: containerized mode builds `--no-install-project`, # so a packaged project's own import would fail inside its image. assert "[build-system]" not in rendered +def test_pyproject_does_not_depend_on_the_engine() -> None: + """The engine is the host's uv tool, not a project dependency — a + scaffolded lock carries only what the analysis itself imports.""" + assert "lightcone-cli" not in templates.pyproject(name="my-analysis") + + def test_python_version_pins_the_running_interpreter() -> None: """No engine constant — a new project pins the python the researcher actually has, rather than one lc would have to download.""" @@ -42,56 +47,13 @@ def test_python_version_pins_the_running_interpreter() -> None: assert templates.python_version() == f"{v.major}.{v.minor}.{v.micro}\n" -def test_requires_python_comes_from_our_own_metadata() -> None: - """Declaring the engine's own bound rather than inventing a second one.""" - from importlib.metadata import metadata - - assert templates.requires_python() == metadata("lightcone-cli")["Requires-Python"] - - -def test_the_ambient_pin_always_satisfies_the_declared_floor() -> None: - """Not a coincidence: lc can only run on an interpreter meeting its own - Requires-Python, so the two can never conflict.""" - from packaging.specifiers import SpecifierSet - - assert templates.python_version().strip() in SpecifierSet(templates.requires_python()) - - -def test_requires_python_falls_back_to_the_running_minor( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A metadata-less install still gets a bound the ambient pin meets.""" - import importlib.metadata - - def _missing(_name: str) -> object: - raise importlib.metadata.PackageNotFoundError - - monkeypatch.setattr(importlib.metadata, "metadata", _missing) +def test_requires_python_is_the_running_minor() -> None: + """The bound and the exact pin come from one interpreter, so the + scaffolded `.python-version` always satisfies `requires-python`.""" v = sys.version_info assert templates.requires_python() == f">={v.major}.{v.minor}" -def test_lightcone_requirement_pins_the_running_version( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Driver and project stay in lockstep — except for dev builds, whose - versions aren't published, so pinning one would make the project's lock - unsolvable.""" - import importlib.metadata - - monkeypatch.setattr(importlib.metadata, "version", lambda _name: "1.2.3") - assert templates.lightcone_requirement() == "lightcone-cli==1.2.3" - - monkeypatch.setattr(importlib.metadata, "version", lambda _name: "1.2.3.dev4+gabc") - assert templates.lightcone_requirement() == "lightcone-cli" - - def _missing(_name: str) -> str: - raise importlib.metadata.PackageNotFoundError - - monkeypatch.setattr(importlib.metadata, "version", _missing) - assert templates.lightcone_requirement() == "lightcone-cli" - - # ---- .gitignore ----------------------------------------------------------- diff --git a/tests/test_worker.py b/tests/test_worker.py index 6accf882..c8840b9a 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -111,6 +111,9 @@ def test_the_manifest_is_complete_before_anything_is_saved(root: Path) -> None: assert manifest.env_version == identity.env_version(root) assert manifest.git_sha == _HEAD[0] and manifest.git_remote == _HEAD[1] assert manifest.hermeticity["mechanism"] + # The engine's version is attestation, not identity: with lc outside + # the project's lock, this field is the record of which engine ran. + assert manifest.lc_version == worker.lc_version() def test_the_recipe_runs_under_the_boundary(root: Path) -> None: @@ -416,6 +419,24 @@ def test_the_module_reruns_unconditionally( assert (root / "results/baseline/first/value.txt").exists() +def test_the_module_converges_the_environment( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A rerun checks out the lock but never `.venv`, and the recipe's own + `uv run --no-sync` silently creates an *empty* environment where none + exists — so the entry point syncs before anything executes, or a fresh + clone's rerun would record the lock's `env_version` over a recipe that + ran in a bare venv.""" + import shutil + + monkeypatch.chdir(root) + shutil.rmtree(root / ".venv") + + assert worker.main(["baseline/first"]) == 0 + assert (root / ".venv").exists() + assert (root / "results/baseline/first/value.txt").read_text() == "one\n" + + def test_the_module_refuses_an_argument_it_cannot_use(root: Path) -> None: assert worker.main([]) == 2 assert worker.main(["first"]) == 2 From a67ddf2f12c7dacfff2c8295fb5bd2b33e048664 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 13:27:29 +0200 Subject: [PATCH 12/23] =?UTF-8?q?Layer=206:=20the=20container=20hatch=20?= =?UTF-8?q?=E2=80=94=20OCI=20runtimes=20as=20the=20exec=20boundary=20(#180?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this is The container hatch, landed whole: declaration, identity, `lc build`, containerized `lc run` and `lc materialize`, manifest v2, and the `lc status` headers. The design decisions were taken interactively and are recorded in CLAUDE.md; the load-bearing ones: - **Engine on host.** The container is the *recipe's* execution world, never the engine's — driver, git, annex, dask and classification all stay the invoked `lc`. This deviates from spec v6.1's full-stack-in-image rule, recorded: v6.1's reason (the host-sync deadlock) is solved by converging `.lightcone/venv` *inside* the image, and the spec's own Perlmutter row ("recipe wrap, step 3 only") is this exact shape. No delegation machinery, no git through a `--userns=keep-id` bind mount, no engine version in the image tag. - **The user never sees a Containerfile.** The whole surface is Modal-shaped TOML — `base` / `apt-install` / `run-commands` / `env`, a closed, fully-hashed key set (`pip_install` deliberately has no equivalent; the lock owns Python). The render exists only in a transient build context; `LABEL io.lightcone.image` keeps the image self-describing. - **The dataset is the image store; runtime stores are caches.** `lc build` saves the image as a docker-archive at `.datalad/environments//image` (the `datalad containers-add` layout), annexed and committed — exact bytes to every clone via `git annex get`, no registry. Execution pins the id read from the archive itself. A dropped archive never substitutes; a rebuild is a new commit under a new id. - **The mount table is the mechanism.** One `OCIBackend` (podman recommended, docker accepted — data-parameterized spellings, one shape) maps the same `Policy` to `:ro`/`:rw` mounts with `--network none`; the attestation (`fs: declared`, `network: denied`) is derived flag-for-flag. No in-container Landlock, no seccomp probe: the engine container never gets the tree `:rw`, so mounts alone express the whole policy. The archive format is chosen so apptainer/singularity become thin layer-7 backends. - **The record stays runtime-neutral through the worker.** Verified against datalad-container's source: templates expand at *record* time and `rerun` executes the literal string, so a bare-recipe `cmd` was rejected. The containerized record adds `extra_inputs: []`, which stock `datalad rerun` fetches through the annex before executing. ## Notable traps found on the way - git-annex routes dotfiles to git regardless of `annex.largefiles` — without a per-add `annex.dotfiles=true` the archive lands as a full git blob, silently. Pinned both ways in `test_dataset.py`. - A standalone `chmod -R /opt` layer copy-on-writes the whole interpreter into every archive, doubling it; the chmod rides inside the layers that write `/opt`. - `boundary.env_argv`'s host-resolved `env` binary need not exist in the image (NixOS), so world-backends apply the overlay natively via `--env` (`contains_prefix`, declared on every backend). - The dirty check now runs *before* the converge in materialize: the image commit would otherwise sweep the user's staged edits. ## Testing - `tests/test_image.py`, `test_sandbox_oci.py` — pure (declaration, document, render ordering, tag sensitivity, mounts, attestation, seam composition). - `tests/test_container.py` — the lifecycle against a stubbed `project._run` (three `ensure_image` strictnesses, refusals mutation-checked, build/save/commit argv). - `tests/test_container_smoke.py` — real podman end to end, gated (`skipif` + `LC_CONTAINER_TESTS_REQUIRED=1` on Linux CI + two guard tests): build → id/label verification → probe (refusal → success → mount denial → loopback/egress) → full materialize with manifest assertions → **`datalad rerun` on a bytes-free clone fetching the archive through the annex and reproducing the output**. - 483 tests pass; ruff and mypy clean; `lc --help` still 65 ms. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012oVCfc4eZdyKqT1zY95Xi2 --------- Co-authored-by: Claude Fable 5 --- .github/workflows/tests.yml | 4 + CLAUDE.md | 340 ++++++++-- pyproject.toml | 4 + src/lightcone/cli/commands.py | 82 ++- src/lightcone/engine/assets.py | 17 +- src/lightcone/engine/container.py | 627 ++++++++++++++++++ src/lightcone/engine/dataset.py | 17 +- src/lightcone/engine/identity.py | 15 +- src/lightcone/engine/image.py | 386 +++++++++++ src/lightcone/engine/materialize.py | 125 +++- src/lightcone/engine/project.py | 91 ++- src/lightcone/engine/run.py | 24 +- src/lightcone/engine/sandbox/boundary.py | 37 +- src/lightcone/engine/sandbox/denial.py | 9 +- src/lightcone/engine/sandbox/landlock.py | 2 + src/lightcone/engine/sandbox/model.py | 27 +- src/lightcone/engine/sandbox/oci.py | 125 ++++ src/lightcone/engine/sandbox/policy.py | 98 ++- src/lightcone/engine/sandbox/seatbelt.py | 2 + .../engine/templates/files/gitattributes.tmpl | 1 + .../engine/templates/files/gitignore.tmpl | 1 + src/lightcone/engine/worker.py | 46 +- tests/conftest.py | 25 + tests/test_cli.py | 45 +- tests/test_container.py | 484 ++++++++++++++ tests/test_container_smoke.py | 309 +++++++++ tests/test_dataset.py | 32 + tests/test_image.py | 283 ++++++++ tests/test_materialize.py | 94 ++- tests/test_project.py | 26 + tests/test_run.py | 11 + tests/test_sandbox_denial.py | 5 + tests/test_sandbox_oci.py | 301 +++++++++ tests/test_worker.py | 24 +- 34 files changed, 3568 insertions(+), 151 deletions(-) create mode 100644 src/lightcone/engine/container.py create mode 100644 src/lightcone/engine/image.py create mode 100644 src/lightcone/engine/sandbox/oci.py create mode 100644 tests/test_container.py create mode 100644 tests/test_container_smoke.py create mode 100644 tests/test_image.py create mode 100644 tests/test_sandbox_oci.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b45d8ee4..8535cd98 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,6 +33,10 @@ jobs: # Landlock or Seatbelt would otherwise report green while testing # nothing at all. LC_SANDBOX_TESTS_REQUIRED: "1" + # Same rule for the container smoke tests, on the runners that carry + # the runtimes (ubuntu ships podman and a running docker daemon). + # macOS runners have no podman machine, so there the suite skips. + LC_CONTAINER_TESTS_REQUIRED: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }} steps: - uses: actions/checkout@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 320d2cdc..5a342a97 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ speculatively. | 3 | ~~The `lc` entrypoint — launcher~~ | ❌ **removed by decision** (2026-08) — see Recorded decisions: the host `lc` is the engine, so there is nothing to delegate to | | 4 | **Fabric** — `lc materialize`, worker sequence, mid-run relock gate | ✅ **done** | | 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | -| 6 | Container hatch — `[tool.lightcone.image]`, generated Containerfile, `lc build` | ⬜ | +| 6 | **Container hatch** — `[tool.lightcone.image]`, `lc build`, OCI runtimes as the exec boundary, the image archived in the dataset | ✅ **done** | | 7 | Venues — Perlmutter, hub/GKE, Cloud Build | ⬜ | | 8 | `lc verify`, WRROC export | ⬜ | @@ -160,9 +160,11 @@ src/lightcone/ # namespace — NO __init__.py │ └── commands.py # lc init, lc run, lc materialize, lc status └── engine/ ├── __init__.py # docstring only - ├── project.py # what a project is: convergence + discovery + ├── project.py # what a project is: convergence, discovery, mode ├── dataset.py # the git + git-annex seam: how a project stores ├── identity.py # env_version, definition_version, the lock scan + ├── image.py # the system layer: declaration, Containerfile, tag — pure + ├── container.py # runtimes, the build, the archived image — impure ├── assets.py # an output: its directory, its manifest, its state ├── plan.py # the spec, read as a graph of tasks ├── worker.py # making one output; also the `python -m` entry point @@ -175,6 +177,7 @@ src/lightcone/ # namespace — NO __init__.py │ ├── boundary.py # detect() + run(): the mechanism-blind half │ ├── landlock.py # Linux backend │ ├── seatbelt.py # macOS backend + │ ├── oci.py # containerized backend: the mount table as mechanism │ └── denial.py # the denial UX └── templates/ # the scaffold's file content ├── __init__.py # loader; a renderer only where there is a value to decide @@ -275,12 +278,15 @@ wrong, and one the type checker cannot catch. The two `*_repair` functions stay named because `_Converger.file` hands them the text alone, so the template name has to be bound before the call site. -**No engine constants for the environment.** The scaffolded -`.python-version` is the interpreter `lc` itself is running on, and -`requires-python` is that interpreter's minor as a floor. Both come from -one place, so they can't conflict, and neither is a number to maintain. -Identity follows the project's files from there: `env_version` hashes -`.python-version`'s bytes, not anything in the engine (spec §3). +**No engine constants for the environment — in direct mode.** The +scaffolded `.python-version` is the interpreter `lc` itself is running +on, and `requires-python` is that interpreter's minor as a floor. Both +come from one place, so they can't conflict, and neither is a number to +maintain. Identity follows the project's files from there: a direct +project's `env_version` hashes `.python-version`'s bytes, not anything +in the engine. The scoping is deliberate: containerized mode's default +base and uv digests *are* engine constants, by spec-§2 design — see the +layer-6 invariants for what moves when they do. **`.gitignore` and `.gitattributes` converge entry-wise, not by marker.** `templates.entries(name)` is a template minus comments and blanks; @@ -556,6 +562,16 @@ the rule for the next. The item records **`blocked`**, not a warning — success and commit nothing — and it names `::`, because `.gitignore` convergence only ever appends and cannot fix this. +**A new managed `.gitattributes` line is always appended at the *end* of +the template.** Repair is append-only, so a project scaffolded under an +older template receives new lines at the bottom of its file — and +`gitattributes_disorder` judges against template order. A line added +anywhere but last makes lc write an order it then reports as blocked, on +every project scaffolded before the line existed. The layer-6 image line +sits after the manifest exemption for exactly this reason (the patterns +are disjoint, so the order is free), and every future line takes the +same slot. + **`.gitattributes` can be *unrepairable*, and that is a second blocked item.** The file is last-match-wins, and a repair only appends — so a project that already carries `results/** annex.largefiles=anything` @@ -577,9 +593,12 @@ regenerated — it identifies the dataset across clones and siblings. Verified, not assumed: `datalad status` recognises a freshly scaffolded project with no `--force` adoption step, and `Dataset('.').id` is the UUID we wrote. The reciprocal is a standing non-goal: **lc never requires -datalad, never imports it, and never parses `.datalad/`.** A researcher -who wants `datalad get`, siblings or RIA stores runs `uv add datalad` in -their own project. +datalad, never imports it, and never parses `.datalad/`.** lc *writes* +under it — the dataset id, and since layer 6 the image archives and the +`datalad.containers.*` config keys — but reads nothing back except +through `git config -f`, and the archive is read as an image, never as +datalad state. A researcher who wants `datalad get`, siblings or RIA +stores runs `uv add datalad` in their own project. **Annexed files are ordinary writable files.** `filter=annex` keeps them unlocked, so an output can be overwritten in place and nothing needs to @@ -982,11 +1001,28 @@ workers pass `--no-sync` — so a lock edited without a sync would leave recipes importing packages the lock does not describe while every manifest recorded the *new* lock's `env_version`. Measured: the recipe imported `packaging 26.3` under a lock saying `24.2`, and uv accepted it silently. -`materialize()` calls `project.sync()` right after the tool and committer -preflight — before the dirty check and the graph — so the state is made -impossible rather than detected. `--check` needs neither: -`env_version` is the lock's bytes, so a drifted `.venv` cannot change what -it answers. +`materialize()` converges right after the dirty check — before the graph +— so the state is made impossible rather than detected. (The dirty check +moved *in front* of the converge with layer 6: in containerized mode the +converge can commit an image archive, and `dataset.save` stages scoped +but commits the whole index, so on a dirty tree the user's staged edits +would be swept into the image commit. Direct mode is order-insensitive — +its sync touches only ignored paths — so both modes run one order.) +`--check` needs neither: `env_version` is the lock's bytes, so a drifted +`.venv` cannot change what it answers. + +**A run fetches its declared inputs; the read-only verbs never do.** +`materialize` batch-runs `git annex get` over the graph's in-tree +declared inputs before anything hashes (driver-side — the storage +invariant that nobody is ever asked to run an annex command by hand), +so a bytes-free clone materializes straight to up-to-date. A failed +fetch is a *warning*, never a refusal: independent tasks still run and +the task whose input is unreachable reports its own failure. `--check` +and `status` stay transfer-free — there an unfetched input is a +reported fact, and the report now says `lc materialize` resolves it. +Out-of-tree inputs are not fetched (no annex holds them — the recorded +weaker promise), and `test_check_mode_never_fetches` pins the read-only +half. **A run takes every core, and there is no flag to say otherwise.** How much of a machine a run may use — and which machine — is one question, and @@ -1117,14 +1153,20 @@ provides that, and for what workers do *not* need (git, git-annex). `schema_version`, `output_id`, `universe_id`, `recipe`, `definition_version`, `env_version`, `data_version`, `decisions`, `input_versions`, `git_sha`, - `git_remote`, `lc_version`, `hermeticity`. Spec §3's longer list — - `uv_version`, `platform`, `python_build`, `worker_runtime`, `image`, + `git_remote`, `lc_version`, `hermeticity` — and, since layer 6, + `image` (`{tag, id, archive, arch}`, defaulting to ``None`` because + that is the true value for a host run — not back-compat machinery, + and `SCHEMA_VERSION` stays 1 pre-release). Spec §3's longer list — + `uv_version`, `platform`, `worker_runtime`, `python_build`, `dpkg_snapshot_sha256`, `sdist_built`, `env_snapshot`, `gpu_driver` — - is either layer 6's or attestation nothing here reads; it lands with the - verb that reads it. -- **`env_version` has three terms, not five.** The - `[tool.lightcone.image]` and `Containerfile.extra` terms are layer 6's, - and hashing an empty shape for them now would be foreshadowing. + is attestation nothing here reads; it lands with the verb that reads + it (`worker_runtime` is additionally derivable from + `hermeticity.mechanism`, so it may never land at all). +- **`env_version` has four terms.** Layer 6 added the image term — + the system layer's identity document, hashed as the literal `null` + for a direct project so the formula stays one formula. (The spec's + separate `Containerfile.extra` term went with the concept; see the + layer-6 decisions.) - **A recipe is handed to `bash -c`.** ASTRA recipes are command lines, not argv, and `bash` is in the exec allowlist — so it is granted by the same rule that grants everything else a recipe may run. @@ -1225,9 +1267,11 @@ from anything a command could return, and it never falls through to running the command unsandboxed. **Layer boundaries showed up as parameters we did *not* add.** -`writable_project`, output-dir write scope, scratch dirs, `in_container()`, -and the `podman*`/`pod*` attestation branches all belong to layers 4 and 6 -and are absent, not stubbed. +`writable_project`, output-dir write scope, and scratch dirs belong to +later layers and are absent, not stubbed. (Layer 6 then landed the +`podman`/`docker` attestation values and the containerized policy shape +— as one keyword on `exec_policy` and one backend, not as the stubs this +rule kept out.) **The macOS profile is vendored, not authored.** `sandbox/profiles/{base,network,platform-defaults}.sbpl` come from the codex CLI @@ -1299,6 +1343,183 @@ last**, after the guard. Get that order wrong and layer 4 materializes on Linux and refuses on macOS, with the golden test still green because it only checks that the guard is present. Verified empirically, not assumed. +## Key Invariants (layer 6) + +**Mode is derived, never configured.** A `[tool.lightcone.image]` table +in `pyproject.toml` *is* the escalation to containerized mode; deleting +it is the way back. `project.mode()` reads presence only; what the table +*means* is `engine/image.py`'s question, so an invalid declaration +refuses where it is consumed, naming the key at fault. + +**The user never sees a Containerfile.** The whole declaration surface is +Modal-shaped TOML — `base` (digest-pinned or the default constant), +`apt-install`, `run-commands`, `env` — a closed set, because every key is +hashed. `pip_install` deliberately has no equivalent: the Python +environment is the lock's business, never the image's, and `run-commands` +is the bounded escape (this deletes spec §2's `Containerfile.extra`). +The render exists only inside a transient build context; the image's +`LABEL io.lightcone.image` carries the identity document, so the archive +stays self-describing without one. + +**The engine never enters the image.** The container is the *recipe's* +execution world: driver, git, annex, dask and classification all stay the +host's `lc`, and exactly two things ever run in-image — the environment +converge (`container.sync`, network on, project `:rw`, host uv cache +mounted, into `.lightcone/venv`) and each recipe/probe exec +(`--network none`). This deviates from spec v6.1's full-stack rule, +recorded: v6.1's reason was the host-sync deadlock, which the +in-container sync solves, and the spec's own Perlmutter row ("recipe +wrap, step 3 only — never the dask worker") is this exact shape. What it +buys: no delegation machinery, no git through a `--userns=keep-id` bind +mount, no engine version in the tag, and one engine per run by +construction. The host `.venv` is inert in containerized mode +(`current_project` does not require it; `lc init` converges no +environment — a host sync of a containerized lock is the deadlock in +miniature); `.lightcone/` is gitignored machine state. + +**The image is the system layer only.** Base + apt (only when +`apt-install` is nonempty — the engine needs nothing from apt, so spec +§2's rule is restored) + the pinned uv + the pinned interpreter into +`/opt/python`. No git, no engine, and **no project file ever enters the +build context** — the context is a scratch directory holding one rendered +Containerfile, which is what makes "code edits never trigger a build" +structural. `bash` is a base-contract check (recipes are `bash -c`), and +each contract violation is a reserved exit code mapped to a refusal +naming the base (43 musl, 44 no bash, 45 no apt), never a raw build log. +The readability `chmod` rides inside the layers that write `/opt` — a +layer of its own would copy-on-write the whole interpreter tree into +every archive, doubling it. + +**The dataset is the image store; runtime stores are caches.** `lc build` +saves the image as a `docker-archive` at +`.datalad/environments//image` — the `datalad containers-add` +layout — annexed and committed, so the exact bytes travel through +`git annex get` with no registry and no credentials. This supersedes the +spec's `dpkg_snapshot_sha256` residue: apt is name-pinned, so a rebuild +months later yields different bytes under the same tag, and the archive +keeps the bytes themselves. The dot-path is the trap: +**git-annex routes dotfiles to git whatever `annex.largefiles` says**, +so `dataset.save` opts in unconditionally (`annex.dotfiles=true`, +per-add like `annex.thin`) — without it the archive, or a `.cache.h5` a +recipe writes into its output directory, lands as a full blob in git, +silently, with every test green. The attributes alone decide from +there (dot-named manifests keep their own exemption); a user's plain +`git add` keeps git-annex's stock behavior. `test_dataset.py` pins both +directions. And because `.gitattributes` is user-authored and lc only +appends, the build **probes the routing with `git check-attr` before +saving** — an archive the attributes would hand to git is a refusal +naming the line, the same probe-don't-assume rule as the `results/` +ignore check. The archive +is the arch it was built for (recorded in the manifest); multi-arch is a +layer-7 item, as is apptainer/singularity — which is *why* the format is +`docker-archive`: all four runtimes consume it, and HPC hosts that +cannot build obtain images through the repository. + +**`ensure_image` is one function with three strictnesses**, and the split +is load-bearing: `lc build` and the materialize preflight may **build + +save + commit** (announced by the CLI, which owns the console — the +engine never prints); the probe and the worker entry point only ever +**find** — a missing archive refuses naming the exact `lc build` +(`lc run` never builds; the worker never commits), unfetched content is +**fetched by lc itself** (`git annex get`, driver-side — the storage +invariant is that nobody is ever asked to run an annex command by hand, +so the refusal is reserved for a fetch with no reachable copy), and an +unloaded image is a silent ` load`. Execution pins the **id** — sha256 of the +archive's config blob, readable with no runtime, the same computation +datalad's docker adapter makes — never a tag, so a retagged store image +cannot substitute. **A dropped archive never substitutes**: a rebuild is +a new archive commit under a new id; old manifests keep naming what ran. + +**Builds and the archive commit happen only on a clean tree, and only +after the graph.** The dirty check runs before `ensure_image` in +materialize, and `lc build` refuses dirt itself: `dataset.save` stages +scoped but commits the whole index, so a build on a dirty tree would +sweep the user's staged edits into the image commit — and the tag +derives from `pyproject.toml`, so the declaration must be committed +before the image it defines. The graph (spec validation, the lock scan) +resolves before the image too, in materialize and in the worker entry +point alike: a refusal over a typo must not cost a minutes-long build or +leave an archive commit behind a failed run, and a failing sync must not +bury "no output `x`". + +**Two identities, one document.** The canonical identity document +(declaration resolved + the uv digest; the interpreter pin deliberately +absent — its raw bytes are already an `env_version` frame) feeds +`env_version` as its fourth frame, `null` for direct projects — so +declaring an image, or an engine release that bumps the default-base or +uv constants, puts containerized outputs `behind`, honestly, while +direct projects never move on an engine release. The **tag** hashes the +rendered Containerfile *and* the document, so a render-only generator +change rebuilds the image without staling anything — accepted residue, +attested by the manifest's image id and the archive's bytes. Read from +`pyproject.toml` only, never `identity._uv_config()` — uv's +"`uv.toml` replaces `[tool.uv]`" rule must not reach this table. + +**The mount table is the mechanism** (`sandbox/oci.py`). The +containerized `exec_policy` shape is the same policy minus the host: no +OS read baseline, no stdlib root, no exec set — the image *is* those, +and everything in it was declared — leaving exactly the paths that +become mounts, project `:ro`, `results/` `:rw`, declared inputs `:ro`, +the private HOME `:rw`, `--tmpfs /tmp`, over a **`--read-only` rootfs**: +without that flag a write outside the declared set *succeeds* into the +container's ephemeral layer and vanishes while the run attests +`fs: declared` — the silent-loss path, closed by making it a denial. +Mounts are **resolved source, declared destination** — the containerized +policy is the one shape that keeps its paths unresolved, because they +are addresses the recipe uses (a symlinked `/data` input resolved on +both sides would leave the container with no `/data` at all). +`--security-opt label=disable`, because SELinux hosts otherwise refuse +every bind read and `:z` would relabel the user's own files. No +in-container Landlock, no seccomp probe, no shim-in-image: the engine +container never gets the tree `:rw`, so mounts alone express the whole +policy, and the attestation (`mechanism: podman|docker`, +`fs: declared`, `network: denied`) is derived flag-for-flag from the +argv — `--network none` is the codebase's one honest `denied`, loopback +intact. One `OCIBackend`, data-parameterized: podman and docker differ +in spellings (`--userns=keep-id`+`--pull=never` vs `--user uid:gid`), +not shape. +Runtime is **host capability** — detected podman → docker, the +`detect()` discipline on a second axis, with `container.backend()` +holding the only mode branch — and never part of any identity. The +containerized `tmp_home` lives under the project's `.lightcone/` rather +than the system temp dir: it is a mount source, and macOS's podman +machine shares the project tree while `/var/folders` arrives empty. + +**The seam learned one asymmetry, `contains_prefix`.** A host mechanism +keeps the `uv run` hop *outside* the wrap (trusted host plumbing); a +backend that is itself a world takes it *inside* — there is no trusted +host plumbing inside a container — and applies the env overlay natively +as `--env` flags, never through `boundary.env_argv`'s host-resolved +`env` binary (NixOS resolves it to a path no Debian image has). Declared +on every backend, so a new mechanism must answer the question. The +container environment is an allowlist: the overlay plus `LC_SANDBOX`, +never the ambient environment. + +**The record stays runtime-neutral, and the worker is its executor.** +`cmd` is still the engine-pinned worker — verified against +datalad-container's source before deciding: `containers-run` expands its +`cmdexec` template at *record* time and `datalad rerun` executes the +literal string (the extension has no rerun hook), so a bare-recipe `cmd` +would rerun on the host, unconfined, with no manifest. Instead the +record is declarative — the task id, the commit's own spec, the +committed archive — and the worker re-resolves container *and runtime* +on the rerun host, keeping the sandbox, the gates and the manifest. The +containerized record adds `extra_inputs: []`, which stock +`datalad rerun` fetches through the annex before executing — the smoke +suite proves the whole claim on a bytes-free clone. `lc build` also +writes `datalad.containers..{image,cmdexec}` so ad-hoc +`datalad containers-run` works for humans, explicitly outside lc's +guarantees. + +**The runtime is resolved once per run and handed down** — the HEAD +discipline, a frozen `container.Runtime` through `worker.materialize` — +because resolving per task could answer differently mid-run. The rerun +entry point resolves its own, as it does HEAD: it *is* the driver of its +one-task run. `lc status` gained the three header lines (mode, image +state, sandbox) — repository facts only, no runtime and no network +required, which is where the denial note and the runtime-missing +refusal point. + ### Recorded decisions - **The engine is the host's uv tool, never a project dependency** @@ -1345,10 +1566,11 @@ only checks that the guard is present. Verified empirically, not assumed. Accepted rather than re-provided: the alternative is hashing an environment lc does not own into artifacts it does, which is the over-sensitivity the `behind` model exists to avoid. - - *Layer 6 note:* the generated image can no longer get its in-image - engine from the lock (`/opt/venv/bin/lc` existed because the pin did). - The Containerfile must install the engine as an explicit layer, and - that version becomes a tag input. + - *Layer 6 resolved its note the other way:* the image gets **no + engine layer at all**. The engine stays on the host in containerized + mode too — the container is the recipe's world, never the engine's — + so the engine version is not a tag input and an lc upgrade still + rebuilds nothing. See the layer-6 invariants. What it buys: a project's resolution is freed from the engine's own pins (no astra-tools/click/rich/distributed/git-annex in any project lock); the git-annex wheel floor gates only the tool install; the eval @@ -1414,6 +1636,31 @@ only checks that the guard is present. Verified empirically, not assumed. (spec §2), so a scaffolded project simply carries a key no code path consults. Don't "fix" this by reconciling the two — a later layer will decide whether the key is dropped upstream, refused, or migrated. +- **Multi-runtime, podman recommended** (2026-08, layer 6 — superseding + an earlier "podman only" plan decision). podman and docker ship + tested; the backend seam and the `docker-archive` format are chosen so + apptainer/singularity become thin layer-7 backends over the same + archive (they exec `docker-archive:` content directly, and HPC hosts + that cannot build obtain images through the annex). Build-capable + (podman|docker) and run-capable (all four, eventually) are therefore + separate questions in `container.py`. docker's daemon is probed at + detection — a CLI with the daemon down is the common broken state. +- **Bare-recipe run records: considered and rejected** (2026-08). + datalad expands container templates at *record* time and `rerun` + executes the literal string — datalad-container has no rerun hook — so + a record whose `cmd` is the recipe replays on the host, unconfined, + with no manifest, and a containerized spelling would freeze one + runtime's flags into git forever. The worker-as-executor keeps the + record declarative and runtime-neutral; mechanical no-lc replay is + layer 8's WRROC export, for which the manifest + archive now carry + everything. +- **Single-arch archives, recorded residue** (layer 6 → layer 7). The + committed archive is the architecture it was built for, and the + manifest says which. An Apple-silicon build cannot serve an amd64 + venue; solving that (arch-suffixed archive paths, `--platform` + cross-builds) is layer 7's, with the venue that makes it real. Old + archives accumulate in annex history; reclaiming them is the user's + `git annex unused`/`drop` — no GC verb. ### Recorded deviations from the spec @@ -1448,15 +1695,12 @@ only checks that the guard is present. Verified empirically, not assumed. `--require-sandbox` / `--no-sandbox` / `--sandbox-debug` are all absent: there is no hatch to escape the sandbox, so there is nothing for the flags to switch. The verb takes a command and nothing else. -- **`lc run` does not detect containerized mode.** The - `[tool.lightcone.image]` refusal was removed with the rest of the - future-facing surface; a system-layer declaration is currently inert, - like `astra.yaml`'s `container:` key. - **The denial's remedies are only what works today** — `uv add` for a - Python package, the ASTRA input declaration for data, and "output goes - in `results/`" plus `tempfile.mkdtemp()` for a write denial. Nothing in - a denial message names a verb, flag, or declaration that does not - exist. + Python package, the system layer (`apt-install` + the containerize + note, real since layer 6), the ASTRA input declaration for data, and + "output goes in `results/`" plus `tempfile.mkdtemp()` for a write + denial. Nothing in a denial message names a verb, flag, or declaration + that does not exist. - **`Attestation` has no serializer of its own.** An earlier draft carried a `to_manifest()` with no caller — deleted, because "no dead code" applies to this layer's own conveniences too. It is persisted by @@ -1521,6 +1765,8 @@ only checks that the guard is present. Verified empirically, not assumed. | Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | | Change how a project stores bytes | `src/lightcone/engine/dataset.py` + `templates/files/gitattributes.tmpl` | Every command through `project._run`; test it against a real annex (`real_tools`) | | Change how an output is identified | `src/lightcone/engine/identity.py` + `tests/test_identity.py` | Sensitivity tests both ways: what must move the hash, and what must not | +| Change what the image is made of | `src/lightcone/engine/image.py` + `tests/test_image.py` | Pure; structure-and-ordering tests, never byte goldens; every declaration key is hashed | +| Change how images are built, stored or entered | `src/lightcone/engine/container.py` (+ `sandbox/oci.py` for the exec argv) + `tests/test_container.py` | Everything through `project._run`; keep the three `ensure_image` strictnesses; runtime differences are spellings inside `OCIBackend`, never new shapes | | Change when an output is remade | `src/lightcone/engine/assets.py` + `tests/test_assets.py` | One `classify()`, two callers; `--check` differs by one input value, never by logic. Ask first whether the thing that moved *contradicts* the project or is a *circumstance* — the second is `behind`, not `stale` | | Change how the spec becomes a graph | `src/lightcone/engine/plan.py` + `tests/test_plan.py` | Ask `astra.resolve`; if the answer is missing, the fix is a PR to astra-tools. Anything ambiguous is a `ProjectError`, never a guess | | Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | @@ -1595,6 +1841,24 @@ The sandbox suite splits along the seam, which is what makes it cheap: - `tests/test_run.py` — what `lc run` decides *before* it execs: the current-directory project check, declared inputs, the uv hop. Nothing spawns. +- `tests/test_image.py` — **pure**: the declaration, the document, the + render's structure and ordering, tag sensitivity both ways, and the + `env_version` integration. `tests/test_sandbox_oci.py` — **pure**: the + mount table, the argv spellings, the attestation, and the + `contains_prefix` composition through a recorded fake `Popen`. +- `tests/test_container.py` — the image lifecycle against a **stubbed** + `project._run` that models each runtime command's observable effect + (a `save` writes a structurally real docker-archive, a `load` marks + the id present), so every refusal and every strictness is asserted on + recorded argv with nothing spawned. +- `tests/test_container_smoke.py` — **the runtime's answer**, gated like + the enforcement suite: skips without a runtime, + `LC_CONTAINER_TESTS_REQUIRED=1` on Linux CI turns the skip into a hard + failure, and two tests cover the guard itself. Parameterized over the + runtimes present. It builds a real image, commits a real archive into + a real annex, materializes through the real Dask cluster, and runs a + real `datalad rerun` on a bytes-free clone — the record's whole claim. + Each denial sits beside its mutation check. Note the autouse `tools` fixture stubs `engine.project._run` only — sandbox tests spawn real processes deliberately, and are the one place in diff --git a/pyproject.toml b/pyproject.toml index 07c41c31..f18e9aaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,3 +94,7 @@ follow_untyped_imports = true [tool.pytest.ini_options] testpaths = ["tests"] +# The container smoke tests leave ~200 MB image archives in each test's +# tmp_path; keeping every passing run's directories exhausts a tmpfs. +tmp_path_retention_policy = "failed" +tmp_path_retention_count = 1 diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index f67c60d5..d9db40c1 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -177,6 +177,62 @@ def run(command: tuple[str, ...]) -> None: sys.exit(128 - code if code < 0 else code) +# ============================================================================= +# lc build +# ============================================================================= + + +@main.command() +@click.option( + "--json", + "as_json", + is_flag=True, + help="Emit the result as JSON on stdout.", +) +def build(as_json: bool) -> None: + """Build the project's system-layer image, and commit it. + + Containerized projects only — a project containerizes by declaring + [tool.lightcone.image] in pyproject.toml. The image is saved into the + repository (.datalad/environments/) as versioned content, so clones + obtain the exact bytes with `git annex get` instead of rebuilding. + Idempotent: an image that is already built and committed is left + alone. + """ + from lightcone.engine import container as engine_container + from lightcone.engine.project import current_project + + root = current_project() + state, tag, _ = engine_container.image_state(root) + if state == "direct": + if as_json: + click.echo(json.dumps({"mode": "direct"})) + else: + _console().print( + "direct mode — no image to build; declare [bold]\\[tool.lightcone.image][/bold] " + "in pyproject.toml to containerize this project." + ) + return + if state == "absent" and not as_json: + _console().print(f"building [bold]{tag}[/bold] — this can take minutes") + runtime, action = engine_container.build(root) + if as_json: + click.echo( + json.dumps( + { + "mode": "containerized", + "tag": runtime.image_tag, + "id": runtime.image_id, + "archive": runtime.archive, + "action": action, + } + ) + ) + return + verb = "Built and committed" if action == "built" else "Already built —" + _console().print(f"[green]✓[/green] {verb} {runtime.image_tag} ({runtime.archive})") + + # ============================================================================= # lc materialize # ============================================================================= @@ -226,10 +282,23 @@ def materialize( records the environment and the commit that produced it. Pass --refresh to remake those too. """ + from lightcone.engine import container as engine_container from lightcone.engine import materialize as engine from lightcone.engine.project import current_project root = current_project() + if not check_only and not as_json: + # The engine never prints, and the build it may be about to run + # can take minutes — so the one place that owns the console says + # so before handing over. Conditional mood, deliberately: the + # engine's own refusals (a dirty tree, an invalid spec) come + # first and cost no build, so this must promise nothing. + state, tag, _ = engine_container.image_state(root) + if state == "absent": + _console().print( + f"image absent — the run rebuilds [bold]{tag}[/bold] first " + "(this can take minutes)" + ) if check_only: report = engine.check(root, targets, refresh=refresh) else: @@ -278,9 +347,20 @@ def status(as_json: bool) -> None: click.echo(json.dumps(report.as_dict(), indent=2)) return + lines = [f" mode: {report.mode}"] + if report.image is not None: + tag, state = report.image["tag"], report.image["state"] + described = { + "present": "built", + "absent": "needs build — run `lc build`", + "unfetched": "content not in this clone — the next build or run fetches it", + }[state] + lines.append(f" image: {tag} — {described}") + lines.append(f" sandbox: {report.sandbox}") + lines.append("") marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} width = max((len(o.output) for o in report.outputs), default=0) - lines = [ + lines += [ # The commit gets a column of its own, for every state and not only # the interesting ones: "which code made this" is the question the # verb exists to answer, and it has an answer for a current output diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index ef808996..f161fb37 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -107,12 +107,12 @@ def data_version(path: Path) -> str: being in this clone, in either of the shapes that takes. """ if path.is_symlink() and not path.exists(): - _refuse_unfetched(path) + require_fetched(path) if not path.exists(): raise FileNotFoundError(path) h = hashlib.sha256() if path.is_file(): - _refuse_unfetched(path) + require_fetched(path) h.update(b"file:") _feed(h, path) return f"sha256:{h.hexdigest()}" @@ -130,7 +130,7 @@ def data_version(path: Path) -> str: if p.name not in _HASH_EXCLUDE and (p.is_file() or (p.is_symlink() and not p.exists())) ] for p in sorted(files, key=lambda x: x.relative_to(path).as_posix()): - _refuse_unfetched(p) + require_fetched(p) h.update(b"path:") h.update(p.relative_to(path).as_posix().encode()) h.update(b"\0data:") @@ -175,7 +175,7 @@ def of(self, path: Path) -> str: return known -def _refuse_unfetched(path: Path) -> None: +def require_fetched(path: Path) -> None: """Refuse a file whose content this clone does not hold. An annexed file takes one of two shapes, and a researcher can convert @@ -187,7 +187,7 @@ def _refuse_unfetched(path: Path) -> None: without the content simply dangles. Args: - path: A file about to be hashed. + path: A file whose bytes are about to be used. Raises: ContentNotFetchedError: If *path* is either shape without its @@ -255,6 +255,13 @@ class Manifest: lc_version: str #: What the sandbox actually enforced, as the boundary attested it. hermeticity: dict[str, Any] + #: The image the recipe ran in — ``{tag, id, archive, arch}`` — or + #: ``None`` on the host. Defaulted, and that is not back-compat + #: machinery: ``None`` is the *true* value for every manifest a + #: container-less engine wrote, and without the default those + #: manifests would read as absent and the whole project would go + #: stale over a field that changes nothing about the bytes. + image: dict[str, Any] | None = None schema_version: int = field(default=SCHEMA_VERSION) def as_dict(self) -> dict[str, Any]: diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py new file mode 100644 index 00000000..3a68c9b6 --- /dev/null +++ b/src/lightcone/engine/container.py @@ -0,0 +1,627 @@ +"""Container runtimes: building the system layer, storing it, entering it. + +The dataset is the image store; runtime-local stores are caches. An image +is built once, saved as a ``docker-archive`` at +``.datalad/environments//image``, and committed — annexed bytes that +travel through ``git annex get`` like any other project content. Every +machine that runs the project *loads* that archive into whatever runtime +it has, so the bytes that made an output are the bytes a rerun enters, +name-pinned apt notwithstanding. A dropped archive never substitutes: a +rebuild is a new archive under a new id, never the old reference. + +Runtime is host capability, not project state — podman is preferred, +docker accepted — and the archive format is the one all of podman, +docker, and (later, on HPC) apptainer/singularity consume, which is why +build-capable and run-capable are separate questions. + +Every command goes through :func:`~lightcone.engine.project._run`, the +seam the whole engine shares, so the suite never spawns a runtime. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import sys +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, cast + +from lightcone.engine import assets, dataset, image, project, sandbox +from lightcone.engine.project import ProjectError, _check_call + + +@dataclass(frozen=True) +class Runtime: + """The execution world one run enters, resolved once by whoever owns it. + + The driver resolves it and hands it to every task — the same + discipline as the run's HEAD read — because resolving per task could + answer differently mid-run. The rerun entry point resolves its own, + because it *is* the driver of its one-task run. + """ + + root: Path + mode: Literal["direct", "containerized"] + #: Where the project environment lives: ``.venv``, or the in-image + #: ``.lightcone/venv``. + env_dir: Path + #: ``podman`` or ``docker``; empty in direct mode. + runtime: str = "" + image_tag: str = "" + #: The image id (bare hex of its config blob) — execution pins on + #: this, never the tag, so a retagged image cannot substitute. + image_id: str = "" + #: The architecture the archive was built for. + arch: str = "" + + @property + def archive(self) -> str: + """The committed archive, project-relative — what the run record's + ``extra_inputs`` names. Derived through :func:`image.archive_path` + so this cannot become a second spelling of the layout.""" + return image.archive_path(self.root, self.image_tag).relative_to(self.root).as_posix() + + def manifest_image(self) -> dict[str, str] | None: + """This world as the manifest's ``image`` field; ``None`` on the + host. Beside the data, so a field added here cannot be forgotten + at the write site — the ``asdict(attestation)`` discipline.""" + if self.mode == "direct": + return None + return { + "tag": self.image_tag, + "id": self.image_id, + "archive": self.archive, + "arch": self.arch, + } + + +def runtime_for_run(root: Path, *, build: bool) -> Runtime: + """Resolve the execution world, converging the image where allowed. + + The three image checks are repository questions first and runtime + questions second: archive committed, content fetched (through the + annex, by lc itself), loaded into the local store. Only the first + check's miss differs by caller — *build* is true for ``lc build`` and + the materialize preflight, which may build and commit on a tree their + own dirty check just proved clean; everything else (the probe, the + rerun entry point) refuses naming the exact ``lc build``, because + ``lc run`` never builds and the worker never commits. + + Args: + root: The project root. + build: Whether a missing archive may be built and committed. + + Returns: + The resolved runtime; a direct-mode one costs a TOML read. + + Raises: + ProjectError: If no runtime is usable, the archive is missing and + *build* is false, its content cannot be fetched from any + reachable copy, or the build fails. + """ + if project.mode(root) == "direct": + return Runtime(root=root, mode="direct", env_dir=project.env_dir(root)) + + name = runtime_name(root) + tag = image.tag(root) + archive = image.archive_path(root, tag) + if not _committed(archive): + if not build: + raise ProjectError( + f"the system-layer image `{tag}` has not been built — this verb " + "never builds one. Run `lc build` first." + ) + _build(root, name, tag, archive) + _fetch(root, archive) + image_id, arch = archive_identity(archive) + if not _loaded(root, name, image_id): + _check_call([name, "load", "-i", str(archive)], cwd=root) + return Runtime( + root=root, + mode="containerized", + env_dir=project.env_dir(root), + runtime=name, + image_tag=tag, + image_id=image_id, + arch=arch, + ) + + +def _fetch(root: Path, archive: Path) -> None: + """Bring the archive's bytes into this clone, if they are elsewhere. + + lc fetches its own artifact rather than printing a git-annex command — + the storage invariant is that nobody is ever asked to run one by hand. + Covers both annexed shapes of absent content (the pointer file and the + dangling symlink); the refusal is reserved for a fetch that genuinely + cannot happen, with git-annex's own reason. + + Raises: + ProjectError: If no reachable copy could supply the content. + """ + relative = archive.relative_to(root).as_posix() + try: + assets.require_fetched(archive) + return + except assets.ContentNotFetchedError: + pass + got = project._run(["git", "annex", "get", "--", relative], cwd=root) + if got.returncode != 0: + raise ProjectError( + f"the image archive `{relative}` is not in this clone, and fetching it " + f"failed:\n{got.stderr.strip()}\n" + "It needs a reachable remote that holds the content — or rebuild a new " + "image with `lc build`." + ) + assets.require_fetched(archive) + + +def _committed(archive: Path) -> bool: + """Whether the repository carries the archive, in either annexed shape. + + A dangling symlink *is* a committed archive (a locked clone without + the content), so the naive ``exists()`` misreads it as never built + and tells the user to rebuild an image the repository already has. + """ + return archive.exists() or archive.is_symlink() + + +def runtime_name(root: Path) -> str: + """Detect which container runtime this host offers. + + podman first — rootless podman needs no daemon and no group — then + docker, whose CLI without a reachable daemon is probed rather than + trusted, because `docker` on PATH with the daemon down is the common + broken state and "cannot connect to the socket" mid-run is a worse + message than this one. + + Args: + root: The project root, for the probe's working directory. + + Returns: + ``"podman"`` or ``"docker"``. + + Raises: + ProjectError: If neither is usable. + """ + name = runtime_hint() + if name == "podman": + _machine_preflight(root) + elif name == "docker": + if project._run(["docker", "info"], cwd=root).returncode != 0: + raise ProjectError( + "docker is installed but its daemon is not reachable — start it " + "(or install podman, which needs no daemon), then retry. " + "`lc status` shows what this project needs." + ) + else: + raise ProjectError( + "this project is containerized and needs a container runtime: install " + "podman (recommended: https://podman.io/docs/installation) or docker. " + "`lc status` shows what this project needs." + ) + return name + + +def backend(runtime: Runtime) -> sandbox.Backend: + """Pick the exec boundary for a resolved runtime. + + The only *mode* branch above the sandbox seam, mirroring + ``sandbox.detect()``'s only *platform* branch: containerized mode is + entered through the OCI backend, whose mount table is the + enforcement; direct mode probes the host as it always has. + + Args: + runtime: A resolved runtime. + + Returns: + The backend the exec goes through. + """ + if runtime.mode == "direct": + return sandbox.detect() + from lightcone.engine.sandbox.oci import OCIBackend + + # `--pull=never` beside the uid flags rather than inside them: it is + # a pull policy (a typo'd reference must fail, not fetch), podman's + # spelling only, and filing it under uid mapping is where the next + # reader would not look. + pull = ("--pull=never",) if runtime.runtime == "podman" else () + return OCIBackend( + runtime=cast(Literal["podman", "docker"], runtime.runtime), + image_id=runtime.image_id, + root=runtime.root, + user_flags=(*uid_flags(runtime.runtime), *pull), + ) + + +def build(root: Path) -> tuple[Runtime, str]: + """Converge the system-layer image, the whole of ``lc build``. + + Refuses a dirty tree before anything else: the archive is committed, + ``dataset.save`` stages scoped but commits the whole index, and the + tag derives from ``pyproject.toml`` — so the declaration must be + committed before the image it defines, and nothing of the user's may + be swept into the image commit. + + Args: + root: The project root. + + Returns: + The resolved runtime, and one word for what happened: ``built`` + if this call built and committed the archive, ``present`` if it + was already there. + + Raises: + ProjectError: On a direct-mode project, a dirty tree, a missing + committer identity, or a failed build. + """ + if project.mode(root) == "direct": + raise ProjectError( + "direct mode — no image to build; declare [tool.lightcone.image] " + "in pyproject.toml to containerize this project." + ) + project.require_git() + project.require_git_annex() + dataset.require_committer(root) + if dataset.status(root): + raise ProjectError( + "uncommitted changes — the image is derived from pyproject.toml and " + "committed into the repository, so `lc build` needs the tree to say " + "what it is building from. Commit first, then re-run `lc build`." + ) + existed = image_state(root)[0] != "absent" + return runtime_for_run(root, build=True), ("present" if existed else "built") + + +def runtime_hint() -> str: + """Name the runtime a run here would pick, or empty. Never a refusal. + + For ``lc status``'s header, which reports rather than gates — a + missing runtime is a fact there, not a failure, and the daemon probe + is skipped because a header must not cost a subprocess. + + Returns: + ``"podman"``, ``"docker"``, or ``""``. + """ + for name in ("podman", "docker"): + if shutil.which(name): + return name + return "" + + +def image_state(root: Path) -> tuple[str, str, str]: + """Report where the project's image stands, without a runtime. + + Repository facts only, so ``lc status`` and the CLI's pre-build + announcement work on hosts with no runtime at all. + + Args: + root: The project root. + + Returns: + ``(state, tag, archive)`` — state is ``direct``, ``absent`` + (never built), ``unfetched`` (committed, content elsewhere) or + ``present``; archive is the project-relative path a remedy can + name, carried here so no renderer respells the layout. + """ + if project.mode(root) == "direct": + return ("direct", "", "") + tag = image.tag(root) + archive = image.archive_path(root, tag) + relative = archive.relative_to(root).as_posix() + if not _committed(archive): + return ("absent", tag, relative) + try: + assets.require_fetched(archive) + except assets.ContentNotFetchedError: + return ("unfetched", tag, relative) + return ("present", tag, relative) + + +def sync(root: Path, runtime: Runtime) -> list[str]: + """Converge ``.lightcone/venv`` inside the image. The containerized twin + of ``project.sync``. + + The one container run that gets the network and a writable project + mount — converge once, then execute without writing to the + environment, the same discipline as direct mode. The host's uv cache + is mounted at its identical path, so a complete environment + materializes from cache hits in about a second; the cache location is + ``uv cache dir``'s answer, never a guess, because that is uv's own + resolution of env, config and platform. + + Args: + root: The project root. + runtime: A resolved containerized runtime. + + Returns: + Whatever uv warned about, lifted out of its progress output. + + Raises: + ProjectError: If uv fails inside the container. + """ + asked = project._run(["uv", "cache", "dir"], cwd=root) + if asked.returncode != 0: + raise ProjectError(f"`uv cache dir` failed:\n{asked.stderr.strip()}") + cache = asked.stdout.strip() + argv = [ + runtime.runtime, "run", "--rm", "--entrypoint", "", + # Same reason as the exec boundary's flag: SELinux hosts refuse + # bind reads from container_t, and relabeling user data is worse. + "--security-opt", "label=disable", + *uid_flags(runtime.runtime), + "-v", f"{root}:{root}:rw", + "-v", f"{cache}:{cache}:rw", + "--env", f"UV_CACHE_DIR={cache}", + "--env", f"UV_PROJECT_ENVIRONMENT={runtime.env_dir}", + "-w", str(root), + runtime.image_id, + # The same sync `project.sync` runs, spelled once for both modes. + "uv", *project._SYNC_ARGS, "--project", str(root), + ] # fmt: skip + return _check_call(argv, cwd=root) + + +def converge(runtime: Runtime) -> list[str]: + """Make the environment match the lock, whichever world this is. + + The one spelling of the mode dispatch, so the entry points that + converge (materialize, the rerun worker) cannot drift apart. The + probe deliberately does not call this in direct mode — its syncing + ``uv run`` hop *is* its converge, documented at the call site. + + Args: + runtime: The resolved runtime. + + Returns: + Whatever uv warned about. + """ + if runtime.mode == "direct": + return project.sync(runtime.root) + return sync(runtime.root, runtime) + + +def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: + """Build the exec policy for a resolved runtime. + + The one place the ``env_dir``/``containerized`` pair is assembled — + two settings that must always agree, projected from the single value + every caller already holds. Lives here rather than in the policy + module so the mechanism-free policy layer never learns what a + ``Runtime`` is. + + Args: + runtime: The resolved runtime. + read_paths: Declared inputs, as :func:`sandbox.exec_policy` takes. + + Returns: + The policy for this world. + """ + return sandbox.exec_policy( + runtime.root, + read_paths=read_paths, + env_dir=runtime.env_dir, + containerized=runtime.mode == "containerized", + ) + + +def archive_identity(path: Path) -> tuple[str, str]: + """Read an archive's image id and architecture, with no runtime. + + The id is the sha256 of the image's config blob — the same value + ``podman inspect`` reports and the same computation datalad's docker + adapter makes — so execution can pin by id before anything is loaded. + + Args: + path: A ``docker-archive`` file with its content present. + + Returns: + ``(bare hex id, architecture)``. + + Raises: + ProjectError: If the file is not a readable docker-archive. + """ + try: + with tarfile.open(path) as tar: + listing = tar.extractfile("manifest.json") + assert listing is not None # a member, not a directory + config_name = json.load(listing)[0]["Config"] + blob = tar.extractfile(config_name) + assert blob is not None + config = blob.read() + except (OSError, KeyError, IndexError, json.JSONDecodeError, tarfile.TarError) as e: + raise ProjectError( + f"{path} is not a readable image archive ({e}) — it was committed by " + "`lc build`; rebuild it with `lc build` after removing the file." + ) from e + return ( + hashlib.sha256(config).hexdigest(), + str(json.loads(config).get("architecture", "")), + ) + + +def uid_flags(runtime: str) -> list[str]: + """The flags that keep files written through a mount owned by the user. + + podman maps the invoking uid into the container (``--userns=keep-id``); + docker has no equivalent spelling, so the container simply runs as the + invoking uid — without which a rootful docker writes root-owned files + into ``results/`` that the host's git cannot manage. The missing + passwd entry that leaves behind is covered by the private-HOME + overlay every exec already gets. + + Args: + runtime: ``"podman"`` or ``"docker"``. + + Returns: + The argv fragment. + """ + if runtime == "podman": + return ["--userns=keep-id"] + return ["--user", f"{os.getuid()}:{os.getgid()}"] + + +# ============================================================================= +# Building +# ============================================================================= + + +def _build(root: Path, runtime: str, tag: str, archive: Path) -> None: + """Build the image, save it as *archive*, and commit it. + + The build context is an empty scratch directory holding only the + rendered Containerfile — no project file ever enters it, which is + what makes "code edits never trigger a build" structural rather than + observed. The archive commit is scoped to the environment directory + plus ``.datalad/config`` (the ``datalad containers-run`` interop + keys), and the caller has already proven the tree clean. + """ + relative = archive.relative_to(root).as_posix() + # The archive is committed, so its routing must be checked *before* + # the bytes exist: with `.gitattributes` not sending it to the annex + # — a user-authored file lc only ever appends to — a several-hundred- + # MB blob would land in git itself, silently, and every clone would + # carry it forever. The same probe-don't-assume rule as the ignore + # check on `results/`. + routed = dataset._git(["check-attr", "annex.largefiles", "--", relative], cwd=root) + if "anything" not in routed: + raise ProjectError( + "the image archive would be committed to git itself instead of the " + f"annex — .gitattributes does not route `{relative}`. Add the line\n" + " .datalad/environments/*/image annex.largefiles=anything\n" + "(`lc init` repairs this), then re-run." + ) + + with tempfile.TemporaryDirectory(prefix="lc-build-") as context: + containerfile = Path(context) / "Containerfile" + containerfile.write_text(image.containerfile(root)) + proc = project._run( + [runtime, "build", "-t", tag, "-f", str(containerfile), context], cwd=root + ) + if proc.returncode != 0: + raise ProjectError(_build_failure(tag, proc.stderr)) + + archive.parent.mkdir(parents=True, exist_ok=True) + # Saved beside its final name and renamed into place: a save that + # dies midway must not leave a partial archive that the dirty-tree + # refusal would then tell the user to commit. + partial = archive.parent / "image.partial" + save_format = ["--format", "docker-archive"] if runtime == "podman" else [] + try: + _check_call([runtime, "save", *save_format, "-o", str(partial), tag], cwd=root) + except ProjectError: + partial.unlink(missing_ok=True) + raise + partial.replace(archive) + + for key, value in ( + ("image", relative), + # Doubled braces survive datalad's record-time `.format`, so + # `{pwd}` is substituted at run time like any datalad command. + # The template names the runtime that built the image — best- + # effort interop for humans, outside lc's guarantees either way. + ( + "cmdexec", + f"{runtime} run --rm -v {{{{pwd}}}}:{{{{pwd}}}} -w {{{{pwd}}}} " + "docker-archive:{img} {cmd}", + ), + ): + dataset._git( + ["config", "-f", ".datalad/config", f"datalad.containers.{tag}.{key}", value], + cwd=root, + ) + dataset.save( + root, + [archive.parent, root / ".datalad" / "config"], + f"Add the system-layer image {tag}", + ) + + +def _build_failure(tag: str, stderr: str) -> str: + """Turn a build log into the refusal it means. Never the raw log.""" + for line in stderr.splitlines(): + if "Unable to locate package" in line: + package = line.rsplit(" ", 1)[-1] + return ( + f"no apt package named `{package}` — check the name in " + f"[tool.lightcone.image] apt-install (search with " + f"`apt-cache search {package}`)." + ) + # Anchored on the failing instruction as well as the code: the error + # names the STEP that died, and a user's own run-command exiting 43 + # (curl does) must not be diagnosed as a musl base. + contract = { + ("43", "ldd --version"): "the base image is musl-based — manylinux wheels and " + "uv-managed interpreters need glibc; use a Debian-family `base`.", + ("44", "command -v bash"): "the base image has no bash, and recipes run through " + "`bash -c` — use a base that carries bash, or a Debian-family `base`.", + ("45", "command -v apt-get"): "the base image has no apt, and `apt-install` is " + "declared — use a Debian-family `base`, or move the packages to `run-commands`.", + } + for (code, instruction), message in contract.items(): + if re.search(rf"exit (status|code):? {code}\b", stderr) and instruction in stderr: + return message + tail = "\n".join(stderr.strip().splitlines()[-15:]) + return f"building `{tag}` failed:\n{tail}" + + +# ============================================================================= +# The local store, and the podman machine +# ============================================================================= + + +def _loaded(root: Path, runtime: str, image_id: str) -> bool: + """Whether the runtime's local store already holds *image_id*.""" + probe = ["image", "exists" if runtime == "podman" else "inspect", image_id] + return project._run([runtime, *probe], cwd=root).returncode == 0 + + +def _machine_preflight(root: Path) -> None: + """Refuse a macOS podman machine that cannot mount what a run needs. + + podman on macOS runs inside a Linux VM, and a bind mount whose source + is outside the VM's shared directories arrives *empty* — no error, + just a project with nothing in it. Linux needs none of this. + """ + if sys.platform != "darwin": + return + proc = project._run(["podman", "machine", "inspect"], cwd=root) + if proc.returncode != 0: + raise ProjectError( + "containerized mode on macOS runs in a podman machine, and there is " + "none — one-time setup:\n podman machine init\n podman machine start" + ) + try: + machine = json.loads(proc.stdout)[0] + state = str(machine.get("State", "")) + mounts = [str(m.get("Source", "")) for m in machine.get("Mounts", [])] + except (json.JSONDecodeError, IndexError, KeyError, TypeError): + return # an unreadable inspect is not a refusal; the mount will speak + if state and state != "running": + # `inspect` succeeds on a stopped machine, so without this the + # preflight passes and the run dies later on a raw connection + # error, far from the one-command fix. + raise ProjectError( + f"the podman machine is {state}, not running — start it:\n" + " podman machine start" + ) + shared = [m for m in mounts if m] + if not any(str(root) == m or str(root).startswith(m.rstrip("/") + "/") for m in shared): + # An empty share list refuses too: a bind whose source the VM + # does not share arrives *empty*, no error — a project with + # nothing in it. (Declared inputs outside the tree are not + # checked here; their mounts carry the same risk, recorded.) + raise ProjectError( + f"{root} is outside the podman machine's shared directories " + f"({', '.join(shared) or 'none'}), so its bind mount would arrive " + f"empty. Share it:\n podman machine stop\n" + f" podman machine set --volume {root}\n podman machine start" + ) + + diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index cab0cc96..4449de0d 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -209,6 +209,18 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: `git add` and whose tools very much do open files for update, so it is passed per-add and never written to the repository's config. + ``annex.dotfiles`` is set for the same reason ``annex.thin`` is: + without it, git-annex routes any file under a dot-directory to git + *regardless* of what ``annex.largefiles`` says — so the image archive + under ``.datalad/environments/``, or a ``.cache.h5`` a recipe writes + into its output directory, would land as a full blob in git, + silently, with every test green, and every clone would carry the + bytes forever. With it, ``annex.largefiles`` alone decides — which is + the storage policy the ``.gitattributes`` template already states, + dot-named manifests staying in git through their own exemption. + Per-add and never written to the repository's config, so a user's + own ``git add`` keeps git-annex's stock behavior. + Args: directory: The repository root. paths: What to stage, absolute or repository-relative. @@ -218,7 +230,10 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: False if there was nothing to commit. """ relative = [_rel(directory, p) for p in paths] - _git(["-c", "annex.thin=true", "add", "-A", "--", *relative], cwd=directory) + _git( + ["-c", "annex.thin=true", "-c", "annex.dotfiles=true", "add", "-A", "--", *relative], + cwd=directory, + ) if _git_ok(["diff", "--cached", "--quiet"], cwd=directory): return False _git(["commit", "-q", "-m", message], cwd=directory) diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py index a0e0d1c8..505444c8 100644 --- a/src/lightcone/engine/identity.py +++ b/src/lightcone/engine/identity.py @@ -8,8 +8,9 @@ the spec describes, so keeping it would be mislabelling it. ``env_version`` is the environment's identity: the lock's bytes, the -interpreter pin's bytes, and the settings that decide *which artifacts* -``uv sync`` materializes from that lock. It is deliberately +interpreter pin's bytes, the settings that decide *which artifacts* +``uv sync`` materializes from that lock, and — for a containerized +project — the system layer's identity document. It is deliberately over-sensitive — raw lock bytes, so a comment reflow moves it — because the alternative is a parse that silently disagrees with uv about what the lock means. @@ -47,6 +48,7 @@ from pathlib import Path from typing import Any +from lightcone.engine import image from lightcone.engine.project import ProjectError #: The closed, audited list of uv settings that change *which artifacts* a @@ -75,9 +77,11 @@ def env_version(root: Path) -> str: """Compute the environment identity of a project. ``sha256(uv.lock bytes ‖ .python-version bytes ‖ canonical - install-settings JSON)``, length-framed. Read fresh on every call: this - is also the mid-run gate's baseline, and a cached value would check - nothing. + install-settings JSON ‖ canonical image identity document)``, + length-framed. A direct-mode project hashes the literal ``null`` for + the image term, so this stays one formula rather than two. Read fresh + on every call: this is also the mid-run gate's baseline, and a cached + value would check nothing. Args: root: The project root. @@ -100,6 +104,7 @@ def env_version(root: Path) -> str: _frame(h, "uv.lock", lock.read_bytes()) _frame(h, "python-version", pin.read_bytes()) _frame(h, "install-settings", _install_settings(root).encode()) + _frame(h, "image", (image.identity_document(root) or "null").encode()) return f"sha256:{h.hexdigest()}" diff --git a/src/lightcone/engine/image.py b/src/lightcone/engine/image.py new file mode 100644 index 00000000..b7a6f8ae --- /dev/null +++ b/src/lightcone/engine/image.py @@ -0,0 +1,386 @@ +"""The system layer: what a containerized project declares, and its identity. + +A project escalates to containerized mode by declaring a +``[tool.lightcone.image]`` table in ``pyproject.toml`` — the table's +presence *is* the escalation, and it is the whole user-facing surface. +Nobody ever writes or sees a Containerfile: the render exists only inside +a transient build context, and everything about the image is derived from +the declaration plus this module's constants. + +The surface is deliberately shaped like Modal's image builders, as TOML: +``base`` (``from_registry``), ``apt-install`` (``apt_install``), +``run-commands`` (``run_commands``) and ``env`` (``env``). There is no +``pip_install`` equivalent on purpose — the Python environment is the +lock's business, never the image's. ``run-commands`` is the bounded +escape for anything apt cannot say. + +Two derived identities, for two different questions. The **identity +document** is canonical JSON of the resolved declaration plus the +generator's pinned inputs; it feeds ``env_version``, so a system-layer +change puts outputs behind. The **tag** additionally hashes the rendered +Containerfile text, so a generator change rebuilds the image even when +the document is unchanged. Code is an input to neither — the image +carries no project files, so a code edit can never trigger a build. + +This module is pure: no subprocess, and no filesystem beyond reading +``pyproject.toml`` and ``.python-version``. + +The render is assembled by hand rather than through a library, checked +rather than assumed (2026-08): no established Containerfile-*authoring* +library exists — the maintained ones are Engine clients, and the +generators are small templating wrappers — and everything hard here is +the validation and the identity, which none of them carry. Assembling +lines from a validated model is also what the ecosystem's own generators +(Modal, repo2docker) do. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from lightcone.engine.project import ProjectError + +#: The default base image. Digest-pinned to the multi-arch manifest list, +#: so the reference is one string on every architecture and the image +#: stays a pure function of the repository plus the engine. An engine +#: release that bumps this moves both the tag and ``env_version`` for +#: containerized projects — the system layer genuinely changed. +#: +#: Vanilla debian-slim, deliberately *without* Python — the same family +#: as Modal's ``debian_slim()`` default but not the same content: Modal's +#: carries their own Python build, while here the base never supplies the +#: interpreter. uv installs the exact ``.python-version`` pin during the +#: build, so "which python ran" is identical under any base — swap in a +#: CUDA image and the interpreter, and its identity, do not move. +DEFAULT_BASE = ( + "docker.io/library/debian:bookworm-slim" + "@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241" +) + +#: Where the pinned uv binary is copied from. Manifest-list digest, so the +#: text is architecture-independent while each build gets its own arch. +UV_IMAGE = ( + "ghcr.io/astral-sh/uv:0.12.5" + "@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1" +) + +#: Where the pinned interpreter is installed inside the image. +PYTHON_INSTALL_DIR = "/opt/python" + +#: The closed key set. Closed because every key is hashed into identity — +#: a key that did nothing would still move ``env_version``, and one that +#: did something unhashed would be an identity hole. +_KEYS = ("base", "apt-install", "run-commands", "env") + + +@dataclass(frozen=True) +class Declaration: + """A parsed ``[tool.lightcone.image]`` table, defaults resolved.""" + + #: The digest-pinned base reference — declared, or :data:`DEFAULT_BASE`. + base: str + apt_install: tuple[str, ...] + run_commands: tuple[str, ...] + env: tuple[tuple[str, str], ...] + + +def declaration(root: Path) -> Declaration | None: + """Read the project's system-layer declaration. + + Args: + root: The project root. + + Returns: + The parsed declaration, or ``None`` for a direct-mode project. + + Raises: + ProjectError: If the table carries an unknown key, a wrong type, + or a ``base`` that is not digest-pinned. Raised at parse time + so the refusal fires on every verb that reads identity, not + just at build time. + """ + table = _table(root) + if table is None: + return None + if unknown := sorted(set(table) - set(_KEYS)): + raise ProjectError( + f"[tool.lightcone.image] has no key `{unknown[0]}` — the surface is " + f"{', '.join(f'`{k}`' for k in _KEYS)}, and every key is part of the " + "environment's identity, so nothing unrecognised can be carried along." + ) + base = table.get("base", DEFAULT_BASE) + if not isinstance(base, str) or not base: + raise ProjectError("[tool.lightcone.image] `base` must be an image reference string.") + if "@sha256:" not in base: + raise ProjectError( + f"[tool.lightcone.image] `base` is not digest-pinned: `{base}`. A tag can " + "move under the project, so the image would stop being a pure function of " + "the repository. Pin it: `base = \"@sha256:\"` " + "(find the digest with `podman image inspect` after a pull)." + ) + if re.search(r"\s", base): + raise ProjectError(f"[tool.lightcone.image] `base` is not an image reference: `{base}`.") + return Declaration( + base=base, + apt_install=tuple(sorted(_strings(table, "apt-install"))), + run_commands=tuple(_strings(table, "run-commands")), + env=tuple(sorted(_env(table).items())), + ) + + +def identity_document(root: Path) -> str | None: + """Build the canonical JSON that is the image's identity. + + The resolved declaration plus the generator's pinned inputs. Every + key is emitted whether or not the project set it — the + install-settings discipline: a project relying on a default and one + spelling it out are the same environment only until the default + changes. The interpreter pin is deliberately absent: its raw bytes + are already a frame of ``env_version``, and the rendered + Containerfile carries it into the tag. + + Args: + root: The project root. + + Returns: + The document, or ``None`` for a direct-mode project. + """ + declared = declaration(root) + return None if declared is None else _document(declared) + + +def _document(declared: Declaration) -> str: + """*declared* as its canonical JSON.""" + return json.dumps( + { + "apt": list(declared.apt_install), + "base": declared.base, + "env": dict(declared.env), + "run": list(declared.run_commands), + "uv": UV_IMAGE, + }, + sort_keys=True, + separators=(",", ":"), + ) + + +def containerfile(root: Path) -> str: + """Render the Containerfile for this project's system layer. + + Generated and transient — it exists only inside a build context and + is never written into the project. The layering is fixed here, never + user-ordered: base, contract checks, apt, the pinned uv, the pinned + interpreter, then the declared ``env`` and ``run-commands``. The + contract checks turn a base that cannot work into a pointed refusal + (via their reserved exit codes) instead of a downstream mystery. + + Args: + root: The project root, which must be containerized. + + Returns: + The Containerfile text. + + Raises: + ProjectError: If the project is direct-mode, the declaration is + invalid, or ``.python-version`` is missing. + """ + declared = declaration(root) + if declared is None: + raise ProjectError(f"{root} declares no [tool.lightcone.image] — nothing to build.") + return _render(root, declared) + + +def _render(root: Path, declared: Declaration) -> str: + """The Containerfile for *declared* — one parse, however it is reached.""" + pin = root / ".python-version" + if not pin.is_file(): + raise ProjectError( + f"{root}: no .python-version — the image bakes the exact interpreter, " + "so the pin is an input to it; run `lc init` to scaffold one." + ) + version = pin.read_text().strip() + # One version token: the pin splices into the install layer's RUN + # line, and a multi-line or annotated file would splice instructions. + if not re.match(r"^[A-Za-z0-9.+@-]+$", version): + raise ProjectError( + f"{root}/.python-version: `{version!r}` is not a single interpreter " + "version — the image bakes exactly one; run `lc init` to repin." + ) + + lines = [ + f"FROM {declared.base}", + # The contract checks, each a reserved exit code the builder maps + # to a refusal naming the base: 43 musl, 44 no bash, 45 no apt. + "RUN if ldd --version 2>&1 | grep -qi musl; then exit 43; fi", + "RUN command -v bash >/dev/null || exit 44", + ] + if declared.apt_install: + lines += [ + "RUN command -v apt-get >/dev/null || exit 45", + "RUN apt-get update && apt-get install -y --no-install-recommends " + + " ".join(declared.apt_install) + + " && rm -rf /var/lib/apt/lists/*", + ] + lines += [ + f"COPY --from={UV_IMAGE} /uv /usr/local/bin/uv", + # The chmod rides in the same layer as the install: rootless uid + # mapping runs as an arbitrary user, so everything lc bakes must + # be world-readable — and a chmod layer of its own would + # copy-on-write every byte it touches, doubling the interpreter + # in every archive. + f"RUN UV_PYTHON_INSTALL_DIR={PYTHON_INSTALL_DIR} uv python install " + + version + + " && chmod -R a+rX /opt", + ] + lines += [f"ENV {key}={_quoted(value)}" for key, value in declared.env] + lines += [f"RUN {command}" for command in declared.run_commands] + if declared.run_commands: + # Only when the user's own layers exist: whatever they put in + # /opt needs the same readability, at the price of re-copying it. + lines.append("RUN chmod -R a+rX /opt") + lines += [ + # Set after `uv python install`, which needs the download the + # final environment then forbids. `never` makes a missing + # interpreter a loud error at run time instead of a silent fetch. + f"ENV UV_PYTHON_INSTALL_DIR={PYTHON_INSTALL_DIR} UV_PYTHON_DOWNLOADS=never", + f"LABEL io.lightcone.image={_quoted(_document(declared))}", + ] + return "\n".join(lines) + "\n" + + +def tag(root: Path) -> str: + """Derive the image tag from everything that goes into the image. + + The rendered Containerfile *and* the identity document, framed as a + JSON list so a boundary between them cannot shift. The render is an + input so a generator change rebuilds; the document alone is what + ``env_version`` sees. + + Args: + root: The project root, which must be containerized. + + Returns: + ``lc-env-<16 hex>``. + """ + declared = declaration(root) + if declared is None: + raise ProjectError(f"{root} declares no [tool.lightcone.image] — nothing to tag.") + framed = json.dumps([_render(root, declared), _document(declared)]) + return "lc-env-" + hashlib.sha256(framed.encode()).hexdigest()[:16] + + +def archive_path(root: Path, image_tag: str) -> Path: + """Locate the committed image archive for *image_tag*. + + The one spelling of where an image lives in the dataset — the same + layout ``datalad containers-add`` uses, so the archive is versioned + project state that travels through the annex. + + Args: + root: The project root. + image_tag: The image tag, from :func:`tag`. + + Returns: + ``/.datalad/environments//image``. + """ + return root / ".datalad" / "environments" / image_tag / "image" + + +def _table(root: Path) -> dict[str, Any] | None: + """The raw ``[tool.lightcone.image]`` table, or ``None`` if absent. + + Read from ``pyproject.toml`` only — never through the uv-config + reader, whose "a ``uv.toml`` replaces ``[tool.uv]``" rule is about + uv's own settings and must not reach this table. + """ + path = root / "pyproject.toml" + if not path.is_file(): + return None + try: + parsed = tomllib.loads(path.read_text()) + except tomllib.TOMLDecodeError as e: + raise ProjectError(f"{path}: invalid TOML: {e}") from e + table = parsed.get("tool", {}).get("lightcone", {}).get("image") + if table is None: + return None + if not isinstance(table, dict): + raise ProjectError("[tool.lightcone.image] must be a table.") + return table + + +#: What an apt package name may contain (Debian source/package charset). +#: Anything else is joined verbatim into a shell line by the apt layer, +#: so the closed charset is what keeps `apt-install` a list of *names* +#: rather than a second `run-commands`. +_APT_NAME = re.compile(r"^[a-z0-9][a-z0-9.+-]*$") + +#: A shell-safe environment variable name. Anything else hits Docker's +#: legacy `ENV key value` parse and silently defines the wrong variable. +_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def _strings(table: dict[str, Any], key: str) -> list[str]: + """A list-of-strings key, defaulting empty. No control characters: + every value here is interpolated into a Containerfile line, and a + newline would splice an instruction of its own into an + identity-hashed surface.""" + value = table.get(key, []) + if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value): + raise ProjectError(f"[tool.lightcone.image] `{key}` must be a list of strings.") + for entry in value: + _plain(key, entry) + if key == "apt-install": + for name in value: + if not _APT_NAME.match(name): + raise ProjectError( + f"[tool.lightcone.image] apt-install: `{name}` is not an apt " + "package name — names are lowercase letters, digits, `.`, `+` " + "and `-`. A command belongs in `run-commands`." + ) + return value + + +def _env(table: dict[str, Any]) -> dict[str, str]: + """The ``env`` key, defaulting empty. Keys must be shell-safe names + and values control-character-free, for the reason `_strings` gives.""" + value = table.get("env", {}) + if not isinstance(value, dict) or not all( + isinstance(k, str) and isinstance(v, str) for k, v in value.items() + ): + raise ProjectError("[tool.lightcone.image] `env` must be a table of strings.") + for key, entry in value.items(): + if not _ENV_KEY.match(key): + raise ProjectError( + f"[tool.lightcone.image] env: `{key}` is not an environment " + "variable name (letters, digits and `_`, not starting with a digit)." + ) + _plain("env", entry) + return value + + +def _plain(key: str, value: str) -> None: + """Refuse control characters in a value bound for a Containerfile line.""" + if any(ord(c) < 0x20 or ord(c) == 0x7F for c in value): + raise ProjectError( + f"[tool.lightcone.image] `{key}` values cannot contain control " + "characters — each renders into a single Containerfile line." + ) + + +def _quoted(value: str) -> str: + """Quote a value for a Containerfile ``ENV``/``LABEL`` line. + + ``$`` is escaped along with the quoting characters, because these + lines undergo build-time variable expansion — measured: an unescaped + ``cost$5`` bakes as ``cost``, and a ``$`` inside the identity LABEL + silently corrupts the document the image carries. Declared ``env`` + values are therefore *literals*, never expansions. + """ + escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("$", "\\$") + return f'"{escaped}"' diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 90087a65..a14ae5cb 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -41,7 +41,7 @@ from pathlib import Path from typing import Any, Protocol -from lightcone.engine import assets, dataset, identity, plan, project, worker +from lightcone.engine import assets, container, dataset, identity, plan, project, worker from lightcone.engine.plan import Graph, Key, Task from lightcone.engine.project import ProjectError @@ -191,7 +191,8 @@ def _classified( report.warnings.append( "reported as out of date because their content is not in this " f"clone, not because they changed: {', '.join(sorted(unfetched))}. " - "Fetch with `git annex get `." + "`lc materialize` fetches declared inputs before it decides " + "anything, so there this resolves itself." ) return classified @@ -287,6 +288,17 @@ class StatusReport: outputs: list[OutputStatus] = field(default_factory=list) #: The lock scan's prose, and inputs this clone has not fetched. warnings: list[str] = field(default_factory=list) + #: The three header facts nothing else surfaces: which world this + #: project executes in, where its image stands, and what would + #: enforce a run on this host. This is where the denial note and the + #: runtime-missing refusal point. + mode: str = "direct" + #: ``{tag, state}`` for a containerized project; ``None`` in direct + #: mode. ``state`` is repository fact only — ``present``, ``absent`` + #: or ``unfetched`` — so status needs no runtime and no network. + image: dict[str, str] | None = None + #: One line naming the enforcement a run here would get. + sandbox: str = "" @property def counts(self) -> dict[str, int]: @@ -303,6 +315,9 @@ def as_dict(self) -> dict[str, Any]: The counts, then every output, then the warnings. """ return { + "mode": self.mode, + "image": self.image, + "sandbox": self.sandbox, "counts": self.counts, "outputs": [output.as_dict() for output in self.outputs], "warnings": self.warnings, @@ -329,6 +344,11 @@ def status(root: Path) -> StatusReport: """ report = MaterializeReport() result = StatusReport() + result.mode = project.mode(root) + state, tag, archive = container.image_state(root) + if state != "direct": + result.image = {"tag": tag, "state": state, "archive": archive} + result.sandbox = _sandbox_line(result.mode) for key, verdict, manifest in _classified(root, [], report, refresh=False): result.outputs.append( OutputStatus( @@ -343,6 +363,27 @@ def status(root: Path) -> StatusReport: return result +def _sandbox_line(mode: str) -> str: + """One line naming the enforcement a run on this host would get. + + The prose restates each backend's constant attestation, because + `Backend.attest` needs a built policy and a status header must not + build one. Keep it in step with the `attest` implementations — the + manifests, which record the real thing, are always authoritative. + """ + if mode == "containerized": + if runtime := container.runtime_hint(): + return f"{runtime} (fs: declared, network: denied)" + return "no container runtime — install podman (or docker)" + from lightcone.engine import sandbox + + found = sandbox.detect().capability + if found.kind == "none": + detail = f" — {found.detail}" if found.detail else "" + return f"none{detail}; runs record `fs: open`" + return f"{found.kind} (fs: declared, network: allowed)" + + # ============================================================================= # Executing # ============================================================================= @@ -373,17 +414,29 @@ def materialize( project.require_git() project.require_git_annex() dataset.require_committer(root) - # Before anything else: workers pass `--no-sync`, so this is the only - # place on a run's path where the environment is made to match the - # lock. (A rerun does not come through here; its entry point syncs.) report = MaterializeReport() - report.warnings.extend(f"uv: {w}" for w in project.sync(root)) + # The dirty check comes before anything that writes: the image + # converge below *commits*, and `dataset.save` stages scoped but + # commits the whole index — on a dirty tree the user's staged edits + # would be swept into the image commit. if changes := dataset.status(root): raise ProjectError(_dirty(root, changes)) - + # The graph — and with it the spec validation and the lock scan — + # before the image: a refusal here must not cost a minutes-long + # build, and must not leave an archive commit behind a run that + # "failed" on a typo in the spec. graph, env_version = _graph(root, targets, report) if not graph.tasks: return report + _fetch_inputs(root, graph, report) + # Materialize is one of the two verbs allowed to build the image (the + # other is `lc build`); the probe and the rerun entry point only find + # one. Resolved once, then handed to every task — the HEAD discipline. + runtime = container.runtime_for_run(root, build=True) + # Converge the environment: workers pass `--no-sync`, so this is the + # only place on a run's path where it is made to match the lock. (A + # rerun does not come through here; its entry point converges too.) + report.warnings.extend(f"uv: {w}" for w in container.converge(runtime)) dsid = dataset.dataset_id(root) # Read once, for every task: the driver commits each output as it lands, @@ -412,11 +465,12 @@ def materialize( head, versions, refresh, + runtime, *[pending[dep] for dep in task.depends_on], key=_name(key), ) for result in scheduler.completed(list(pending.values())): - _consume(root, graph.tasks[result.key], result, dsid, report) + _consume(root, graph.tasks[result.key], result, dsid, runtime, report) outstanding.pop(result.key, None) finally: # Whatever never reported — an interrupt, a dead cluster — left a @@ -429,7 +483,12 @@ def materialize( def _consume( - root: Path, task: Task, result: worker.TaskResult, dsid: str, report: MaterializeReport + root: Path, + task: Task, + result: worker.TaskResult, + dsid: str, + runtime: container.Runtime, + report: MaterializeReport, ) -> None: """Record one finished task, and commit or undo what it left on disk.""" name = _name(task.key) @@ -440,7 +499,7 @@ def _consume( report.notes.extend([f"{name}:", *lines]) if result.status == "ok": - dataset.save(root, [task.output_dir], run_record(root, task, dsid)) + dataset.save(root, [task.output_dir], run_record(root, task, dsid, runtime)) report.made.append(name) return @@ -540,12 +599,47 @@ def cluster_for_run() -> Iterator[Scheduler]: yield _Dask(client) +def _fetch_inputs(root: Path, graph: Graph, report: MaterializeReport) -> None: + """Bring declared inputs' bytes into this clone before anything hashes. + + lc fetches rather than telling anyone to — the storage invariant — + and only here: ``--check`` and ``status`` are read-only verbs that + must not start network transfers, so there an unfetched input stays a + reported fact. In-tree inputs only, because an absolute input outside + the repository has no annex to fetch it from (the recorded weaker + promise). Unconditional rather than probed: ``git annex get`` on + content already present is a fast no-op, and one batch invocation + beats a detection walk. + + A failed fetch is a *warning*, never a refusal: independent tasks + still run, and the task whose input is genuinely unreachable reports + its own failure with the real error — per-task reporting is most of + what owning the loop buys. + """ + declared = sorted( + { + plan.declared_path(root, path) + for task in graph.tasks.values() + for name, path in task.inputs.items() + if name not in task.produced_by and root in path.parents + } + ) + if not declared: + return + got = project._run(["git", "annex", "get", "--", *declared], cwd=root) + if got.returncode != 0: + report.warnings.append( + "some declared inputs could not be fetched into this clone — tasks " + f"needing them will report it:\n{got.stderr.strip()}" + ) + + # ============================================================================= # The commit # ============================================================================= -def run_record(root: Path, task: Task, dsid: str) -> str: +def run_record(root: Path, task: Task, dsid: str, runtime: container.Runtime) -> str: """Build the commit message for one materialized output. A ``[DATALAD RUNCMD]`` record — datalad's format, not ours, so all of @@ -565,6 +659,13 @@ def run_record(root: Path, task: Task, dsid: str) -> str: root: The project root. task: The output that was made. dsid: The dataset's UUID, which ``rerun`` reads. + runtime: The run's execution world. A containerized run lists its + committed image archive under ``extra_inputs``, which + ``datalad rerun`` fetches before executing — so the worker + enters the exact bytes that made the output, on whatever + runtime the rerun host has. The ``cmd`` itself stays + runtime-neutral for the same reason: the worker is the + executor that resolves the container at rerun time. Returns: The full commit message, subject and record. @@ -583,6 +684,8 @@ def run_record(root: Path, task: Task, dsid: str) -> str: "outputs": [plan.declared_path(root, task.output_dir)], "pwd": ".", } + if runtime.mode == "containerized": + info["extra_inputs"] = [runtime.archive] body = json.dumps(info, indent=1, sort_keys=True, ensure_ascii=False) return ( f"[DATALAD RUNCMD] {task.output_id} [{task.universe_id}]\n\n" diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index b8304e0e..a139f245 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -12,6 +12,7 @@ from dataclasses import asdict, dataclass, field from functools import partial from pathlib import Path +from typing import Literal from astra.scaffold import create_boilerplate @@ -238,12 +239,20 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: lambda: _uv(c, ["lock"], directory=directory), is_current=lambda: _lock_is_current(directory), ) - c.item( - ".venv", - (directory / ".venv").exists(), - lambda: _uv(c, _SYNC_ARGS, directory=directory), - is_current=lambda: _env_is_current(directory), - ) + # Locking is resolution and works on the bare host in both modes; a + # *sync* of a containerized project is the host-sync deadlock in + # miniature — the lock's system-level dependencies (the reason the + # project containerized at all) are not on the host, so the sync + # fails and `--check` would report unconverged forever. The + # environment converge in containerized mode belongs to the verbs, + # which run it inside the image. + if mode(directory) == "direct": + c.item( + ".venv", + (directory / ".venv").exists(), + lambda: _uv(c, _SYNC_ARGS, directory=directory), + is_current=lambda: _env_is_current(directory), + ) return c.report @@ -320,13 +329,16 @@ def uv_prefix(directory: Path, *, sync: bool) -> list[str]: def sync(directory: Path) -> list[str]: - """Make ``.venv`` match ``uv.lock``. + """Make the host ``.venv`` match ``uv.lock``. Direct mode's converge. - Both entry points that execute recipes call this before they start, + Both entry points that execute recipes converge before they start, rather than checking and refusing: workers pass ``--no-sync``, so nothing else would notice a lock edited without a sync, and a manifest recording an environment the recipe did not run under is the identity - model saying something untrue. + model saying something untrue. The containerized twin is + ``container.sync``, which runs the same uv command inside the image — + callers pick by :func:`mode`, because only they know whether an image + has been ensured. Args: directory: The project root. @@ -475,6 +487,55 @@ def _can_ask_git(directory: Path) -> bool: return directory.is_dir() and _in_repository(directory) +def mode(directory: Path) -> Literal["direct", "containerized"]: + """Read which execution world this project declares. + + Derived, never configured: a ``[tool.lightcone.image]`` table in + ``pyproject.toml`` *is* the escalation to containerized mode, and + deleting it is the way back. Presence only — what the table means is + :mod:`~lightcone.engine.image`'s question, asked by the verbs that + consume it, so an invalid declaration refuses there with the line at + fault rather than here with none. + + Args: + directory: The project root. + + Returns: + ``"containerized"`` if the table is declared, else ``"direct"`` — + including for a directory with no ``pyproject.toml`` yet, which is + what lets convergence scaffold one. + + Raises: + ProjectError: If ``pyproject.toml`` is not valid TOML. + """ + # image.py is the table's one reader; imported lazily because it + # imports ProjectError from here. + from lightcone.engine import image + + return "direct" if image._table(directory) is None else "containerized" + + +def env_dir(directory: Path) -> Path: + """Locate the project environment for this project's mode. + + The one spelling of where the environment lives. Direct mode syncs + ``.venv`` on the host; containerized mode syncs ``.lightcone/venv`` + *inside* the image, against the baked interpreter — so its symlinks + dangle on the host, deliberately, and the host ``.venv`` (if any) is + inert. Two directories, because sharing one means every sync flips it + between worlds and breaks whichever side is not looking. + + Args: + directory: The project root. + + Returns: + The environment prefix for the current mode. + """ + if mode(directory) == "containerized": + return directory / ".lightcone" / "venv" + return directory / ".venv" + + def project_name(directory: Path) -> str: """Derive a project name from a directory name. @@ -524,7 +585,10 @@ def current_project(directory: Path | None = None) -> Path: The question every verb that runs something asks, and the stronger of the two: a ``.venv`` that is not there is not something these callers - are going to create. + are going to create. In containerized mode the host ``.venv`` is + inert and never required — the environment those verbs enter is + ``.lightcone/venv``, which their own converge creates inside the + image, so only what the repository carries is asked for. Args: directory: Defaults to the working directory. @@ -533,9 +597,12 @@ def current_project(directory: Path | None = None) -> Path: The resolved project root. Raises: - ProjectError: If any of the environment is absent. + ProjectError: If any of the required environment is absent. """ - return _project_root(directory, _ENVIRONMENT_FILES) + resolved = (directory or Path.cwd()).resolve() + if mode(resolved) == "containerized": + return _project_root(resolved, _DECLARED_FILES) + return _project_root(resolved, _ENVIRONMENT_FILES) def _project_root(directory: Path | None, required: Sequence[str]) -> Path: diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index d152f2a8..e406909d 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -18,13 +18,19 @@ from pathlib import Path from typing import Any -from lightcone.engine import sandbox +from lightcone.engine import container, sandbox from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv, uv_prefix def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: """Run a command in the project environment, inside the boundary. + A containerized probe never builds the image — it finds one, or + refuses naming the exact ``lc build`` — and converges the in-image + environment before executing, which is the same promise the direct + probe makes through its syncing ``uv run`` hop: the environment a + probe describes is one it just converged. + Args: project: The project root. command: The argv to run. Required — there is deliberately no bare @@ -38,14 +44,24 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: require_uv() spec = read_spec(project) - built = sandbox.exec_policy(project, read_paths=input_paths(project, spec)) + runtime = container.runtime_for_run(project, build=False) + if runtime.mode == "containerized": + # The probe's converge. Direct mode's is the syncing hop below — + # the deliberate exception to `container.converge`, because there + # the hop itself is what converges. + container.sync(project, runtime) + + built = container.policy_for(runtime, input_paths(project, spec)) with sandbox.scope(built) as policy: return sandbox.run( - sandbox.detect(), + container.backend(runtime), policy, list(command), cwd=project, - prefix=uv_prefix(project, sync=True), + # The direct hop converges; the containerized one must not — + # the converge above already did, and the exec runs with the + # network denied, where a sync cannot. + prefix=uv_prefix(project, sync=runtime.mode == "direct"), # Same reason as convergence: this uv invocation names its # project explicitly, so an environment activated elsewhere # is never what we mean — and uv says so, once per run, in diff --git a/src/lightcone/engine/sandbox/boundary.py b/src/lightcone/engine/sandbox/boundary.py index 8ccf4660..2d719565 100644 --- a/src/lightcone/engine/sandbox/boundary.py +++ b/src/lightcone/engine/sandbox/boundary.py @@ -45,6 +45,8 @@ class Unavailable: """ capability: Capability = field(default_factory=lambda: Capability(kind="none")) + #: No mechanism, so nothing owns the prefix either. + contains_prefix: bool = False def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: """Return *argv* unchanged — there is no mechanism to wrap with.""" @@ -143,15 +145,22 @@ def run( cwd: Where to run it. env: The environment for everything outside the rewrite. The policy's own overlay is applied inside it, not merged here. - prefix: Spawned *outside* the rewrite — how a caller says "wrap - the command, not this". Used for the ``uv run`` hop, since - uv's config and caches are trusted plumbing. + prefix: The ``uv run`` hop. For a host mechanism it is spawned + *outside* the rewrite — uv's config and caches are trusted + plumbing. For a backend that is itself a world + (``contains_prefix``), it goes *inside*: there is no trusted + host plumbing inside a container, and the env overlay is + that backend's to apply natively rather than through a + host-resolved ``env``. Returns: The exit code, what was actually enforced, and any lines the caller should print verbatim. """ - wrapped = [*prefix, *backend.wrap(policy, [*env_argv(policy), *argv])] + if backend.contains_prefix: + wrapped = backend.wrap(policy, [*prefix, *argv]) + else: + wrapped = [*prefix, *backend.wrap(policy, [*env_argv(policy), *argv])] attestation = backend.attest(policy) # `policy.env` is deliberately **not** merged here: it went inside # the wrap, above, via :func:`env_argv`. Everything *outside* the @@ -183,14 +192,26 @@ def run( # module eagerly, and the shim drags ctypes in for one integer. from lightcone._sandbox_exec import SETUP_FAILURE_EXIT - if returncode == SETUP_FAILURE_EXIT: - # The shim's reserved code: the sandbox could not be *built*. Say - # so plainly — the trailer would otherwise point the user at their - # own command's permissions for a failure that is entirely ours. + if returncode == SETUP_FAILURE_EXIT and attestation.mechanism == "landlock": + # The shim's reserved code — meaningful only where the shim ran: + # under any other mechanism a command legitimately exiting 97 is + # just a failed command, and blaming lc for it would misattribute + # at the one place nobody is watching. Say the real case plainly — + # the trailer would otherwise point the user at their own + # command's permissions for a failure that is entirely ours. notes.append( "lc could not set up the sandbox (see above) — this is an lc " "problem, not your command's" ) + elif returncode == 125 and backend.contains_prefix: + # The runtimes reserve 125 for their own failures (a bad flag, a + # vanished mount source): the command never ran, so the denial + # heuristics have nothing to say about it. + notes.append( + f"the container runtime failed before the command ran (see " + f"above, `{attestation.mechanism}` exit 125) — this is a " + "runtime problem, not your command's" + ) elif returncode != 0 and attestation.mechanism != "none": from lightcone.engine.sandbox import denial diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py index 8a4f3bb6..713a8b3e 100644 --- a/src/lightcone/engine/sandbox/denial.py +++ b/src/lightcone/engine/sandbox/denial.py @@ -147,8 +147,15 @@ def _render_tool(path: Path) -> list[str]: return _message( f"cannot execute {path}", [ - " if this is a tool the command needs, declare it in the environment:", + " if this is a Python tool, declare it in the environment:", f" uv add ", + "", + " if this is a system tool, declare it in the system layer:", + " [tool.lightcone.image]", + f' apt-install = [""]', + " (apt package names — unsure? try: apt-cache search )", + " note: this containerizes the project — podman (or docker) required —", + " and puts every materialized output behind.", ], ) diff --git a/src/lightcone/engine/sandbox/landlock.py b/src/lightcone/engine/sandbox/landlock.py index 70a5b189..ed439a75 100644 --- a/src/lightcone/engine/sandbox/landlock.py +++ b/src/lightcone/engine/sandbox/landlock.py @@ -73,6 +73,8 @@ class LandlockBackend: capability: Capability = field(default_factory=capability) interpreter: str = sys.executable + #: The uv hop stays outside the wrap: trusted host plumbing. + contains_prefix: bool = False def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: """Rewrite *argv* to run under the Landlock shim. diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index abd3ce30..86959f9a 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -75,7 +75,7 @@ def grants(self, path: Path, roots: tuple[Path, ...]) -> bool: class Capability: """What enforcement this host can provide, as probed.""" - kind: Literal["landlock", "seatbelt", "none"] + kind: Literal["landlock", "seatbelt", "podman", "docker", "none"] landlock_abi: int | None = None #: Why, when ``kind`` is ``none``. Reaches the user — a downgrade is #: never silent. @@ -87,14 +87,15 @@ class Attestation: """The hermeticity record for one exec. Derived from the flags actually applied, never from the mechanism - matrix's expectations. ``network`` is ``allowed`` on every path here - by recorded decision: lc applies no network restriction, and saying - so is the honest value. + matrix's expectations. ``network`` is ``allowed`` wherever lc applied + no restriction — the direct-mode mechanisms — and ``denied`` only + where a flag actually denied it, which today is the OCI backend's + ``--network none``. """ - mechanism: Literal["landlock", "seatbelt", "none"] + mechanism: Literal["landlock", "seatbelt", "podman", "docker", "none"] fs: Literal["declared", "open"] - network: Literal["allowed"] = "allowed" + network: Literal["allowed", "denied"] = "allowed" landlock_abi: int | None = None exec_allowlist_version: int | None = None @@ -108,6 +109,20 @@ class Backend(Protocol): testable on a host that cannot run it. """ + @property + def contains_prefix(self) -> bool: + """Whether the wrap owns the whole command line, prefix included. + + A host mechanism restricts the command and leaves the ``uv run`` + hop outside as trusted host plumbing; a backend that is itself a + *world* (a container) has no trusted host plumbing inside it — + uv is part of what is being entered — so the seam hands it the + prefix too, and the env overlay becomes the backend's to apply + natively. Declared on every backend rather than defaulted at the + call site, so a new mechanism must answer the question. + """ + ... + @property def capability(self) -> Capability: """What this backend probed on this host. diff --git a/src/lightcone/engine/sandbox/oci.py b/src/lightcone/engine/sandbox/oci.py new file mode 100644 index 00000000..b23e2940 --- /dev/null +++ b/src/lightcone/engine/sandbox/oci.py @@ -0,0 +1,125 @@ +"""The containerized backend: the mount table is the mechanism. + +One backend for podman and docker, data-parameterized — the two differ +in spellings (how the invoking uid is kept, whether pulling must be +forbidden), not in shape. The policy's path sets map one-to-one onto +mounts: ``read`` becomes ``:ro``, ``write`` becomes ``:rw``, and the +image itself is the OS baseline and the exec set — everything present in +it was declared, which is why the containerized policy carries no +baseline and no exec tier to translate. + +Unlike the host mechanisms, this wrap owns the whole command line +(``contains_prefix``): the ``uv run`` hop executes *inside* the world +being entered, and the env overlay is applied through the runtime's own +``--env`` rather than a host-resolved ``env`` binary — whose path (NixOS +keeps it under ``/run/current-system/sw``) need not exist in the image. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from lightcone.engine.sandbox.boundary import SANDBOX_ENV +from lightcone.engine.sandbox.model import Attestation, Capability, Policy + + +@dataclass(frozen=True) +class OCIBackend: + """A container runtime, expressed as an argv rewrite.""" + + #: ``podman`` or ``docker`` — also what the attestation names. + runtime: Literal["podman", "docker"] + #: The image id (bare hex). Execution pins on the id, never a tag, so + #: a retagged image in the local store can never substitute. + image_id: str + #: The project root — the container's working directory. + root: Path + #: How this runtime keeps mount writes owned by the invoking user, + #: resolved by the caller so the wrap stays a pure function of its + #: fields (``--userns=keep-id`` / ``--user uid:gid``). + user_flags: tuple[str, ...] = () + contains_prefix: bool = True + + @property + def capability(self) -> Capability: + """What this backend enforces with, named by the runtime.""" + return Capability(kind=self.runtime) + + def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: + """Rewrite *argv* into a container invocation of itself. + + Pure: no temporary files, no file descriptors, no global state. + Read roots mount ``:ro`` before write roots mount ``:rw``, so a + writable directory nested in a read-only tree lands in the order + the runtimes resolve natively. ``/tmp`` is a fresh tmpfs (the + policy's ``tmp_home`` is a bind inside it), ``/dev/shm`` gets a + real size because the 64 MB default is a scientific-workload + footgun, and the environment is an allowlist — the policy's + overlay as ``--env``, never the ambient environment. + + Args: + policy: What the command may touch, as mounts. + argv: The command, run prefix included. + + Returns: + The rewritten command. + """ + # Resolved source, declared destination: the host bind must name + # the real file, while the recipe addresses the path the analysis + # declared — a symlinked `/data` input mounted at its target + # would leave the container with no `/data` at all. Residue, + # recorded: a declared input living under `/tmp` is shadowed by + # the tmpfs on runtimes that mount over it — the same class of + # host-layout collision `_write_roots` documents for direct mode. + mounts = [f"--volume={path.resolve()}:{path}:ro" for path in policy.read] + mounts += [f"--volume={path.resolve()}:{path}:rw" for path in policy.write] + overlay = [f"--env={k}={v}" for k, v in sorted(policy.env.items())] + return [ + self.runtime, "run", "--rm", + "--network", "none", + "--entrypoint", "", + # The rootfs is read-only so a write outside the declared set + # is a loud denial rather than bytes vanishing with the + # container — without this, `mkdir /output` *succeeds* into + # the ephemeral layer and the run attests `fs: declared` + # over silently lost output. + "--read-only", + # SELinux hosts (Fedora/RHEL, podman's home turf) would + # otherwise refuse every bind read from container_t. Label + # separation is disabled rather than relabeling (`:z`), which + # would rewrite the user's own file contexts on disk. + "--security-opt", "label=disable", + *self.user_flags, + *mounts, + "--tmpfs", "/tmp:rw,exec", + "--shm-size", "1g", + "--workdir", str(self.root), + *overlay, + f"--env={SANDBOX_ENV}={self.runtime}", + self.image_id, + *argv, + ] # fmt: skip + + def attest(self, policy: Policy) -> Attestation: + """Report what the wrapped command will actually have enforced. + + Every value is a flag in :meth:`wrap`'s output: the mounts plus + the read-only rootfs bound the filesystem to the declared set, + and ``--network none`` is a real denial — the one place in the + codebase that honestly says ``denied`` (loopback stays intact, + which is the meaning of it). + + Args: + policy: The policy being wrapped. + + Returns: + The record written with every output. + """ + return Attestation( + mechanism=self.runtime, + fs="declared", + network="denied", + ) diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py index 6943b36f..d3e7fbf1 100644 --- a/src/lightcone/engine/sandbox/policy.py +++ b/src/lightcone/engine/sandbox/policy.py @@ -54,6 +54,11 @@ #: profile is listed too or nothing but `/bin/sh` ever resolves there. _UTILITY_PATH = "/usr/local/bin:/usr/bin:/bin:/run/current-system/sw/bin" +#: The ``PATH`` tail inside a containerized exec. The image's own FHS — +#: never :data:`_UTILITY_PATH`, whose NixOS entry names a directory no +#: Debian-family image has. +_IMAGE_PATH = "/usr/local/bin:/usr/bin:/bin" + def utility(name: str) -> Path | None: """Resolve an allowlisted tool from the fixed search path. @@ -169,7 +174,13 @@ def utility(name: str) -> Path | None: } -def exec_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: +def exec_policy( + project: Path, + *, + read_paths: Sequence[Path] = (), + env_dir: Path | None = None, + containerized: bool = False, +) -> Policy: """Build what a sandboxed command may touch. The tree is read-only apart from ``results/``, so ``lc run`` and a @@ -183,9 +194,19 @@ def exec_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: leaves a manifest that is self-consistent and wrong, which no checksum can see. + The containerized shape is the same policy with the host stripped + out: the *image* is the OS baseline and the exec set — everything + present in it was declared — so the path sets carry only the project + world, which is exactly what the OCI backend turns into mounts. One + builder for both, so a probe still gets what a recipe gets per mode. + Args: project: The project root, granted read. read_paths: Declared inputs outside the tree. + env_dir: The project environment prefix; defaults to the host + ``.venv``. + containerized: Build the mount-shaped policy instead of the host + one. Returns: The policy. ``results/`` is granted only if it exists — a policy @@ -193,11 +214,36 @@ def exec_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: the caller owns removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). """ - tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-")).resolve() + env_dir = env_dir if env_dir is not None else project / ".venv" + # The containerized HOME lives under the project's own (gitignored) + # `.lightcone/`, not the system temp dir: it is a mount source, and + # on macOS the podman machine shares the project's tree while the + # host's /var/folders temp roots arrive empty. + if containerized: + parent = project / ".lightcone" + parent.mkdir(parents=True, exist_ok=True) + tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-", dir=parent)).resolve() + else: + tmp_home = Path(tempfile.mkdtemp(prefix="lc-home-")).resolve() for sub in _HOME_LAYOUT.values(): (tmp_home / sub).mkdir(parents=True, exist_ok=True) - python = _venv_python(project) + if containerized: + # Declared spellings, not realpaths — the one shape that keeps + # its paths unresolved. These become mount *destinations*, and a + # recipe addresses the declared path: resolving here would mount + # a symlinked `/data/catalog.h5` at its target and leave the + # container with no `/data` at all. (The backend resolves the + # *source* side itself.) + return Policy( + read=_declared([project, *read_paths]), + write=_declared([tmp_home, project / "results"]), + execute=(), + tmp_home=tmp_home, + env=home_overlay(tmp_home, env_dir, containerized=True), + ) + + python = _venv_python(env_dir) # EXECUTE on the interpreter *file*; READ on the install root beside # it, for the stdlib. See :func:`_venv_python` and :func:`_stdlib_root`. stdlib = _stdlib_root(python) @@ -207,9 +253,9 @@ def exec_policy(project: Path, *, read_paths: Sequence[Path] = ()) -> Policy: return Policy( read=read, write=write, - execute=_existing(_exec_set(project, python)), + execute=_existing(_exec_set(env_dir, python)), tmp_home=tmp_home, - env=home_overlay(tmp_home, project), + env=home_overlay(tmp_home, env_dir), ) @@ -229,7 +275,7 @@ def _write_roots(project: Path) -> list[Path]: return [root for root in roots if not resolved.is_relative_to(root)] -def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: +def home_overlay(tmp_home: Path, env_dir: Path, *, containerized: bool = False) -> dict[str, str]: """Point ``HOME`` and friends at a fresh private directory. The real ``$HOME`` is neither readable nor writable inside the @@ -245,12 +291,16 @@ def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: Args: tmp_home: The per-run directory to point at. - project: The project, whose ``.venv/bin`` fronts ``PATH``. + env_dir: The project environment whose ``bin`` fronts ``PATH``. + containerized: Use the image's own FHS as the ``PATH`` tail + rather than the host allowlist search path, and pin uv to the + in-image environment — the ``uv run`` hop executes inside the + container, and this is how it finds ``.lightcone/venv``. Returns: The environment overlay the boundary applies inside the wrap. """ - return { + overlay = { "HOME": str(tmp_home), # The search path *is* the exec set. Without this the command # resolves tools through the host's ambient PATH while the policy @@ -258,20 +308,25 @@ def home_overlay(tmp_home: Path, project: Path) -> dict[str, str]: # whose PATH fronts another copy (homebrew's bash on macOS, say) # the sandbox denies `bash` itself, and the message blames the # user's command for lc's own incoherence. - "PATH": os.pathsep.join([str(project / ".venv" / "bin"), _UTILITY_PATH]), + "PATH": os.pathsep.join( + [str(env_dir / "bin"), _IMAGE_PATH if containerized else _UTILITY_PATH] + ), **{k: str(tmp_home / v) for k, v in _HOME_LAYOUT.items()}, } + if containerized: + overlay["UV_PROJECT_ENVIRONMENT"] = str(env_dir) + return overlay -def _venv_python(project: Path) -> Path | None: +def _venv_python(env_dir: Path) -> Path | None: """The realpath of the venv's interpreter, if there is one. - Resolved, because ``.venv/bin/python`` is a symlink and Landlock - evaluates the target. What gets granted on it is - :func:`_exec_set`'s decision, and its install root is separately a - read root (:func:`exec_policy`) for the standard library beside it. + Resolved, because ``bin/python`` is a symlink and Landlock evaluates + the target. What gets granted on it is :func:`_exec_set`'s decision, + and its install root is separately a read root (:func:`exec_policy`) + for the standard library beside it. """ - python = project / ".venv" / "bin" / "python" + python = env_dir / "bin" / "python" return python.resolve() if python.exists() else None @@ -297,7 +352,7 @@ def _stdlib_root(python: Path | None) -> list[Path]: return [] if Path.home().resolve().is_relative_to(root) else [root] -def _exec_set(project: Path, python: Path | None) -> list[Path]: +def _exec_set(env_dir: Path, python: Path | None) -> list[Path]: """The two exec tiers: the environment, and the utility allowlist. Grants are per *file* for the utilities, never per directory: @@ -318,7 +373,7 @@ def _exec_set(project: Path, python: Path | None) -> list[Path]: runnable and silently outrank this whole allowlist. """ paths: list[Path] = [] - bin_dir = project / ".venv" / "bin" + bin_dir = env_dir / "bin" if bin_dir.is_dir(): # Per *file*, never the directory, for the same reason `/usr/bin` # is: a directory grant is a grant on whatever the directory holds @@ -387,6 +442,15 @@ def _loader_patterns() -> dict[str, set[str]]: return grouped +def _declared(paths: Iterable[Path]) -> tuple[Path, ...]: + """Drop what is not there, de-duplicate, keep the declared spelling.""" + found: dict[Path, None] = {} + for path in paths: + if Path(path).exists(): + found.setdefault(Path(path), None) + return tuple(found) + + def _existing(paths: Iterable[Path]) -> tuple[Path, ...]: """Resolve, drop what is not there, de-duplicate, keep order. diff --git a/src/lightcone/engine/sandbox/seatbelt.py b/src/lightcone/engine/sandbox/seatbelt.py index e4eb093b..a737de0e 100644 --- a/src/lightcone/engine/sandbox/seatbelt.py +++ b/src/lightcone/engine/sandbox/seatbelt.py @@ -106,6 +106,8 @@ class SeatbeltBackend: """Seatbelt, expressed as an argv rewrite.""" capability: Capability = field(default_factory=lambda: Capability(kind="seatbelt")) + #: The uv hop stays outside the wrap: trusted host plumbing. + contains_prefix: bool = False def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: """Rewrite *argv* to run under ``sandbox-exec``. diff --git a/src/lightcone/engine/templates/files/gitattributes.tmpl b/src/lightcone/engine/templates/files/gitattributes.tmpl index ffc33bdc..a5cb916b 100644 --- a/src/lightcone/engine/templates/files/gitattributes.tmpl +++ b/src/lightcone/engine/templates/files/gitattributes.tmpl @@ -12,3 +12,4 @@ results/** annex.largefiles=anything data/** annex.largefiles=anything **/.lightcone-manifest.json annex.largefiles=nothing +.datalad/environments/*/image annex.largefiles=anything diff --git a/src/lightcone/engine/templates/files/gitignore.tmpl b/src/lightcone/engine/templates/files/gitignore.tmpl index 8b1c3d8d..cfb929ac 100644 --- a/src/lightcone/engine/templates/files/gitignore.tmpl +++ b/src/lightcone/engine/templates/files/gitignore.tmpl @@ -4,4 +4,5 @@ __pycache__/ .ipynb_checkpoints/ .DS_Store .venv/ +.lightcone/ _build/ diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index 0eb80c93..2b06e34b 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -35,13 +35,12 @@ from pathlib import Path from typing import Literal -from lightcone.engine import assets, dataset, identity, plan, sandbox +from lightcone.engine import assets, container, dataset, identity, plan, sandbox from lightcone.engine.plan import Key, Task from lightcone.engine.project import ( ProjectError, child_env, declared_project, - sync, uv_prefix, ) @@ -97,6 +96,7 @@ def materialize( head: Head, versions: assets.Versions, refresh: bool, + runtime: container.Runtime, *upstream: TaskResult, ) -> TaskResult: """Make *task* if it needs making. What Dask submits, once per task. @@ -115,6 +115,9 @@ def materialize( driver because it commits as outputs land and HEAD moves. versions: The run's content-hash memo for declared inputs. refresh: Whether to remake an output that is merely behind. + runtime: The execution world, resolved once by the driver — the + same discipline as *head*, because resolving per task could + answer differently mid-run. *upstream: The results of this task's dependencies, arriving as the futures it was given — which is what makes Dask the scheduler rather than a loop here. @@ -123,7 +126,7 @@ def materialize( What happened. Never raises. """ try: - return _materialize(root, task, env_version, head, versions, refresh, upstream) + return _materialize(root, task, env_version, head, versions, refresh, runtime, upstream) except Exception as e: # the contract is that this function returns return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") @@ -135,6 +138,7 @@ def _materialize( head: Head, versions: assets.Versions, refresh: bool, + runtime: container.Runtime, upstream: tuple[TaskResult, ...], ) -> TaskResult: reported = {u.key: u for u in upstream if u.usable} @@ -155,7 +159,7 @@ def _materialize( inputs=inputs, ) if verdict.calls_for_a_remake(refresh=refresh): - return execute(root, task, env_version, inputs, head=head) + return execute(root, task, env_version, inputs, head=head, runtime=runtime) # Left alone, so the bytes on disk stand. Their *recorded* digest, # never a recomputed one: on a clone that has fetched no annex content @@ -179,6 +183,7 @@ def execute( input_versions: Mapping[str, str], *, head: Head, + runtime: container.Runtime, ) -> TaskResult: """Run *task*'s recipe and record what it produced. @@ -195,6 +200,9 @@ def execute( input_versions: Each declared input's content identity, recorded in the manifest as the chain. head: The run's ``(commit sha, origin URL)``. + runtime: The execution world the recipe enters — the host under + the platform's mechanism, or the project image behind its + mount table. Returns: ``ok`` with the output's ``data_version``, or ``failed``. Commits @@ -214,10 +222,10 @@ def execute( task.output_dir.mkdir(parents=True) read_paths = [p for p in task.inputs.values() if p.exists()] - policy = sandbox.exec_policy(root, read_paths=read_paths) + policy = container.policy_for(runtime, read_paths) with sandbox.scope(policy): outcome = sandbox.run( - sandbox.detect(), + container.backend(runtime), policy, [_SHELL, "-c", task.recipe], cwd=root, @@ -256,6 +264,7 @@ def execute( git_remote=remote, lc_version=lc_version(), hermeticity=asdict(outcome.attestation), + image=runtime.manifest_image(), ), ) except (OSError, ProjectError) as e: @@ -319,6 +328,11 @@ def main(argv: list[str]) -> int: ``--locked``, so a lock that no longer matches ``pyproject.toml`` is a loud refusal rather than a quiet relock. + The image, by contrast, is *found*, never built: the run record lists + the committed archive in ``extra_inputs``, so ``datalad rerun`` has + fetched the exact bytes before this runs — and a build would be a + commit, which nothing here makes. + Args: argv: One argument, ``/``. @@ -333,17 +347,25 @@ def main(argv: list[str]) -> int: universe_id, _, output_id = argv[0].partition("/") try: root = declared_project() - sync(root) - env_version = identity.env_version(root) + # The graph — and with it the task lookup — before any converge: + # a typo'd target must cost nothing and mask nothing, and a + # failing sync must not bury "no output `x`" under its own error. graph = plan.build(root) if (task := graph.tasks.get((universe_id, output_id))) is None: print(f"no output `{argv[0]}` in this project", file=sys.stderr) return 2 - # This one-task run reads HEAD for itself, because it *is* the - # driver here — the rule is that the run's commit is read once by - # whoever owns the run, not that a worker never reads it. + # This one-task run resolves its own runtime and HEAD, because it + # *is* the driver here — the rule is that each is read once by + # whoever owns the run, not that a worker never reads them. + runtime = container.runtime_for_run(root, build=False) + container.converge(runtime) result = execute( - root, task, env_version, _from_disk(task), head=dataset.head(root) + root, + task, + identity.env_version(root), + _from_disk(task), + head=dataset.head(root), + runtime=runtime, ) except ProjectError as e: print(f"error: {e}", file=sys.stderr) diff --git a/tests/conftest.py b/tests/conftest.py index 1a2edef6..ac602a51 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,7 @@ from __future__ import annotations import shutil +import subprocess import textwrap from collections.abc import Callable from pathlib import Path @@ -200,3 +201,27 @@ def uv_calls(calls: list[list[str]]) -> list[list[str]]: def probes(calls: list[list[str]]) -> list[list[str]]: """Just the read-only ``--check`` probes.""" return [c for c in uv_calls(calls) if "--check" in c] + + +@pytest.fixture(scope="session") +def engine_dist(tmp_path_factory: pytest.TempPathFactory) -> tuple[str, Path]: + """Build the engine under test into a wheel the pinned record can find. + + The record pins ``lightcone-cli==`` and the suite's build is not + published, so the rerun's ephemeral environment is pointed here via + ``UV_FIND_LINKS`` — which is what lets the suite execute the same + record shape every real commit carries, rather than a test-only one. + + Returns: + The wheel's exact version, and the directory serving it. + """ + dist = tmp_path_factory.mktemp("engine-dist") + subprocess.run( + ["uv", "build", "--wheel", "--out-dir", str(dist)], + cwd=Path(__file__).parent.parent, + check=True, + capture_output=True, + text=True, + ) + wheel = next(dist.glob("*.whl")) + return wheel.name.split("-")[1], dist diff --git a/tests/test_cli.py b/tests/test_cli.py index 349b647d..28fe0dc4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,9 +27,9 @@ def test_help_advertises_exactly_the_verbs_that_work(runner: CliRunner) -> None: """`lc --help` advertises only verbs that work — advertising others before they do would be a lie.""" result = runner.invoke(main, ["--help"]) - for verb in ("init", "materialize", "run", "status"): + for verb in ("init", "build", "materialize", "run", "status"): assert f" {verb}" in result.output - for verb in ("verify", "build", "export"): + for verb in ("verify", "export"): assert f" {verb}" not in result.output @@ -515,6 +515,44 @@ def test_status_shows_each_output_its_state_and_its_commit( assert fragment in result.output +def test_status_headers_answer_mode_image_and_sandbox( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The three facts nothing else surfaces — where the denial note and + the runtime-missing refusal both point.""" + from lightcone.engine.materialize import StatusReport + + report = _report() + assert isinstance(report, StatusReport) + report.mode = "containerized" + report.image = { + "tag": "lc-env-0123456789abcdef", + "state": "absent", + "archive": ".datalad/environments/lc-env-0123456789abcdef/image", + } + report.sandbox = "podman (fs: declared, network: denied)" + _status_stub(monkeypatch, report) + + output = runner.invoke(main, ["status"]).output + + assert "mode: containerized" in output + assert "lc-env-0123456789abcdef" in output + assert "needs build" in output and "lc build" in output + assert "podman" in output + + +def test_build_on_a_direct_project_is_an_explanatory_no_op( + runner: CliRunner, project: Path +) -> None: + """Not an error: `lc build` answers "what would building do here", + and for a direct project the answer names the escalation.""" + result = runner.invoke(main, ["build"]) + + assert result.exit_code == 0 + assert "direct mode" in result.output + assert "[tool.lightcone.image]" in result.output + + def test_status_exits_zero_even_with_stale_outputs( runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -534,6 +572,9 @@ def test_status_json_is_machine_readable( result = runner.invoke(main, ["status", "--json"]) assert json.loads(result.output) == { + "mode": "direct", + "image": None, + "sandbox": "", "counts": {"current": 1, "behind": 1, "stale": 1}, "outputs": [ { diff --git a/tests/test_container.py b/tests/test_container.py new file mode 100644 index 00000000..09d66c87 --- /dev/null +++ b/tests/test_container.py @@ -0,0 +1,484 @@ +"""Tests for `lightcone.engine.container` — the image lifecycle, stubbed. + +Every runtime command goes through `project._run`, so these tests hand it +a fake that models each command's observable effect and records every +argv — the same discipline as convergence's `tools` fixture. What a real +runtime does with the argv is `test_container_smoke.py`'s question. +""" + +from __future__ import annotations + +import io +import json +import shutil +import tarfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from lightcone.engine import container, image, project +from lightcone.engine.project import ProjectError + +_TABLE = '[tool.lightcone.image]\napt-install = ["bc"]\n' +_CONFIG = b'{"architecture":"amd64","os":"linux"}' + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + project_dir = tmp_path / "analysis" + project_dir.mkdir() + (project_dir / "pyproject.toml").write_text( + '[project]\nname = "analysis"\nversion = "0.1.0"\n' + 'requires-python = ">=3.11"\ndependencies = []\n' + _TABLE + ) + (project_dir / ".python-version").write_text("3.12.11\n") + return project_dir + + +def _write_archive(path: Path, config: bytes = _CONFIG) -> str: + """Write a minimal but structurally real docker-archive at *path*.""" + path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(path, "w") as tar: + for name, data in ( + ("abc.json", config), + ("manifest.json", json.dumps([{"Config": "abc.json", "RepoTags": []}]).encode()), + ): + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + import hashlib + + return hashlib.sha256(config).hexdigest() + + +@pytest.fixture +def fake(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """A podman-having host: records argv, models each command's effect.""" + calls: list[list[str]] = [] + loaded: set[str] = set() + + def run(argv: list[str], *, cwd: Path) -> MagicMock: + calls.append(list(argv)) + if argv[0] in ("podman", "docker"): + if argv[1] == "image": # the loaded probe, both spellings + return MagicMock(returncode=0 if argv[3] in loaded else 1) + if argv[1] == "load": + loaded.add(container.archive_identity(Path(argv[3]))[0]) + return MagicMock(returncode=0, stdout="", stderr="") + if argv[1] == "save": + _write_archive(Path(argv[argv.index("-o") + 1])) + return MagicMock(returncode=0, stdout="", stderr="") + return MagicMock(returncode=0, stdout="", stderr="") + if argv[:2] == ["uv", "cache"]: + return MagicMock(returncode=0, stdout="/home/user/.cache/uv\n", stderr="") + if argv[:3] == ["git", "diff", "--cached"]: + return MagicMock(returncode=1) # something staged: commits proceed + if argv[:2] == ["git", "check-attr"]: + return MagicMock( + returncode=0, stdout=f"{argv[-1]}: annex.largefiles: anything\n", stderr="" + ) + if argv[:3] == ["git", "annex", "get"]: + _write_archive(cwd / argv[-1]) # the fetch's observable effect + return MagicMock(returncode=0, stdout="", stderr="") + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(project, "_run", run) + monkeypatch.setattr(shutil, "which", lambda name, path=None: f"/usr/bin/{name}") + return calls + + +def _argvs(calls: list[list[str]], *head: str) -> list[list[str]]: + return [c for c in calls if c[: len(head)] == list(head)] + + +# ---- runtime detection ------------------------------------------------------ + + +def test_podman_is_preferred(root: Path, fake: list[list[str]]) -> None: + assert container.runtime_name(root) == "podman" + + +def test_docker_without_its_daemon_is_a_refusal( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """`docker` on PATH with the daemon down is the common broken state; + 'cannot connect to the socket' mid-run is a worse message.""" + monkeypatch.setattr( + shutil, "which", lambda name, path=None: f"/usr/bin/{name}" if name == "docker" else None + ) + monkeypatch.setattr( + project, "_run", lambda argv, *, cwd: MagicMock(returncode=1, stdout="", stderr="") + ) + with pytest.raises(ProjectError, match="daemon"): + container.runtime_name(root) + + +def test_no_runtime_is_a_refusal_naming_both( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(shutil, "which", lambda name, path=None: None) + with pytest.raises(ProjectError, match="podman"): + container.runtime_name(root) + + +# ---- the three strictnesses ------------------------------------------------- + + +def test_a_direct_project_resolves_without_touching_a_runtime( + tmp_path: Path, fake: list[list[str]] +) -> None: + plain = tmp_path / "plain" + plain.mkdir() + (plain / "pyproject.toml").write_text('[project]\nname = "p"\nversion = "0"\n') + + runtime = container.runtime_for_run(plain, build=False) + + assert runtime.mode == "direct" + assert runtime.env_dir == plain / ".venv" + assert fake == [] + + +def test_a_missing_archive_refuses_unless_the_caller_may_build( + root: Path, fake: list[list[str]] +) -> None: + """`lc run` never builds and the worker never writes git — and the + mutation check: the same state under a build-allowed caller succeeds.""" + with pytest.raises(ProjectError, match="lc build"): + container.runtime_for_run(root, build=False) + assert _argvs(fake, "podman", "build") == [] + assert _argvs(fake, "git", "commit") == [] + + runtime = container.runtime_for_run(root, build=True) + + assert runtime.mode == "containerized" + assert runtime.image_tag == image.tag(root) + assert container.runtime_for_run(root, build=False).image_id == runtime.image_id + + +def test_unfetched_archive_content_is_fetched_by_lc_itself( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """Nobody is ever asked to run a git-annex command by hand — the + storage invariant. lc gets its own artifact; the refusal is reserved + for a fetch with no reachable copy, which is the second half.""" + archive = image.archive_path(root, image.tag(root)) + archive.parent.mkdir(parents=True) + archive.write_bytes(b"/annex/objects/SHA256E-s323--abc\n") # the pointer shape + + runtime = container.runtime_for_run(root, build=False) + + assert runtime.image_id + (got,) = [c for c in fake if c[:3] == ["git", "annex", "get"]] + assert got[-1] == f".datalad/environments/{image.tag(root)}/image" + + archive.write_bytes(b"/annex/objects/SHA256E-s323--abc\n") # unfetched again + inner = project._run + + def failing(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:3] == ["git", "annex", "get"]: + return MagicMock(returncode=1, stdout="", stderr="no reachable copy") + return inner(argv, cwd=cwd) + + monkeypatch.setattr(project, "_run", failing) + with pytest.raises(ProjectError, match="fetching it failed"): + container.runtime_for_run(root, build=False) + + +def test_a_present_archive_is_loaded_once_and_reused( + root: Path, fake: list[list[str]] +) -> None: + expected = _write_archive(image.archive_path(root, image.tag(root))) + + first = container.runtime_for_run(root, build=False) + second = container.runtime_for_run(root, build=False) + + assert first.image_id == expected and second.image_id == expected + assert first.arch == "amd64" + assert first.archive == f".datalad/environments/{image.tag(root)}/image" + assert len(_argvs(fake, "podman", "load")) == 1 # the second run hit the store + + +# ---- building --------------------------------------------------------------- + + +def test_the_build_context_holds_only_the_containerfile( + root: Path, fake: list[list[str]] +) -> None: + """No project file ever enters the context — what makes 'code edits + never trigger a build' structural rather than observed.""" + container.runtime_for_run(root, build=True) + + (build,) = _argvs(fake, "podman", "build") + context = Path(build[-1]) + assert not context.is_relative_to(root) + containerfile = Path(build[build.index("-f") + 1]) + assert containerfile.name == "Containerfile" + assert build[build.index("-t") + 1] == image.tag(root) + + +def test_the_build_saves_and_commits_the_archive(root: Path, fake: list[list[str]]) -> None: + """The dataset is the image store: the archive and the datalad + containers config land in one scoped commit.""" + runtime = container.runtime_for_run(root, build=True) + + (save,) = _argvs(fake, "podman", "save") + # Saved beside its final name and renamed into place, so a save that + # dies midway leaves no partial archive for the dirty refusal to + # tell the user to commit. + archive = image.archive_path(root, runtime.image_tag) + assert save[save.index("-o") + 1] == str(archive.parent / "image.partial") + assert archive.is_file() and not (archive.parent / "image.partial").exists() + assert "--format" in save and save[save.index("--format") + 1] == "docker-archive" + configured = {c[-2] for c in _argvs(fake, "git", "config", "-f", ".datalad/config")} + assert f"datalad.containers.{runtime.image_tag}.image" in configured + assert f"datalad.containers.{runtime.image_tag}.cmdexec" in configured + (add,) = [c for c in fake if c[0] == "git" and "add" in c] + assert f".datalad/environments/{runtime.image_tag}" in " ".join(add) + # The dot-path routing: without it the archive is a full blob in git. + assert "annex.dotfiles=true" in add + assert len(_argvs(fake, "git", "commit")) == 1 + + +def test_an_unrouted_archive_refuses_before_building( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """The archive is committed, and `.gitattributes` is a user-authored + file lc only appends to — so an archive it does not route to the + annex would land in git as a several-hundred-MB blob, silently. The + probe fires before any build is paid for. Mutation check: the `fake` + fixture's routed answer is what every other build test passes with.""" + original = project._run + + def unrouted(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:2] == ["git", "check-attr"]: + return MagicMock( + returncode=0, stdout=f"{argv[-1]}: annex.largefiles: unspecified\n", stderr="" + ) + return original(argv, cwd=cwd) + + monkeypatch.setattr(project, "_run", unrouted) + with pytest.raises(ProjectError, match="annex.largefiles=anything"): + container.runtime_for_run(root, build=True) + assert _argvs(fake, "podman", "build") == [] + + +def test_a_bad_apt_name_is_parsed_out_of_the_build_log( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + _failing_build("E: Unable to locate package texlive-latex-bass", monkeypatch) + with pytest.raises(ProjectError, match="texlive-latex-bass"): + container.runtime_for_run(root, build=True) + + +def _failing_build(stderr: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Route only the build through failure; everything else keeps the + `fake` fixture's modelled answers (the routing probe included).""" + from lightcone.engine import project as project_module + + inner = project_module._run + + def failing(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:2] == ["podman", "build"]: + return MagicMock(returncode=1, stdout="", stderr=stderr) + return inner(argv, cwd=cwd) + + monkeypatch.setattr(project_module, "_run", failing) + + +@pytest.mark.parametrize( + ("code", "instruction", "expected"), + [ + ("43", "RUN if ldd --version 2>&1 | grep -qi musl", "glibc"), + ("44", "RUN command -v bash >/dev/null", "bash"), + ("45", "RUN command -v apt-get >/dev/null", "apt"), + ], +) +def test_a_contract_violation_names_the_base_not_the_log( + root: Path, + fake: list[list[str]], + monkeypatch: pytest.MonkeyPatch, + code: str, + instruction: str, + expected: str, +) -> None: + _failing_build( + f'Error: building at STEP "{instruction}": while running runtime: ' + f"exit status {code}", + monkeypatch, + ) + with pytest.raises(ProjectError, match=expected): + container.runtime_for_run(root, build=True) + + +def test_a_run_command_exiting_a_contract_code_is_not_misdiagnosed( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """curl exits 43 for its own reasons; blaming the base for it would + point the user away from their own failing command. The anchor is + the failing instruction, not the code alone.""" + _failing_build( + 'Error: building at STEP "RUN curl -fsSL https://example.org/tool.tar": ' + "exit status 43", + monkeypatch, + ) + with pytest.raises(ProjectError, match="curl") as raised: + container.runtime_for_run(root, build=True) + assert "musl" not in str(raised.value) + + +def test_lc_build_refuses_a_dirty_tree( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """The archive commit takes the whole index with it, and the tag + derives from pyproject.toml — the declaration commits first.""" + + def dirty(argv: list[str], *, cwd: Path) -> MagicMock: + if argv[:2] == ["git", "status"]: + return MagicMock(returncode=0, stdout=" M pyproject.toml\n", stderr="") + return MagicMock(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(project, "_run", dirty) + with pytest.raises(ProjectError, match="[Cc]ommit"): + container.build(root) + + +def test_lc_build_is_idempotent(root: Path, fake: list[list[str]]) -> None: + _, first = container.build(root) + _, second = container.build(root) + assert (first, second) == ("built", "present") + assert len(_argvs(fake, "podman", "build")) == 1 + + +def test_lc_build_on_a_direct_project_says_so(tmp_path: Path, fake: list[list[str]]) -> None: + plain = tmp_path / "plain" + plain.mkdir() + (plain / "pyproject.toml").write_text('[project]\nname = "p"\nversion = "0"\n') + with pytest.raises(ProjectError, match="direct mode"): + container.build(plain) + + +# ---- the containerized converge --------------------------------------------- + + +def test_sync_runs_uv_inside_the_image_with_the_host_cache( + root: Path, fake: list[list[str]] +) -> None: + runtime = container.runtime_for_run(root, build=True) + fake.clear() + + container.sync(root, runtime) + + (sync,) = _argvs(fake, "podman", "run") + assert f"{root}:{root}:rw" in " ".join(sync) + assert "/home/user/.cache/uv:/home/user/.cache/uv:rw" in " ".join(sync) + assert f"UV_PROJECT_ENVIRONMENT={root / '.lightcone' / 'venv'}" in " ".join(sync) + assert "--userns=keep-id" in sync + assert runtime.image_id in sync + tail = sync[sync.index(runtime.image_id) + 1 :] + assert tail[:2] == ["uv", "sync"] + assert "--locked" in tail and "--exact" in tail and "--compile-bytecode" in tail + + +# ---- the podman machine (macOS) --------------------------------------------- + + +def _darwin(monkeypatch: pytest.MonkeyPatch) -> None: + import sys + + monkeypatch.setattr(container.sys, "platform", "darwin") + assert sys.platform # the global module object is patched; reverted after + + +def test_no_podman_machine_is_a_refusal_naming_the_setup( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + _darwin(monkeypatch) + monkeypatch.setattr( + project, "_run", lambda argv, *, cwd: MagicMock(returncode=1, stdout="", stderr="") + ) + with pytest.raises(ProjectError, match="podman machine init"): + container.runtime_name(root) + + +def test_a_project_outside_the_machine_shares_is_a_refusal( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """A bind mount from outside the VM's shared directories arrives + *empty* — no error, just a project with nothing in it — so the + preflight names the exact `podman machine set`. The mutation check: + a share that covers the project passes.""" + _darwin(monkeypatch) + + def machine(shares: list[str]) -> None: + inspect = json.dumps( + [{"State": "running", "Mounts": [{"Source": s} for s in shares]}] + ) + monkeypatch.setattr( + project, + "_run", + lambda argv, *, cwd: MagicMock(returncode=0, stdout=inspect, stderr=""), + ) + + machine(["/Users"]) + with pytest.raises(ProjectError, match="podman machine set"): + container.runtime_name(root) + + # A machine sharing nothing at all is the same refusal, not a pass. + machine([]) + with pytest.raises(ProjectError, match="podman machine set"): + container.runtime_name(root) + + machine([str(root.parent)]) + assert container.runtime_name(root) == "podman" + + +def test_a_stopped_machine_is_a_refusal_naming_start( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """`podman machine inspect` succeeds on a stopped machine — without + the state check the preflight passes and the run dies later on a raw + connection error, far from the one-command fix.""" + _darwin(monkeypatch) + inspect = json.dumps([{"State": "stopped", "Mounts": [{"Source": "/Users"}]}]) + monkeypatch.setattr( + project, + "_run", + lambda argv, *, cwd: MagicMock(returncode=0, stdout=inspect, stderr=""), + ) + with pytest.raises(ProjectError, match="podman machine start"): + container.runtime_name(root) + + +# ---- state for status and the CLI ------------------------------------------- + + +def test_image_state_reports_repository_facts_only(root: Path, fake: list[list[str]]) -> None: + tag = image.tag(root) + relative = f".datalad/environments/{tag}/image" + assert container.image_state(root) == ("absent", tag, relative) + + archive = image.archive_path(root, tag) + archive.parent.mkdir(parents=True) + archive.write_bytes(b"/annex/objects/SHA256E-s1--abc\n") + assert container.image_state(root)[0] == "unfetched" + + _write_archive(archive) + assert container.image_state(root)[0] == "present" + assert not any(c[0] in ("podman", "docker") for c in fake) + + +def test_archive_identity_matches_the_runtime_id_computation(tmp_path: Path) -> None: + """The same sha256-of-the-config-blob podman and docker report, and + the value the smoke test compares against a real `podman inspect`.""" + config = b'{"architecture":"arm64"}' + expected = _write_archive(tmp_path / "image", config) + found, arch = container.archive_identity(tmp_path / "image") + assert found == expected + assert arch == "arm64" + + +def test_a_garbled_archive_is_a_pointed_refusal(tmp_path: Path) -> None: + (tmp_path / "image").write_bytes(b"not a tar at all" * 4096) + with pytest.raises(ProjectError, match="lc build"): + container.archive_identity(tmp_path / "image") diff --git a/tests/test_container_smoke.py b/tests/test_container_smoke.py new file mode 100644 index 00000000..89e26c97 --- /dev/null +++ b/tests/test_container_smoke.py @@ -0,0 +1,309 @@ +"""The runtime's answer — the one file that can say the container layer works. + +Everything here builds a real image, saves a real archive into a real +annexed repository, and runs real commands through a real runtime's mount +table. Gated exactly like the sandbox enforcement suite: hosts without a +runtime skip, and ``LC_CONTAINER_TESTS_REQUIRED=1`` (set in Linux CI) +turns that skip into a hard failure, with two tests covering the guard +itself — an unfailing guard is worse than none. + +Parameterized over the runtimes actually present, because a leak or an +asymmetry only docker catches is still a bug. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path + +import pytest + +from lightcone.engine import assets, container, dataset, image +from lightcone.engine import materialize as engine +from lightcone.engine import run as engine_run +from lightcone.engine.project import ProjectError, child_env + +REQUIRED_ENV = "LC_CONTAINER_TESTS_REQUIRED" + +_SPEC = """ +version: "0.0.13" +name: analysis + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: sums + type: metric + recipe: + command: echo "2+2" | bc > {output}/sum.txt +""" + + +def _available() -> list[str]: + """The runtimes this host can actually run, probed once at collection.""" + found = [] + if shutil.which("podman"): + found.append("podman") + try: + if ( + shutil.which("docker") + and subprocess.run( + ["docker", "info"], capture_output=True, timeout=10 + ).returncode + == 0 + ): + found.append("docker") + except subprocess.TimeoutExpired: + pass # a hung daemon at collection time must not hang the suite + return found + + +_RUNTIMES = _available() + + +def _gate(available: list[str]) -> None: + if available: + return + if os.environ.get(REQUIRED_ENV): + pytest.fail( + f"{REQUIRED_ENV} is set but this host has no container runtime. " + "Container tests must not be skipped on CI." + ) + pytest.skip("no container runtime here") + + +@pytest.fixture(params=_RUNTIMES or [""]) +def runtime(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> str: + """One available runtime, with the others hidden from detection.""" + _gate(_RUNTIMES) + name = str(request.param) + real_which = shutil.which + monkeypatch.setattr( + shutil, + "which", + lambda tool, path=None: None + if tool in ("podman", "docker") and tool != name + else real_which(tool, path=path), + ) + return name + + +@pytest.fixture +def cproject(analysis: Callable[..., Path]) -> Path: + """A real committed project, containerized after its scaffold commit — + the way a real project escalates: edit pyproject, commit, build.""" + root = analysis(_SPEC, files={"data/catalog.fits": "stars\n"}) + text = (root / "pyproject.toml").read_text() + (root / "pyproject.toml").write_text( + text + '\n[tool.lightcone.image]\napt-install = ["bc"]\n' + ) + dataset.save(root, [root], "containerize") + return root + + +def _inspect_id(runtime: str, image_id: str) -> str: + argv = { + "podman": ["podman", "image", "inspect", "--format", "{{.Id}}", image_id], + "docker": ["docker", "image", "inspect", "--format", "{{.Id}}", image_id], + }[runtime] + out = subprocess.run(argv, capture_output=True, text=True, check=True).stdout.strip() + return out.removeprefix("sha256:") + + +# ---- lc build --------------------------------------------------------------- + + +def test_build_commits_the_exact_bytes_and_leaves_the_tree_clean( + runtime: str, cproject: Path +) -> None: + resolved, action = container.build(cproject) + + assert action == "built" + archive = image.archive_path(cproject, resolved.image_tag) + assert archive.is_file() + assert not dataset.status(cproject), "the archive commit left the tree dirty" + # The user never sees a Containerfile — not in the tree, not beside + # the archive. + assert not list(cproject.rglob("Containerfile")) + # The id read from the archive is the id the runtime computed. + assert resolved.image_id == _inspect_id(runtime, resolved.image_id) + # And the image carries its own identity. + label = subprocess.run( + [ + runtime, "image", "inspect", + "--format", '{{index .Config.Labels "io.lightcone.image"}}', + resolved.image_id, + ], # fmt: skip + capture_output=True, + text=True, + check=True, + ).stdout.strip() + assert json.loads(label) == json.loads(image.identity_document(cproject) or "") + + _, again = container.build(cproject) + assert again == "present" + + +# ---- lc run: the probe ------------------------------------------------------ + + +def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: + """One image, the probe's whole contract — each denial beside the + mutation check that proves the same reach works where it should. + + `lc run` never builds: the first probe refuses naming `lc build`, and + succeeding after one is the mutation check. The mounts are the + mechanism: an undeclared host file simply is not there (while the + host itself reads it fine). And `--network none` is a real denial + with loopback intact — the meaning of `network: denied`.""" + with pytest.raises(ProjectError, match="lc build"): + engine_run.probe(cproject, ["bc", "--version"]) + + container.build(cproject) + outcome = engine_run.probe(cproject, ["bc", "--version"]) + assert outcome.returncode == 0 + assert outcome.attestation.mechanism == runtime + assert outcome.attestation.network == "denied" + assert outcome.attestation.fs == "declared" + + outside = Path.home() / ".lc-smoke-outside.txt" + outside.write_text("host secret\n") + try: + denied = engine_run.probe(cproject, ["cat", str(outside)]) + assert denied.returncode != 0 + assert outside.read_text() == "host secret\n" # the host itself can + finally: + outside.unlink() + + # The rootfs is read-only: a write outside the declared set is a + # loud denial, not bytes vanishing with the container. The mutation + # check is the same write into results/, which must succeed. + rootfs = engine_run.probe(cproject, ["bash", "-c", "mkdir /output"]) + assert rootfs.returncode != 0 + results = engine_run.probe(cproject, ["bash", "-c", "touch results/probe-write"]) + assert results.returncode == 0 + (cproject / "results" / "probe-write").unlink() + + loopback = engine_run.probe( + cproject, + ["python", "-c", 'import socket; socket.socket().bind(("127.0.0.1", 0))'], + ) + assert loopback.returncode == 0 + + egress = engine_run.probe( + cproject, + [ + "python", "-c", + "import socket, urllib.request; socket.setdefaulttimeout(3); " + 'urllib.request.urlopen("http://1.1.1.1")', + ], # fmt: skip + ) + assert egress.returncode != 0 + + +# ---- lc materialize --------------------------------------------------------- + + +def test_materialize_end_to_end_in_the_image(runtime: str, cproject: Path) -> None: + """The whole layer at once: the driver builds and commits the image, + converges the in-image environment, runs the recipe behind the mount + table, records the runtime facts, and leaves the tree clean — through + the real Dask cluster, not a stub.""" + report = engine.materialize(cproject, []) + + assert report.ok, report.warnings + assert report.made == ["baseline/sums"] + assert (cproject / "results/baseline/sums/sum.txt").read_text() == "4\n" + assert not dataset.status(cproject) + + manifest = assets.read(cproject / "results/baseline/sums") + assert manifest is not None + assert manifest.hermeticity["mechanism"] == runtime + assert manifest.hermeticity["network"] == "denied" + assert manifest.hermeticity["fs"] == "declared" + assert manifest.image is not None + assert manifest.image["id"] == _inspect_id(runtime, manifest.image["id"]) + assert manifest.image["archive"] == f".datalad/environments/{manifest.image['tag']}/image" + + # The record lists the archive, so a rerun's `datalad get` fetches + # the exact bytes before the worker runs. Parsed rather than matched + # as text, because the record is datalad's format, not ours. + record = dataset._git(["log", "-1", "--format=%B"], cwd=cproject) + body = record.partition("=== Do not change lines below ===")[2].partition("^^^")[0] + assert json.loads(body)["extra_inputs"] == [manifest.image["archive"]] + + +def test_a_rerun_on_a_clone_fetches_the_archive_and_reproduces( + runtime: str, + cproject: Path, + tmp_path: Path, + engine_dist: tuple[str, Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The record's whole claim, on the machine that matters: a clone with + no annex content and no local image. `datalad rerun` fetches the + archive through the annex (it is in `extra_inputs`), the worker loads + it and syncs `.lightcone/venv` in-image, and the output reproduces.""" + pytest.importorskip("datalad") + if runtime != _RUNTIMES[0]: + # The rerun is a subprocess and detects the host's preferred + # runtime for itself — a monkeypatch cannot reach it, so running + # this under any other parameter would silently retest the first. + pytest.skip("the rerun subprocess always uses the host's preferred runtime") + version, dist = engine_dist + monkeypatch.setattr(engine, "_engine_requirement", lambda: f"lightcone-cli=={version}") + report = engine.materialize(cproject, []) + assert report.ok, report.warnings + original = assets.read(cproject / "results/baseline/sums") + assert original is not None + + clone = tmp_path / "clone" + dataset._git(["clone", "-q", str(cproject), str(clone)], cwd=tmp_path) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=clone) + dataset._git(["annex", "init", "-q", "clone"], cwd=clone) + # No annex content: the archive here is a pointer until rerun gets it. + with pytest.raises(assets.ContentNotFetchedError): + assets.require_fetched(clone / str(original.image["archive"])) # type: ignore[index] + + proc = subprocess.run( + [sys.executable, "-c", "from datalad.api import rerun; rerun('HEAD')"], + cwd=clone, + capture_output=True, + text=True, + env={**child_env(), "UV_FIND_LINKS": str(dist)}, + ) + + assert proc.returncode == 0, proc.stderr + rerun = assets.read(clone / "results/baseline/sums") + assert rerun is not None + assert rerun.data_version == original.data_version + assert not dataset.status(clone) + + +# ---- the guard on this file itself ----------------------------------------- + + +def test_the_ci_guard_fails_rather_than_skipping(monkeypatch: pytest.MonkeyPatch) -> None: + """The one test here that must pass everywhere: CI cannot go green by + skipping the rest. Without this, `LC_CONTAINER_TESTS_REQUIRED` is a + comment.""" + monkeypatch.setenv(REQUIRED_ENV, "1") + with pytest.raises(pytest.fail.Exception, match="must not be skipped"): + _gate([]) + + +def test_without_the_guard_a_runtimeless_host_skips(monkeypatch: pytest.MonkeyPatch) -> None: + """And off CI it stays a skip — a laptop without podman should run the + rest of the suite, not fail it.""" + monkeypatch.delenv(REQUIRED_ENV, raising=False) + with pytest.raises(pytest.skip.Exception): + _gate([]) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index ccfafe0e..734b4597 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -94,6 +94,38 @@ def test_a_plain_git_add_annexes_content_by_itself(repo: Path) -> None: assert _annexed(repo, output / "fit.csv") +def test_dot_paths_follow_the_storage_policy_not_annex_defaults(repo: Path) -> None: + """git-annex routes any file under a dot-directory to git whatever + `annex.largefiles` says, unless the add opts in — so the image + archive under `.datalad/environments/`, or a `.cache.h5` a recipe + writes into results/, would land as a full blob in git, silently, + and every clone would carry the bytes forever. `save` opts in + unconditionally, so the attributes alone decide: archives and dot + outputs reach the annex, dot-named manifests keep their exemption. + The mutation check is a *plain* `git add` of the same shape, which + keeps git-annex's stock dotfile behavior.""" + archive = repo / ".datalad" / "environments" / "lc-env-abc" / "image" + archive.parent.mkdir(parents=True) + archive.write_bytes(b"pretend image bytes\n" * 64) + output = repo / "results" / "baseline" / "fit" + output.mkdir(parents=True) + (output / ".cache.h5").write_bytes(b"intermediate\n" * 64) + (output / ".lightcone-manifest.json").write_text("{}\n") + + dataset.save(repo, [archive.parent, output], "routed") + + assert _annexed(repo, archive) + assert _annexed(repo, output / ".cache.h5") + assert not _annexed(repo, output / ".lightcone-manifest.json") + + plain = repo / ".datalad" / "environments" / "lc-env-def" / "image" + plain.parent.mkdir(parents=True) + plain.write_bytes(b"pretend image bytes\n" * 64) + dataset._git(["add", "-A", "--", ".datalad/environments/lc-env-def"], cwd=repo) + dataset._git(["commit", "-q", "-m", "plain add"], cwd=repo) + assert not _annexed(repo, plain) + + def test_analysis_code_stays_in_git_and_stays_writable(repo: Path) -> None: """The default `annex.largefiles=nothing` is what keeps `filter=annex` from routing source files into the annex along with the data.""" diff --git a/tests/test_image.py b/tests/test_image.py new file mode 100644 index 00000000..1eafcf3c --- /dev/null +++ b/tests/test_image.py @@ -0,0 +1,283 @@ +"""Tests for `lightcone.engine.image` — the declaration and the identity. + +Containerfile assertions are about structure and ordering, never bytes: +this repo keeps no golden fixtures, and a byte-level test would pin the +render's prose rather than the properties that matter — what layers +exist, in what order, derived from which declaration. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from lightcone.engine import identity, image, project +from lightcone.engine.project import ProjectError + +_PIN = "3.12.11\n" + + +def _project(root: Path, table: str = "") -> Path: + root.mkdir(parents=True, exist_ok=True) + (root / "pyproject.toml").write_text( + '[project]\nname = "analysis"\nversion = "0.1.0"\n' + 'requires-python = ">=3.11"\ndependencies = []\n' + table + ) + (root / ".python-version").write_text(_PIN) + (root / "uv.lock").write_text("version = 1\n") + return root + + +_DECLARED = """ +[tool.lightcone.image] +apt-install = ["r-base-core", "bc"] +run-commands = ["curl -fsSL https://example.org/tool.tar | tar x -C /opt/tool"] +env = { R_LIBS_SITE = "/opt/rlibs" } +""" + + +# ---- the declaration -------------------------------------------------------- + + +def test_no_table_is_direct_mode(tmp_path: Path) -> None: + root = _project(tmp_path / "p") + assert image.declaration(root) is None + assert image.identity_document(root) is None + assert project.mode(root) == "direct" + + +def test_an_empty_table_is_the_escalation(tmp_path: Path) -> None: + """The table's presence is the whole trigger — a project may + containerize purely for the default system layer.""" + root = _project(tmp_path / "p", "[tool.lightcone.image]\n") + declared = image.declaration(root) + assert declared is not None + assert declared.base == image.DEFAULT_BASE + assert declared.apt_install == () + assert project.mode(root) == "containerized" + + +def test_the_key_surface_is_closed(tmp_path: Path) -> None: + """Every key is hashed, so a key nothing implements cannot ride along.""" + root = _project(tmp_path / "p", "[tool.lightcone.image]\npip-install = [\"numpy\"]\n") + with pytest.raises(ProjectError, match="pip-install"): + image.declaration(root) + + +def test_a_tag_only_base_is_refused(tmp_path: Path) -> None: + root = _project(tmp_path / "p", '[tool.lightcone.image]\nbase = "debian:bookworm-slim"\n') + with pytest.raises(ProjectError, match="digest"): + image.declaration(root) + + +@pytest.mark.parametrize( + "table", + [ + '[tool.lightcone.image]\napt-install = "bc"\n', + "[tool.lightcone.image]\nrun-commands = [1]\n", + "[tool.lightcone.image]\nenv = { K = 1 }\n", + "[tool.lightcone.image]\nbase = 3\n", + ], +) +def test_a_wrong_type_is_refused(tmp_path: Path, table: str) -> None: + with pytest.raises(ProjectError): + image.declaration(_project(tmp_path / "p", table)) + + +@pytest.mark.parametrize( + "table", + [ + # An apt "name" that is really a command reaches a raw RUN line. + '[tool.lightcone.image]\napt-install = ["foo; rm -rf /"]\n', + # A key with a space hits Docker's legacy `ENV key value` parse + # and silently defines the wrong variable. + '[tool.lightcone.image]\nenv = { "A B" = "v" }\n', + # A newline in a value splices a Containerfile instruction of its + # own into an identity-hashed surface. + '[tool.lightcone.image]\nenv = { K = "a\\nUSER root" }\n', + '[tool.lightcone.image]\nrun-commands = ["true\\nUSER root"]\n', + '[tool.lightcone.image]\nbase = "a b@sha256:0000"\n', + ], +) +def test_values_that_cannot_render_as_one_line_are_refused(tmp_path: Path, table: str) -> None: + """Everything in the declaration is interpolated into Containerfile + lines, so structural validation is what keeps the closed surface + closed — a value that smuggles an instruction is not a value.""" + with pytest.raises(ProjectError): + image.declaration(_project(tmp_path / "p", table)) + + +def test_a_multiline_interpreter_pin_is_refused(tmp_path: Path) -> None: + """The pin splices into the install layer's RUN line.""" + root = _project(tmp_path / "p", "[tool.lightcone.image]\n") + (root / ".python-version").write_text("3.12.11\nUSER root\n") + with pytest.raises(ProjectError, match="single interpreter"): + image.containerfile(root) + + +def test_reads_pyproject_only_never_the_uv_config(tmp_path: Path) -> None: + """A `uv.toml` replaces `[tool.uv]` — uv's rule about uv's own + settings, which must not reach this table.""" + root = _project(tmp_path / "p", _DECLARED) + (root / "uv.toml").write_text("no-build = true\n") + declared = image.declaration(root) + assert declared is not None and declared.apt_install == ("bc", "r-base-core") + + +# ---- the identity document -------------------------------------------------- + + +def test_the_document_is_canonical(tmp_path: Path) -> None: + root = _project(tmp_path / "p", _DECLARED) + document = image.identity_document(root) + assert document is not None + parsed = json.loads(document) + assert parsed["apt"] == ["bc", "r-base-core"] # sorted into the identity + assert parsed["base"] == image.DEFAULT_BASE + assert parsed["env"] == {"R_LIBS_SITE": "/opt/rlibs"} + assert parsed["uv"] == image.UV_IMAGE + # One spelling: re-serializing canonically reproduces the text. + assert document == json.dumps(parsed, sort_keys=True, separators=(",", ":")) + + +def test_absent_keys_hash_as_their_empty_shape(tmp_path: Path) -> None: + """A project relying on a default and one spelling it out are the same + environment only until the default changes — so the keys are always + emitted, the install-settings discipline.""" + spelled = _project( + tmp_path / "a", + f'[tool.lightcone.image]\nbase = "{image.DEFAULT_BASE}"\n' + "apt-install = []\nrun-commands = []\nenv = {}\n", + ) + bare = _project(tmp_path / "b", "[tool.lightcone.image]\n") + assert image.identity_document(spelled) == image.identity_document(bare) + + +# ---- the Containerfile ------------------------------------------------------ + + +def test_the_render_layers_in_the_generator_order(tmp_path: Path) -> None: + root = _project(tmp_path / "p", _DECLARED) + lines = image.containerfile(root).splitlines() + + assert lines[0] == f"FROM {image.DEFAULT_BASE}" + order = [ + next(i for i, line in enumerate(lines) if marker in line) + for marker in ( + "exit 43", # the glibc contract + "exit 44", # the bash contract + "exit 45", # the apt contract, present because apt-install is + "apt-get install", + f"COPY --from={image.UV_IMAGE}", + "uv python install 3.12.11", + "ENV R_LIBS_SITE=", + "RUN curl -fsSL", + "UV_PYTHON_DOWNLOADS=never", + "LABEL io.lightcone.image=", + ) + ] + assert order == sorted(order), "the fixed layering moved" + # The readability chmod rides inside the layers that write /opt — a + # layer of its own would copy-on-write the whole interpreter tree + # into every archive. + assert "uv python install 3.12.11 && chmod -R a+rX /opt" in image.containerfile(root) + + +def test_apt_layers_exist_only_when_packages_are_declared(tmp_path: Path) -> None: + """apt is required iff `apt-install` is nonempty — the engine and the + dataset stay on the host, so lc itself needs nothing from apt.""" + root = _project(tmp_path / "p", "[tool.lightcone.image]\n") + text = image.containerfile(root) + assert "apt-get" not in text + assert "exit 45" not in text + + +def test_the_render_needs_the_interpreter_pin(tmp_path: Path) -> None: + root = _project(tmp_path / "p", _DECLARED) + (root / ".python-version").unlink() + with pytest.raises(ProjectError, match=".python-version"): + image.containerfile(root) + + +def test_the_render_refuses_a_direct_project(tmp_path: Path) -> None: + with pytest.raises(ProjectError, match="lightcone.image"): + image.containerfile(_project(tmp_path / "p")) + + +def test_env_values_are_quoted_and_dollar_is_literal(tmp_path: Path) -> None: + """The label is JSON full of double quotes, an env value may hold + spaces, and `$` undergoes build-time expansion — measured, an + unescaped `cost$5` bakes as `cost`. Declared values are literals.""" + root = _project( + tmp_path / "p", + '[tool.lightcone.image]\nenv = { OPTS = "-a \\"b\\" c", PRICE = "cost$5" }\n', + ) + text = image.containerfile(root) + assert 'ENV OPTS="-a \\"b\\" c"' in text + assert 'ENV PRICE="cost\\$5"' in text + assert 'LABEL io.lightcone.image="{' in text + + +# ---- the tag ---------------------------------------------------------------- + + +def test_every_declared_key_moves_the_tag(tmp_path: Path) -> None: + tags = { + name: image.tag(_project(tmp_path / name, table)) + for name, table in { + "bare": "[tool.lightcone.image]\n", + "apt": '[tool.lightcone.image]\napt-install = ["bc"]\n', + "run": '[tool.lightcone.image]\nrun-commands = ["true"]\n', + "env": '[tool.lightcone.image]\nenv = { K = "v" }\n', + }.items() + } + assert len(set(tags.values())) == len(tags) + + +def test_the_interpreter_pin_moves_the_tag(tmp_path: Path) -> None: + """The pin is baked into the image, so it is an input to the tag even + though the identity document deliberately omits it.""" + a = _project(tmp_path / "a", "[tool.lightcone.image]\n") + b = _project(tmp_path / "b", "[tool.lightcone.image]\n") + (b / ".python-version").write_text("3.13.1\n") + assert image.tag(a) != image.tag(b) + + +def test_identical_declarations_are_one_tag_wherever_they_live(tmp_path: Path) -> None: + a = _project(tmp_path / "somewhere" / "a", _DECLARED) + b = _project(tmp_path / "elsewhere" / "b", _DECLARED) + assert image.tag(a) == image.tag(b) + assert image.archive_path(a, image.tag(a)) == ( + a / ".datalad" / "environments" / image.tag(a) / "image" + ) + + +# ---- env_version integration ------------------------------------------------ + + +def test_declaring_an_image_moves_env_version(tmp_path: Path) -> None: + """The system layer is part of what a recipe ran under, so declaring + one puts every output behind — the model working, not a bug.""" + direct = _project(tmp_path / "a") + containerized = _project(tmp_path / "b", "[tool.lightcone.image]\n") + assert identity.env_version(direct) != identity.env_version(containerized) + + +def test_every_image_key_moves_env_version(tmp_path: Path) -> None: + bare = identity.env_version(_project(tmp_path / "a", "[tool.lightcone.image]\n")) + with_apt = identity.env_version( + _project(tmp_path / "b", '[tool.lightcone.image]\napt-install = ["bc"]\n') + ) + assert bare != with_apt + + +def test_other_lightcone_tables_do_not_move_env_version(tmp_path: Path) -> None: + """Only the image table is the environment; a sibling table under + `[tool.lightcone]` is somebody else's future.""" + plain = identity.env_version(_project(tmp_path / "a")) + with_sibling = identity.env_version( + _project(tmp_path / "b", '[tool.lightcone.something-else]\nkey = "v"\n') + ) + assert plain == with_sibling diff --git a/tests/test_materialize.py b/tests/test_materialize.py index e5c01f9c..a091da94 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -687,6 +687,76 @@ def _clone(root: Path, into: Path) -> Path: return clone +_FETCH_SPEC = """ +version: "0.0.13" +name: analysis + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: copy + type: metric + inputs: [catalog] + recipe: + command: cat {inputs.catalog} > {output}/copy.txt +""" + + +def test_a_bytes_free_clone_fetches_its_inputs_and_is_up_to_date( + analysis: Callable[..., Path], inline: None, tmp_path: Path +) -> None: + """lc fetches declared inputs rather than telling anyone to — a clone + holding only pointers materializes straight to up-to-date, hashing + the same bytes the origin recorded. Without the fetch this run + *failed*: the worker's hash refused the pointer file.""" + root = analysis(_FETCH_SPEC, files={"data/catalog.fits": "stars\n"}) + assert engine.materialize(root, []).ok + + clone = _clone(root, tmp_path) + with pytest.raises(assets.ContentNotFetchedError): + assets.data_version(clone / "data" / "catalog.fits") + + again = engine.materialize(clone, []) + + assert again.up_to_date, (again.failed, again.warnings) + assert assets.data_version(clone / "data" / "catalog.fits") # the bytes came + assert not dataset.status(clone) + + +def test_an_unreachable_input_is_a_warning_and_a_per_task_failure( + analysis: Callable[..., Path], inline: None, tmp_path: Path +) -> None: + """A failed fetch must not refuse the whole run — independent tasks + still run, and the task whose input is unreachable reports its own + failure. Reaching the state honestly: clone, then delete the origin + the annex would fetch from.""" + root = analysis(_FETCH_SPEC, files={"data/catalog.fits": "stars\n"}) + assert engine.materialize(root, []).ok + clone = _clone(root, tmp_path) + dataset._git(["remote", "remove", "origin"], cwd=clone) + + report = engine.materialize(clone, []) + + assert not report.ok + assert report.failed == ["baseline/copy"] + assert any("could not be fetched" in w for w in report.warnings) + + +def test_check_mode_never_fetches(root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`--check` and `status` are read-only verbs: an unfetched input is a + reported fact there, never a network transfer.""" + + def refuse(*args: Any) -> None: + raise AssertionError("check mode fetched") + + monkeypatch.setattr(engine, "_fetch_inputs", refuse) + engine.check(root, []) + engine.status(root) + + def test_a_drifted_environment_is_made_to_match_before_anything_runs( root: Path, inline: None ) -> None: @@ -712,30 +782,6 @@ def test_a_drifted_environment_is_made_to_match_before_anything_runs( assert project_mod._env_is_current(root) -@pytest.fixture(scope="session") -def engine_dist(tmp_path_factory: pytest.TempPathFactory) -> tuple[str, Path]: - """Build the engine under test into a wheel the pinned record can find. - - The record pins ``lightcone-cli==`` and the suite's build is not - published, so the rerun's ephemeral environment is pointed here via - ``UV_FIND_LINKS`` — which is what lets the suite execute the same - record shape every real commit carries, rather than a test-only one. - - Returns: - The wheel's exact version, and the directory serving it. - """ - dist = tmp_path_factory.mktemp("engine-dist") - subprocess.run( - ["uv", "build", "--wheel", "--out-dir", str(dist)], - cwd=Path(__file__).parent.parent, - check=True, - capture_output=True, - text=True, - ) - wheel = next(dist.glob("*.whl")) - return wheel.name.split("-")[1], dist - - def test_the_recorded_command_reproduces_the_output( root: Path, inline: None, diff --git a/tests/test_project.py b/tests/test_project.py index 459cc1ad..6630393a 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -122,6 +122,32 @@ def test_converge_scaffolds_no_environment_escalation(tmp_path: Path) -> None: assert "[tool.lightcone.image]" not in (project / "pyproject.toml").read_text() +def test_a_containerized_project_converges_no_host_venv( + tmp_path: Path, tools: list[list[str]] +) -> None: + """The host sync is the host-sync deadlock in miniature: the lock's + system-level dependencies — the reason the project containerized at + all — are not on the host, so a host `uv sync` fails and `--check` + would report unconverged forever. The verbs converge the environment + inside the image instead; init owes neither podman nor minutes.""" + project = tmp_path / "proj" + converge(project) + text = (project / "pyproject.toml").read_text() + (project / "pyproject.toml").write_text( + text + '\n[tool.lightcone.image]\napt-install = ["bc"]\n' + ) + shutil.rmtree(project / ".venv") + + report = converge(project) + + assert ".venv" not in [*report.created, *report.repaired, *report.unchanged] + assert report.converged + assert not (project / ".venv").exists() + # The lock still converges: locking is resolution, which the bare + # host can do in both modes. + assert "uv.lock" in report.unchanged + + def test_converge_scaffolds_no_src_directory(tmp_path: Path) -> None: """astra stopped creating it (astra-tools#100) and so do we: the boilerplate's `python src/main.py` is a TODO placeholder, and git drops diff --git a/tests/test_run.py b/tests/test_run.py index 265e861d..8334c2d6 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -78,6 +78,17 @@ def test_an_unbuilt_project_is_told_to_build_it(project: Path) -> None: assert "lc init" in str(raised.value) +def test_a_containerized_project_needs_no_host_venv(project: Path) -> None: + """The host `.venv` is inert in containerized mode — the environment + the verbs enter is `.lightcone/venv`, created by their own converge + inside the image — so a clone is runnable without it.""" + (project / ".venv").rmdir() + (project / "pyproject.toml").write_text( + '[project]\nname = "proj"\n\n[tool.lightcone.image]\napt-install = ["bc"]\n' + ) + assert current_project(project) == project.resolve() + + def test_the_wrong_directory_is_told_to_move_not_to_scaffold(tmp_path: Path) -> None: """The complaint that produced this split: standing in `$HOME`, being told to run `lc init` is advice to scaffold a project in your home diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py index 3bb215ec..f38dfbbf 100644 --- a/tests/test_sandbox_denial.py +++ b/tests/test_sandbox_denial.py @@ -69,6 +69,11 @@ def test_an_undeclared_tool_is_named_with_its_remedy(policy: Policy, project: Pa joined = "\n".join(lines) assert f"cannot execute {tool}" in joined assert "uv add" in joined + # The system-layer remedy is real now — `lc build` exists — so the + # standing cap on remedies makes naming it mandatory, not optional. + assert "[tool.lightcone.image]" in joined + assert "apt-install" in joined + assert "behind" in joined def test_an_undeclared_data_file_gets_the_astra_snippet( diff --git a/tests/test_sandbox_oci.py b/tests/test_sandbox_oci.py new file mode 100644 index 00000000..6947a791 --- /dev/null +++ b/tests/test_sandbox_oci.py @@ -0,0 +1,301 @@ +"""Tests for the OCI backend — the mount table as the mechanism. + +Pure, and run on every OS: the wrap is a function of the policy and the +backend's fields, so the argv a containerized recipe would get is checked +here with nothing spawned and no runtime installed. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from lightcone.engine.sandbox import boundary, exec_policy +from lightcone.engine.sandbox.boundary import Unavailable +from lightcone.engine.sandbox.model import Policy +from lightcone.engine.sandbox.oci import OCIBackend + +_IMAGE_ID = "956ea01f6c5b94522bedc346c9646f81d0707b2a00b2a9ed8b4e5b6a8d2d00d1" + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + project = tmp_path / "analysis" + (project / "results").mkdir(parents=True) + (project / "data").mkdir() + (project / "data" / "catalog.fits").write_text("stars\n") + return project + + +@pytest.fixture +def policy(root: Path) -> Policy: + built = exec_policy( + root, + read_paths=[root / "data" / "catalog.fits"], + env_dir=root / ".lightcone" / "venv", + containerized=True, + ) + yield built + import shutil + + shutil.rmtree(built.tmp_home, ignore_errors=True) + + +def _backend(root: Path, runtime: str = "podman") -> OCIBackend: + flags = ("--userns=keep-id", "--pull=never") if runtime == "podman" else ("--user", "1000:1000") + return OCIBackend( + runtime=runtime, # type: ignore[arg-type] + image_id=_IMAGE_ID, + root=root, + user_flags=flags, + ) + + +# ---- the containerized policy shape ---------------------------------------- + + +def test_the_containerized_policy_is_the_project_world_only(root: Path, policy: Policy) -> None: + """The image is the OS baseline and the exec set — everything present + in it was declared — so the path sets carry only what becomes mounts. + Declared spellings, deliberately unresolved: they become mount + *destinations*, and the recipe addresses the declared path.""" + assert all(str(p).startswith(str(root)) for p in policy.read) + assert policy.execute == () + assert root in policy.read + assert root / "results" in policy.write + assert policy.tmp_home in policy.write + + +def test_the_containerized_home_lives_under_the_project(root: Path, policy: Policy) -> None: + """It is a mount source, and on macOS the podman machine shares the + project's tree while the host's temp roots arrive empty.""" + assert policy.tmp_home.is_relative_to((root / ".lightcone").resolve()) + + +def test_the_overlay_points_uv_at_the_image_environment(root: Path, policy: Policy) -> None: + """The `uv run` hop executes inside the container; this is how it + finds `.lightcone/venv` instead of inventing a `.venv`.""" + env_dir = root / ".lightcone" / "venv" + assert policy.env["UV_PROJECT_ENVIRONMENT"] == str(env_dir) + assert policy.env["PATH"].startswith(str(env_dir / "bin")) + # The PATH tail is the image's own FHS, never the host allowlist + # search path — whose NixOS entry no Debian-family image has. + assert "/run/current-system" not in policy.env["PATH"] + + +# ---- the wrap --------------------------------------------------------------- + + +def test_wrap_is_pure(root: Path, policy: Policy, tmp_path: Path) -> None: + before = set(tmp_path.rglob("*")) + backend = _backend(root) + assert backend.wrap(policy, ["true"]) == backend.wrap(policy, ["true"]) + assert set(tmp_path.rglob("*")) == before + + +def test_reads_mount_ro_and_writes_mount_rw(root: Path, policy: Policy) -> None: + argv = _backend(root).wrap(policy, ["true"]) + assert f"--volume={root.resolve()}:{root}:ro" in argv + results = root / "results" + assert f"--volume={results.resolve()}:{results}:rw" in argv + catalog = root / "data" / "catalog.fits" + assert f"--volume={catalog.resolve()}:{catalog}:ro" in argv + # Read mounts land before write mounts, so the writable results + # directory nests over the read-only tree the way the runtimes + # resolve natively. + assert argv.index(f"--volume={root.resolve()}:{root}:ro") < argv.index( + f"--volume={results.resolve()}:{results}:rw" + ) + + +def test_a_symlinked_input_mounts_at_its_declared_path(root: Path, tmp_path: Path) -> None: + """The HPC case: `/data` is a symlink into a shared store. The bind's + source must be the real file, but the destination is the path the + analysis declared — resolving both would leave the container with no + `/data` at all and the recipe's literal path ENOENT.""" + store = tmp_path / "store" + store.mkdir() + (store / "catalog.h5").write_text("stars\n") + link = tmp_path / "data-link" + link.symlink_to(store) + declared = link / "catalog.h5" + + built = exec_policy( + root, + read_paths=[declared], + env_dir=root / ".lightcone" / "venv", + containerized=True, + ) + try: + argv = _backend(root).wrap(built, ["true"]) + assert f"--volume={store / 'catalog.h5'}:{declared}:ro" in argv + finally: + import shutil + + shutil.rmtree(built.tmp_home, ignore_errors=True) + + +def test_the_rootfs_is_read_only_and_labels_are_disabled(root: Path, policy: Policy) -> None: + """`--read-only`: without it a write outside the declared set + *succeeds* into the container's ephemeral layer and vanishes, while + the run attests `fs: declared` — the silent-loss path. And + `label=disable`: SELinux hosts otherwise refuse every bind read.""" + argv = _backend(root).wrap(policy, ["true"]) + assert "--read-only" in argv + assert "--security-opt" in argv + assert argv[argv.index("--security-opt") + 1] == "label=disable" + + +def test_execution_pins_the_image_by_id_never_a_tag(root: Path, policy: Policy) -> None: + argv = _backend(root).wrap(policy, ["bash", "-c", "true"]) + assert _IMAGE_ID in argv + assert argv[argv.index(_IMAGE_ID) + 1 :] == ["bash", "-c", "true"] + assert not any("lc-env-" in part for part in argv) + + +def test_the_network_is_denied_by_flag(root: Path, policy: Policy) -> None: + argv = _backend(root).wrap(policy, ["true"]) + assert "--network" in argv and argv[argv.index("--network") + 1] == "none" + + +def test_runtimes_differ_only_in_their_spellings(root: Path, policy: Policy) -> None: + podman = _backend(root, "podman").wrap(policy, ["true"]) + docker = _backend(root, "docker").wrap(policy, ["true"]) + assert "--userns=keep-id" in podman and "--pull=never" in podman + assert "--user" in docker and "1000:1000" in docker + strip = { + "--userns=keep-id", "--pull=never", "--user", "1000:1000", "podman", "docker", + "--env=LC_SANDBOX=podman", "--env=LC_SANDBOX=docker", + } # fmt: skip + assert [a for a in podman if a not in strip] == [a for a in docker if a not in strip] + + +def test_the_environment_is_an_allowlist_never_ambient( + root: Path, policy: Policy, monkeypatch: pytest.MonkeyPatch +) -> None: + """A secret in the invoking shell must never reach the container.""" + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "hunter2") + argv = _backend(root).wrap(policy, ["true"]) + assert not any("hunter2" in part or "AWS_SECRET" in part for part in argv) + for key, value in policy.env.items(): + assert f"--env={key}={value}" in argv + assert "--env=LC_SANDBOX=podman" in argv + + +def test_no_host_resolved_env_binary_in_the_argv(root: Path, policy: Policy) -> None: + """The overlay travels as `--env` flags: a host path for `env` (NixOS + keeps it under /run/current-system/sw) need not exist in the image, + and argv[0] dying there would blame the user's command.""" + argv = _backend(root).wrap(policy, ["true"]) + assert not any(part.endswith("/env") for part in argv) + + +# ---- the attestation -------------------------------------------------------- + + +def test_the_attestation_is_derived_from_the_flags(root: Path, policy: Policy) -> None: + for runtime in ("podman", "docker"): + attested = _backend(root, runtime).attest(policy) + assert attested.mechanism == runtime + assert attested.fs == "declared" + assert attested.network == "denied" + assert attested.landlock_abi is None + + +# ---- the seam's composition ------------------------------------------------- + + +class _Recorder: + """A Popen stand-in that records the argv and exits as told.""" + + def __init__(self, returncode: int = 0) -> None: + self.argv: list[str] | None = None + self.returncode = returncode + + def __call__(self, argv: list[str], **kwargs: Any) -> Any: + self.argv = list(argv) + code = self.returncode + + class _Proc: + import io + + stderr = io.StringIO("") + returncode = code + + def wait(self) -> int: + return code + + return _Proc() + + +def test_a_world_backend_takes_the_prefix_inside( + root: Path, policy: Policy, monkeypatch: pytest.MonkeyPatch +) -> None: + """In a container there is no trusted host plumbing: the `uv run` hop + is part of the world being entered, so it lands after the image in + the argv rather than in front of the runtime.""" + recorder = _Recorder() + monkeypatch.setattr(subprocess, "Popen", recorder) + + boundary.run( + _backend(root), + policy, + ["bash", "-c", "true"], + cwd=root, + env={}, + prefix=["uv", "run", "--locked", "--no-sync", "--project", str(root), "--"], + ) + + assert recorder.argv is not None + assert recorder.argv[0] == "podman" + assert recorder.argv[recorder.argv.index(_IMAGE_ID) + 1 :] == [ + "uv", "run", "--locked", "--no-sync", "--project", str(root), "--", + "bash", "-c", "true", + ] # fmt: skip + + +def test_a_host_backend_keeps_the_prefix_outside( + root: Path, policy: Policy, monkeypatch: pytest.MonkeyPatch +) -> None: + """The existing composition, pinned: uv's config and caches are + trusted plumbing outside a host mechanism's rewrite.""" + recorder = _Recorder() + monkeypatch.setattr(subprocess, "Popen", recorder) + + boundary.run( + Unavailable(), + policy, + ["bash", "-c", "true"], + cwd=root, + env={}, + prefix=["uv", "run", "--"], + ) + + assert recorder.argv is not None + assert recorder.argv[:3] == ["uv", "run", "--"] + + +def test_exit_97_is_the_shims_only_under_landlock( + root: Path, policy: Policy, monkeypatch: pytest.MonkeyPatch +) -> None: + """97 is the shim's reserved code, and there is no shim in a + container — a recipe legitimately exiting 97 must not be told lc + could not set up the sandbox.""" + monkeypatch.setattr(subprocess, "Popen", _Recorder(returncode=97)) + outcome = boundary.run(_backend(root), policy, ["true"], cwd=root, env={}) + assert not any("could not set up" in note for note in outcome.notes) + + +def test_exit_125_names_the_runtime_not_the_command( + root: Path, policy: Policy, monkeypatch: pytest.MonkeyPatch +) -> None: + """The runtimes reserve 125 for their own failures — the command + never ran, so neither the denial heuristics nor the trailer should + point at it.""" + monkeypatch.setattr(subprocess, "Popen", _Recorder(returncode=125)) + outcome = boundary.run(_backend(root), policy, ["true"], cwd=root, env={}) + assert any("runtime failed before the command ran" in note for note in outcome.notes) + assert not any("ran under the lc sandbox" in note for note in outcome.notes) diff --git a/tests/test_worker.py b/tests/test_worker.py index c8840b9a..de53cbfd 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -21,7 +21,7 @@ import pytest -from lightcone.engine import assets, identity, plan, worker +from lightcone.engine import assets, container, identity, plan, worker from lightcone.engine.sandbox import Unavailable from lightcone.engine.worker import TaskResult @@ -62,13 +62,25 @@ def _task(root: Path, output_id: str) -> plan.Task: _HEAD = ("0123456789abcdef", "https://example/analysis.git") +def _runtime(root: Path) -> container.Runtime: + """The direct-mode runtime the driver would resolve for these projects.""" + return container.runtime_for_run(root, build=False) + + def _make( root: Path, output_id: str, *upstream: TaskResult, refresh: bool = False ) -> TaskResult: """Run one task the way Dask would, handed its upstream results.""" task = _task(root, output_id) return worker.materialize( - root, task, identity.env_version(root), _HEAD, assets.Versions(), refresh, *upstream + root, + task, + identity.env_version(root), + _HEAD, + assets.Versions(), + refresh, + _runtime(root), + *upstream, ) @@ -236,7 +248,10 @@ def test_a_stale_file_does_not_survive_a_rebuild(root: Path) -> None: output = root / "results/baseline/first" (output / "leftover.txt").write_text("from a previous run\n") - worker.execute(root, _task(root, "first"), identity.env_version(root), {}, head=_HEAD) + worker.execute( + root, _task(root, "first"), identity.env_version(root), {}, head=_HEAD, + runtime=_runtime(root), + ) assert not (output / "leftover.txt").exists() assert (output / "value.txt").exists() @@ -295,7 +310,8 @@ def test_an_environment_that_moved_under_the_run_is_refused(root: Path) -> None: """A manifest may not claim an environment that had already been edited by the time the recipe ran.""" result = worker.execute( - root, _task(root, "first"), "sha256:from-another-run", {}, head=_HEAD + root, _task(root, "first"), "sha256:from-another-run", {}, head=_HEAD, + runtime=_runtime(root), ) assert result.status == "failed" From 735f4a770f889055a7e96f121f5b7bf63215f0c8 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 15:43:40 +0200 Subject: [PATCH 13/23] =?UTF-8?q?Layer=207:=20venues=20=E2=80=94=20SLURM?= =?UTF-8?q?=20in-allocation=20execution=20and=20podman-hpc=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first HPC venue, NERSC-focused, in two commits. ## The SLURM venue (7a) `lc materialize` inside a SLURM allocation now spans every node it was granted: `cluster_for_run()` detects `SLURM_JOB_ID`, binds the Dask scheduler in the driver process to `SLURMD_NODENAME`, and launches one worker per node with a single `srun` on the driver's own interpreter — so driver and workers are the identical installation off the shared filesystem, which is all a worker needs (`lightcone.engine` importable at the driver's version; no git, no annex). Detected, never configured: the allocation *is* the resource declaration, the same shape as mode derivation. No new dependency. ``` salloc -N 4 -C cpu -q regular -t 2:00:00 lc materialize # runs across all 4 nodes # or, fire-and-forget from a login node: sbatch -N 4 -C cpu -q regular -t 2:00:00 --wrap 'lc materialize' ``` A NERSC login node refuses `lc materialize` first thing, naming both commands above; `--check`, `status` and `lc run` work anywhere. A dead srun reports its own exit code; teardown retires workers so srun ends silently. A dask-jobqueue submission venue, if ever wanted, is one more branch in the same ladder plus the config table it genuinely needs. The suite fakes the *host*, never the code: SLURM env vars plus a bash stub for srun drive the real graph through the real detection → bind → launch → teardown path in CI, with real worker processes. A new `LocalCluster(processes=True)` test pins the cross-process pickling contract the recorded decision asked layer 7 to re-verify. ## podman-hpc (7b) A containerized project now runs on NERSC: podman-hpc lands as a spelling inside the existing `OCIBackend` — no new shape — plus one step, `podman-hpc migrate `, placed outside the load branch (a fresh build never loads, yet compute nodes only see migrated images) and run driver-side once for all nodes. Detection prefers the wrapper over bare podman (whose node-local store is exactly what compute nodes cannot see); it is build-capable, making a login node the source of matching-arch archives. A foreign-arch archive refuses before the load, naming both arches and the fix. apptainer/singularity stay deferred; CLAUDE.md records the design headroom they need (Runtime stays facts; honest `network: allowed` for a runtime that cannot deny; the exit-125 note is a podman/docker-family fact). ## Verification - 519 tests pass locally (`uv run pytest`), ruff and mypy clean. - The container smoke suite gains podman-hpc as a parametrization that fires only where the site tool exists — running it on a Perlmutter login node is the spike vehicle; the checklist (srun `--overlap`, `--network none` on compute, `--module` mounts vs `fs: declared`, migrate-by-id, Landlock in the SLES LSM list, Lustre annex timings) is recorded in CLAUDE.md. Deviation from the approved plan, recorded: the plan called for two PRs; repo convention is one PR per layer (#175, #180), so both parts land here as separate commits. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012oVCfc4eZdyKqT1zY95Xi2 --------- Co-authored-by: Claude Fable 5 --- CLAUDE.md | 193 +++++++++++++- src/lightcone/engine/container.py | 124 ++++++--- src/lightcone/engine/materialize.py | 54 +++- src/lightcone/engine/sandbox/model.py | 4 +- src/lightcone/engine/sandbox/oci.py | 15 +- src/lightcone/engine/venue.py | 326 +++++++++++++++++++++++ src/lightcone/engine/worker.py | 5 +- tests/conftest.py | 23 ++ tests/test_container.py | 133 +++++++++- tests/test_container_smoke.py | 11 +- tests/test_materialize.py | 25 ++ tests/test_sandbox_oci.py | 20 +- tests/test_venue.py | 359 ++++++++++++++++++++++++++ 13 files changed, 1222 insertions(+), 70 deletions(-) create mode 100644 src/lightcone/engine/venue.py create mode 100644 tests/test_venue.py diff --git a/CLAUDE.md b/CLAUDE.md index 5a342a97..533dfeab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,7 +53,7 @@ speculatively. | 4 | **Fabric** — `lc materialize`, worker sequence, mid-run relock gate | ✅ **done** | | 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | | 6 | **Container hatch** — `[tool.lightcone.image]`, `lc build`, OCI runtimes as the exec boundary, the image archived in the dataset | ✅ **done** | -| 7 | Venues — Perlmutter, hub/GKE, Cloud Build | ⬜ | +| 7 | **Venues** — SLURM in-allocation execution, login guard, podman-hpc | 🔶 **landed; Perlmutter spike pending** — hub/GKE and Cloud Build deferred to their own layer | | 8 | `lc verify`, WRROC export | ⬜ | `lc status` landed with the invalidation model rather than at layer 8: @@ -1415,7 +1415,8 @@ layer-7 item, as is apptainer/singularity — which is *why* the format is `docker-archive`: all four runtimes consume it, and HPC hosts that cannot build obtain images through the repository. -**`ensure_image` is one function with three strictnesses**, and the split +**`runtime_for_run(root, *, build)` is one function with two +strictnesses**, and the split is load-bearing: `lc build` and the materialize preflight may **build + save + commit** (announced by the CLI, which owns the console — the engine never prints); the probe and the worker entry point only ever @@ -1431,7 +1432,7 @@ cannot substitute. **A dropped archive never substitutes**: a rebuild is a new archive commit under a new id; old manifests keep naming what ran. **Builds and the archive commit happen only on a clean tree, and only -after the graph.** The dirty check runs before `ensure_image` in +after the graph.** The dirty check runs before `runtime_for_run` in materialize, and `lc build` refuses dirt itself: `dataset.save` stages scoped but commits the whole index, so a build on a dirty tree would sweep the user's staged edits into the image commit — and the tag @@ -1520,6 +1521,168 @@ state, sandbox) — repository facts only, no runtime and no network required, which is where the denial note and the runtime-missing refusal point. +## Key Invariants (layer 7) + +**The venue is detected, never configured, and only in one place.** +`cluster_for_run()` is the whole ladder — a SLURM allocation +(`SLURM_JOB_ID` set) spans every node it was granted, anything else is +the local machine — and nothing outside that function asks where a run +executes. The allocation *is* the resource declaration: the user already +answered every sizing question at `salloc`/`sbatch`, so lc adds no venue +config surface, no report field, and no status line (a venue is host +state, and the status header is repository facts). The venue speaks only +when it refuses, and those refusals carry the job facts (nodes expected +vs connected, srun's exit code). A future submission-model venue +(dask-jobqueue) is one more branch in this ladder plus the config table +it genuinely needs — nothing else changes. + +**The allocation branch is `venue.slurm_client()`**: a scheduler in the +driver process bound to `SLURMD_NODENAME` (the default loopback bind is +unreachable from peer nodes), one `srun --overlap --ntasks= +--ntasks-per-node=1` launching `sys.executable -m +distributed.cli.dask_worker` — the driver's own interpreter, the tool +env on the shared filesystem, so driver and workers are the identical +installation and `-m` cannot resolve to a different install the way a +PATH-found `dask` can. One worker process per node with +`--nthreads=` (tasks block in `subprocess.wait()` with the GIL +released — the local branch's own rationale), `--no-nanny` (srun won't +relaunch either; the nanny logs a spurious death on every clean +retirement), `--memory-limit 0` (the real work is in subprocesses behind +the exec boundary, so Dask's memory manager could only pause workers +over phantom numbers), `--death-timeout 60` (a worker whose driver died +exits instead of holding its node to walltime), and `--local-directory` +on a **literal `/tmp`** — never the project tree, explicit so ambient +`DASK_TEMPORARY_DIRECTORY` cannot point it there, and literal rather +than the driver's `tempfile.gettempdir()` because a site prolog can +scope `TMPDIR` to the node or job step that set it, leaving a +driver-resolved path absent on the allocation's other nodes. The +driver's node +hosts a worker too, and a single-node allocation takes the same srun +path — one path means the venue is exercised on the cheapest allocation. + +**The srun child is the one documented exception to `project._run`.** +That seam is run-to-completion capture; this child lives as long as the +run, and its stderr must reach the terminal live (srun's own errors are +the user's to see as they happen). It is a `subprocess.Popen` with the +rationale at the call site, and the fake-srun tests keep the argv +inspectable. Worker connection is a poll loop rather than +`wait_for_workers`, so a dead srun is reported as *its exit code* +immediately, not as a timeout two minutes later; teardown retires +workers first (they exit 0, srun ends silently — killing srun prints +"srun: forcing job termination" on every clean run) and escalates +wait → terminate → kill, bounded, never a hang. A leaked `SLURM_JOB_ID` +with no srun on PATH is a loud refusal, never a silent local fallback — +and the other leak shapes get the same treatment: a non-integer SLURM +count refuses naming the variable (`venue._int_env`), and a +`SLURMD_NODENAME` that does not resolve becomes a `ProjectError` naming +the bind rather than the raw `socket.gaierror` (distributed wraps it in +a `RuntimeError`, so the constructor catches both). + +**The login guard is materialize-scoped, table-driven, and comes +first.** `venue.require_compute_node()` refuses iff a known center's +marker is in the environment and `SLURM_JOB_ID` is not (a marker is set +on compute nodes too; the allocation is what distinguishes them), +naming that center's copy-pasteable `salloc` and `sbatch --wrap 'lc +materialize'` commands. The centers live in `venue._SITES` — one row +each: name, marker variable, and the center's own allocation spellings, +**verified against the center's documentation, never guessed** (the +remedies rule). NERSC is the seeded row; supporting another center is +one row, and nothing else moves — the conftest scrub derives its marker +list from the table, and `test_a_new_center_is_one_table_row` pins that +the row alone drives the message. The guard is the first line of +`materialize()`, before even the tool checks: the allocation is the +remedy with queue latency, so the user submits it first and fixes +whatever later refusals name while waiting. The rerun entry point +(`worker.main`) is guarded too — a rerun executes a recipe, and the +record's `cmd` is how recipes reach a login node without `lc` in the +command line. `check()`, `status()` and `lc run` never call it — a login +node is exactly where "where does this project stand" gets asked. + +**Containerized runs assume a homogeneous allocation**, and the one +part of that lc can check is *enforced*: a multi-node allocation with a +containerized project **refuses** unless the runtime's image store spans +nodes — `container._SHARED_STORE_RUNTIMES`, a positively-stated fact +like `_PODMAN_FAMILY`, holding podman-hpc alone — because podman's and +docker's stores are node-local: the image loads on the driver's node +only, and every task scheduled elsewhere would fail at `run ` with +`--pull=never` forbidding the fetch. The check sits in `materialize()` +before the runtime resolves (a refusal must not cost an image build, so +it asks `runtime_hint()`, letting a wholly missing runtime reach +`runtime_for_run`'s own refusal) and not in `runtime_for_run` (the rerun +entry point shares that, and a rerun is a one-node run wherever it +sits). The rest stays a +recorded assumption: every node offers the same runtime binary and sees +the shared-filesystem project tree. podman-hpc is what makes multi-node +true on NERSC — `migrate` squashes the image to the shared filesystem, +where every compute node runs it. + +**podman-hpc is a spelling, not a shape.** It rides the existing +`OCIBackend` (standard podman flags, `--userns=keep-id`, +`--pull=never`), joins the podman family in `uid_flags`, `_loaded` +(`image exists`) and `_build` (`save --format docker-archive`) — the +set stated once as `container._PODMAN_FAMILY` and asked positively at +every site, so a future runtime falls *outside* it by default instead +of inheriting podman behavior through a `!= "docker"` back door — and +adds +exactly one step: `podman-hpc migrate ` in `runtime_for_run`, +**outside the load branch** — after a fresh `lc build` the image is +already in the store and the load never runs, but compute nodes only see +migrated images. Migrate runs driver-side wherever `runtime_for_run` +runs (login node at `lc build`, materialize preflight, the rerun entry +point); one migrate serves all nodes. Detection order is +**podman-hpc → podman → docker**: a site installs the wrapper precisely +because plain podman's node-local overlay store is invisible to compute +nodes, so where both are on PATH the bare sibling is the broken one. +Presence-only detection (no daemon, no machine); the no-runtime refusal +still names only podman/docker — podman-hpc is site-installed, never a +user remedy. It is build-capable (it wraps `podman build`; NERSC login +nodes are where a matching-arch archive comes from), so all three +runtimes are both build- and run-capable and a capability predicate +would be dead code — it lands with the first run-only runtime. + +**The architecture gate refuses before the load.** A wrong-arch +`load` *succeeds* and then dies as `exec format error` deep inside a +recipe — so `_require_arch` compares the archive's recorded arch against +`platform.machine()` (via the closed `_OCI_ARCH` map) before anything is +loaded, naming both arches and the fix: build on a matching host (on +NERSC, a login node), commit, push, pull. Ignorance is not a mismatch — +an archive that does not say, or an unmapped host machine, passes. +Recorded residue: an arm64 mac that *could* emulate an amd64 archive is +refused too (emulation is unattested ten-times-slower execution), and +full multi-arch (arch-suffixed archive paths, `--platform` cross-builds) +remains open, with the venue that makes it real. + +**Design headroom for a daemonless runtime (apptainer/singularity), +deferred but load-bearing**: `Runtime` stays facts (root/mode/name/ +tag/id/arch), never mechanism — the committed→fetched→loaded ladder +stays name-branched inside `runtime_for_run`, and a store-less runtime's +analogue is a one-time archive→SIF conversion into gitignored +`.lightcone/` cache keyed by the same runtime-independent config-blob +id (the reason `docker-archive` stays the store format). +`container.backend()` stays the single construction point; a future +world-backend is one dataclass in `sandbox/` plus one branch there. A +runtime that cannot deny network must attest `network: allowed` with no +denial flag emitted — no consumer may assume containerized ⇒ denied, +and `_sandbox_line`'s containerized prose is the one place that +assumption lives today. `boundary.run`'s exit-125 note is a +podman/docker-family fact, not a `contains_prefix` fact — it becomes +mechanism-keyed when a non-OCI backend lands. + +**Pending the one-time Perlmutter spike** (run `tests/ +test_container_smoke.py` on a login node, then one materialize through +`sbatch`; record findings here): `--overlap` and `--cpus-per-task` +behavior inside salloc/sbatch steps; `nidXXXXXX` resolution from peer +nodes (else `--interface hsn0`); `SLURM_CPUS_ON_NODE` on a CPU node +(128 vs 256 hyperthreads); cold-Lustre `distributed` import vs the +120 s worker wait; `podman-hpc migrate` accepting a bare image id and +re-running cheaply; `--network none` on compute nodes (the +`network: denied` attestation hangs on it); podman-hpc `--module` +site-injected mounts vs the honesty of `fs: declared` (the one item +that could add a flag); whether Landlock is in the SLES boot LSM list +(either answer is handled — a host without it attests `fs: open` with +the downgrade note); `git annex add`/`get` timings on `$SCRATCH` and +`$CFS` (the recorded Lustre residue). + ### Recorded decisions - **The engine is the host's uv tool, never a project dependency** @@ -1543,12 +1706,12 @@ refusal point. worker process actually needs is `lightcone.engine` importable at a matching version — pickle serializes `worker.materialize` by reference and `Task`/`Versions`/`TaskResult` by class reference. - Verified by hand (2026-08-20), not pinned by the suite — a full - materialization through a `LocalCluster(processes=True)`: the - current code works unchanged across process boundaries, and workers + Pinned by the suite since layer 7 + (`test_a_processes_cluster_fits_through_the_seam`, a full + materialization through a `LocalCluster(processes=True)`): the + code works unchanged across process boundaries, and workers need **no git and no git-annex** (the driver owns git alone; - `data_version` is pure file hashing). The shipped cluster is - threads; re-verify when layer 7 makes processes real. So per venue: on HPC the tool env lives on the shared + `data_version` is pure file hashing). So per venue: on HPC the tool env lives on the shared filesystem and the venue launches workers on the driver's own interpreter (`sys.executable` — dask-jobqueue's `python=`), which makes driver and workers the identical installation; in containerized @@ -1766,11 +1929,12 @@ refusal point. | Change how a project stores bytes | `src/lightcone/engine/dataset.py` + `templates/files/gitattributes.tmpl` | Every command through `project._run`; test it against a real annex (`real_tools`) | | Change how an output is identified | `src/lightcone/engine/identity.py` + `tests/test_identity.py` | Sensitivity tests both ways: what must move the hash, and what must not | | Change what the image is made of | `src/lightcone/engine/image.py` + `tests/test_image.py` | Pure; structure-and-ordering tests, never byte goldens; every declaration key is hashed | -| Change how images are built, stored or entered | `src/lightcone/engine/container.py` (+ `sandbox/oci.py` for the exec argv) + `tests/test_container.py` | Everything through `project._run`; keep the three `ensure_image` strictnesses; runtime differences are spellings inside `OCIBackend`, never new shapes | +| Change how images are built, stored or entered | `src/lightcone/engine/container.py` (+ `sandbox/oci.py` for the exec argv) + `tests/test_container.py` | Everything through `project._run`; keep `runtime_for_run`'s two strictnesses; runtime differences are spellings inside `OCIBackend`, never new shapes | | Change when an output is remade | `src/lightcone/engine/assets.py` + `tests/test_assets.py` | One `classify()`, two callers; `--check` differs by one input value, never by logic. Ask first whether the thing that moved *contradicts* the project or is a *circumstance* — the second is `behind`, not `stale` | | Change how the spec becomes a graph | `src/lightcone/engine/plan.py` + `tests/test_plan.py` | Ask `astra.resolve`; if the answer is missing, the fix is a PR to astra-tools. Anything ambiguous is a `ProjectError`, never a guess | | Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | | Change what a run commits | `src/lightcone/engine/materialize.py` + `tests/test_materialize.py` | The driver owns git alone; the tree ends as clean as it started | +| Change where a run executes | `src/lightcone/engine/venue.py` + `materialize.cluster_for_run` + `tests/test_venue.py` | One detection ladder, in `cluster_for_run` alone; venues are detected, never configured; test by faking the host (env vars + a stub srun), never the code | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | | Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | | Change what a sandboxed command may touch | `sandbox/policy.py` + `tests/test_sandbox_policy.py` | Path sets only — no mechanism ever leaks in here | @@ -1841,6 +2005,17 @@ The sandbox suite splits along the seam, which is what makes it cheap: - `tests/test_run.py` — what `lc run` decides *before* it execs: the current-directory project check, declared inputs, the uv hop. Nothing spawns. +- `tests/test_venue.py` — the venue's surface is ambient (env vars, an + srun on PATH), so the suite fakes the *host*, never the code: SLURM + variables set deliberately, a bash stub standing in for srun, and the + end-to-end tests run a real graph through the real detection, bind, + launch and teardown path on any machine — real worker processes, no + SLURM anywhere. `SLURMD_NODENAME=127.0.0.1` keeps the scheduler bind + hermetic against CI DNS. No required-vs-skip gating: nothing depends + on host capability. The `venue_env` autouse fixture in conftest scrubs + the venue variables suite-wide — without it the whole suite fails on a + NERSC login node (the guard) and the real-cluster test would srun + across a live allocation. - `tests/test_image.py` — **pure**: the declaration, the document, the render's structure and ordering, tag sensitivity both ways, and the `env_version` integration. `tests/test_sandbox_oci.py` — **pure**: the diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py index 3a68c9b6..347b44a7 100644 --- a/src/lightcone/engine/container.py +++ b/src/lightcone/engine/container.py @@ -9,10 +9,10 @@ name-pinned apt notwithstanding. A dropped archive never substitutes: a rebuild is a new archive under a new id, never the old reference. -Runtime is host capability, not project state — podman is preferred, -docker accepted — and the archive format is the one all of podman, -docker, and (later, on HPC) apptainer/singularity consume, which is why -build-capable and run-capable are separate questions. +Runtime is host capability, not project state — podman-hpc where a site +provides it, podman preferred, docker accepted — and every one of them +consumes the same ``docker-archive``, which is what lets the repository +stay the one store. Every command goes through :func:`~lightcone.engine.project._run`, the seam the whole engine shares, so the suite never spawns a runtime. @@ -23,6 +23,7 @@ import hashlib import json import os +import platform import re import shutil import sys @@ -35,6 +36,18 @@ from lightcone.engine import assets, dataset, image, project, sandbox from lightcone.engine.project import ProjectError, _check_call +#: The runtimes that are podman underneath and share its spellings — +#: the one statement of the set, spelled positively everywhere it is +#: asked, so a future runtime falls outside it by default rather than +#: inheriting podman behavior through a `!= "docker"` back door. +_PODMAN_FAMILY = ("podman", "podman-hpc") + +#: The runtimes whose image store every node of an allocation can see — +#: podman-hpc's migrate squashes the image to the shared filesystem. +#: podman's and docker's overlay stores are node-local, which is what +#: the multi-node materialize refusal stands on. +_SHARED_STORE_RUNTIMES = ("podman-hpc",) + @dataclass(frozen=True) class Runtime: @@ -51,7 +64,7 @@ class Runtime: #: Where the project environment lives: ``.venv``, or the in-image #: ``.lightcone/venv``. env_dir: Path - #: ``podman`` or ``docker``; empty in direct mode. + #: ``podman``, ``podman-hpc`` or ``docker``; empty in direct mode. runtime: str = "" image_tag: str = "" #: The image id (bare hex of its config blob) — execution pins on @@ -120,8 +133,17 @@ def runtime_for_run(root: Path, *, build: bool) -> Runtime: _build(root, name, tag, archive) _fetch(root, archive) image_id, arch = archive_identity(archive) + _require_arch(root, archive, arch) # before the load — see its docstring if not _loaded(root, name, image_id): _check_call([name, "load", "-i", str(archive)], cwd=root) + if name == "podman-hpc": + # Compute nodes run only migrated images — the squashed copy on + # the shared filesystem — never the login node's overlay store. + # Outside the load branch, because after a fresh `lc build` the + # image is already in the store and the load never runs. + # Unconditional: nothing here can ask whether the squashed copy + # already exists, so re-migrate cost is podman-hpc's to bound. + _check_call([name, "migrate", image_id], cwd=root) return Runtime( root=root, mode="containerized", @@ -175,20 +197,25 @@ def _committed(archive: Path) -> bool: def runtime_name(root: Path) -> str: """Detect which container runtime this host offers. - podman first — rootless podman needs no daemon and no group — then - docker, whose CLI without a reachable daemon is probed rather than - trusted, because `docker` on PATH with the daemon down is the common - broken state and "cannot connect to the socket" mid-run is a worse - message than this one. + podman-hpc first: a site installs the wrapper precisely because plain + podman does not work on its compute nodes — the node-local overlay + store is invisible there, only the wrapper's migrated copies run — so + where both are on PATH, the bare sibling is the broken one. Then + podman — rootless, no daemon, no group — then docker, whose CLI + without a reachable daemon is probed rather than trusted, because + `docker` on PATH with the daemon down is the common broken state and + "cannot connect to the socket" mid-run is a worse message than this + one. podman-hpc is presence-only: no daemon to probe, and no machine + (it is a Linux-site tool). Args: root: The project root, for the probe's working directory. Returns: - ``"podman"`` or ``"docker"``. + ``"podman-hpc"``, ``"podman"`` or ``"docker"``. Raises: - ProjectError: If neither is usable. + ProjectError: If none is usable. """ name = runtime_hint() if name == "podman": @@ -200,7 +227,7 @@ def runtime_name(root: Path) -> str: "(or install podman, which needs no daemon), then retry. " "`lc status` shows what this project needs." ) - else: + elif not name: raise ProjectError( "this project is containerized and needs a container runtime: install " "podman (recommended: https://podman.io/docs/installation) or docker. " @@ -225,15 +252,15 @@ def backend(runtime: Runtime) -> sandbox.Backend: """ if runtime.mode == "direct": return sandbox.detect() - from lightcone.engine.sandbox.oci import OCIBackend + from lightcone.engine.sandbox.oci import OCIBackend, OCIRuntime # `--pull=never` beside the uid flags rather than inside them: it is - # a pull policy (a typo'd reference must fail, not fetch), podman's - # spelling only, and filing it under uid mapping is where the next - # reader would not look. - pull = ("--pull=never",) if runtime.runtime == "podman" else () + # a pull policy (a typo'd reference must fail, not fetch), the podman + # family's spelling only, and filing it under uid mapping is where + # the next reader would not look. + pull = ("--pull=never",) if runtime.runtime in _PODMAN_FAMILY else () return OCIBackend( - runtime=cast(Literal["podman", "docker"], runtime.runtime), + runtime=cast(OCIRuntime, runtime.runtime), image_id=runtime.image_id, root=runtime.root, user_flags=(*uid_flags(runtime.runtime), *pull), @@ -287,9 +314,9 @@ def runtime_hint() -> str: is skipped because a header must not cost a subprocess. Returns: - ``"podman"``, ``"docker"``, or ``""``. + ``"podman-hpc"``, ``"podman"``, ``"docker"``, or ``""``. """ - for name in ("podman", "docker"): + for name in ("podman-hpc", "podman", "docker"): if shutil.which(name): return name return "" @@ -446,23 +473,58 @@ def archive_identity(path: Path) -> tuple[str, str]: ) +#: `platform.machine()` spellings mapped onto the OCI architecture names +#: an archive's config records. Closed deliberately: an unmapped host +#: cannot be refused on ignorance. +_OCI_ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} + + +def _require_arch(root: Path, archive: Path, arch: str) -> None: + """Refuse an archive built for another architecture. + + The archive is single-arch — it is the exact bytes that ran, which is + the point of committing it — so a host on the other architecture gets + a refusal naming the fix, not an emulated run ten times slower than + the attestation implies, and not an `exec format error` deep inside + a recipe. Skipped when either side is unknown. + + Args: + root: The project root, for naming the archive. + archive: The committed archive. + arch: Its recorded architecture; empty means unknown. + + Raises: + ProjectError: On a mismatch. + """ + host = _OCI_ARCH.get(platform.machine().lower(), "") + if not host or not arch or host == arch: + return + relative = archive.relative_to(root).as_posix() + raise ProjectError( + f"the committed image archive `{relative}` was built for {arch} and this " + f"host is {host} — it cannot run here. Run `lc build` on a host whose " + "architecture matches (on NERSC, a login node), commit and push; then " + "`git pull` here." + ) + + def uid_flags(runtime: str) -> list[str]: """The flags that keep files written through a mount owned by the user. - podman maps the invoking uid into the container (``--userns=keep-id``); - docker has no equivalent spelling, so the container simply runs as the - invoking uid — without which a rootful docker writes root-owned files - into ``results/`` that the host's git cannot manage. The missing - passwd entry that leaves behind is covered by the private-HOME - overlay every exec already gets. + The podman family maps the invoking uid into the container + (``--userns=keep-id``); docker has no equivalent spelling, so the + container simply runs as the invoking uid — without which a rootful + docker writes root-owned files into ``results/`` that the host's git + cannot manage. The missing passwd entry that leaves behind is covered + by the private-HOME overlay every exec already gets. Args: - runtime: ``"podman"`` or ``"docker"``. + runtime: ``"podman"``, ``"podman-hpc"`` or ``"docker"``. Returns: The argv fragment. """ - if runtime == "podman": + if runtime in _PODMAN_FAMILY: return ["--userns=keep-id"] return ["--user", f"{os.getuid()}:{os.getgid()}"] @@ -512,7 +574,7 @@ def _build(root: Path, runtime: str, tag: str, archive: Path) -> None: # dies midway must not leave a partial archive that the dirty-tree # refusal would then tell the user to commit. partial = archive.parent / "image.partial" - save_format = ["--format", "docker-archive"] if runtime == "podman" else [] + save_format = ["--format", "docker-archive"] if runtime in _PODMAN_FAMILY else [] try: _check_call([runtime, "save", *save_format, "-o", str(partial), tag], cwd=root) except ProjectError: @@ -578,7 +640,7 @@ def _build_failure(tag: str, stderr: str) -> str: def _loaded(root: Path, runtime: str, image_id: str) -> bool: """Whether the runtime's local store already holds *image_id*.""" - probe = ["image", "exists" if runtime == "podman" else "inspect", image_id] + probe = ["image", "exists" if runtime in _PODMAN_FAMILY else "inspect", image_id] return project._run([runtime, *probe], cwd=root).returncode == 0 diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index a14ae5cb..e5ebc228 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -41,7 +41,7 @@ from pathlib import Path from typing import Any, Protocol -from lightcone.engine import assets, container, dataset, identity, plan, project, worker +from lightcone.engine import assets, container, dataset, identity, plan, project, venue, worker from lightcone.engine.plan import Graph, Key, Task from lightcone.engine.project import ProjectError @@ -406,10 +406,14 @@ def materialize( blocked, plus the boundary's notes and the lock scan's warnings. Raises: - ProjectError: If a required tool or git's committer identity is - missing, the tree has uncommitted changes, or the lock cannot - be audited. + ProjectError: If this is a login node, a required tool or git's + committer identity is missing, the tree has uncommitted + changes, or the lock cannot be audited. """ + # First, because its remedy is the one with queue latency: the user + # can submit the allocation and fix anything the later refusals name + # while waiting for it. + venue.require_compute_node() project.require_uv() project.require_git() project.require_git_annex() @@ -429,6 +433,25 @@ def materialize( if not graph.tasks: return report _fetch_inputs(root, graph, report) + # Before the runtime resolves, because the refusal must not cost an + # image build: a containerized graph can span an allocation only if + # every node can see the image — the hint suffices, since which + # stores span nodes is `container._SHARED_STORE_RUNTIMES`'s fact and + # a wholly missing runtime gets `runtime_for_run`'s own refusal. Off + # the driver's node a task would otherwise fail to find an image + # `--pull=never` forbids it to fetch. + if ( + (nodes := venue.allocation_nodes()) > 1 + and project.mode(root) == "containerized" + and (name := container.runtime_hint()) + and name not in container._SHARED_STORE_RUNTIMES + ): + raise ProjectError( + f"this allocation spans {nodes} nodes and `{name}`'s image store is " + "node-local, so recipes scheduled on the other nodes would not find " + "the image. Use a single-node allocation, or a system whose runtime " + "shares images across nodes (NERSC's podman-hpc)." + ) # Materialize is one of the two verbs allowed to build the image (the # other is `lc build`); the probe and the rerun entry point only find # one. Resolved once, then handed to every task — the HEAD discipline. @@ -573,20 +596,27 @@ def completed(self, handles: list[Any]) -> Iterator[worker.TaskResult]: @contextmanager def cluster_for_run() -> Iterator[Scheduler]: - """Open a scheduler for one run. The seam venues will land behind. + """Open a scheduler for one run — the venue ladder, and nothing else. Every core, with no knob to say otherwise: how much of a machine a run - may use, and which machine, is one question and it belongs to the - declaration of an execution backend. + may use, and which machine, is one question, and the venue answers it — + a SLURM allocation spans every node it was granted, and the local + machine is the whole of itself. Detected, never configured, and only + here: nothing outside this function asks where a run executes. - Threads rather than processes — every task's real work happens in a - subprocess behind the exec boundary, so a worker spends its time in - ``wait()`` with the GIL released, and a threaded cluster costs no - interpreter startup and no pickling of results. + Threads rather than processes on the local branch — every task's real + work happens in a subprocess behind the exec boundary, so a worker + spends its time in ``wait()`` with the GIL released, and a threaded + cluster costs no interpreter startup and no pickling of results. The + allocation branch runs one such worker per node. Yields: - A scheduler bound to a local Dask cluster, closed on exit. + A scheduler bound to a Dask cluster, closed on exit. """ + if venue.allocation_nodes(): + with venue.slurm_client() as client: + yield _Dask(client) + return from distributed import Client, LocalCluster with LocalCluster( # type: ignore[no-untyped-call] diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index 86959f9a..ea4fdb39 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -75,7 +75,7 @@ def grants(self, path: Path, roots: tuple[Path, ...]) -> bool: class Capability: """What enforcement this host can provide, as probed.""" - kind: Literal["landlock", "seatbelt", "podman", "docker", "none"] + kind: Literal["landlock", "seatbelt", "podman", "docker", "podman-hpc", "none"] landlock_abi: int | None = None #: Why, when ``kind`` is ``none``. Reaches the user — a downgrade is #: never silent. @@ -93,7 +93,7 @@ class Attestation: ``--network none``. """ - mechanism: Literal["landlock", "seatbelt", "podman", "docker", "none"] + mechanism: Literal["landlock", "seatbelt", "podman", "docker", "podman-hpc", "none"] fs: Literal["declared", "open"] network: Literal["allowed", "denied"] = "allowed" landlock_abi: int | None = None diff --git a/src/lightcone/engine/sandbox/oci.py b/src/lightcone/engine/sandbox/oci.py index b23e2940..3cd07f5c 100644 --- a/src/lightcone/engine/sandbox/oci.py +++ b/src/lightcone/engine/sandbox/oci.py @@ -1,8 +1,8 @@ """The containerized backend: the mount table is the mechanism. -One backend for podman and docker, data-parameterized — the two differ -in spellings (how the invoking uid is kept, whether pulling must be -forbidden), not in shape. The policy's path sets map one-to-one onto +One backend for podman, podman-hpc and docker, data-parameterized — they +differ in spellings (how the invoking uid is kept, whether pulling must +be forbidden), not in shape. The policy's path sets map one-to-one onto mounts: ``read`` becomes ``:ro``, ``write`` becomes ``:rw``, and the image itself is the OS baseline and the exec set — everything present in it was declared, which is why the containerized policy carries no @@ -25,13 +25,18 @@ from lightcone.engine.sandbox.boundary import SANDBOX_ENV from lightcone.engine.sandbox.model import Attestation, Capability, Policy +#: The runtimes this backend can speak for — the one statement of the +#: set, so the type does not get hand-copied out of step at its uses. +OCIRuntime = Literal["podman", "docker", "podman-hpc"] + @dataclass(frozen=True) class OCIBackend: """A container runtime, expressed as an argv rewrite.""" - #: ``podman`` or ``docker`` — also what the attestation names. - runtime: Literal["podman", "docker"] + #: ``podman``, ``podman-hpc`` or ``docker`` — also what the + #: attestation names. + runtime: OCIRuntime #: The image id (bare hex). Execution pins on the id, never a tag, so #: a retagged image in the local store can never substitute. image_id: str diff --git a/src/lightcone/engine/venue.py b/src/lightcone/engine/venue.py new file mode 100644 index 00000000..9acdcbda --- /dev/null +++ b/src/lightcone/engine/venue.py @@ -0,0 +1,326 @@ +"""Where a run executes: the venue a materialization finds itself on. + +A venue is host state, never project state — nothing here reads the +project or enters any identity. The one venue beyond the local machine is +a SLURM allocation, detected rather than configured: the user already +declared every resource question to SLURM (`salloc -N4 …`), so the +allocation *is* the declaration, and lc's job is to span it — one Dask +worker per allocated node, launched with a single `srun`, all connected +to a scheduler living in the driver process. + +Workers run the driver's own interpreter (`sys.executable -m`), which on +an HPC system is the lc tool environment on the shared filesystem — so +driver and workers are the identical installation, which is all a worker +process needs: `lightcone.engine` importable at the driver's version. +Workers need no git and no git-annex; the driver owns git alone. + +If the driver dies uncleanly, workers exit on their own (death timeout) +and the allocation's walltime is the backstop; whatever the interrupted +run left behind meets the next run's dirty-tree refusal, which names the +`results/` paths to discard — that is the designed recovery, not a +watchdog. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any + +from lightcone.engine.project import ProjectError + +#: How long the allocation's workers get to connect before the run +#: refuses. Generous because the first import of `distributed` from a +#: cold parallel filesystem is seconds, not milliseconds. +_WORKER_WAIT = 120.0 + +#: Grace given to srun to end on its own once the workers are retired, +#: before the terminate/kill escalation. +_REAP_GRACE = 20.0 + + +@dataclass(frozen=True) +class _Site: + """One HPC center whose login nodes must not run recipes. + + A row is the whole cost of knowing a center: the variable its + systems put in every environment, and the center's own allocation + spellings for the refusal's remedies — copy-pasteable, so they are + verified against the center's documentation, never guessed. + """ + + #: The center's name, as the refusal states it. + name: str + #: The environment variable whose presence identifies the center's + #: machines — set on login and compute nodes alike; an active + #: allocation (SLURM_JOB_ID) is what tells the two apart. + marker: str + #: An interactive allocation, with concrete numbers the user edits. + salloc: str + #: The batch form of the same request; the guard appends ``--wrap``. + sbatch: str + + +#: The centers the guard knows. Supporting another is one row here — +#: nothing else changes, including the test-suite scrub, which derives +#: its variable list from this table. +_SITES = ( + _Site( + name="NERSC", + marker="NERSC_HOST", + salloc="salloc --nodes=1 --constraint=cpu --qos=interactive --time=02:00:00", + sbatch="sbatch --nodes=1 --constraint=cpu --qos=regular --time=02:00:00", + ), +) + + +def require_compute_node(command: str = "lc materialize") -> None: + """Refuse to execute recipes on a known HPC center's login node. + + A login node is for editing and submitting, not computing — and every + node of an allocation becomes a worker, so the remedy is to run the + same command inside one. A center's marker is set on compute nodes + too; an active allocation (SLURM_JOB_ID) is what distinguishes them. + + Args: + command: What to name in the refusal — the command whose recipes + would have run here. + + Raises: + ProjectError: On a known center's login node, naming that + center's salloc and sbatch commands to run instead. + """ + if "SLURM_JOB_ID" in os.environ: + return + site = next((s for s in _SITES if s.marker in os.environ), None) + if site is None: + return + raise ProjectError( + f"{command} executes recipes on compute nodes, and this is a " + f"{site.name} login node ({site.marker} is set with no SLURM " + "allocation active).\n" + "\n" + "Get an allocation and run it there:\n" + "\n" + " interactive:\n" + f" {site.salloc}\n" + f" {command}\n" + "\n" + " batch (from the project root):\n" + f" {site.sbatch} \\\n" + f" --wrap '{command}'\n" + "\n" + "lc materialize --check, lc status and lc run work anywhere." + ) + + +def allocation_nodes() -> int: + """How many nodes the surrounding SLURM allocation holds; 0 outside one. + + Returns: + The node count, or 0 when this process is not inside an + allocation. + + Raises: + ProjectError: If the allocation's node count is not a number. + """ + if "SLURM_JOB_ID" not in os.environ: + return 0 + return _int_env("SLURM_JOB_NUM_NODES", _int_env("SLURM_NNODES", 1)) + + +def _int_env(name: str, default: int) -> int: + """Read a SLURM count, refusing garbage rather than tracebacking. + + SLURM writes plain integers into the variables read here; anything + else is a hand-set or mangled environment — the same leak class as + ``SLURM_JOB_ID`` without an srun, and it gets the same treatment. + """ + value = os.environ.get(name) + if not value: + return default + try: + return int(value) + except ValueError: + raise ProjectError( + f"{name}={value!r} is not a number — this does not look like a real " + "SLURM allocation. If the variable leaked in from outside, unset it." + ) from None + + +@contextmanager +def slurm_client() -> Iterator[Any]: + """Span the SLURM allocation this process is running inside. + + The scheduler lives here, in the driver, bound to this node's + SLURM-canonical hostname so workers on the allocation's other nodes + can reach it — the default loopback bind cannot be. One `srun` + launches one worker per node; the driver's own node hosts a worker + too, because the driver's footprint is small against a node and + excluding it would waste one. + + Yields: + A connected Dask client with every node's worker registered. + + Raises: + ProjectError: If srun is missing, exits before the workers + connect, or the workers do not all connect in time. + """ + from distributed import Client, LocalCluster + + if shutil.which("srun") is None: + raise ProjectError( + "SLURM_JOB_ID is set but srun is not on PATH, so lc cannot launch " + "workers across the allocation. If the variable leaked in from " + "outside — a container, a copied environment — unset it to run on " + "this machine alone." + ) + nodes = allocation_nodes() + cpus = _int_env("SLURM_CPUS_ON_NODE", os.cpu_count() or 1) + host = os.environ.get("SLURMD_NODENAME") or socket.gethostname() + + try: + cluster = LocalCluster( # type: ignore[no-untyped-call] + n_workers=0, + host=host, + dashboard_address=None, + ) + except (OSError, RuntimeError) as e: + # SLURM's NodeName is an alias, not a promise of a resolvable + # hostname — a site where it differs from NodeHostname would + # otherwise die here as a raw traceback (distributed wraps the + # socket.gaierror in a RuntimeError of its own). + raise ProjectError( + f"cannot start the run's scheduler bound to `{host}` ({e}). The " + "usual cause is a node's SLURM name that did not resolve to an " + "address this process can bind — workers on the allocation's " + "other nodes could not have reached it either." + ) from e + with cluster, Client(cluster) as client: # type: ignore[no-untyped-call] + # Not `project._run`, deliberately: that seam is run-to-completion + # capture, and this child lives as long as the run — and its + # stderr must reach the terminal live, because srun's own errors + # (bad step, drained node) are the user's to see as they happen. + env = dict(os.environ) + env.setdefault("DASK_LOGGING__DISTRIBUTED", "warning") + # Literal `/tmp`, not the driver's resolved tempdir: a site + # prolog can scope TMPDIR to the node or job step that set + # it, and a driver-side path baked into every worker's argv + # would then be absent on the allocation's other nodes. + proc = subprocess.Popen( + _srun_argv(cluster.scheduler_address, nodes, cpus, "/tmp"), + env=env, + ) + try: + _await_workers(client, proc, nodes) + yield client + finally: + _wind_down(client, proc) + + +def _srun_argv(scheduler: str, nodes: int, cpus: int, scratch: str) -> list[str]: + """Build the one srun invocation that spans the allocation. + + Args: + scheduler: The driver-side scheduler's address. + nodes: Allocated node count — one worker task per node. + cpus: Threads per worker; tasks block in ``subprocess.wait()`` + with the GIL released, so threads carry a whole node. + scratch: Node-local directory for the worker's own state — never + the project tree, and explicit so ambient Dask configuration + cannot point it there. + + Returns: + The argv, ready for Popen. + """ + return [ + "srun", + # Inside salloc's interactive step a plain srun can wait forever + # for the resources that step already holds. + "--overlap", + f"--ntasks={nodes}", + "--ntasks-per-node=1", + # Without it the step is entitled to one core and the worker's + # threads are bound to it. + f"--cpus-per-task={cpus}", + # The driver's own interpreter — the tool environment on the + # shared filesystem — so driver and workers are the identical + # installation. `-m` cannot resolve to some other install the + # way a PATH-found `dask` can. + sys.executable, + "-m", + "distributed.cli.dask_worker", + scheduler, + "--nthreads", + str(cpus), + "--nworkers", + "1", + "--no-dashboard", + # srun will not relaunch the task, so an auto-restart nanny adds + # nothing and logs a spurious death on every clean retirement. + "--no-nanny", + # A worker whose driver is gone exits instead of holding its node + # to walltime. + "--death-timeout", + "60", + # The real work happens in subprocesses behind the exec boundary, + # whose memory Dask cannot see — its manager could only ever + # pause a worker over phantom numbers. + "--memory-limit", + "0", + "--local-directory", + scratch, + ] + + +def _await_workers(client: Any, proc: subprocess.Popen[bytes], nodes: int) -> None: + """Wait until every node's worker is registered. + + A poll loop rather than ``wait_for_workers`` so that a dead srun is + reported as srun's own exit code, not as a timeout two minutes later. + """ + deadline = time.monotonic() + _WORKER_WAIT + while True: + connected = len(client.scheduler_info()["workers"]) + if connected >= nodes: + return + if (code := proc.poll()) is not None: + raise ProjectError( + f"srun exited with code {code} before the allocation's workers " + f"connected ({connected} of {nodes} had) — its error is above." + ) + if time.monotonic() >= deadline: + raise ProjectError( + f"expected {nodes} dask workers (one per allocated node); " + f"{connected} connected within {int(_WORKER_WAIT)}s." + ) + time.sleep(0.5) + + +def _wind_down(client: Any, proc: subprocess.Popen[bytes]) -> None: + """Retire the workers, then reap srun — gracefully first. + + Retirement makes each worker exit 0, so srun ends silently; killing + srun instead prints "srun: forcing job termination" on every clean + run. The escalation below it is for the runs that were not clean. + """ + try: + client.retire_workers(close_workers=True, remove=True) + except Exception: + pass + try: + proc.wait(timeout=_REAP_GRACE) + except subprocess.TimeoutExpired: + proc.terminate() + try: + proc.wait(timeout=_REAP_GRACE / 2) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index 2b06e34b..92937598 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -35,7 +35,7 @@ from pathlib import Path from typing import Literal -from lightcone.engine import assets, container, dataset, identity, plan, sandbox +from lightcone.engine import assets, container, dataset, identity, plan, sandbox, venue from lightcone.engine.plan import Key, Task from lightcone.engine.project import ( ProjectError, @@ -346,6 +346,9 @@ def main(argv: list[str]) -> int: universe_id, _, output_id = argv[0].partition("/") try: + # A rerun executes a recipe, so it is gated the way materialize + # is: compute nodes, never a NERSC login node. + venue.require_compute_node("datalad rerun ") root = declared_project() # The graph — and with it the task lookup — before any converge: # a typo'd target must cost nothing and mask nothing, and a diff --git a/tests/conftest.py b/tests/conftest.py index ac602a51..4badb112 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,29 @@ def runner() -> CliRunner: return CliRunner() +@pytest.fixture(autouse=True) +def venue_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip the host's venue out of the suite's environment. + + On a known center's login node every materialize test would otherwise + meet the login guard, and inside an allocation the real-cluster test + would launch srun across it. The site markers come from the guard's + own table, so a center added there is scrubbed here for free; the + venue tests set these back deliberately. + """ + from lightcone.engine import venue + + for name in ( + *(site.marker for site in venue._SITES), + "SLURM_JOB_ID", + "SLURMD_NODENAME", + "SLURM_JOB_NUM_NODES", + "SLURM_NNODES", + "SLURM_CPUS_ON_NODE", + ): + monkeypatch.delenv(name, raising=False) + + @pytest.fixture(autouse=True) def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: """Fake every external tool convergence shells out to, so the suite is diff --git a/tests/test_container.py b/tests/test_container.py index 09d66c87..ad7a8a1b 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -54,13 +54,19 @@ def _write_archive(path: Path, config: bytes = _CONFIG) -> str: @pytest.fixture def fake(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: - """A podman-having host: records argv, models each command's effect.""" + """A podman-having host: records argv, models each command's effect. + + Every tool name resolves except ``podman-hpc`` — a site wrapper no + laptop has, and it would win detection everywhere. The host's + architecture is pinned to the test archives' ``amd64`` so the arch + gate answers the same on every CI machine. + """ calls: list[list[str]] = [] loaded: set[str] = set() def run(argv: list[str], *, cwd: Path) -> MagicMock: calls.append(list(argv)) - if argv[0] in ("podman", "docker"): + if argv[0] in ("podman", "docker", "podman-hpc"): if argv[1] == "image": # the loaded probe, both spellings return MagicMock(returncode=0 if argv[3] in loaded else 1) if argv[1] == "load": @@ -84,10 +90,35 @@ def run(argv: list[str], *, cwd: Path) -> MagicMock: return MagicMock(returncode=0, stdout="", stderr="") monkeypatch.setattr(project, "_run", run) - monkeypatch.setattr(shutil, "which", lambda name, path=None: f"/usr/bin/{name}") + monkeypatch.setattr( + shutil, + "which", + lambda name, path=None: None if name == "podman-hpc" else f"/usr/bin/{name}", + ) + monkeypatch.setattr(container.platform, "machine", lambda: "x86_64") return calls +@pytest.fixture +def hpc( + fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> list[list[str]]: + """The same host with the site's podman-hpc wrapper installed too. + + A wrap around `fake`'s stub rather than a replacement, so it changes + exactly one fact — anything else `fake` hides stays hidden here. + """ + inner = shutil.which + monkeypatch.setattr( + shutil, + "which", + lambda name, path=None: "/usr/bin/podman-hpc" + if name == "podman-hpc" + else inner(name, path), + ) + return fake + + def _argvs(calls: list[list[str]], *head: str) -> list[list[str]]: return [c for c in calls if c[: len(head)] == list(head)] @@ -482,3 +513,99 @@ def test_a_garbled_archive_is_a_pointed_refusal(tmp_path: Path) -> None: (tmp_path / "image").write_bytes(b"not a tar at all" * 4096) with pytest.raises(ProjectError, match="lc build"): container.archive_identity(tmp_path / "image") + + +# ---- podman-hpc ------------------------------------------------------------- + + +def test_podman_hpc_is_preferred_where_present(root: Path, hpc: list[list[str]]) -> None: + """Where the site wrapper exists, the bare podman beside it is the one + whose store compute nodes cannot see.""" + assert container.runtime_hint() == "podman-hpc" + assert container.runtime_name(root) == "podman-hpc" + + +def test_podman_hpc_loads_then_migrates(root: Path, hpc: list[list[str]]) -> None: + expected = _write_archive(image.archive_path(root, image.tag(root))) + + runtime = container.runtime_for_run(root, build=False) + + assert runtime.runtime == "podman-hpc" + assert _argvs(hpc, "podman-hpc", "image") == [["podman-hpc", "image", "exists", expected]] + assert len(_argvs(hpc, "podman-hpc", "load")) == 1 + assert _argvs(hpc, "podman-hpc", "migrate") == [["podman-hpc", "migrate", expected]] + + +def test_migrate_runs_even_when_the_store_already_holds_the_image( + root: Path, hpc: list[list[str]] +) -> None: + """The migrate lives outside the load branch: a store hit (or a fresh + build) must still put the squashed copy where compute nodes look.""" + _write_archive(image.archive_path(root, image.tag(root))) + + container.runtime_for_run(root, build=False) + container.runtime_for_run(root, build=False) + + assert len(_argvs(hpc, "podman-hpc", "load")) == 1 + assert len(_argvs(hpc, "podman-hpc", "migrate")) == 2 + + +def test_podman_hpc_builds_saves_and_migrates(root: Path, hpc: list[list[str]]) -> None: + """The wrapper is build-capable — NERSC login nodes are where the + matching-arch archive comes from — and a fresh build still migrates.""" + runtime, verdict = container.build(root) + + assert verdict == "built" + assert len(_argvs(hpc, "podman-hpc", "build")) == 1 + (save,) = _argvs(hpc, "podman-hpc", "save") + assert save[2:4] == ["--format", "docker-archive"] + assert _argvs(hpc, "podman-hpc", "migrate") == [["podman-hpc", "migrate", runtime.image_id]] + + +def test_podman_hpc_keeps_the_uid_and_forbids_pulling(root: Path, hpc: list[list[str]]) -> None: + _write_archive(image.archive_path(root, image.tag(root))) + runtime = container.runtime_for_run(root, build=False) + + assert container.uid_flags("podman-hpc") == ["--userns=keep-id"] + backend = container.backend(runtime) + assert backend.capability.kind == "podman-hpc" + assert "--pull=never" in backend.user_flags # type: ignore[attr-defined] + assert "--userns=keep-id" in backend.user_flags # type: ignore[attr-defined] + + +# ---- the architecture gate -------------------------------------------------- + + +def test_a_foreign_arch_archive_refuses_before_loading( + root: Path, fake: list[list[str]] +) -> None: + """A wrong-arch load *succeeds* and then dies as `exec format error` + deep inside a recipe — so the refusal comes first, naming the fix.""" + _write_archive( + image.archive_path(root, image.tag(root)), b'{"architecture":"arm64","os":"linux"}' + ) + + with pytest.raises(ProjectError, match="built for arm64 and this host is amd64"): + container.runtime_for_run(root, build=False) + + assert _argvs(fake, "podman", "load") == [] + + +def test_the_matching_arch_loads(root: Path, fake: list[list[str]]) -> None: + _write_archive(image.archive_path(root, image.tag(root))) + + assert container.runtime_for_run(root, build=False).arch == "amd64" + assert len(_argvs(fake, "podman", "load")) == 1 + + +def test_an_unknown_arch_is_not_refused( + root: Path, fake: list[list[str]], monkeypatch: pytest.MonkeyPatch +) -> None: + """Ignorance is not a mismatch — an archive that does not say, or a + host machine outside the map, passes rather than refusing blind.""" + _write_archive(image.archive_path(root, image.tag(root)), b'{"os":"linux"}') + container.runtime_for_run(root, build=False) + + monkeypatch.setattr(container.platform, "machine", lambda: "riscv64") + _write_archive(image.archive_path(root, image.tag(root))) + container.runtime_for_run(root, build=False) diff --git a/tests/test_container_smoke.py b/tests/test_container_smoke.py index 89e26c97..76b61d4c 100644 --- a/tests/test_container_smoke.py +++ b/tests/test_container_smoke.py @@ -48,8 +48,15 @@ def _available() -> list[str]: - """The runtimes this host can actually run, probed once at collection.""" + """The runtimes this host can actually run, probed once at collection. + + podman-hpc exists only where a site installed it (NERSC login nodes), + so its parametrization never fires in CI — running this suite on such + a host is the site spike. + """ found = [] + if shutil.which("podman-hpc"): + found.append("podman-hpc") if shutil.which("podman"): found.append("podman") try: @@ -90,7 +97,7 @@ def runtime(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> shutil, "which", lambda tool, path=None: None - if tool in ("podman", "docker") and tool != name + if tool in ("podman-hpc", "podman", "docker") and tool != name else real_which(tool, path=path), ) return name diff --git a/tests/test_materialize.py b/tests/test_materialize.py index a091da94..9934f07c 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -867,6 +867,31 @@ def test_a_real_cluster_still_fits_through_the_seam(root: Path) -> None: assert not dataset.status(root) +def test_a_processes_cluster_fits_through_the_seam( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Workers in other processes — the shape every venue beyond one + machine has. Pins that the unit crosses the boundary by reference + (`worker.materialize`, `Task`, `Versions`, `TaskResult`) and that + results travel back whole.""" + + @contextmanager + def processes() -> Iterator[engine._Dask]: + from distributed import Client, LocalCluster + + with LocalCluster( # type: ignore[no-untyped-call] + n_workers=2, threads_per_worker=1, processes=True, dashboard_address=None + ) as cluster: + with Client(cluster) as client: # type: ignore[no-untyped-call] + yield engine._Dask(client) + + monkeypatch.setattr(engine, "cluster_for_run", processes) + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert not dataset.status(root) + + # ---- the report ------------------------------------------------------------ diff --git a/tests/test_sandbox_oci.py b/tests/test_sandbox_oci.py index 6947a791..c39bda95 100644 --- a/tests/test_sandbox_oci.py +++ b/tests/test_sandbox_oci.py @@ -45,7 +45,11 @@ def policy(root: Path) -> Policy: def _backend(root: Path, runtime: str = "podman") -> OCIBackend: - flags = ("--userns=keep-id", "--pull=never") if runtime == "podman" else ("--user", "1000:1000") + flags = ( + ("--user", "1000:1000") + if runtime == "docker" + else ("--userns=keep-id", "--pull=never") + ) return OCIBackend( runtime=runtime, # type: ignore[arg-type] image_id=_IMAGE_ID, @@ -164,13 +168,19 @@ def test_the_network_is_denied_by_flag(root: Path, policy: Policy) -> None: def test_runtimes_differ_only_in_their_spellings(root: Path, policy: Policy) -> None: podman = _backend(root, "podman").wrap(policy, ["true"]) docker = _backend(root, "docker").wrap(policy, ["true"]) + hpc = _backend(root, "podman-hpc").wrap(policy, ["true"]) assert "--userns=keep-id" in podman and "--pull=never" in podman assert "--user" in docker and "1000:1000" in docker + # The site wrapper is podman's argv with only the runtime word swapped. + swap = {"podman": "podman-hpc", "--env=LC_SANDBOX=podman": "--env=LC_SANDBOX=podman-hpc"} + assert hpc == [swap.get(a, a) for a in podman] strip = { - "--userns=keep-id", "--pull=never", "--user", "1000:1000", "podman", "docker", - "--env=LC_SANDBOX=podman", "--env=LC_SANDBOX=docker", + "--userns=keep-id", "--pull=never", "--user", "1000:1000", + "podman", "docker", "podman-hpc", + "--env=LC_SANDBOX=podman", "--env=LC_SANDBOX=docker", "--env=LC_SANDBOX=podman-hpc", } # fmt: skip - assert [a for a in podman if a not in strip] == [a for a in docker if a not in strip] + p, d, h = ([a for a in argv if a not in strip] for argv in (podman, docker, hpc)) + assert p == d == h def test_the_environment_is_an_allowlist_never_ambient( @@ -197,7 +207,7 @@ def test_no_host_resolved_env_binary_in_the_argv(root: Path, policy: Policy) -> def test_the_attestation_is_derived_from_the_flags(root: Path, policy: Policy) -> None: - for runtime in ("podman", "docker"): + for runtime in ("podman", "docker", "podman-hpc"): attested = _backend(root, runtime).attest(policy) assert attested.mechanism == runtime assert attested.fs == "declared" diff --git a/tests/test_venue.py b/tests/test_venue.py new file mode 100644 index 00000000..d75ab01c --- /dev/null +++ b/tests/test_venue.py @@ -0,0 +1,359 @@ +"""Tests for `lightcone.engine.venue` — the SLURM allocation, and the guard. + +The venue's whole surface is ambient: environment variables and an srun +on PATH. So the suite fakes the *host* rather than the code — SLURM +variables set deliberately, and a bash stub standing in for srun that +launches the worker command locally — and the end-to-end tests run the +real graph through the real detection, bind, launch and teardown path on +any machine. +""" + +from __future__ import annotations + +import os +import textwrap +from collections.abc import Callable +from pathlib import Path + +import pytest + +from lightcone.engine import dataset, venue +from lightcone.engine import materialize as engine +from lightcone.engine.project import ProjectError + +_SPEC = """ +version: "0.0.13" +name: analysis + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: first + type: metric + decisions: [method] + recipe: + command: echo {decisions.method} > {output}/value.txt + + - id: second + type: report + inputs: [first] + recipe: + command: cat {inputs.first}/value.txt > {output}/copy.txt + +decisions: + method: + label: Method + default: alpha + options: + alpha: {label: alpha} + beta: {label: beta} +""" + +_UNIVERSE = "id: baseline\ndecisions:\n method: alpha\n" + + +@pytest.fixture +def root(analysis: Callable[..., Path]) -> Path: + return analysis(_SPEC, universes={"baseline": _UNIVERSE}) + + +def _allocation(monkeypatch: pytest.MonkeyPatch, *, nodes: int = 1) -> None: + """The environment salloc leaves on the head compute node.""" + monkeypatch.setenv("SLURM_JOB_ID", "12345") + monkeypatch.setenv("SLURM_JOB_NUM_NODES", str(nodes)) + monkeypatch.setenv("SLURM_CPUS_ON_NODE", "2") + # The literal IP keeps the scheduler bind hermetic against CI DNS. + monkeypatch.setenv("SLURMD_NODENAME", "127.0.0.1") + + +def _stub_srun(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, body: str) -> None: + """Put a bash srun with *body* on PATH.""" + stubs = tmp_path / "stubs" + stubs.mkdir(exist_ok=True) + stub = stubs / "srun" + stub.write_text(f"#!/usr/bin/env bash\n{textwrap.dedent(body)}") + stub.chmod(0o755) + monkeypatch.setenv("PATH", f"{stubs}{os.pathsep}{os.environ['PATH']}") + + +#: What a real srun does from lc's point of view: drop the step flags, +#: honor ``--ntasks``, run the worker command. +_FAITHFUL = """ +ntasks=1 +while [[ $1 == --* ]]; do + case "$1" in --ntasks=*) ntasks="${1#--ntasks=}";; esac + shift +done +pids=() +for _ in $(seq "$ntasks"); do + "$@" & + pids+=($!) +done +wait "${pids[@]}" +""" + + +# ---- the login guard -------------------------------------------------------- + + +def test_a_login_node_refuses_with_both_commands(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NERSC_HOST", "perlmutter") + + with pytest.raises(ProjectError) as err: + venue.require_compute_node() + + message = str(err.value) + assert "salloc" in message + assert "sbatch" in message + assert "--wrap 'lc materialize'" in message + + +def test_the_guard_fires_before_anything_else( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Even a directory that is not a project refuses as a login node — the + allocation is the remedy with queue latency, so it comes first.""" + monkeypatch.setenv("NERSC_HOST", "perlmutter") + + with pytest.raises(ProjectError, match="login node"): + engine.materialize(tmp_path, []) + + +def test_an_allocation_passes_the_guard(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NERSC_HOST", "perlmutter") + monkeypatch.setenv("SLURM_JOB_ID", "12345") + + venue.require_compute_node() + + +def test_a_host_no_site_marker_names_passes_the_guard() -> None: + venue.require_compute_node() + + +def test_a_new_center_is_one_table_row(monkeypatch: pytest.MonkeyPatch) -> None: + """The guard is data: a center lands as its marker plus its own + verified allocation spellings, and the message carries them.""" + monkeypatch.setattr( + venue, + "_SITES", + ( + venue._Site( + name="Fictional Computing", + marker="FCC_HOST", + salloc="salloc --partition=work --time=01:00:00", + sbatch="sbatch --partition=work --time=01:00:00", + ), + ), + ) + monkeypatch.setenv("FCC_HOST", "cluster9") + + with pytest.raises(ProjectError) as err: + venue.require_compute_node() + + message = str(err.value) + assert "Fictional Computing login node" in message + assert "FCC_HOST" in message + assert "salloc --partition=work --time=01:00:00" in message + assert "sbatch --partition=work --time=01:00:00" in message + + monkeypatch.setenv("SLURM_JOB_ID", "12345") + venue.require_compute_node() # inside an allocation the same site passes + + +def test_the_read_only_verbs_work_on_a_login_node( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`--check` and `status` are for finding out where a project stands, + and a login node is exactly where that question gets asked.""" + monkeypatch.setenv("NERSC_HOST", "perlmutter") + + assert set(engine.check(root, []).planned) == {"baseline/first", "baseline/second"} + assert engine.status(root).counts["stale"] == 2 + + +# ---- the srun invocation ---------------------------------------------------- + + +def test_the_srun_argv_spans_the_allocation() -> None: + import sys + + argv = venue._srun_argv("tcp://10.0.0.1:8786", 4, 128, "/scratch") + + assert argv == [ + "srun", + "--overlap", + "--ntasks=4", + "--ntasks-per-node=1", + "--cpus-per-task=128", + sys.executable, + "-m", + "distributed.cli.dask_worker", + "tcp://10.0.0.1:8786", + "--nthreads", "128", + "--nworkers", "1", + "--no-dashboard", + "--no-nanny", + "--death-timeout", "60", + "--memory-limit", "0", + "--local-directory", "/scratch", + ] # fmt: skip + + +def test_slurm_without_srun_refuses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A leaked SLURM_JOB_ID — a container, a copied environment — must be + a loud refusal, never a silent fall back to running locally.""" + _allocation(monkeypatch) + monkeypatch.setenv("PATH", str(tmp_path)) + + with pytest.raises(ProjectError, match="srun is not on PATH"): + with venue.slurm_client(): + pass + + +# ---- the allocation, end to end --------------------------------------------- + + +def test_a_run_spans_the_allocation( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The whole venue path — detection, scheduler bind, worker launch, + a real graph in a real worker process, teardown — with only srun faked.""" + _allocation(monkeypatch) + _stub_srun(tmp_path, monkeypatch, _FAITHFUL) + + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert (root / "results/baseline/second/copy.txt").read_text() == "alpha\n" + assert not dataset.status(root) + + +def test_every_allocated_node_gets_a_worker( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two declared nodes, a stub honoring ``--ntasks`` — completion proves + the run waited for both workers and used the count it was granted.""" + _allocation(monkeypatch, nodes=2) + _stub_srun(tmp_path, monkeypatch, _FAITHFUL) + + report = engine.materialize(root, []) + + assert report.made == ["baseline/first", "baseline/second"] + assert not dataset.status(root) + + +def test_workers_that_never_connect_refuse_and_reap_srun( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _allocation(monkeypatch) + pidfile = tmp_path / "srun.pid" + _stub_srun(tmp_path, monkeypatch, f"echo $$ > {pidfile}\nexec sleep 60\n") + monkeypatch.setattr(venue, "_WORKER_WAIT", 2.0) + monkeypatch.setattr(venue, "_REAP_GRACE", 0.5) + + with pytest.raises(ProjectError, match=r"expected 1 dask worker"): + with venue.slurm_client(): + pass + + pid = int(pidfile.read_text()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + + +def test_a_dead_srun_reports_its_exit_code( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """srun failing is srun's error, reported as such — not a two-minute + wait that ends in a timeout blaming the workers.""" + _allocation(monkeypatch) + _stub_srun(tmp_path, monkeypatch, "exit 3\n") + monkeypatch.setattr(venue, "_REAP_GRACE", 0.5) + + with pytest.raises(ProjectError, match="exited with code 3"): + with venue.slurm_client(): + pass + + +# ---- the fixes the first review asked for ----------------------------------- + + +def test_slurm_counts_that_are_not_numbers_refuse(monkeypatch: pytest.MonkeyPatch) -> None: + """A mangled count is the same leak class as SLURM_JOB_ID without an + srun, and gets the same curated refusal — never a ValueError traceback.""" + monkeypatch.setenv("SLURM_CPUS_ON_NODE", "72(x2)") + with pytest.raises(ProjectError, match="SLURM_CPUS_ON_NODE"): + venue._int_env("SLURM_CPUS_ON_NODE", 1) + + monkeypatch.setenv("SLURM_JOB_ID", "12345") + monkeypatch.setenv("SLURM_JOB_NUM_NODES", "four") + with pytest.raises(ProjectError, match="SLURM_JOB_NUM_NODES"): + venue.allocation_nodes() + + +def test_allocation_nodes_answers_zero_outside_an_allocation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert venue.allocation_nodes() == 0 + monkeypatch.setenv("SLURM_JOB_ID", "12345") + assert venue.allocation_nodes() == 1 # in one, count unstated + monkeypatch.setenv("SLURM_JOB_NUM_NODES", "3") + assert venue.allocation_nodes() == 3 + + +def test_an_unresolvable_node_name_is_a_refusal_not_a_traceback( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """SLURM's NodeName is an alias, not a promise of a hostname.""" + _allocation(monkeypatch) + monkeypatch.setenv("SLURMD_NODENAME", "no-such-node.invalid") + _stub_srun(tmp_path, monkeypatch, "exit 0\n") + + with pytest.raises(ProjectError, match="did not resolve"): + with venue.slurm_client(): + pass + + +def test_a_multi_node_allocation_refuses_a_node_local_image_store( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """podman's and docker's stores are node-local; only podman-hpc's + migrate makes an image visible to the allocation's other nodes. + Mutation check: the same state under podman-hpc reaches runtime + resolution — the gate itself is what stands between them.""" + from lightcone.engine import container + + text = (root / "pyproject.toml").read_text() + (root / "pyproject.toml").write_text(text + '\n[tool.lightcone.image]\napt-install = ["bc"]\n') + dataset.save(root, [root], "containerize") + _allocation(monkeypatch, nodes=2) + monkeypatch.setattr(container, "runtime_hint", lambda: "podman") + + with pytest.raises(ProjectError, match="node-local"): + engine.materialize(root, []) + + monkeypatch.setattr(container, "runtime_hint", lambda: "podman-hpc") + + def reached(r: Path, *, build: bool) -> None: + raise ProjectError("reached runtime resolution") + + monkeypatch.setattr(container, "runtime_for_run", reached) + with pytest.raises(ProjectError, match="reached runtime resolution"): + engine.materialize(root, []) + + +def test_the_rerun_entry_point_is_guarded_like_materialize( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A rerun executes a recipe, so a NERSC login node refuses it too.""" + from lightcone.engine import worker + + monkeypatch.setenv("NERSC_HOST", "perlmutter") + + assert worker.main(["baseline/first"]) == 2 + err = capsys.readouterr().err + assert "login node" in err and "salloc" in err From efbbbf346f2ff47fc7d30d1a28fe1357fb99ca12 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 18:43:06 +0200 Subject: [PATCH 14/23] =?UTF-8?q?Layer=208:=20the=20publication=20view=20?= =?UTF-8?q?=E2=80=94=20the=20RO-Crate=20converged,=20integrity=20as=20a=20?= =?UTF-8?q?status=20fact=20(#182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The last layer of the rebuild table, rescoped during planning from the spec's "`lc verify`, WRROC export" to two mechanisms with **no new verbs**: **The project is the crate.** `ro-crate-metadata.json` lives at the project root as a derived artifact, converged by `lc materialize` the way `uv.lock` is: rendered from repository state, string-compared, committed alone only on a difference. Deposit is `git archive`/`datalad export-archive` on a repository that is already an RO-Crate — nothing copied, no bundle, no `lc export`. Maintenance is derived, never configured: a declared `[project].license` turns it on (RO-Crate requires one; materialize must not refuse science over a missing key, and inventing CC-BY-4.0 — the pre-rebuild default — asserts terms over someone's data). Absence is one report line. **Integrity is a status fact, not a verb.** The hole layer 4 recorded — a hand-edited-and-committed output reads `current` forever because a skip returns the recorded digest (the agent-forged-file scenario) — is closed by history, not hashing: every output is committed, so a hand edit *requires* a commit, and `dataset.last_writer` names it. `lc status` flags any output whose directory's last commit is not its own run record, with sha, author, date and a `git show` remedy. O(history), answers on a bytes-free clone, still exits 0. ## The crate itself - `engine/crate.py`, the one new module — a **pure** builder over `plan.build()` and the manifests; git enters only as an injected `writer` callable. - Targets **Provenance Run Crate 0.5 structurally, not vacuously** (the pre-rebuild exporter passed REQUIRED only because the SHACL shapes are vacuous over an empty target class): outputs sharing a manifest `git_sha` are one run — the driver's single HEAD read makes run identity free — giving `OrganizeAction` + workflow-level `CreateAction` per run, `ControlAction` per execution, `HowToStep` per output id. - The workflow-run `@context` makes `containerImage`/`sha256`/`ContainerImage` real vocabulary (the old exporter emitted them as undefined terms JSON-LD silently drops). The committed image archive is one entity: `File` + `ContainerImage`, config-blob id and payload together. - The manifest stays canonical: in the crate as a `File`, `subjectOf` its `Dataset`; `env_version`/`definition_version`/`hermeticity` are deliberately not transliterated. - Deterministic by construction: sorted iteration, `sort_keys`, no clock (`datePublished` = newest manifest `finished_at`, overriding rocrate's stamp-now default), no `mimetypes` (it reads `/etc/mime.types`, which would make the render host-dependent). - The crate's `Person` is the author of each output's **saving** commit via `last_writer` — never the manifest's `git_sha`, which is the commit the run *started* at. ## Validation Official `rocrate-validator` against the Provenance profile, on a real materialized project: **REQUIRED clean**; RECOMMENDED failures pinned as a *set* (`test_crate_smoke.py::_FLOOR`) of checks the crate cannot truthfully satisfy (the workflow's id is its in-crate path and the shape wants `^http`; an annex-stored image has no registry; lc knows no publisher/affiliation). A new failure is a regression; a disappearing one is the floor to shrink. Gated with `LC_CRATE_TESTS_REQUIRED=1` on every CI runner — the validator is a dev dep, so none may skip. ## Also in here - The worker stamps `started_at`/`finished_at` (millisecond ISO 8601 — crate consumers parse `endTime` with at most three fractional digits). Attestation like `lc_version`; `classify` never reads them; `SCHEMA_VERSION` stays 1. - `materialize.run_subject` is now the one spelling of the run record's subject, shared by `run_record` (composer) and the foreign-write check (comparator) — byte-identical to the previous inline string, and the datalad parser + real `datalad rerun` tests all pass. - Recorded rejections, with reasons in CLAUDE.md: no `lc verify` (O(bytes), blind on unfetched outputs, no lifecycle moment enforcing it; `git annex fsck` owns object corruption), no `lc export`, and the honest residues stated (a forged run-record subject defeats the history check; a manifest lc itself mis-recorded is the one case only a rehash would catch; the rerun entry point does not regenerate the crate). - One trap the suite found live: forging a result **in place** dirties its byte-identical sibling through the shared thin annex object — the recorded thin hazard demonstrating itself. Tests unlink first; CLAUDE.md records it. ## Test plan - 556 passed (31 new), ruff and mypy clean, `LC_CRATE_TESTS_REQUIRED=1` locally. - New: `test_crate.py` (pure, 14 tests, no git — determinism, run grouping, the Provenance edges, ContainerImage, Person, license spellings), `test_crate_smoke.py` (real materialize → real validator, two guard tests), `last_writer` against real repos incl. the enclosing-repository case, the foreign-write fact end-to-end incl. a bytes-free clone, crate convergence (created once / idempotent re-run commits nothing / license added later / license removed keeps the file / spec-dropped output excluded and named). - Manual e2e: `lc init` → license → materialize (crate lands in its own trailing commit) → re-materialize (no commit) → validator (0 REQUIRED) → forge + commit → `lc status` names the foreign commit. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01Q9ikHqeMA9tTX1Jrx5DdNL --------- Co-authored-by: Claude Fable 5 --- .github/workflows/tests.yml | 3 + CLAUDE.md | 139 +++++- pyproject.toml | 10 +- src/lightcone/cli/commands.py | 10 +- src/lightcone/engine/assets.py | 36 +- src/lightcone/engine/crate.py | 654 ++++++++++++++++++++++++++++ src/lightcone/engine/dataset.py | 49 +++ src/lightcone/engine/materialize.py | 173 +++++++- src/lightcone/engine/worker.py | 26 +- tests/conftest.py | 31 +- tests/test_assets.py | 36 +- tests/test_cli.py | 37 ++ tests/test_crate.py | 328 ++++++++++++++ tests/test_crate_smoke.py | 164 +++++++ tests/test_dataset.py | 49 +++ tests/test_materialize.py | 208 ++++++++- tests/test_worker.py | 1 + 17 files changed, 1902 insertions(+), 52 deletions(-) create mode 100644 src/lightcone/engine/crate.py create mode 100644 tests/test_crate.py create mode 100644 tests/test_crate_smoke.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8535cd98..632ea2ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,9 @@ jobs: # the runtimes (ubuntu ships podman and a running docker daemon). # macOS runners have no podman machine, so there the suite skips. LC_CONTAINER_TESTS_REQUIRED: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }} + # And for the crate validation: rocrate-validator is in the dev + # group, so every runner has it and none may skip. + LC_CRATE_TESTS_REQUIRED: "1" steps: - uses: actions/checkout@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 533dfeab..6675c45c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ speculatively. | 5 | **Sandbox layer** — Landlock / Seatbelt, exec-shim, denial UX, `lc run` | ✅ **done** | | 6 | **Container hatch** — `[tool.lightcone.image]`, `lc build`, OCI runtimes as the exec boundary, the image archived in the dataset | ✅ **done** | | 7 | **Venues** — SLURM in-allocation execution, login guard, podman-hpc | 🔶 **landed; Perlmutter spike pending** — hub/GKE and Cloud Build deferred to their own layer | -| 8 | `lc verify`, WRROC export | ⬜ | +| 8 | **Publication view** — the RO-Crate converged by materialize, foreign writes stale by history; **no `lc verify`, no `lc export`, by decision** | ✅ **done** | `lc status` landed with the invalidation model rather than at layer 8: once an output can be *behind*, something has to say which ones are, and @@ -165,6 +165,7 @@ src/lightcone/ # namespace — NO __init__.py ├── identity.py # env_version, definition_version, the lock scan ├── image.py # the system layer: declaration, Containerfile, tag — pure ├── container.py # runtimes, the build, the archived image — impure + ├── crate.py # the publication view: the repo as an RO-Crate — pure ├── assets.py # an output: its directory, its manifest, its state ├── plan.py # the spec, read as a graph of tasks ├── worker.py # making one output; also the `python -m` entry point @@ -1156,7 +1157,11 @@ provides that, and for what workers do *not* need (git, git-annex). `git_remote`, `lc_version`, `hermeticity` — and, since layer 6, `image` (`{tag, id, archive, arch}`, defaulting to ``None`` because that is the true value for a host run — not back-compat machinery, - and `SCHEMA_VERSION` stays 1 pre-release). Spec §3's longer list — + and `SCHEMA_VERSION` stays 1 pre-release), and since layer 8 + `started_at` / `finished_at` (ISO 8601 UTC, **millisecond** precision + because RO-Crate consumers parse `endTime` with at most three + fractional digits; attestation like `lc_version`, defaulted `""`, + never read by `classify`). Spec §3's longer list — `uv_version`, `platform`, `worker_runtime`, `python_build`, `dpkg_snapshot_sha256`, `sdist_built`, `env_snapshot`, `gpu_driver` — is attestation nothing here reads; it lands with the verb that reads @@ -1683,6 +1688,125 @@ that could add a flag); whether Landlock is in the SLES boot LSM list the downgrade note); `git annex add`/`get` timings on `$SCRATCH` and `$CFS` (the recorded Lustre residue). +## Key Invariants (layer 8) + +**The project is the crate, and materialize maintains it — there is no +export verb.** `ro-crate-metadata.json` sits at the project root as a +derived artifact, converged by `lc materialize` the way `uv.lock` is: +`engine/crate.py` renders it from repository state, the hook +(`materialize._converge_crate`) string-compares it against the tree, and +only a difference is written and committed — alone, in its own trailing +commit, which `datalad rerun` skips harmlessly as "no command". Deposit +is `git archive` / `datalad export-archive` on a repository that is +already a crate; nothing is copied and there is no bundle directory. +The rerun entry point deliberately does not regenerate it (it is one +task's executor, not the driver), so the crate lags until the next +materialize — recorded residue. + +**Publication intent is derived, never configured.** A +`[project].license` in `pyproject.toml` turns crate maintenance on — +RO-Crate *requires* a license, materialize must not refuse to run +science over a missing key, and inventing CC-BY-4.0 (the pre-rebuild +default) asserts terms over someone's data. Absent ⇒ no crate and one +report line; removed later ⇒ the file is left (it is in committed +history either way) and the line says it is no longer maintained. The +same shape as `[tool.lightcone.image]` deriving containerized mode. + +**The document is a pure function of repository state, and that is what +makes convergence sound.** The clock never enters it: `datePublished` is +the newest manifest `finished_at` (the spec file's own last-commit date +for a never-materialized project) and **must override rocrate's +default**, which stamps construction time — the old exporter carried a +`datePublished` its code never set. Entities are built in sorted order +and serialized with `sort_keys`; +`test_rendering_twice_at_the_same_state_is_byte_identical` pins it. The +render is also injected-pure: `dataset.last_writer` comes in as a +callable and `dsid` as a value, so `tests/test_crate.py` runs with no +git at all. + +**Run identity comes free from `git_sha`.** The driver reads HEAD once +per run and hands it down, so every output of one materialize carries +the same sha — grouping manifests by it *is* grouping by run, and that +is what makes the Provenance profile expressible with no new manifest +field: one `OrganizeAction` + workflow-level `CreateAction` per run, a +`ControlAction` per output execution, a `HowToStep` per output id +(deduped across universes: a step is spec structure, an action is one +execution — exactly how the multiverse maps on). + +**The crate's `Person` is the author of the output's *saving* commit, +via `last_writer` — never the manifest's `git_sha`**, which is the +commit the run *started* at and can be someone else's. No `--author` +flag, no env var, no config probe: `require_committer` already +guarantees the commit has an author. + +**The manifest stays canonical; the crate does not transliterate it.** +`env_version`, `definition_version` and `hermeticity` get no schema.org +spelling — the manifest itself is in the crate as a `File`, `subjectOf` +its output's `Dataset`, so nothing is lost and no vocabulary is +invented. What does get real vocabulary comes from the workflow-run +`@context` (`https://w3id.org/ro/terms/workflow-run`), without which +`containerImage`, `sha256` and `ContainerImage` are undefined terms +JSON-LD drops on expansion — the pre-rebuild exporter's silent failure. +The committed archive is one entity, `["File", "ContainerImage"]`, +identity (`sha256` = config-blob id) and payload together. + +**The validator floor is pinned as a set, not a count.** +`tests/test_crate_smoke.py` materializes a real project and runs the +official `rocrate-validator` (dev dependency) against Provenance Run +Crate 0.5: REQUIRED must be clean, and RECOMMENDED failures must stay +within `_FLOOR` — checks the crate cannot truthfully satisfy (the +workflow's id is its in-crate path and the shape wants `^http`; an +annex-stored image has no registry; lc knows no publisher and no +affiliation). A new failure is a regression, a disappearing one is the +floor to shrink. Gated like the container smoke suite: +`LC_CRATE_TESTS_REQUIRED=1` in CI turns the no-validator skip into a +failure, two tests cover the guard. + +**A foreign write is `stale`, everywhere, and it is history, not +hashing.** The hole layer 4 recorded: a skip returns the *recorded* +digest, so a hand-edited-and-committed output (the agent-forged-file +scenario) reads `current` forever. The check: every output is +committed, so a hand edit requires a commit, and `dataset.last_writer` +names it — an output is cleanly written iff the commit that last +touched its directory carries its own run-record subject. +`materialize.datalad_run_subject` is the one spelling of that subject +(named for whose format it is — datalad's, matched by `rerun`'s regex), +shared by `run_record` (the composer) and `_foreign_write` (the +comparator), because two strings here would drift. A hit is a +*contradiction*, not a circumstance — the manifest no longer describes +the bytes — so it classifies `stale` and **the next run remakes it**, +the same philosophy as the dirty refusal's path split: `results/` is +lc's to write, and a committed hand edit is wreckage with a commit +message. Three coherent surfaces, one walk: `lc status` reports it +stale (still exit 0; `OutputStatus.foreign_write` carries the foreign +commit's sha — what prose cannot give a machine consumer), `--check` +plans it, and materialize's driver answers the history question up +front and hands each worker the offending `dataset.LastWrite` — +workers have no git, the HEAD discipline on a third value. History +enters `assets.classify` as **one more input value**, exactly like +check mode's sentinel: computed by whoever has git, handed in as the +*fact*, and the rule stays pure — the verdict's prose is classify's, +like every other why, so `calls_for_a_remake` remains the one bool +that turns a state into an action, with no `foreign or …` re-spelled +at any call site. `last_writer` answers "cannot say" as +empty, never an error — a read-only verb must not refuse over an +unborn HEAD or a stripped `.git`. A full-rehash `lc verify` was +considered and rejected: O(bytes), blind on unfetched outputs (history +answers on a bytes-free clone), and with no lifecycle moment enforcing +it. What it leaves to existing tools, on purpose: uncommitted edits +are a dirty tree, annex object corruption (the thin-write hazard +included) is `git annex fsck`, and a manifest lc itself mis-recorded +is the accepted residue only a rehash would catch. A commit that +*forges* the run-record subject defeats the check — as a regenerated +hash would defeat a rehash; the threat model is accidental damage and +shortcuts, not adversaries (spec §7's line). + +**Forging in a test must break the hard link first.** Results are +committed thin, so two byte-identical outputs share one annex object — +an in-place `write_text` on one dirties the other, which is the +recorded thin hazard demonstrating itself. `test_materialize._forge` +unlinks before writing; a new tampering test should too. + ### Recorded decisions - **The engine is the host's uv tool, never a project dependency** @@ -1935,6 +2059,8 @@ the downgrade note); `git annex add`/`get` timings on `$SCRATCH` and | Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | | Change what a run commits | `src/lightcone/engine/materialize.py` + `tests/test_materialize.py` | The driver owns git alone; the tree ends as clean as it started | | Change where a run executes | `src/lightcone/engine/venue.py` + `materialize.cluster_for_run` + `tests/test_venue.py` | One detection ladder, in `cluster_for_run` alone; venues are detected, never configured; test by faking the host (env vars + a stub srun), never the code | +| Change what the crate says | `src/lightcone/engine/crate.py` + `tests/test_crate.py` | Pure builder: sorted iteration, no clock, git injected as `writer`; structure tests, never byte goldens — the one byte-level claim is render-twice-identical. The validator floor lives in `tests/test_crate_smoke.py::_FLOOR` | +| Change how a foreign write is detected | `dataset.last_writer` + `materialize._foreign_write` + `tests/test_dataset.py` | History, never hashing; `datalad_run_subject` is the one spelling of the record's subject; a foreign write classifies `stale` in every verb | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | | Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | | Change what a sandboxed command may touch | `sandbox/policy.py` + `tests/test_sandbox_policy.py` | Path sets only — no mechanism ever leaks in here | @@ -2034,6 +2160,15 @@ The sandbox suite splits along the seam, which is what makes it cheap: a real annex, materializes through the real Dask cluster, and runs a real `datalad rerun` on a bytes-free clone — the record's whole claim. Each denial sits beside its mutation check. +- `tests/test_crate.py` — **pure**: fixture manifests on `tmp_path`, a + hand-built graph, a stub `writer` — no git anywhere. Structure and + ordering only; the single byte-level assertion is + render-twice-identical, the property convergence rests on. + `tests/test_crate_smoke.py` — **the validator's answer**, gated with + `LC_CRATE_TESTS_REQUIRED=1` (all CI runners — the validator is a dev + dependency, so none may skip): a real materialize, then the official + `rocrate-validator` against the Provenance profile, REQUIRED clean and + RECOMMENDED pinned to the recorded `_FLOOR` set. Note the autouse `tools` fixture stubs `engine.project._run` only — sandbox tests spawn real processes deliberately, and are the one place in diff --git a/pyproject.toml b/pyproject.toml index f18e9aaf..025c4675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "rich>=13.0", "git-annex>=10.2026", "distributed>=2026.7", + "rocrate>=0.15", ] [dependency-groups] @@ -38,6 +39,7 @@ dev = [ "ruff", "mypy", "datalad", + "roc-validator>=0.11.3", ] docs = [ "zensical>=0.0.33", @@ -88,10 +90,16 @@ explicit_package_bases = true mypy_path = "src" [[tool.mypy.overrides]] -module = ["astra.*"] +module = ["astra.*", "rocrate.*"] ignore_missing_imports = true follow_untyped_imports = true +# rocrate ships no type information, and engine/crate.py is a client of +# little else — one module-scoped relaxation beats an ignore on every call. +[[tool.mypy.overrides]] +module = ["lightcone.engine.crate"] +disallow_untyped_calls = false + [tool.pytest.ini_options] testpaths = ["tests"] # The container smoke tests leave ~200 MB image archives in each test's diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index d9db40c1..631ab1da 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -360,11 +360,13 @@ def status(as_json: bool) -> None: lines.append("") marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} width = max((len(o.output) for o in report.outputs), default=0) + # The commit gets a column of its own, for every state and not only + # the interesting ones: "which code made this" is the question the + # verb exists to answer, and it has an answer for a current output + # too. A foreign write arrives as an ordinary stale with its message + # in `why`, so this one path covers it; the dedicated field exists + # for machine consumers of `--json`. lines += [ - # The commit gets a column of its own, for every state and not only - # the interesting ones: "which code made this" is the question the - # verb exists to answer, and it has an answer for a current output - # too. f" {marks[o.status]} {o.status:<8} {o.output:<{width}} " f"{o.git_sha[:7] or '—':<7}" + (f" [dim]{o.why}[/dim]" if o.why else "") for o in report.outputs diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index f161fb37..92175ac0 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -22,10 +22,14 @@ from collections.abc import Mapping from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from lightcone.engine.project import ProjectError +if TYPE_CHECKING: + # A type only: history is git's, and this module runs no git. + from lightcone.engine.dataset import LastWrite + MANIFEST_FILENAME = ".lightcone-manifest.json" SCHEMA_VERSION = 1 @@ -255,6 +259,13 @@ class Manifest: lc_version: str #: What the sandbox actually enforced, as the boundary attested it. hermeticity: dict[str, Any] + #: When the recipe entered and left the boundary, ISO 8601 UTC with + #: millisecond precision. Attestation, like ``lc_version``: outside + #: both hashes, never a rebuild signal — and defaulted empty because + #: that is the true value for a manifest written before the fields + #: existed, not back-compat machinery. + started_at: str = "" + finished_at: str = "" #: The image the recipe ran in — ``{tag, id, archive, arch}`` — or #: ``None`` on the host. Defaulted, and that is not back-compat #: machinery: ``None`` is the *true* value for every manifest a @@ -395,12 +406,18 @@ def classify( env_version: str, manifest: Manifest | None, inputs: Mapping[str, str | None], + foreign: LastWrite | None = None, ) -> Verdict: """Decide what an output is, relative to the project as it now stands. The only place this rule lives. Values rather than objects, because - the two callers arrive at them differently and must not diverge in - anything else. + the callers arrive at them differently and must not diverge in + anything else — history included: whether the output's directory was + last written by its own run record is git's to answer, so it arrives + here as a value computed by whoever has git (the driver), and the + rule stays pure. The *prose* for every verdict lives here too, which + is why this takes the offending commit rather than a finished + sentence. Args: definition_version: What the spec currently says this output is. @@ -410,6 +427,11 @@ def classify( for an input the caller has already decided will be remade — check mode's sentinel, meaning "this is going to change", since it cannot know whether a rebuild is byte-identical. + foreign: The commit that last wrote the output's directory, when + it was not the output's own run record; ``None`` when clean. + A hit is a contradiction — the manifest no longer describes + the bytes — so it is ``stale``, though an output stale on + its definition or inputs keeps that more actionable reason. Returns: ``stale``, ``behind`` or ``current``, with the reason for the @@ -417,6 +439,14 @@ def classify( """ if (reason := _stale(definition_version, manifest, inputs)) is not None: return Verdict("stale", str(reason)) + if foreign is not None: + return Verdict( + "stale", + f'last changed by {foreign.sha[:7]} ("{foreign.subject}", {foreign.author}, ' + f"{foreign.date}) rather than its run record, so the manifest no longer " + f"describes these bytes — the next run remakes it; inspect first with " + f"`git show {foreign.sha[:7]}`", + ) assert manifest is not None # `_stale` returns a reason when it is None if manifest.env_version != env_version: # The sentence says what happened; *where* it happened is the diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py new file mode 100644 index 00000000..a3ead986 --- /dev/null +++ b/src/lightcone/engine/crate.py @@ -0,0 +1,654 @@ +"""The publication view: the repository described as a Workflow Run RO-Crate. + +The project *is* the crate. ``ro-crate-metadata.json`` sits at the root +and describes what the repository already holds — the spec, the lock, the +universes, each materialized output and the run that made it — so a +deposit is ``git archive`` on the repository, not an export step that +copies bytes. lc's manifests stay the canonical record; the crate is the +same facts in schema.org vocabulary, readable by archives and viewers +that will never run ``lc``. + +The document is a pure function of repository state: :func:`render` +takes the graph, the license, and a way to ask git who last wrote each +output, and returns bytes that are identical when nothing changed. That +is what lets ``lc materialize`` converge the file the way it converges +``uv.lock`` — compare, and commit only a difference — with no verb for a +human to remember. + +Profiles targeted: Process, Workflow and Provenance Run Crate 0.5, plus +Workflow RO-Crate 1.0. The Provenance layer is structural, not claimed: +one ``lc materialize`` invocation is one ``OrganizeAction`` (outputs of a +run share their manifests' ``git_sha``, because the driver reads HEAD +once and hands it down), each output's execution is a ``CreateAction`` +steered by a ``ControlAction``, and the spec's outputs are the +workflow's ``HowToStep`` s. +""" + +from __future__ import annotations + +import json +import tomllib +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from rocrate.model import ContextEntity +from rocrate.model.computerlanguage import ComputerLanguage +from rocrate.rocrate import ROCrate + +from lightcone.engine import assets, plan +from lightcone.engine.dataset import LastWrite +from lightcone.engine.plan import Graph, Key +from lightcone.engine.project import SPEC_FILENAME + +CRATE_FILENAME = "ro-crate-metadata.json" + +#: The vocabulary the run-level facts come from. Without it in the +#: ``@context``, terms like ``containerImage`` and ``sha256`` are +#: undefined and silently dropped on JSON-LD expansion — typed but inert. +_WORKFLOW_RUN_CONTEXT = "https://w3id.org/ro/terms/workflow-run" + +#: What the root conforms to, each also declared as a CreativeWork in the +#: graph — consumers resolve the reference inside the crate, not on the +#: network. +_PROFILES = ( + ("https://w3id.org/ro/wfrun/process/0.5", "Process Run Crate", "0.5"), + ("https://w3id.org/ro/wfrun/workflow/0.5", "Workflow Run Crate", "0.5"), + ("https://w3id.org/ro/wfrun/provenance/0.5", "Provenance Run Crate", "0.5"), + ("https://w3id.org/workflowhub/workflow-ro-crate/1.0", "Workflow RO-Crate", "1.0"), +) + +#: The files that pin what a run installed — one spelling, so the crate's +#: data entities and every action's ``object`` list cannot drift apart. +_ENVIRONMENT = ("pyproject.toml", "uv.lock", ".python-version") + + +def license_of(root: Path) -> str: + """Read the project's declared license out of ``pyproject.toml``. + + Presence is what turns crate maintenance on: RO-Crate requires a + license, a run must not refuse over a missing key, and inventing one + would assert terms over someone's data — so declaring + ``[project].license`` is declaring the intent to publish. + + Args: + root: The project root. + + Returns: + The license as declared — an SPDX expression, a URL, free text, + or a file path for the table forms — or empty when undeclared. + """ + try: + data = tomllib.loads((root / "pyproject.toml").read_text()) + except (OSError, tomllib.TOMLDecodeError): + return "" + declared = data.get("project", {}).get("license") + if isinstance(declared, str): + return declared + if isinstance(declared, dict): + return str(declared.get("text") or declared.get("file") or "") + return "" + + +def render( + root: Path, + graph: Graph, + *, + license: str, + dsid: str, + writer: Callable[[Path], LastWrite], +) -> str: + """Build the crate document for the project as it stands. + + Pure given its arguments: reads the tree, runs nothing, and returns + identical bytes for identical repository state — timestamps come from + manifests and commits, never from the clock, and entities are built + in sorted order. The writer is asked about each path at most once. + + Args: + root: The project root. + graph: The full task graph — every universe, every output. + license: The declared license, from :func:`license_of`. + dsid: The dataset UUID, the namespace absolute entity ids are + minted under so they are stable across clones. + writer: Answers "which commit last touched this path" — + :func:`dataset.last_writer` bound to the root, injected so + the builder stays free of git. + + Returns: + The ``ro-crate-metadata.json`` text, trailing newline included. + """ + build = _Builder(root, graph, license, dsid, writer) + return build.document() + + +def _control_id(key: Key) -> str: + return f"#control-{key[0]}-{key[1]}" + + +class _Builder: + """One render: accumulates entities into a ``ROCrate``, in one order.""" + + def __init__( + self, + root: Path, + graph: Graph, + license: str, + dsid: str, + writer: Callable[[Path], LastWrite], + ) -> None: + from astra.helpers import load_yaml + + self.root = root + self.graph = graph + self.license = license + self.writer = writer + self.crate = ROCrate() + self.crate.metadata.extra_contexts.append(_WORKFLOW_RUN_CONTEXT) + #: Every materialized task, sorted: the one iteration order. + self.made: list[tuple[Key, assets.Manifest]] = sorted( + ( + (key, manifest) + for key, task in graph.tasks.items() + if (manifest := assets.read(task.output_dir)) is not None + ), + key=lambda pair: pair[0], + ) + self.made_keys = {key for key, _ in self.made} + self.persons: dict[str, str] = {} # email/name → @id + self.agents: dict[Key, str] = {} # each action's Person @id, or "" + self.images: set[str] = set() # archive paths already added + self.engines: set[str] = set() # engine @ids already added + self.actions: list[str] = [] # action @ids, for the root's mentions + #: Loop invariants, asked once: the uuid5 namespace every minted + #: id lives under, the project's one recorded remote, the spec's + #: header and last commit, and which decisions it declares. + self.namespace = uuid.uuid5(uuid.NAMESPACE_URL, dsid) + remotes = {m.git_remote for _, m in self.made if m.git_remote} + self.remote = remotes.pop() if len(remotes) == 1 else "" + loaded = load_yaml(root / SPEC_FILENAME) + self.spec: dict[str, Any] = dict(loaded) if isinstance(loaded, dict) else {} + self.title = str(self.spec.get("name") or root.name) + self.spec_write = writer(root / SPEC_FILENAME) + self.parameters = {d for t in graph.tasks.values() for d in t.decisions} + + def document(self) -> str: + """Assemble the graph and serialize it, deterministically.""" + workflow = self._workflow() + self._steps_and_tools(workflow) + self._environment_files() + for key, manifest in self.made: + self._dataset(key, manifest) + for key, manifest in self.made: + self._control(key, self._action(key, manifest)) + self._runs(workflow) + self._root() + text: str = json.dumps( + self.crate.metadata.generate(), indent=1, sort_keys=True, ensure_ascii=False + ) + return text + "\n" + + # ----- the workflow and its structure ----- + + def _workflow(self) -> Any: + # Before `add_workflow` rewrites the descriptor's conformsTo: 1.1 + # is what RO-Crate validators key on, 1.2 is the context actually + # emitted — the crate is structurally both, and saying only the + # newer one reads as conforming to neither. + self.crate.metadata["conformsTo"] = [ + {"@id": "https://w3id.org/ro/crate/1.1"}, + {"@id": "https://w3id.org/ro/crate/1.2"}, + ] + lang = ComputerLanguage( + self.crate, + "#astra", + properties={ + "@type": "ComputerLanguage", + "name": "ASTRA", + "alternateName": "Agentic Schema for Transparent Research Analysis", + "url": "https://pypi.org/project/astra-tools/", + }, + ) + self.crate.add(lang) + workflow = self.crate.add_workflow( + self.root / SPEC_FILENAME, + SPEC_FILENAME, + main=True, + lang=lang, + properties={ + # `HowTo` is what licenses the `step` list below — the + # Provenance profile's requirement, not decoration. + "@type": ["File", "SoftwareSourceCode", "ComputationalWorkflow", "HowTo"], + "name": self.title, + "encodingFormat": "application/yaml", + }, + ) + if description := self.spec.get("description"): + workflow["description"] = str(description) + workflow["conformsTo"] = { + "@id": "https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE" + } + workflow["license"] = self._license_ref() + if self.remote: + workflow["url"] = self.remote + if self.spec_write: + # The spec's version is the commit that last changed it — the + # only version an astra.yaml actually has. + workflow["version"] = self.spec_write.sha[:7] + workflow["dateCreated"] = self.spec_write.date + workflow["creator"] = { + "@id": self._person(self.spec_write.author, self.spec_write.email) + } + for decision in sorted(self.parameters): + parameter = ContextEntity( + self.crate, + f"#param-{decision}", + properties={ + "@type": "FormalParameter", + "name": decision, + # ASTRA decisions arrive rendered, as strings. + "additionalType": "Text", + }, + ) + self.crate.add(parameter) + workflow.append_to("input", parameter) + return workflow + + def _steps_and_tools(self, workflow: Any) -> None: + output_ids = sorted({output_id for _, output_id in self.graph.tasks}) + for position, output_id in enumerate(output_ids): + tool = ContextEntity( + self.crate, + self._tool_id(output_id), + properties={ + "@type": "SoftwareApplication", + "name": output_id, + "description": f"the recipe of output `{output_id}`", + }, + ) + if self.remote: + tool["url"] = self.remote + if self.spec_write: + tool["softwareVersion"] = self.spec_write.sha[:7] + self.crate.add(tool) + step = ContextEntity( + self.crate, + f"#step-{output_id}", + properties={ + "@type": "HowToStep", + "position": position, + "workExample": {"@id": tool.id}, + }, + ) + self.crate.add(step) + workflow.append_to("step", step) + # The Provenance profile's one MUST on the workflow itself: + # the tools it orchestrates are its parts. + workflow.append_to("hasPart", tool) + + def _tool_id(self, output_id: str) -> str: + """An id for one output's recipe — under the project's own + repository URL when it has one (an http URI, which is what the + profiles prefer for an application), a ``urn:uuid`` otherwise.""" + if self.remote: + return f"{self.remote}#tool/{output_id}" + return self._uri(f"tool/{output_id}") + + # ----- the data entities ----- + + def _environment_files(self) -> None: + """The lock and its companions — what every run consumed.""" + universes = sorted( + path.relative_to(self.root).as_posix() + for path in (self.root / "universes").glob("*.yaml") + ) + for name in (*_ENVIRONMENT, *universes): + if (self.root / name).is_file(): + self._file(name) + if (self.root / "README.md").is_file(): + readme = self._file("README.md") + readme["about"] = {"@id": "./"} + + def _file(self, name: str) -> Any: + properties: dict[str, Any] = {} + if fmt := _format_of(name): + properties["encodingFormat"] = fmt + return self.crate.add_file(self.root / name, name, properties=properties) + + def _dataset_id(self, key: Key) -> str: + """One output directory's crate id — :func:`plan.declared_path`'s + answer, never a second spelling of the results layout.""" + return plan.declared_path(self.root, self.graph.tasks[key].output_dir) + "/" + + def _dataset(self, key: Key, manifest: assets.Manifest) -> None: + universe_id, output_id = key + dataset_id = self._dataset_id(key) + entity = self.crate.add_dataset(self.graph.tasks[key].output_dir, dataset_id) + entity["name"] = f"{output_id} (universe {universe_id})" + entity["description"] = f"output `{output_id}` materialized under `{universe_id}`" + entity["version"] = manifest.data_version + manifest_file = self._file(f"{dataset_id}{assets.MANIFEST_FILENAME}") + manifest_file["about"] = {"@id": dataset_id} + entity["subjectOf"] = {"@id": manifest_file.id} + + # ----- the runs ----- + + def _action(self, key: Key, manifest: assets.Manifest) -> str: + universe_id, output_id = key + write = self.writer(self.graph.tasks[key].output_dir) + self.agents[key] = self._person(write.author, write.email) if write else "" + properties: dict[str, Any] = { + "@type": "CreateAction", + "name": f"run of `{output_id}` in universe `{universe_id}`", + # The *recorded* recipe, not the graph's: the action states + # what ran, and the spec may have moved since — the manifest + # is the canonical record of the execution. + "description": manifest.recipe, + "instrument": {"@id": self._tool_id(output_id)}, + "result": [{"@id": self._dataset_id(key)}], + "actionStatus": "http://schema.org/CompletedActionStatus", + "object": self._objects(key, manifest), + } + if manifest.started_at: + properties["startTime"] = manifest.started_at + if manifest.finished_at: + properties["endTime"] = manifest.finished_at + if self.agents[key]: + properties["agent"] = {"@id": self.agents[key]} + if manifest.image is not None: + properties["containerImage"] = {"@id": self._image(manifest.image)} + action = ContextEntity( + self.crate, self._uri(f"action/{universe_id}/{output_id}"), properties + ) + self.crate.add(action) + self.actions.append(str(action.id)) + return str(action.id) + + def _objects(self, key: Key, manifest: assets.Manifest) -> list[dict[str, str]]: + task = self.graph.tasks[key] + refs = [{"@id": name} for name in _ENVIRONMENT if (self.root / name).is_file()] + for name in sorted(task.inputs): + upstream = task.produced_by.get(name) + if upstream is not None: + if upstream in self.made_keys: + refs.append({"@id": self._dataset_id(upstream)}) + continue + refs.append({"@id": self._external(name, task.inputs[name], manifest)}) + # The *recorded* decisions: the values the recipe actually ran + # under, whatever the spec says today. A decision the workflow no + # longer declares gets no exampleOfWork — there is no parameter + # for it to exemplify. + for decision in sorted(manifest.decisions): + properties: dict[str, Any] = { + "@type": "PropertyValue", + "name": decision, + "value": manifest.decisions[decision], + } + if decision in self.parameters: + properties["exampleOfWork"] = {"@id": f"#param-{decision}"} + value = ContextEntity( + self.crate, f"#value-{key[0]}-{key[1]}-{decision}", properties + ) + self.crate.add(value) + refs.append({"@id": value.id}) + return refs + + def _external(self, name: str, path: Path, manifest: assets.Manifest) -> str: + """A declared input the spec points at, in or out of the tree. + + In-or-out is :func:`plan.declared_path`'s answer — relative + inside the tree, absolute outside it — never a second spelling + of that rule here: two copies of one path rule is how the first + one shipped a bug. + """ + declared = plan.declared_path(self.root, path) + in_tree = not Path(declared).is_absolute() + entity_id = declared if in_tree else Path(declared).as_uri() + recorded = manifest.input_versions.get(name, "") + digest = recorded.removeprefix("sha256:") if recorded.startswith("sha256:") else "" + if (existing := self.crate.dereference(entity_id)) is not None: + # Two manifests can testify to different bytes for one input + # — a half-rebuilt project. Publishing either digest would + # contradict a manifest in the same crate, so publish + # neither; the manifests themselves keep the full story. + if digest and existing.get("sha256") not in (None, digest): + existing.pop("sha256") + return entity_id + properties: dict[str, Any] = {"@type": "File", "name": declared} + if fmt := _format_of(declared): + properties["encodingFormat"] = fmt + if digest: + properties["sha256"] = digest + if in_tree: + self.crate.add_file(path, declared, properties=properties) + else: + # Outside the repository: recorded by content, not stored + # in it — the layer's stated weaker promise, so a context + # entity rather than a data entity the crate cannot hold. + self.crate.add(ContextEntity(self.crate, entity_id, properties)) + return entity_id + + def _control(self, key: Key, action_id: str) -> None: + universe_id, output_id = key + self.crate.add( + ContextEntity( + self.crate, + _control_id(key), + properties={ + "@type": "ControlAction", + "name": f"orchestration of `{output_id}` in universe `{universe_id}`", + "instrument": {"@id": f"#step-{output_id}"}, + "object": {"@id": action_id}, + }, + ) + ) + + def _runs(self, workflow: Any) -> None: + """One ``OrganizeAction`` per run — outputs sharing a ``git_sha`` + were made by one ``lc materialize``, the driver's one HEAD read.""" + runs: dict[str, list[tuple[Key, assets.Manifest]]] = {} + for key, manifest in self.made: + runs.setdefault(manifest.git_sha, []).append((key, manifest)) + for sha in sorted(runs): + group = runs[sha] + # The engine *that run* recorded, never the one installed + # here: the render must be a function of repository state, + # or two collaborators on different lc versions re-commit + # the crate at each other forever — and an lc upgrade is + # recorded as moving nothing. One run has one engine by + # construction; every manifest of the group agrees. + engine = self._engine(group[0][1].lc_version) + times = sorted(t for _, m in group for t in (m.started_at, m.finished_at) if t) + run_action = ContextEntity( + self.crate, + self._uri(f"run/{sha}"), + properties={ + "@type": "CreateAction", + "name": f"lc materialize at {sha[:7] or 'unknown commit'}", + "description": ( + f"one materialization run, starting from commit {sha or '(unrecorded)'}" + ), + "instrument": {"@id": workflow.id}, + "result": [{"@id": self._dataset_id(key)} for key, _ in group], + "actionStatus": "http://schema.org/CompletedActionStatus", + }, + ) + if times: + run_action["startTime"] = times[0] + run_action["endTime"] = times[-1] + if agent := self.agents.get(group[0][0], ""): + run_action["agent"] = {"@id": agent} + self.crate.add(run_action) + self.actions.append(str(run_action.id)) + organize = ContextEntity( + self.crate, + f"#organize-{sha[:7] or 'unknown'}", + properties={ + "@type": "OrganizeAction", + "name": f"scheduling of the run at {sha[:7] or 'unknown commit'}", + "instrument": {"@id": engine}, + "object": [{"@id": _control_id(key)} for key, _ in group], + "result": {"@id": run_action.id}, + }, + ) + self.crate.add(organize) + + def _engine(self, version: str) -> str: + """The engine one run's manifests attest, added once per version. + + Its release page is both the id and the ``url`` — the one address + derivable from repository state alone; anything read off the + installed engine would differ between collaborators' hosts. + """ + engine_id = ( + f"https://pypi.org/project/lightcone-cli/{version}/" + if version + else "#lightcone-cli" + ) + if engine_id not in self.engines: + properties: dict[str, Any] = { + "@type": "SoftwareApplication", + "name": "lightcone-cli", + } + if version: + properties["softwareVersion"] = version + properties["url"] = engine_id + self.crate.add(ContextEntity(self.crate, engine_id, properties)) + self.engines.add(engine_id) + return engine_id + + def _image(self, image: dict[str, Any]) -> str: + """The committed archive: identity and payload as one entity.""" + archive = str(image.get("archive") or "") + if archive not in self.images: + properties: dict[str, Any] = { + "@type": ["File", "ContainerImage"], + "name": str(image.get("tag") or archive), + "tag": str(image.get("tag") or ""), + "encodingFormat": "application/x-tar", + # The archive is `docker-archive` format wherever it runs — + # podman, docker and podman-hpc all consume it as one. + "additionalType": {"@id": "https://w3id.org/ro/terms/workflow-run#DockerImage"}, + } + if str(image.get("id") or "").startswith("sha256:"): + properties["sha256"] = str(image["id"]).removeprefix("sha256:") + self.crate.add_file(self.root / archive, archive, properties=properties) + self.images.add(archive) + return archive + + # ----- shared entities and the root ----- + + def _person(self, author: str, email: str) -> str: + person_key = email or author + if person_key not in self.persons: + person_id = f"mailto:{email}" if email else f"#person-{len(self.persons)}" + self.crate.add( + ContextEntity( + self.crate, + person_id, + properties={"@type": "Person", "name": author, "email": email}, + ) + ) + self.persons[person_key] = person_id + return self.persons[person_key] + + def _root(self) -> None: + root = self.crate.root_dataset + root["name"] = self.title + root["description"] = str( + self.spec.get("description") + or f"ASTRA analysis `{self.title}`, materialized by lightcone-cli." + ) + root["license"] = self._license_ref() + # The newest recorded instant, never the clock: the document must + # be a pure function of repository state, or every render is a + # fresh diff and convergence commits forever. + stamps = sorted(m.finished_at for _, m in self.made if m.finished_at) + root["datePublished"] = stamps[-1] if stamps else (self.spec_write.date or "1970-01-01") + root["conformsTo"] = [{"@id": profile_id} for profile_id, _, _ in _PROFILES] + for profile_id, name, version in _PROFILES: + self.crate.add( + ContextEntity( + self.crate, + profile_id, + properties={ + "@type": "CreativeWork", + "name": f"{name} {version}", + "version": version, + }, + ) + ) + if self.persons: + root["author"] = [ + {"@id": person_id} for person_id in sorted(self.persons.values()) + ] + if self.actions: + root["mentions"] = [{"@id": action_id} for action_id in self.actions] + + def _license_ref(self) -> Any: + """The root's license value, always a linkable entity. + + A path in the tree becomes a File data entity; a URL becomes a + CreativeWork at that URL; anything else — an SPDX id, an SPDX + expression, free text — becomes a *local* CreativeWork carrying + the declared string. Deliberately never a minted spdx.org URL: + the declaration is not validated against the SPDX list, and a + fabricated dead URL in a document built for archives is worse + than a local entity. Idempotent: `add` replaces by id, so the + workflow and the root may both ask. + """ + if (self.root / self.license).is_file(): + self._file(self.license) + return {"@id": self.license} + license_id = ( + self.license + if self.license.startswith(("http://", "https://")) + else "#license" + ) + self.crate.add( + ContextEntity( + self.crate, + license_id, + properties={"@type": "CreativeWork", "name": self.license}, + ) + ) + return {"@id": license_id} + + def _uri(self, kind: str) -> str: + """Mint an absolute, deterministic id under the dataset's UUID. + + ``urn:uuid`` from the dsid namespace, so the same entity gets the + same id in every clone and every render — and an absolute URI is + what the profiles ask of an application id. + """ + return f"urn:uuid:{uuid.uuid5(self.namespace, kind)}" + + +#: A closed suffix → media type map, deliberately not ``mimetypes``: +#: that module reads the host's own tables (``/etc/mime.types``), which +#: would make the rendered document differ between machines — and the +#: document must be a pure function of repository state. Unknown stays +#: unknown; empty means the entity simply carries no ``encodingFormat``. +_FORMATS = { + ".toml": "application/toml", + ".lock": "application/toml", + ".yaml": "application/yaml", + ".yml": "application/yaml", + ".json": "application/json", + ".md": "text/markdown", + ".python-version": "text/plain", + ".txt": "text/plain", + ".csv": "text/csv", + ".fits": "image/fits", + ".h5": "application/x-hdf5", + ".hdf5": "application/x-hdf5", + ".parquet": "application/vnd.apache.parquet", + ".png": "image/png", + ".pdf": "application/pdf", +} + + +def _format_of(name: str) -> str: + """A media type for *name* from the closed map — empty when unknown.""" + return _FORMATS.get(Path(name).suffix or Path(name).name, "") diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index 4449de0d..a59f73ad 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -19,10 +19,26 @@ from __future__ import annotations from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path from lightcone.engine import project + +@dataclass(frozen=True) +class LastWrite: + """The commit that last touched a path — empty fields mean "cannot + say", and the whole record is falsy then.""" + + sha: str = "" + subject: str = "" + author: str = "" + email: str = "" + date: str = "" + + def __bool__(self) -> bool: + return bool(self.sha) + # ============================================================================= # The repository # ============================================================================= @@ -192,6 +208,39 @@ def dataset_id(directory: Path) -> str: return found.stdout.strip() if found.returncode == 0 else "" +def last_writer(directory: Path, path: Path) -> LastWrite: + """Find the commit that last touched *path*. + + Run from the project root with a relative pathspec, so it answers + about the project's own subdirectory even inside an enclosing + repository — and since nothing parses paths out of its output, no + prefix handling is needed. + + "Cannot say" is the empty answer, never an error: the callers are + ``lc status`` and the crate render, and a read-only verb must not + refuse a project over an unborn HEAD, a deposit stripped of its + ``.git``, or a host without git — states such projects are actually + in. + + Args: + directory: The project root. + path: The path to ask about, absolute or repository-relative. + + Returns: + The commit, falsy-empty when none has touched the path — or when + git cannot answer at all. + """ + argv = ["log", "-1", "--format=%H%x00%s%x00%an%x00%ae%x00%as", "--", _rel(directory, path)] + try: + proc = project._run(["git", *argv], cwd=directory) + except OSError: + return LastWrite() + out = str(proc.stdout or "").strip("\n") + if proc.returncode != 0 or not out: + return LastWrite() + return LastWrite(*out.split("\0")) + + def save(directory: Path, paths: Iterable[Path], message: str) -> bool: """Commit *paths*. diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index e5ebc228..12fea41a 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -32,6 +32,7 @@ from __future__ import annotations +import functools import json import os import re @@ -134,7 +135,7 @@ def check(root: Path, targets: Sequence[str], *, refresh: bool = False) -> Mater read, or a target matches nothing. """ report = MaterializeReport() - for key, verdict, _ in _classified(root, targets, report, refresh=refresh): + for key, verdict, _, _ in _classified(root, targets, report, refresh=refresh): name = _name(key) if verdict.calls_for_a_remake(refresh=refresh): report.planned[name] = verdict.why @@ -147,7 +148,7 @@ def check(root: Path, targets: Sequence[str], *, refresh: bool = False) -> Mater def _classified( root: Path, targets: Sequence[str], report: MaterializeReport, *, refresh: bool -) -> list[tuple[Key, assets.Verdict, assets.Manifest | None]]: +) -> list[tuple[Key, assets.Verdict, assets.Manifest | None, dataset.LastWrite | None]]: """Classify every task in topological order, reading nothing but disk. The walk both read-only modes share, so there is one answer to "what @@ -166,9 +167,10 @@ def _classified( decides whether their dependents see the sentinel. Returns: - One ``(key, verdict, manifest)`` per task, upstream first. + One ``(key, verdict, manifest, foreign write)`` per task, upstream + first. """ - graph, env_version = _graph(root, targets, report) + graph, env_version, _ = _graph(root, targets, report) versions = assets.Versions() would_run: set[Key] = set() unfetched: set[str] = set() @@ -177,15 +179,20 @@ def _classified( for key in graph.order(): task = graph.tasks[key] manifest = assets.read(task.output_dir) + # History is git's to answer, so it enters the one classification + # rule as a value — computed here, where git lives, exactly as + # the worker's driver computes it for a run. + foreign = None if manifest is None else _foreign_write(root, key) verdict = assets.classify( definition_version=task.definition_version, env_version=env_version, manifest=manifest, inputs=_predicted(root, task, would_run, versions, unfetched), + foreign=foreign, ) if verdict.calls_for_a_remake(refresh=refresh): would_run.add(key) - classified.append((key, verdict, manifest)) + classified.append((key, verdict, manifest, foreign)) if unfetched: report.warnings.append( @@ -271,6 +278,17 @@ class OutputStatus: git_sha: str #: Its content identity, or empty if it was never materialized. data_version: str + #: Empty when the commit that last touched the output's directory is + #: its own run record. Otherwise that foreign commit's sha — the + #: agent-forged-file fact, which also makes the output read `stale`: + #: the manifest no longer describes the bytes, and a skip would + #: return the recorded digest forever. The sha rather than prose, + #: so a machine consumer gets what `why`'s sentence cannot carry. + #: History-based on purpose: it costs no hashing and answers on a + #: clone that holds none of the bytes. What it leaves to existing + #: tools: uncommitted edits are a dirty tree, and annex object + #: corruption is `git annex fsck`. + foreign_write: str = "" def as_dict(self) -> dict[str, Any]: """Return the record as JSON-ready data. @@ -349,7 +367,7 @@ def status(root: Path) -> StatusReport: if state != "direct": result.image = {"tag": tag, "state": state, "archive": archive} result.sandbox = _sandbox_line(result.mode) - for key, verdict, manifest in _classified(root, [], report, refresh=False): + for key, verdict, manifest, foreign in _classified(root, [], report, refresh=False): result.outputs.append( OutputStatus( output=_name(key), @@ -357,12 +375,24 @@ def status(root: Path) -> StatusReport: why=verdict.why, git_sha=manifest.git_sha if manifest else "", data_version=manifest.data_version if manifest else "", + foreign_write=foreign.sha if foreign else "", ) ) result.warnings = report.warnings return result +def _foreign_write(root: Path, key: Key) -> dataset.LastWrite | None: + """Find the commit that last touched *key*'s directory, unless it is + the output's own run record — then ``None``, the clean answer. The + fact only: the verdict's prose is `classify`'s, like every other + why.""" + write = dataset.last_writer(root, assets.output_dir(root, *key)) + if not write or write.subject == datalad_run_subject(key): + return None + return write + + def _sandbox_line(mode: str) -> str: """One line naming the enforcement a run on this host would get. @@ -429,8 +459,13 @@ def materialize( # before the image: a refusal here must not cost a minutes-long # build, and must not leave an archive commit behind a run that # "failed" on a typo in the spec. - graph, env_version = _graph(root, targets, report) + graph, env_version, full = _graph(root, targets, report) + dsid = dataset.dataset_id(root) if not graph.tasks: + # No tasks is not no project: a spec whose outputs were all + # dropped still has a crate describing them, and this is its + # only maintainer. + _converge_crate(root, report, full, dsid) return report _fetch_inputs(root, graph, report) # Before the runtime resolves, because the refusal must not cost an @@ -461,7 +496,6 @@ def materialize( # rerun does not come through here; its entry point converges too.) report.warnings.extend(f"uv: {w}" for w in container.converge(runtime)) - dsid = dataset.dataset_id(root) # Read once, for every task: the driver commits each output as it lands, # so HEAD moves during the run, and a per-task read would give later # manifests a commit this run created — nondeterministically, depending @@ -471,6 +505,20 @@ def materialize( # declared input shared by several outputs — or by one output across # several universes — is the same bytes every time it is asked for. versions = assets.Versions() + # The history question is the driver's to answer — workers have no + # git, by design — so each task is told up front whether its + # directory was last written by something other than its own run + # record. A foreign write contradicts the manifest, and a worker that + # trusted the recorded digest would skip the output forever. Guarded + # on the manifest's presence, as `_classified` is: without one the + # answer is dead — the output is remade regardless — and each ask is + # a git process. + foreign = { + key: _foreign_write(root, key) + if (task.output_dir / assets.MANIFEST_FILENAME).is_file() + else None + for key, task in graph.tasks.items() + } outstanding: dict[Key, Task] = dict(graph.tasks) try: with cluster_for_run() as scheduler: @@ -488,6 +536,7 @@ def materialize( head, versions, refresh, + foreign[key], runtime, *[pending[dep] for dep in task.depends_on], key=_name(key), @@ -502,6 +551,7 @@ def materialize( # survive. for task in outstanding.values(): dataset.restore(root, [task.output_dir]) + _converge_crate(root, report, full, dsid) return report @@ -664,6 +714,83 @@ def _fetch_inputs(root: Path, graph: Graph, report: MaterializeReport) -> None: ) +# ============================================================================= +# The publication view +# ============================================================================= + + +def _converge_crate(root: Path, report: MaterializeReport, full: Graph, dsid: str) -> None: + """Bring ``ro-crate-metadata.json`` in line with the repository. + + A derived artifact, converged the way ``uv.lock`` is: rendered from + repository state, compared, and committed only on a difference — so + an idempotent re-run commits nothing and nobody has to remember a + verb. Maintenance is derived from publication intent: a declared + ``[project].license`` turns it on, the same shape as + ``[tool.lightcone.image]`` deriving containerized mode. Runs after + the consume loop, driver-side, on the full graph rather than the + run's targets — the crate describes the project, not one invocation. + (The rerun entry point does not come through here: it is one task's + executor, so the crate lags until the next materialize.) + + Contained on purpose: by the time this runs every output is already + committed, so a failure here becomes a warning and the report still + reaches the user — the crate is the publication view, not the run. + An interrupt between the write and its commit restores the file, so + the tree ends as clean as the loop left it; and the save runs even + when the text already matches, healing a previously interrupted + converge whose write survived uncommitted. + + Args: + root: The project root. + report: Collects the one maintenance line and any warnings. + full: The whole-project graph — never a run's narrowed one. + dsid: The dataset's UUID, already read by the caller. + """ + # Deferred so `lc status` and `--check` never import rocrate — the + # crate is the one materialize-only dependency on their shared path. + from lightcone.engine import crate + + spdx = crate.license_of(root) + path = root / crate.CRATE_FILENAME + if not spdx: + report.warnings.append( + f"pyproject.toml no longer declares [project].license, so " + f"{crate.CRATE_FILENAME} is no longer maintained" + if path.exists() + else "no [project].license in pyproject.toml, so no RO-Crate " + "publication view is maintained — declare one to enable it" + ) + return + for directory in sorted((root / "results").glob("*/*/")): + key = (directory.parent.name, directory.name) + if key not in full.tasks and assets.read(directory) is not None: + report.warnings.append( + f"{plan.declared_path(root, directory)} has a manifest but the " + "spec no longer declares it, so it is not in the publication view" + ) + try: + document = crate.render( + root, + full, + license=spdx, + dsid=dsid, + writer=functools.partial(dataset.last_writer, root), + ) + if not (path.is_file() and path.read_text() == document): + path.write_text(document) + try: + dataset.save(root, [path], "Update the RO-Crate publication view") + except BaseException: + # An interrupt or a git failure between the write and its + # commit must not leave a file lc wrote for the next run's + # dirty refusal to blame on the user. + dataset.restore(root, [path]) + raise + except Exception as e: + report.warnings.append(f"the RO-Crate publication view could not be updated: {e}") + + # ============================================================================= # The commit # ============================================================================= @@ -718,13 +845,26 @@ def run_record(root: Path, task: Task, dsid: str, runtime: container.Runtime) -> info["extra_inputs"] = [runtime.archive] body = json.dumps(info, indent=1, sort_keys=True, ensure_ascii=False) return ( - f"[DATALAD RUNCMD] {task.output_id} [{task.universe_id}]\n\n" + f"{datalad_run_subject(task.key)}\n\n" "=== Do not change lines below ===\n" f"{body}\n" "^^^ Do not change lines above ^^^" ) +def datalad_run_subject(key: Key) -> str: + """The subject line of *key*'s ``[DATALAD RUNCMD]`` commit — datalad's + format, which its ``rerun`` matches with a regex, not a spelling of + ours to adjust. + + One function, shared with the foreign-write check: an output is + cleanly written iff the commit that last touched its directory + carries exactly this subject, so the composer and the comparator must + not be two strings that can drift apart. + """ + return f"[DATALAD RUNCMD] {key[1]} [{key[0]}]" + + def _engine_requirement() -> str: """Build the requirement that reconstructs the running engine. @@ -770,12 +910,16 @@ def _repository_url() -> str: # ============================================================================= -def _graph(root: Path, targets: Sequence[str], report: MaterializeReport) -> tuple[Graph, str]: +def _graph( + root: Path, targets: Sequence[str], report: MaterializeReport +) -> tuple[Graph, str, Graph]: """The tasks a run covers, and the environment they will be compared to. The lock scan runs here, once, for both modes: what it refuses is a dependency whose bytes the lock does not pin, which makes every hash - below it a claim nobody can check. + below it a claim nobody can check. The unnarrowed graph is returned + beside the narrowed one, because the crate describes the whole + project and must not pay a second spec validation to see it. """ scan = identity.scan_lock(root) if scan.refusals: @@ -797,9 +941,8 @@ def _graph(root: Path, targets: Sequence[str], report: MaterializeReport) -> tup ) env_version = identity.env_version(root) - graph = plan.build(root) - if targets: - graph = graph.closure(graph.resolve(list(targets))) + full = plan.build(root) + graph = full.closure(full.resolve(list(targets))) if targets else full # A declared input outside the project is hashed into the manifest like # any other, so a change to it still cascades — but it is not in the @@ -818,7 +961,7 @@ def _graph(root: Path, targets: Sequence[str], report: MaterializeReport) -> tup "not stored in it, so a commit cannot restore them: " + ", ".join(sorted(outside)) ) - return graph, env_version + return graph, env_version, full def _name(key: Key) -> str: diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index 92937598..bbed7208 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -32,6 +32,7 @@ import sys from collections.abc import Mapping from dataclasses import asdict, dataclass +from datetime import UTC, datetime from pathlib import Path from typing import Literal @@ -96,6 +97,7 @@ def materialize( head: Head, versions: assets.Versions, refresh: bool, + foreign: dataset.LastWrite | None, runtime: container.Runtime, *upstream: TaskResult, ) -> TaskResult: @@ -115,6 +117,10 @@ def materialize( driver because it commits as outputs land and HEAD moves. versions: The run's content-hash memo for declared inputs. refresh: Whether to remake an output that is merely behind. + foreign: The commit that last wrote the output's directory in + place of its own run record, or ``None`` — answered by the + driver, because history is git's and workers have no git; + handed to the one classification rule, where it is `stale`. runtime: The execution world, resolved once by the driver — the same discipline as *head*, because resolving per task could answer differently mid-run. @@ -126,7 +132,9 @@ def materialize( What happened. Never raises. """ try: - return _materialize(root, task, env_version, head, versions, refresh, runtime, upstream) + return _materialize( + root, task, env_version, head, versions, refresh, foreign, runtime, upstream + ) except Exception as e: # the contract is that this function returns return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") @@ -138,6 +146,7 @@ def _materialize( head: Head, versions: assets.Versions, refresh: bool, + foreign: dataset.LastWrite | None, runtime: container.Runtime, upstream: tuple[TaskResult, ...], ) -> TaskResult: @@ -157,6 +166,7 @@ def _materialize( env_version=env_version, manifest=manifest, inputs=inputs, + foreign=foreign, ) if verdict.calls_for_a_remake(refresh=refresh): return execute(root, task, env_version, inputs, head=head, runtime=runtime) @@ -223,6 +233,7 @@ def execute( read_paths = [p for p in task.inputs.values() if p.exists()] policy = container.policy_for(runtime, read_paths) + started_at = _now() with sandbox.scope(policy): outcome = sandbox.run( container.backend(runtime), @@ -232,6 +243,7 @@ def execute( prefix=uv_prefix(root, sync=False), env=child_env(), ) + finished_at = _now() if outcome.returncode != 0: return TaskResult( @@ -264,6 +276,8 @@ def execute( git_remote=remote, lc_version=lc_version(), hermeticity=asdict(outcome.attestation), + started_at=started_at, + finished_at=finished_at, image=runtime.manifest_image(), ), ) @@ -277,6 +291,16 @@ def execute( return TaskResult(task.key, "ok", data_version=data_version, notes=outcome.notes) +def _now() -> str: + """The current instant, as a manifest timestamp. + + Milliseconds, not microseconds: RO-Crate consumers parse + ``schema:endTime`` with at most three fractional digits, and the + manifest is where that string is minted. + """ + return datetime.now(UTC).isoformat(timespec="milliseconds") + + def _gate(root: Path, env_version: str) -> str: """Empty while the environment is still the one the run started in.""" if identity.env_version(root) == env_version: diff --git a/tests/conftest.py b/tests/conftest.py index 4badb112..9be56af2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,8 @@ import shutil import subprocess import textwrap -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from pathlib import Path from unittest.mock import MagicMock @@ -112,6 +113,34 @@ def fake_which(name: str, path: str | None = None) -> str | None: return calls +class _Inline: + """Run a graph in this thread, in the order it was submitted. + + Submission is topological, so a dependent is submitted only after its + upstreams have already run — which means the "handles" it is passed + are the upstream results themselves, exactly what the worker expects. + """ + + def submit(self, fn: Callable[..., object], *args: object, key: str) -> object: + return fn(*args) + + def completed(self, handles: list[object]) -> Iterator[object]: + yield from handles + + +@pytest.fixture +def inline(monkeypatch: pytest.MonkeyPatch) -> None: + """Replace the Dask cluster with an in-thread scheduler — the one + monkeypatch point `cluster_for_run` exists to be.""" + from lightcone.engine import materialize + + @contextmanager + def fake() -> Iterator[_Inline]: + yield _Inline() + + monkeypatch.setattr(materialize, "cluster_for_run", fake) + + @pytest.fixture def real_tools(monkeypatch: pytest.MonkeyPatch) -> None: """Opt out of :func:`tools`, and run the real git and git-annex. diff --git a/tests/test_assets.py b/tests/test_assets.py index ed0345e7..ad6a07cd 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -22,7 +22,7 @@ import pytest -from lightcone.engine import assets +from lightcone.engine import assets, dataset from lightcone.engine.assets import Manifest, classify, data_version, output_dir from lightcone.engine.project import ProjectError @@ -277,6 +277,11 @@ def test_writing_a_manifest_replaces_the_previous_one_whole(tmp_path: Path) -> N _ENV = "sha256:env" +_TWEAK = dataset.LastWrite( + "8d31f00" + "0" * 33, "tweak colors", "Ada", "ada@example.org", "2026-08-19" +) + + def _classify(**overrides: object) -> assets.Verdict: """Classify against the manifest `_manifest()` builds, unchanged.""" call: dict[str, object] = { @@ -385,6 +390,35 @@ def test_stale_wins_over_behind() -> None: assert verdict.calls_for_a_remake(refresh=False) +def test_a_foreign_write_is_stale_and_the_prose_is_composed_here() -> None: + """History enters the one rule as a value — the caller with git hands + over the offending commit, and a hit is a contradiction: the manifest + no longer describes the bytes. The sentence, like every other why, + is this module's.""" + verdict = _classify(foreign=_TWEAK) + assert verdict.status == "stale" + assert verdict.calls_for_a_remake(refresh=False) + assert "8d31f00" in verdict.why and "tweak colors" in verdict.why + assert "git show 8d31f00" in verdict.why + + +def test_a_definition_stale_output_keeps_its_own_why_over_a_foreign_write() -> None: + """Both call for the same remake, and the definition drift is the more + actionable reason to report.""" + verdict = _classify(definition_version="sha256:other", foreign=_TWEAK) + assert verdict.status == "stale" + assert "tweak colors" not in verdict.why + + +def test_a_foreign_write_wins_over_behind() -> None: + """A behind output is not wrong; a foreign-written one is — reporting + "left alone" about something the run is about to remake is the one + wrong answer, the same rule as stale-over-behind.""" + verdict = _classify(env_version="sha256:moved", foreign=_TWEAK) + assert verdict.status == "stale" + assert "tweak colors" in verdict.why + + def test_an_environment_that_did_not_move_is_not_behind() -> None: """The negative half of the sensitivity pair: `behind` has to be off by default, or every output reports it forever and the signal is dead.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index 28fe0dc4..eec31b5c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -553,6 +553,40 @@ def test_build_on_a_direct_project_is_an_explanatory_no_op( assert "[tool.lightcone.image]" in result.output +def test_status_renders_a_foreign_write_as_a_stale_output( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A foreign write arrives as a stale with its message in `why`, so + the one rendering path covers it — and it does not change the exit + code: status reports.""" + from lightcone.engine.materialize import OutputStatus, StatusReport + + message = ( + 'last changed by 8d31f00 ("tweak colors", Ada, 2026-08-19) rather ' + "than its run record, so the manifest no longer describes these " + "bytes — the next run remakes it; inspect first with `git show 8d31f00`" + ) + report = StatusReport( + outputs=[ + OutputStatus( + "baseline/first", + "stale", + message, + "3f2a1c8ffff", + "sha256:one", + foreign_write="8d31f00" + "0" * 33, + ), + ] + ) + _status_stub(monkeypatch, report) + + result = runner.invoke(main, ["status"]) + + assert result.exit_code == 0 + assert "stale" in result.output + assert "8d31f00" in result.output + + def test_status_exits_zero_even_with_stale_outputs( runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -583,6 +617,7 @@ def test_status_json_is_machine_readable( "why": "", "git_sha": "3f2a1c8ffff", "data_version": "sha256:one", + "foreign_write": "", }, { "output": "baseline/second", @@ -590,6 +625,7 @@ def test_status_json_is_machine_readable( "why": "made under an earlier environment", "git_sha": "3f2a1c8ffff", "data_version": "sha256:two", + "foreign_write": "", }, { "output": "baseline/third", @@ -597,6 +633,7 @@ def test_status_json_is_machine_readable( "why": "the input `first` changed", "git_sha": "", "data_version": "", + "foreign_write": "", }, ], "warnings": [], diff --git a/tests/test_crate.py b/tests/test_crate.py new file mode 100644 index 00000000..e25679db --- /dev/null +++ b/tests/test_crate.py @@ -0,0 +1,328 @@ +"""Tests for `lightcone.engine.crate` — the publication view, rendered pure. + +Everything here is disk-only: fixture manifests under ``tmp_path``, a +hand-built graph, and a stub for "who last wrote this path" — no git, no +subprocess. Structure and ordering are asserted, never byte goldens; the +one byte-level claim is determinism, because convergence-by-materialize +rests on rendering twice at the same state yielding identical text. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from lightcone.engine import assets, crate +from lightcone.engine.dataset import LastWrite +from lightcone.engine.plan import Graph, Key, Task + +_DSID = "4b7b5c1e-0000-4000-8000-000000000000" + +Writer = Callable[[Path], LastWrite] + + +@pytest.fixture +def project(tmp_path: Path) -> Path: + root = tmp_path / "demo" + (root / "universes").mkdir(parents=True) + (root / "data").mkdir() + (root / "astra.yaml").write_text("name: demo\ndescription: A demo analysis.\n") + (root / "universes" / "baseline.yaml").write_text("id: baseline\n") + (root / "universes" / "alt.yaml").write_text("id: alt\n") + (root / "pyproject.toml").write_text('[project]\nname = "demo"\nlicense = "MIT"\n') + (root / "uv.lock").write_text("version = 1\n") + (root / ".python-version").write_text("3.12.5\n") + (root / "data" / "catalog.csv").write_text("a,b\n") + return root + + +def _made( + root: Path, + universe_id: str, + output_id: str, + *, + git_sha: str, + image: dict[str, str] | None = None, + inputs: dict[str, str] | None = None, + finished_at: str = "2026-08-19T10:05:00.000+00:00", +) -> Path: + directory = root / "results" / universe_id / output_id + directory.mkdir(parents=True) + (directory / "out.txt").write_text(f"{universe_id}/{output_id}\n") + assets.write( + directory, + assets.Manifest( + output_id=output_id, + universe_id=universe_id, + recipe=f"make {output_id}", + definition_version="sha256:def", + env_version="sha256:env", + data_version=f"sha256:{universe_id}-{output_id}", + decisions={"method": "alpha"}, + input_versions=inputs or {}, + git_sha=git_sha, + git_remote="https://github.com/example/demo.git", + lc_version="0.4.2", + hermeticity={"mechanism": "landlock"}, + started_at="2026-08-19T10:00:00.000+00:00", + finished_at=finished_at, + image=image, + ), + ) + return directory + + +def _graph(root: Path, universes: tuple[str, ...] = ("baseline",)) -> Graph: + tasks: dict[Key, Task] = {} + for universe_id in universes: + first_dir = root / "results" / universe_id / "first" + tasks[(universe_id, "first")] = Task( + universe_id, + "first", + first_dir, + "make first", + {"catalog": root / "data" / "catalog.csv"}, + {}, + {"method": "alpha"}, + "sha256:def", + ) + tasks[(universe_id, "second")] = Task( + universe_id, + "second", + root / "results" / universe_id / "second", + "make second", + {"first": first_dir}, + {"first": (universe_id, "first")}, + {"method": "alpha"}, + "sha256:def", + ) + return Graph(tasks) + + +def _writer(path: Path) -> LastWrite: + return LastWrite("a" * 40, "irrelevant", "Ada Lovelace", "ada@example.org", "2026-08-19") + + +def _render(root: Path, graph: Graph, writer: Writer = _writer) -> dict[str, Any]: + text = crate.render(root, graph, license="MIT", dsid=_DSID, writer=writer) + loaded = json.loads(text) + assert isinstance(loaded, dict) + return loaded + + +def _entities(document: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {entity["@id"]: entity for entity in document["@graph"]} + + +# ---- determinism, the property convergence rests on ------------------------ + + +def test_rendering_twice_at_the_same_state_is_byte_identical(project: Path) -> None: + _made(project, "baseline", "first", git_sha="aaa111") + _made(project, "baseline", "second", git_sha="aaa111") + graph = _graph(project) + + first = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) + second = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) + + assert first == second + + +def test_the_clock_never_enters_the_document(project: Path) -> None: + """`datePublished` is the newest recorded instant — rocrate's own + default stamps the current time, and this pins the override.""" + _made(project, "baseline", "first", git_sha="aaa111") + _made( + project, + "baseline", + "second", + git_sha="aaa111", + finished_at="2026-08-20T09:00:00.000+00:00", + ) + + root_entity = _entities(_render(project, _graph(project)))["./"] + + assert root_entity["datePublished"] == "2026-08-20T09:00:00.000+00:00" + + +# ---- the workflow-run context and the profiles ----------------------------- + + +def test_the_context_carries_the_workflow_run_vocabulary(project: Path) -> None: + """Without it, `containerImage` and `sha256` are undefined terms that + JSON-LD silently drops on expansion — typed but inert.""" + _made(project, "baseline", "first", git_sha="aaa111") + document = _render(project, _graph(project)) + + assert "https://w3id.org/ro/terms/workflow-run" in document["@context"] + + +def test_the_root_conforms_to_every_claimed_profile_and_declares_each( + project: Path, +) -> None: + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + claimed = {ref["@id"] for ref in entities["./"]["conformsTo"]} + assert "https://w3id.org/ro/wfrun/provenance/0.5" in claimed + for profile in claimed: + assert entities[profile]["@type"] == "CreativeWork" + + +# ---- the Provenance layer, structurally ------------------------------------ + + +def test_one_step_per_output_and_one_action_per_universe(project: Path) -> None: + """A step is spec structure and an action is one execution — which is + exactly how a multiverse maps onto the Provenance profile.""" + for universe_id in ("baseline", "alt"): + _made(project, universe_id, "first", git_sha="aaa111") + _made(project, universe_id, "second", git_sha="aaa111") + entities = _entities(_render(project, _graph(project, ("baseline", "alt")))) + + steps = [e for e in entities.values() if e["@type"] == "HowToStep"] + actions = [e for e in entities.values() if e["@type"] == "CreateAction"] + workflow = entities["astra.yaml"] + assert len(steps) == 2 + assert {ref["@id"] for ref in workflow["step"]} == {"#step-first", "#step-second"} + # four tool-level actions, plus one workflow-level action for the run + assert len(actions) == 5 + assert "HowTo" in workflow["@type"] + + +def test_outputs_sharing_a_commit_are_one_run(project: Path) -> None: + """The driver reads HEAD once and hands it down, so `git_sha` groups + manifests into runs — the whole Provenance layer's identity.""" + _made(project, "baseline", "first", git_sha="aaa111") + _made(project, "baseline", "second", git_sha="bbb222") + entities = _entities(_render(project, _graph(project))) + + organizers = {i: e for i, e in entities.items() if e["@type"] == "OrganizeAction"} + assert set(organizers) == {"#organize-aaa111", "#organize-bbb222"} + for organizer in organizers.values(): + run = entities[organizer["result"]["@id"]] + assert run["@type"] == "CreateAction" + assert run["instrument"]["@id"] == "astra.yaml" + + +def test_an_action_chains_its_inputs_and_its_environment(project: Path) -> None: + _made(project, "baseline", "first", git_sha="aaa111", inputs={"catalog": "sha256:cafe"}) + _made(project, "baseline", "second", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + actions = { + e["name"]: e for e in entities.values() if e["@type"] == "CreateAction" + } + first = actions["run of `first` in universe `baseline`"] + second = actions["run of `second` in universe `baseline`"] + first_objects = {ref["@id"] for ref in first["object"]} + assert {"uv.lock", ".python-version", "pyproject.toml", "data/catalog.csv"} <= first_objects + assert entities["data/catalog.csv"]["sha256"] == "cafe" + assert "results/baseline/first/" in {ref["@id"] for ref in second["object"]} + assert second["result"] == [{"@id": "results/baseline/second/"}] + assert second["description"] == "make second" + assert entities["results/baseline/second/"]["version"] == "sha256:baseline-second" + + +def test_the_manifest_is_in_the_crate_and_about_its_dataset(project: Path) -> None: + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + manifest = entities["results/baseline/first/.lightcone-manifest.json"] + assert manifest["about"] == {"@id": "results/baseline/first/"} + assert entities["results/baseline/first/"]["subjectOf"] == { + "@id": "results/baseline/first/.lightcone-manifest.json" + } + + +def test_a_containerized_output_names_its_committed_archive(project: Path) -> None: + """Identity and payload as one entity: the archive file in the crate, + carrying the config-blob id the execution pinned.""" + _made( + project, + "baseline", + "first", + git_sha="aaa111", + image={ + "tag": "lc-env-x", + "id": "sha256:beef", + "archive": ".datalad/environments/lc-env-x/image", + "arch": "amd64", + }, + ) + entities = _entities(_render(project, _graph(project))) + + archive = entities[".datalad/environments/lc-env-x/image"] + assert set(archive["@type"]) == {"File", "ContainerImage"} + assert archive["sha256"] == "beef" + action = next(e for e in entities.values() if e["@type"] == "CreateAction") + assert action["containerImage"] == {"@id": ".datalad/environments/lc-env-x/image"} + + +def test_the_person_is_the_saving_commits_author(project: Path) -> None: + """The manifest's `git_sha` is the commit the run *started* at — the + author comes from the commit that saved the output, via the writer.""" + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + person = entities["mailto:ada@example.org"] + assert person["name"] == "Ada Lovelace" + action = next(e for e in entities.values() if e["@type"] == "CreateAction") + assert action["agent"] == {"@id": "mailto:ada@example.org"} + assert {"@id": "mailto:ada@example.org"} in entities["./"]["author"] + + +def test_decision_values_point_back_at_their_parameter(project: Path) -> None: + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + value = entities["#value-baseline-first-method"] + assert value["value"] == "alpha" + assert value["exampleOfWork"] == {"@id": "#param-method"} + workflow = entities["astra.yaml"] + assert {"@id": "#param-method"} in _as_list(workflow["input"]) + + +def test_a_never_materialized_project_still_describes_its_workflow(project: Path) -> None: + """No outputs, no runs — the crate is the workflow and the environment, + dated by the spec's own last commit rather than the clock.""" + entities = _entities(_render(project, _graph(project))) + + assert entities["./"]["datePublished"] == "2026-08-19" + assert not [e for e in entities.values() if e["@type"] == "CreateAction"] + assert "astra.yaml" in entities + + +def test_the_license_is_a_local_entity_never_a_minted_url(project: Path) -> None: + """The declaration is not validated against the SPDX list, so a minted + spdx.org URL could be a fabricated dead link — a local CreativeWork + carries the declared string instead, and a URL license is used as + given.""" + entities = _entities(_render(project, _graph(project))) + + assert entities["./"]["license"] == {"@id": "#license"} + assert entities["#license"] == { + "@id": "#license", + "@type": "CreativeWork", + "name": "MIT", + } + + +def test_license_of_reads_every_spelling(tmp_path: Path) -> None: + cases = { + 'license = "MIT"': "MIT", + 'license = { text = "BSD-3-Clause" }': "BSD-3-Clause", + 'license = { file = "LICENSE" }': "LICENSE", + "": "", + } + for spelling, expected in cases.items(): + (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\n{spelling}\n') + assert crate.license_of(tmp_path) == expected, spelling + + +def _as_list(value: Any) -> list[Any]: + return value if isinstance(value, list) else [value] diff --git a/tests/test_crate_smoke.py b/tests/test_crate_smoke.py new file mode 100644 index 00000000..c7bf0847 --- /dev/null +++ b/tests/test_crate_smoke.py @@ -0,0 +1,164 @@ +"""The validator's answer — the one file that can say the crate conforms. + +A real project is materialized through the real driver, and the committed +``ro-crate-metadata.json`` is handed to the official ``rocrate-validator`` +against the deepest profile the crate claims. Gated exactly like the +container smoke suite: hosts without the validator skip, +``LC_CRATE_TESTS_REQUIRED=1`` turns that skip into a hard failure, and +two tests cover the guard itself — an unfailing guard is worse than none. + +At RECOMMENDED there is a known floor, pinned as a *set* rather than a +count: the workflow's id is the file in the crate and cannot be an http +URI, the image lives in the annex and has no registry, and lc knows no +publisher and no author affiliation. A new failure is a regression; a +disappearing one is the floor to shrink. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path + +import pytest + +from lightcone.engine import dataset +from lightcone.engine import materialize as engine + +REQUIRED_ENV = "LC_CRATE_TESTS_REQUIRED" + +_VALIDATOR = importlib.util.find_spec("rocrate_validator") is not None + +#: The checks the crate cannot truthfully satisfy, by check identifier. +_FLOOR = { + # the workflow's id is its path in the crate, and the shape wants http + "process-run-crate-0.5_5.1", + # the image is bytes in the annex; there is no registry to name + "process-run-crate-0.5_13.2", + # lc knows no publishing organization and no author affiliation + "ro-crate-1.1_22.3", + "ro-crate-1.1_29.2", + "ro-crate-1.1_29.3", +} + +_SPEC = """ +version: "0.0.13" +name: analysis +description: A two-step analysis for the crate smoke test. + +inputs: + - id: catalog + type: data + source: data/catalog.fits + +outputs: + - id: first + type: metric + decisions: [method] + recipe: + command: echo {decisions.method} > {output}/value.txt + + - id: second + type: report + inputs: [first] + recipe: + command: cat {inputs.first}/value.txt > {output}/copy.txt + +decisions: + method: + label: Method + default: alpha + options: + alpha: {label: alpha} + beta: {label: beta} +""" + + +def _gate() -> None: + if _VALIDATOR: + return + if os.environ.get(REQUIRED_ENV): + pytest.fail( + f"{REQUIRED_ENV} is set but rocrate-validator is not importable. " + "Crate validation must not be skipped on CI." + ) + pytest.skip("rocrate-validator is not installed here") + + +def test_the_guard_skips_without_the_validator(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys.modules[__name__], "_VALIDATOR", False) + monkeypatch.delenv(REQUIRED_ENV, raising=False) + + with pytest.raises(pytest.skip.Exception): + _gate() + + +def test_the_guard_fails_when_skipping_is_forbidden( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys.modules[__name__], "_VALIDATOR", False) + monkeypatch.setenv(REQUIRED_ENV, "1") + + with pytest.raises(pytest.fail.Exception): + _gate() + + +def test_a_materialized_crate_validates_against_the_provenance_profile( + analysis: Callable[..., Path], inline: None, tmp_path: Path +) -> None: + _gate() + root = analysis( + _SPEC, + files={"data/catalog.fits": "stars\n", "README.md": "# analysis\n\nA demo.\n"}, + universes={"baseline": "id: baseline\ndecisions:\n method: alpha\n"}, + ) + pyproject = root / "pyproject.toml" + pyproject.write_text(pyproject.read_text() + 'license = "MIT"\n') + # A remote gives the tools http ids; the manifests record origin. + dataset._git( + ["remote", "add", "origin", "https://github.com/example/analysis.git"], cwd=root + ) + dataset.save(root, [root], "license and remote") + + report = engine.materialize(root, []) + assert report.ok and (root / "ro-crate-metadata.json").is_file() + assert not dataset.status(root) + + out = tmp_path / "validation.json" + proc = subprocess.run( + [ + sys.executable, + "-c", + # The console script, run on this interpreter — the same one + # the gate probed for importability. + "from rocrate_validator.cli import cli; cli()", + "validate", + "-p", + "provenance-run-crate", + "-l", + "recommended", + "-f", + "json", + "-o", + str(out), + str(root), + ], + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=600, + ) + result = json.loads(out.read_text()) + issues = result.get("issues") or [] + + required = [i for i in issues if i["severity"] == "REQUIRED"] + assert not required, f"REQUIRED failures:\n{json.dumps(required, indent=1)}" + unexpected = [i for i in issues if i["check"]["identifier"] not in _FLOOR] + assert not unexpected, ( + f"failures beyond the recorded floor (validator exit {proc.returncode}):\n" + f"{json.dumps(unexpected, indent=1)}" + ) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 734b4597..9356c057 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -454,3 +454,52 @@ def test_the_annex_executables_are_ours_to_install() -> None: assert ours.get(name) == value, f"{name} is not re-declared as {value}" + + +# ---- who last wrote a path ------------------------------------------------- + + +def test_last_writer_names_the_commit_that_last_touched_a_path(repo: Path) -> None: + """The foreign-write fact's whole mechanism: every output is committed, + so a hand edit needs a commit, and history names it.""" + out = repo / "results" / "fit" + out.mkdir() + (out / "value.txt").write_text("42\n") + dataset.save(repo, [out], "[DATALAD RUNCMD] fit [baseline]\n\nrecord body") + + write = dataset.last_writer(repo, out) + assert write and write.sha and write.date + assert write.subject == "[DATALAD RUNCMD] fit [baseline]" + assert (write.author, write.email) == ("Test", "t@example.com") + + (out / "value.txt").write_text("curated\n") + dataset.save(repo, [out], "tweak colors") + assert dataset.last_writer(repo, out).subject == "tweak colors" + + +def test_last_writer_of_an_untouched_path_is_empty(repo: Path) -> None: + assert not dataset.last_writer(repo, repo / "results" / "never") + + +def test_last_writer_answers_inside_an_enclosing_repository( + tmp_path: Path, real_tools: None +) -> None: + """A project can sit inside a larger repository, and the answer must be + about the project's own subdirectory — a later commit elsewhere in the + tree must not become every output's last writer.""" + outer = tmp_path / "outer" + project_root = outer / "project" + out = project_root / "results" / "fit" + out.mkdir(parents=True) + (out / "value.txt").write_text("42\n") + dataset.init_git(outer) + for key, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key, value], cwd=outer) + dataset._git(["add", "-A", "."], cwd=outer) + dataset._git(["commit", "-q", "-m", "[DATALAD RUNCMD] fit [baseline]"], cwd=outer) + (outer / "unrelated.txt").write_text("someone else's work\n") + dataset._git(["add", "-A", "."], cwd=outer) + dataset._git(["commit", "-q", "-m", "outer edit"], cwd=outer) + + write = dataset.last_writer(project_root, out) + assert write.subject == "[DATALAD RUNCMD] fit [baseline]" diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 9934f07c..13582add 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -21,6 +21,7 @@ from typing import Any import pytest +from conftest import _Inline from lightcone.engine import assets, dataset, identity from lightcone.engine import materialize as engine @@ -66,30 +67,6 @@ def root(analysis: Callable[..., Path]) -> Path: return analysis(_SPEC, universes={"baseline": _UNIVERSE}) -class _Inline: - """Run the graph in this thread, in the order it was submitted. - - Submission is topological, so a dependent is submitted only after its - upstreams have already run — which means the "handles" it is passed - are the upstream results themselves, exactly what the worker expects. - """ - - def submit(self, fn: Any, *args: Any, key: str) -> Any: - return fn(*args) - - def completed(self, handles: list[Any]) -> Iterator[TaskResult]: - yield from handles - - -@pytest.fixture -def inline(monkeypatch: pytest.MonkeyPatch) -> None: - @contextmanager - def fake() -> Iterator[_Inline]: - yield _Inline() - - monkeypatch.setattr(engine, "cluster_for_run", fake) - - def _commits(root: Path) -> int: return len(dataset._git(["log", "--oneline"], cwd=root).splitlines()) @@ -901,3 +878,186 @@ def test_the_report_is_json_ready(root: Path, inline: None) -> None: assert data["ok"] is True assert data["made"] == ["baseline/first"] + + +# ---- the foreign-write fact ------------------------------------------------ + + +def _forge(path: Path, text: str) -> None: + """Overwrite a committed result the way a hand edit would — unlinking + first, because results are committed thin: an in-place truncate would + rewrite the shared annex object and dirty every file hard-linked to it + (the recorded thin-write hazard, demonstrated by this very test suite + when it forged in place).""" + path.unlink() + path.write_text(text) + + +def test_a_materialized_output_has_no_foreign_write(root: Path, inline: None) -> None: + engine.materialize(root, []) + + assert all(not o.foreign_write for o in engine.status(root).outputs) + + +def test_a_foreign_write_is_stale_and_names_its_commit(root: Path, inline: None) -> None: + """The agent-forged-file fact: a hand-edited-and-committed output would + read `current` forever, because a skip returns the recorded digest — + so a directory last written by anything but its own run record is a + *contradiction*, and contradiction is what `stale` means.""" + engine.materialize(root, []) + forged = root / "results" / "baseline" / "first" / "value.txt" + _forge(forged, "curated by hand\n") + dataset.save(root, [forged.parent], "tweak colors") + + outputs = {o.output: o for o in engine.status(root).outputs} + + assert outputs["baseline/first"].status == "stale" + forged_sha = dataset.last_writer(root, root / "results/baseline/first").sha + assert outputs["baseline/first"].foreign_write == forged_sha + assert "tweak colors" in outputs["baseline/first"].why + assert "git show" in outputs["baseline/first"].why + assert not outputs["baseline/second"].foreign_write + + +def test_check_plans_the_remake_of_a_foreign_written_output( + root: Path, inline: None +) -> None: + """Status and `--check` answer from one walk, so they cannot disagree + about a foreign write — and the gate exits nonzero over it.""" + engine.materialize(root, []) + forged = root / "results" / "baseline" / "first" / "value.txt" + _forge(forged, "curated by hand\n") + dataset.save(root, [forged.parent], "tweak colors") + + report = engine.check(root, []) + + assert any("tweak colors" in why for why in report.planned.values()) + assert not report.up_to_date + + +def test_the_foreign_write_fact_survives_a_bytes_free_clone( + root: Path, inline: None, tmp_path: Path +) -> None: + """History-based on purpose: content changes move pointers in git, so + the fact needs no annex content — where a rehash would have nothing to + hash.""" + engine.materialize(root, []) + forged = root / "results" / "baseline" / "first" / "value.txt" + _forge(forged, "curated by hand\n") + dataset.save(root, [forged.parent], "tweak colors") + clone = _clone(root, tmp_path) + + outputs = {o.output: o for o in engine.status(clone).outputs} + + assert outputs["baseline/first"].status == "stale" + assert "tweak colors" in outputs["baseline/first"].why + assert outputs["baseline/first"].foreign_write + assert not outputs["baseline/second"].foreign_write + + +def test_the_next_run_remakes_a_foreign_written_output(root: Path, inline: None) -> None: + """`results/` is lc's to write — the same philosophy as the dirty-tree + refusal's path split — so a committed hand edit is remade, and the + rebuild's own run record becomes the last writer again.""" + engine.materialize(root, []) + forged = root / "results" / "baseline" / "first" / "value.txt" + _forge(forged, "curated by hand\n") + dataset.save(root, [forged.parent], "tweak colors") + + report = engine.materialize(root, []) + + assert "baseline/first" in report.made + assert forged.read_text() == "alpha\n" # the recipe's bytes, not the hand's + assert all(not o.foreign_write for o in engine.status(root).outputs) + + +# ---- the publication view -------------------------------------------------- + + +def _declare_license(root: Path) -> None: + """Declare publication intent the way a researcher would: one key, + committed like any other edit.""" + pyproject = root / "pyproject.toml" + pyproject.write_text(pyproject.read_text() + 'license = "MIT"\n') + dataset.save(root, [pyproject], "declare a license") + + +def test_an_unlicensed_project_gets_no_crate_and_one_report_line( + root: Path, inline: None +) -> None: + report = engine.materialize(root, []) + + assert not (root / "ro-crate-metadata.json").exists() + assert any("[project].license" in w for w in report.warnings) + + +def test_a_licensed_materialize_converges_the_crate_and_commits_it( + root: Path, inline: None +) -> None: + _declare_license(root) + + engine.materialize(root, []) + + crate_path = root / "ro-crate-metadata.json" + assert crate_path.is_file() + assert not dataset.status(root) # committed, tree exactly as clean as before + assert dataset.last_writer(root, crate_path).subject == "Update the RO-Crate publication view" + graph = json.loads(crate_path.read_text())["@graph"] + types = {e["@id"]: e["@type"] for e in graph} + assert "OrganizeAction" in types.values() + assert types["results/baseline/first/"] == "Dataset" + + +def test_an_idempotent_rerun_commits_nothing(root: Path, inline: None) -> None: + """The document is a pure function of repository state — a re-render at + the same state is a string compare, not a commit.""" + _declare_license(root) + engine.materialize(root, []) + before = _commits(root) + + engine.materialize(root, []) + + assert _commits(root) == before + + +def test_declaring_a_license_later_creates_the_crate_then(root: Path, inline: None) -> None: + engine.materialize(root, []) + assert not (root / "ro-crate-metadata.json").exists() + + _declare_license(root) + engine.materialize(root, []) + + assert (root / "ro-crate-metadata.json").is_file() + + +def test_a_removed_license_stops_maintenance_but_keeps_the_file( + root: Path, inline: None +) -> None: + """The crate is in committed history either way; deleting a file over a + possibly temporary edit is not convergence's call.""" + _declare_license(root) + engine.materialize(root, []) + pyproject = root / "pyproject.toml" + pyproject.write_text(pyproject.read_text().replace('license = "MIT"\n', "")) + dataset.save(root, [pyproject], "drop the license") + + report = engine.materialize(root, []) + + assert (root / "ro-crate-metadata.json").is_file() + assert any("no longer maintained" in w for w in report.warnings) + + +def test_an_output_the_spec_dropped_is_excluded_and_named(root: Path, inline: None) -> None: + _declare_license(root) + engine.materialize(root, []) + spec_path = root / "astra.yaml" + spec = spec_path.read_text() + block = spec[spec.index(" - id: second") : spec.index("\ndecisions:") + 1] + spec_path.write_text(spec.replace(block, "")) + dataset.save(root, [spec_path], "drop the second output") + + report = engine.materialize(root, []) + + assert any("results/baseline/second" in w for w in report.warnings) + document = (root / "ro-crate-metadata.json").read_text() + assert "results/baseline/second/" not in document diff --git a/tests/test_worker.py b/tests/test_worker.py index de53cbfd..014b882f 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -79,6 +79,7 @@ def _make( _HEAD, assets.Versions(), refresh, + None, _runtime(root), *upstream, ) From 8b4e03817aad35313539aa5ffd9ce701c0fbaab3 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 21:30:57 +0200 Subject: [PATCH 15/23] Reactivate the agentic eval against the rebuilt CLI (#183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval workflow and its prompt were written for the pre-rebuild CLI — the Snakemake shim, `lc run --universe`, `ok`/`stale`/`missing` states, the Containerfile + `requirements.txt` container model. With layer 8 landed, this brings the eval back in line with what `lc` actually is. ## `evals/prompt.md` (rewritten) - `lc materialize` is the build verb; `lc run ` is the probe. States are `current`/`behind`/`stale`, `lc status` always exits 0, and `lc materialize --check` is the gate. - Recipes import from the project's own locked environment — packages arrive via `uv add` in the project root (what the denial messages recommend); `uv pip` reaches nothing a recipe sees. - A run refuses a dirty tree, so the agent is told to commit its edits with plain `git add`/`git commit` (the annex filter is transparent — nobody runs a git-annex command). - `results/` is the engine's to write; a hand-placed file is now detected as a foreign write and remade — the old "fails verification" warning got teeth. - The seed spec is stated to be **deliberately incomplete**: the recipe commands carry no placeholders and the dependency contracts are partial, so completing the spec (`{output}`, `{inputs.}`, `{decisions.}`, and the matching `inputs:`/`decisions:` lists) is part of the task. This makes the existing seed's shape an intentional test of spec literacy rather than a leftover. - The recipe template grammar section is kept — verified it still matches astra's own grammar (`astra guide`, astra-tools 0.2.16). ## `.github/workflows/eval.yml` - **Check step**: the old jq expression (`.universes[].outputs[]`, `"ok"`, `"alias"`) matches nothing any verb emits now. Replaced with the designed gate — `lc materialize --check --json | jq -e '.up_to_date and ((.current | length) > 0)'` — which exits nonzero while anything is planned or failed, plus the non-emptiness guard the old check had. `behind` deliberately does not fail the gate. - **Setup step**: no scientific deps are pre-seeded — the project starts empty and declaring what the recipes import (`uv add`, spelled out in the prompt) is part of what the eval measures; the repo-venv `uv pip install` is dropped (it never reached recipes). Setup commits the seed — `lc init` never commits, and the first `lc materialize` would otherwise be refused on a dirty tree before the agent did anything. - **Trigger**: the dormant PR plumbing (comment step, concurrency group) gets its trigger back — `pull_request: types: [labeled]` gated on a `run-eval` label, alongside `workflow_dispatch`. - **Artifacts**: `.snakemake/**` exclusion → `.lightcone/**`; keeping `.git` via `include-hidden-files` is now a feature — the run records and annexed results are the provenance. - **Install**: lc and astra are installed as uv tools — `uv tool install` of the checkout (the commit under test, stamped into the version by hatch-vcs) links `lc` and the four git-annex executables through the mirrored entry points, exercising the exact mechanism a user install relies on; astra-tools is its own tool install, pinned to the branch dependency. No venv is activated, so the runner needs no system git-annex and `VIRTUAL_ENV` is never set. ## Seed + trace prompt - `evals/tasks/snae/astra.yaml` drops `container: Containerfile` — the directive is ignored by recorded decision, and no Containerfile exists to point at. - `trace-analysis-prompt.md` names the sandbox instead of the container environment. ## Verified locally - The seed validates against astra-tools 0.2.16 after the edit. - The full setup + check plumbing runs end-to-end against today's `lc`: `lc init` → overlay → `astra universe generate -n baseline` → `uv add` → seed commit → `lc status` (three outputs, direct mode, landlock) → the `--check` gate correctly failing with all three outputs planned on the unbuilt project. The agent step itself (Claude Code + astra plugin) is unchanged and needs a real dispatch run to shake out — suggest a `workflow_dispatch` on this branch once merged, or label this PR `run-eval`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx --------- Co-authored-by: Claude Fable 5 --- .github/workflows/eval.yml | 53 +++++++++++------ evals/prompt.md | 105 ++++++++++++++++++++++----------- evals/tasks/snae/astra.yaml | 2 - evals/trace-analysis-prompt.md | 2 +- 4 files changed, 107 insertions(+), 55 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 9594e56e..5b24e23e 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -3,10 +3,12 @@ name: Eval # Minimal agentic eval: scaffold a project from the eval seed on the # runner, let Claude Code (with the astra plugin) build it using the # branch's lc/astra, then check the results with astra validate and -# lc status. +# lc materialize --check. on: workflow_dispatch: + pull_request: + types: [labeled] # Only one eval per PR at a time — cancel in-progress runs concurrency: @@ -19,7 +21,7 @@ permissions: jobs: eval: - if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-eval' runs-on: ubuntu-latest # Step-level timeout on the agent step (below) is what actually # bounds the run: a job-level timeout would cancel the always() @@ -33,14 +35,21 @@ jobs: with: python-version: "3.12" enable-cache: true - activate-environment: true - - name: Install lightcone-cli (branch under test) + - name: Install lightcone-cli + astra (uv tools) + # The engine is the host's uv tool — install the branch under + # test the way a user installs a release: `uv tool install`, + # which links `lc` and (via the mirrored entry points) the + # git-annex executables into ~/.local/bin. The commit under test + # is the checkout itself, stamped into the version by hatch-vcs. + # astra's executable belongs to astra-tools, so it is its own + # tool install, pinned to the version the branch depends on. No + # venv is activated: the agent's shell sees the tools exactly as + # an end user's would. run: | - uv sync - # Scientific deps the task's analysis scripts will need — the - # agent can add more with `uv pip install` - uv pip install numpy scipy matplotlib + uv tool install "$GITHUB_WORKSPACE" + uv tool install "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Install Claude Code + astra plugin run: | @@ -69,6 +78,12 @@ jobs: rm -f universes/baseline.yaml astra universe generate -n baseline \ -d 'Default configuration using standard practices' + # No scientific deps are pre-seeded — declaring what the + # recipes import (`uv add`) is part of what the eval measures + # lc materialize refuses a dirty tree — commit the seed so the + # agent starts from a clean one + git add -A + git commit -q -m 'Seed the eval task' - name: Run Claude Code id: agent @@ -91,11 +106,14 @@ jobs: working-directory: ${{ env.PROJECT }} run: | astra validate astra.yaml - lc status --universe baseline - # Fail unless every materializable output is ok - lc status --json | jq -e \ - '[.universes[].outputs[] | select(.status != "alias")] - | length > 0 and all(.status == "ok")' + lc status + # The gate: nothing failed, nothing left to make — and at least + # one output actually got materialized + lc materialize --check --json | jq -e \ + '.up_to_date and ((.current | length) > 0)' + # The publication view: declaring a license and materializing + # once more is part of the task + test -f ro-crate-metadata.json - name: Build report if: always() @@ -135,7 +153,7 @@ jobs: echo '
lc status' echo "" echo '```' - (cd "$PROJECT" && lc status --universe baseline) || echo "unavailable" + (cd "$PROJECT" && lc status) || echo "unavailable" echo '```' echo "" echo '
' @@ -198,11 +216,12 @@ jobs: uses: actions/upload-artifact@v4 with: name: eval-project - # include-hidden-files keeps the .lightcone-manifest.json - # provenance files; the venv and snakemake scratch are excluded + # include-hidden-files keeps the manifests, the run records in + # .git, and the annexed results; the venv and lc's per-run + # machine state are excluded path: | ${{ env.PROJECT }} !${{ env.PROJECT }}/.venv/** - !${{ env.PROJECT }}/.snakemake/** + !${{ env.PROJECT }}/.lightcone/** include-hidden-files: true if-no-files-found: warn diff --git a/evals/prompt.md b/evals/prompt.md index 15a7a9db..c0f948f4 100644 --- a/evals/prompt.md +++ b/evals/prompt.md @@ -8,19 +8,32 @@ This project is driven by two CLIs — use them rather than improvising: `astra validate astra.yaml` checks it against the schema. If an `astra` skill or plugin is available in your environment, load it before reading or editing `astra.yaml` — it documents the full spec format. -- `lc` (lightcone-cli) is the execution layer, a thin shim over Snakemake: - - `lc run --universe baseline` materializes an output (and - anything upstream of it) by running the recipe commands declared in - `astra.yaml`. With no output ids it builds everything. It is - idempotent: re-running only rebuilds what is stale or missing. - - `lc status --universe baseline` reports each output as `ok`, `stale`, - or `missing`; `lc status --json` is the machine-readable form. +- `lc` (lightcone-cli) is the execution layer: + - `lc materialize` makes every output the spec declares, running each + recipe in dependency order and committing each result to git as it + lands, together with a provenance manifest. It refuses to start on + a dirty tree: commit your own edits first, with plain `git add` and + `git commit` — the project's git-annex filter handles large files + transparently, so never run a git-annex command yourself. + - `lc materialize ` (or `/`) narrows + a run to one output and whatever it depends on. Re-running is + idempotent: only what is stale gets remade — an output the spec now + defines differently, or one whose declared inputs changed. + - `lc status` reports each output as `current`, `stale`, or `behind`, + with the commit it was made at; `lc status --json` is the + machine-readable form. It always exits 0. The pass/fail gate is + `lc materialize --check`, which exits 1 while anything still needs + making. + - `lc run ` runs an ad-hoc command in the project + environment under the same isolation a recipe gets — useful for + probing why a recipe would fail. - Outputs land in `results/baseline//`, each with a - `.lightcone-manifest.json` provenance manifest written by the engine. - Files placed in `results/` by hand have no manifest and fail - verification — never write there yourself. - - When `lc run` fails, read the error and the Snakemake log it points - to, fix the script or spec, and re-run. + `.lightcone-manifest.json` manifest written and committed by the + engine. Never write into `results/` yourself: a hand-placed file + has no run record, and the engine detects the foreign write and + remakes the output. + - When a recipe fails, `lc materialize` reports which output failed + and why; fix the script or the spec, commit, and re-run. ## Recipe template grammar @@ -49,37 +62,59 @@ is how the engine orders the build. ## Environment -Recipes and your interactive shell run in two different environments — -keep them straight: - -- **Recipe commands run by `lc run`** may execute inside a container - built from the project's `Containerfile` + `requirements.txt` - (whenever `astra.yaml` declares a `container:` and a runtime is - available). Every package a recipe script imports must therefore be - listed in `requirements.txt` — add it there *before* running, and the - engine rebuilds the content-addressed image automatically. Host-side - installs never reach the container. -- **Your own shell commands** run on the host in an activated uv-managed - virtual environment with numpy, scipy, and matplotlib pre-installed. - For ad-hoc host tools use `uv pip install ` — plain `pip` is - not available in this venv. +Recipes run in the project's own locked environment (`pyproject.toml` + +`uv.lock` + `.venv`), sandboxed: the project tree is read-only apart from +each recipe's own output directory under `results/`, and only declared +tools are executable. + +- The project is managed by uv and starts with **no dependencies**. + Every package a recipe script imports must be declared before + materializing: run `uv add [ ...]` in the project + root (e.g. `uv add numpy scipy`). That updates `pyproject.toml`, + re-locks `uv.lock`, and syncs `.venv` in one step — commit all of it + along with your scripts, like any other edit. +- To remove a package use `uv remove `; to pin a version, + `uv add 'numpy>=2'`. Do **not** use plain `pip` or `uv pip install` — + an install that bypasses the lock reaches nothing a recipe sees. +- A sandbox denial names the path or tool that was denied and the + remedy — follow the remedy rather than working around the sandbox. ## Build loop `astra.yaml` is the single source of truth: inputs, outputs, recipes, and -methodological decisions all live there — read it first. For each output -that needs materializing: +methodological decisions all live there — read it first. The seed spec is +deliberately incomplete: recipe commands do not yet pass their inputs, +decisions, or output directory, and outputs may be missing entries in +their `inputs:` / `decisions:` contracts. Completing the spec is part of +the task. For each output: -1. Read the recipe's `command` to see what script and arguments it expects. +1. Complete the recipe `command` so it references `{output}` and the + `{inputs.}` / `{decisions.}` the computation needs, and + declare everything it references in that output's `inputs:` / + `decisions:` lists. 2. Write the script at the path the command names, parameterizing every decision via argparse — never hardcode option values. -3. Run `lc run --universe baseline` to materialize it through - the engine. -4. Commit progress as you go. +3. Commit your edits, then run `lc materialize` (or + `lc materialize `) to build through the engine. + +Build iteratively from upstream outputs to downstream. `lc status` shows +where every output stands. + +## Publication + +Once every output is materialized, prepare the repository for +publication: + +1. Declare a license in `pyproject.toml`, as an SPDX expression under + `[project]` — e.g. `license = "CC-BY-4.0"`. Declaring one is what + turns publication on: from then on `lc materialize` also maintains + `ro-crate-metadata.json` at the project root, an RO-Crate view of + the project and its provenance. +2. Commit the edit, then run `lc materialize` once more — nothing is + remade, but the crate document is generated and committed. -Build iteratively from upstream outputs to downstream. `lc status ---universe baseline` shows you what's `ok`, `stale`, or `missing` — you're -done when every output shows `ok` and `astra validate astra.yaml` passes. +You're done when `astra validate astra.yaml` and +`lc materialize --check` pass and `ro-crate-metadata.json` exists. Skip plan approval and interactive confirmations — this is an automated eval run. diff --git a/evals/tasks/snae/astra.yaml b/evals/tasks/snae/astra.yaml index 3bdf0fb5..41b9d647 100644 --- a/evals/tasks/snae/astra.yaml +++ b/evals/tasks/snae/astra.yaml @@ -9,8 +9,6 @@ description: | using maximum-likelihood (MAP) point estimation. This provides best-fit cosmological parameters as a building block for a larger analysis. -container: Containerfile - inputs: - id: union21 type: data diff --git a/evals/trace-analysis-prompt.md b/evals/trace-analysis-prompt.md index 1058b5f9..120f4eb0 100644 --- a/evals/trace-analysis-prompt.md +++ b/evals/trace-analysis-prompt.md @@ -8,7 +8,7 @@ Write a section titled exactly `### Confusion & pain points` containing - failed commands or errored tool results, and their root cause - moments where the agent misunderstood the spec format, the CLI - surface, or the execution/container environment + surface, the sandbox, or the execution environment - detours: reverse-engineering source code, probing the environment, redoing work, or fixing things the harness should have provided - workarounds the agent invented that hint at a product gap (these are From bd68cfff8f070582b7152e1ac6866c1fe584c3ed Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:17:49 +0200 Subject: [PATCH 16/23] Docs rewrite 1/5: README, landing page, install, getting started (#185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First pass of the post-rebuild documentation rewrite — the entry points: `README.md`, `docs/index.md`, `docs/user/{index,install,getting-started}.md`. Style deliberately follows the previous versions (tabs, admonitions, numbered steps, the "what just happened" recap); the content is brought to the rebuilt CLI. ## What changed - **README**: quick start shows the real loop (`init` → spec + scripts → `uv add` → commit → `materialize`); the Capabilities section replaces `lc verify` / `lc export wrroc` / Snakemake with what exists — multiverse materialization, provenance by construction (manifests + run records, annex-carried bytes), locked sandboxed execution, derived containerized mode + SLURM, and the license-derived RO-Crate view. - **docs/index.md**: light touch — drops "skills" from the alpha notice, updates the lightcone-cli card's description. - **user/index.md**: quick start updated (commit before materialize, with a sentence on why the commit isn't ceremony); covered-pages list wording refreshed. - **user/install.md**: rewritten. Prerequisites are uv + git; git-annex ships as a wheel with the install (with the platform floor stated honestly, and WSL for Windows); a new "tell git who you are" step (materialize checks identity up front); container runtime section reflects detection (podman-hpc → podman → docker) and the absence of any config file — `~/.lightcone/config.yaml` no longer exists. Upgrade section notes an engine upgrade never invalidates results. - **user/getting-started.md**: new worked example — a line fit over a synthetic dataset with one methodological decision (keep vs clip outliers), swept across `baseline` and `robust` universes, ending with the license declaration and the auto-committed `ro-crate-metadata.json`. It walks through the scaffold, declared inputs and the annex, the dependency contract and placeholders, `uv add`, the clean-tree rule, real `lc materialize` / `lc status` output, incremental rebuilds, and the fresh-clone story. ## Verification - Every command block in the walkthrough was executed against the current `lc` before landing; the quoted console output is captured from that run (slope numbers included). - The fresh-clone claim was tested: a bare clone refuses, `lc init` + `lc materialize` reports up to date with no annex content fetched — the guide says exactly that. - `zensical build` passes; the two remaining link warnings are in frozen pages (`api/snakefile.md`, `hpc/targets.md`) that later passes replace. Next passes: (2) core concepts + cluster/troubleshooting/glossary, (3) CLI + JSON reference, (4) developer corner, (5) nav + workflow refresh. Two cosmetic engine warts observed while verifying, deliberately not touched here: Rich markup swallows the literal `[project]` in the license hint (renders as "no .license in pyproject.toml"), and a stray Dask teardown `TimeoutError` line can appear on stderr after a run. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx --------- Co-authored-by: Claude Fable 5 --- README.md | 34 ++- docs/index.md | 4 +- docs/user/getting-started.md | 401 +++++++++++++++++++++------------- docs/user/index.md | 34 +-- docs/user/install.md | 177 ++++----------- src/lightcone/cli/commands.py | 48 ++-- tests/test_cli.py | 18 ++ 7 files changed, 392 insertions(+), 324 deletions(-) diff --git a/README.md b/README.md index faaf15dc..6483fcf7 100644 --- a/README.md +++ b/README.md @@ -5,22 +5,33 @@ [![PyPI](https://img.shields.io/pypi/v/lightcone-cli?style=flat&color=f8f7f3)](https://pypi.org/project/lightcone-cli/) [![Tests](https://img.shields.io/github/actions/workflow/status/LightconeResearch/lightcone-cli/tests.yml?style=flat&color=darkgreen)](https://github.com/LightconeResearch/lightcone-cli/actions/workflows/tests.yml) - - **lightcone-cli** (`lc`) is the execution layer for [ASTRA](https://astra-spec.org/latest/) (Agentic Schema for Transparent Research Analysis). Describe your analysis in an `astra.yaml` -specification and `lc` takes care of the rest — execution and -provenance. +specification and `lc` takes care of the rest — execution, environments, +and provenance. ## Quick Start +**lightcone-cli** only requires you to have `uv` installed on your environment, and will take care of everything else. See how to install uv here: [https://docs.astral.sh/uv/getting-started/installation](https://docs.astral.sh/uv/getting-started/installation). + +Then to install **lightcone-cli**: +```bash +uv tool install lightcone-cli +``` + +Once the CLI is installed, you can use it to create an ASTRA project and generate outputs like so: + ```bash -uv tool install lightcone-cli # or: pip install lightcone-cli lc init my-analysis cd my-analysis -# describe your analysis in astra.yaml, then: -lc run +# describe your analysis in astra.yaml, write your scripts, +# declare what they import through normal uv interactions: +uv add numpy +# When you are done with your edits, commit: +git add -A && git commit -m "First analysis" +# Use the lightcone CLI to generate your outputs with full provenance tracking +lc materialize ``` ASTRA specs are plain, structured YAML — they work well hand-written or @@ -30,10 +41,11 @@ drafted with any AI coding assistant. ## Capabilities -- **Multiverse analysis** — define methodological decisions with multiple options; `lc` runs your analysis across all defensible paths automatically -- **Provenance integrity** — every output gets a content-addressed manifest; `lc verify` detects tampering or broken chains -- **HPC-ready execution** — Snakemake-backed DAG dispatch with SLURM and container support (Docker, Podman, Apptainer) out of the box -- **Reproducible publishing** — `lc export wrroc` emits a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) bundle ready for Zenodo or WorkflowHub +- **Multiverse analysis** — declare methodological decisions with multiple defensible options; `lc` materializes your analysis across every universe you define +- **Provenance by construction** — every output is committed to git together with a content-addressed manifest and a re-runnable run record; git-annex carries the bytes, so results travel with the repository +- **Locked, isolated execution** — a project's environment is `pyproject.toml` + `uv.lock`; recipes run in it under a sandbox (Landlock on Linux, Seatbelt on macOS) that keeps undeclared files out and stray writes contained +- **Containers and HPC** — declare `[tool.lightcone.image]` and recipes run in a content-addressed image archived in the repository itself; a SLURM allocation is detected and used automatically, every node included +- **Publication view** — declare a license and `lc materialize` maintains an [RO-Crate](https://www.researchobject.org/ro-crate/) of the project and its provenance, ready to archive or deposit → [Full documentation](https://docs.lightconeresearch.org) diff --git a/docs/index.md b/docs/index.md index 725ba7e0..31464072 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ It serves as the machinery that ties an analysis `astra.yaml` specification to a of materialized outputs. !!! warning "Alpha development" - lightcone-cli is in **early alpha**. The CLI, skills, and execution layer are all + lightcone-cli is in **early alpha**. The CLI and the execution layer are still moving — expect breaking changes between minor versions. Bug reports, design challenges, and use cases the tooling doesn't yet cover are exactly what we want to hear at this stage; please open an issue on the @@ -43,7 +43,7 @@ of materialized outputs. - __lightcone-cli__ - The library that ships the `lc` CLI: the workflow execution layer and the provenance integrity layer. Depends on [**astra-tools**][astra-tools], the SDK for working with ASTRA analysis specifications. + The library that ships the `lc` CLI: project scaffolding, locked environments, sandboxed execution, and the provenance layer. Depends on [**astra-tools**][astra-tools], the SDK for working with ASTRA analysis specifications. [:fontawesome-brands-github: Repository][cli]{ .md-button } diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index d16c6d79..3a61d1fd 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -4,19 +4,19 @@ Let's go from nothing on your disk to a working, reproducible analysis. You can read this top to bottom without running anything, or follow along — every command is copy-paste ready. -**What you'll build:** a small two-output analysis that fits a linear model on -a public dataset and sweeps one methodological decision (whether to standardize -features). The result is two universes, `baseline` and `raw`, each with its -own `r2` metric and `fit_plot` figure — a clean comparison ready for a paper -figure. +**What you'll build:** a small two-output analysis that fits a line to a +noisy dataset and sweeps one methodological decision — whether points far +from an initial fit are kept or clipped. The result is two universes, +`baseline` and `robust`, each with its own fitted slope and figure, and a +project that ends published as an [RO-Crate](https://www.researchobject.org/ro-crate/). Make sure you've finished the [install](install.md) first. ## 1. Create a project ```bash -lc init r2-decision-demo -cd r2-decision-demo +lc init line-fit-demo +cd line-fit-demo ``` `lc init` converges the directory to a small, opinionated layout and @@ -24,97 +24,151 @@ stops; it doesn't ask any questions, and it's idempotent — re-running it later only fills in whatever is missing. ``` -r2-decision-demo/ +line-fit-demo/ ├── astra.yaml # the spec — this is where everything lives +├── pyproject.toml # the project's environment: its dependencies… +├── .python-version # …and the exact interpreter, locked by uv +├── uv.lock +├── .venv/ # built from the lock (local, never committed) +├── .git/ # a git repository, with git-annex initialized +├── .gitattributes # the storage policy: what the annex carries ├── .gitignore -├── .git # initialized git repository (skip with --no-git) -├── .venv/ # Python virtual env with the analysis dependencies (skip with --no-venv) -├── .lightcone/ # internal scratchpad — don't edit by hand -├── Containerfile # build instructions for the project container -├── requirements.txt # software dependencies -├── myst.yml # MyST report configuration -├── index.md # template report that references the spec +├── .datalad/ # dataset identity — the project is a DataLad dataset +├── data/ # declared input data lives here +├── results/ # outputs materialize here — lc's to write, not yours ├── universes/ │ └── baseline.yaml # one universe, built from decision defaults -└── results/ - └── README.md # outputs materialize here via `lc run` +├── myst.yml # MyST report configuration +└── index.md # template report that references the spec ``` -The file you'll actually work in: +Two things are worth registering now: -**`astra.yaml`** — the single source of truth for your analysis. Inputs, -outputs, methodological decisions, recipes. Everything else lightcone-cli does -is downstream of this file. The boilerplate from `lc init` has one example -output and an example decision — enough to run `lc run` and see something -materialize, but not yet a real analysis. +- **The project is a git repository.** Every + output `lc` makes is committed together with the code that produced + it; large files ride in git-annex behind the scenes, but you only + ever type ordinary `git add` and `git commit`. +- **The environment is the lock.** `pyproject.toml` + `uv.lock` define + exactly what your recipes can import, and `.venv` is built from them. + You'll add packages with `uv add` in a moment — never `pip install`. -ASTRA specs are plain YAML, designed to be easy for both humans and AI -assistants to write. In this guide you'll write one by hand — it's short. +The file you'll actually work in is **`astra.yaml`** — the single source +of truth for your analysis. Inputs, outputs, methodological decisions, +recipes: everything else lightcone-cli does is downstream of this file. -## 2. Write the spec +## 2. Add the data -Open `astra.yaml` and replace the boilerplate with our analysis: a linear -regression on sklearn's bundled diabetes dataset, with one decision — whether -to standardize features before fitting. +A real project starts from a dataset; ours will generate a small one — +200 points on a line, with a few outliers thrown far off it: -```yaml -version: "0.0.13" # ASTRA spec version — keep what the scaffold wrote -name: "R² with and without feature standardization" -description: "Linear regression on the diabetes dataset, sweeping the standardization choice." -container: Containerfile +```bash +python3 - <<'EOF' +import random +random.seed(0) +rows = ["x,y"] +for _ in range(200): + x = random.uniform(0, 10) + y = 2.5 * x + 1.0 + random.gauss(0, 1.5) + if random.random() < 0.04: + y += random.gauss(0, 15) + rows.append(f"{x:.6f},{y:.6f}") +open("data/points.csv", "w").write("\n".join(rows) + "\n") +EOF +``` -inputs: [] # the diabetes dataset ships with scikit-learn +`data/` is where declared inputs live. When you commit, the +`.gitattributes` policy routes the file's bytes into git-annex +automatically — the file stays an ordinary readable, writable file in +your tree, and the repository stays light. -decisions: - standardize: - label: "Feature standardization" - rationale: "Standardizing changes coefficient scales and can shift R² for ridge-like models." - default: standardized - options: - standardized: { label: "StandardScaler before fit" } - raw: { label: "No preprocessing" } +## 3. Write the spec + +Open `astra.yaml` and replace the boilerplate with our analysis: + +```yaml +version: "0.0.13" # ASTRA schema version — keep what the scaffold wrote +name: "line_fit" +description: | + Fit a straight line to a small synthetic dataset and sweep one + methodological decision: whether points far from an initial fit are + kept or clipped before the final fit. + +inputs: + - id: points + type: data + source: data/points.csv + description: "200 synthetic (x, y) points, a few of them far off the line" outputs: - - id: r2 + - id: fit type: metric - description: "Coefficient of determination on the test split." - decisions: [standardize] + description: "Slope and intercept of the least-squares line" + inputs: [points] + decisions: [outliers] recipe: - command: python src/fit.py --standardize {decisions.standardize} --output {output} + command: python src/fit.py --points {inputs.points} --outliers {decisions.outliers} --output {output} + - id: fit_plot type: figure - description: "Predicted vs true scatter." - inputs: [r2] + description: "The points and the fitted line" + inputs: [points, fit] recipe: - command: python src/plot.py --r2_dir {inputs.r2} --output {output} + command: python src/plot.py --points {inputs.points} --fit {inputs.fit} --output {output} + +decisions: + outliers: + label: "Outlier handling" + rationale: "A few points sit far off the line; keeping or clipping them shifts the slope." + default: keep + options: + keep: + label: "Keep every point" + clip: + label: "Drop points beyond 3 sigma of an initial fit" ``` A few things to notice: -- Each output declares what it depends on: `r2` depends on the - `standardize` decision, `fit_plot` depends on the sibling output `r2`. +- Each output declares its full dependency contract: `fit` depends on + the `points` input and the `outliers` decision; `fit_plot` depends on + `points` and on the sibling output `fit`. That contract is how `lc` + orders the build — and how it knows what to rebuild when something + changes. - Recipes reference those dependencies through placeholders — - `{decisions.standardize}`, `{inputs.r2}`, `{output}` — which `lc run` - expands at execution time. `{output}` is the output's own results - directory. + `{inputs.points}`, `{decisions.outliers}`, `{output}` — which are + expanded at execution time. `{output}` is the output's own results + directory, `results///`; the engine creates it + before the recipe runs. - The decision's options aren't hardcoded anywhere in code; the scripts will take them as command-line arguments. +`universes/baseline.yaml` was scaffolded against the boilerplate spec, +so point it at our decision instead: + +```yaml +id: baseline +description: "Every point kept — the decision defaults." +decisions: + outliers: keep +``` + +Each universe is one complete selection of decision values; its results +materialize to `results///`. + Check the spec is well-formed: ```bash astra validate astra.yaml ``` -(`astra` is the spec-side CLI; it ships with `astra-tools`, a dependency of -lightcone-cli.) +(`astra` is the spec-side CLI; it ships with `astra-tools`, a dependency +of lightcone-cli.) -## 3. Write the scripts +## 4. Write the scripts Two short scripts, in a `src/` directory (`mkdir src` — the scaffold doesn't create it; where code lives is your choice, the recipes above -just happen to point there). First `src/fit.py` — fits the model, -writes the R² metric and the test-set predictions: +just happen to point there). First `src/fit.py`: ```python import argparse @@ -122,33 +176,29 @@ import json from pathlib import Path import numpy as np -from sklearn.datasets import load_diabetes -from sklearn.linear_model import LinearRegression -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler parser = argparse.ArgumentParser() -parser.add_argument("--standardize", choices=["standardized", "raw"], required=True) +parser.add_argument("--points", required=True) +parser.add_argument("--outliers", choices=["keep", "clip"], required=True) parser.add_argument("--output", required=True) args = parser.parse_args() -X, y = load_diabetes(return_X_y=True) -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.25, random_state=0 -) -if args.standardize == "standardized": - scaler = StandardScaler().fit(X_train) - X_train, X_test = scaler.transform(X_train), scaler.transform(X_test) - -model = LinearRegression().fit(X_train, y_train) +x, y = np.loadtxt(args.points, delimiter=",", skiprows=1, unpack=True) +if args.outliers == "clip": + slope, intercept = np.polyfit(x, y, 1) + residuals = y - (slope * x + intercept) + mask = np.abs(residuals) < 3 * residuals.std() + x, y = x[mask], y[mask] +slope, intercept = np.polyfit(x, y, 1) out = Path(args.output) -out.mkdir(parents=True, exist_ok=True) -(out / "r2.json").write_text(json.dumps({"r2": model.score(X_test, y_test)})) -np.savez(out / "predictions.npz", y_true=y_test, y_pred=model.predict(X_test)) +(out / "fit.json").write_text( + json.dumps({"slope": slope, "intercept": intercept, "n_used": len(x)}, indent=2) +) ``` -Then `src/plot.py` — reads the upstream output directory, makes the figure: +Then `src/plot.py` — reads the upstream output's directory, makes the +figure: ```python import argparse @@ -162,131 +212,174 @@ import matplotlib.pyplot as plt import numpy as np parser = argparse.ArgumentParser() -parser.add_argument("--r2_dir", required=True) +parser.add_argument("--points", required=True) +parser.add_argument("--fit", required=True) parser.add_argument("--output", required=True) args = parser.parse_args() -r2_dir = Path(args.r2_dir) -r2 = json.loads((r2_dir / "r2.json").read_text())["r2"] -data = np.load(r2_dir / "predictions.npz") +x, y = np.loadtxt(args.points, delimiter=",", skiprows=1, unpack=True) +fit = json.loads((Path(args.fit) / "fit.json").read_text()) fig, ax = plt.subplots() -ax.scatter(data["y_true"], data["y_pred"], s=12) -ax.set_xlabel("true") -ax.set_ylabel("predicted") -ax.set_title(f"R² = {r2:.3f}") - -out = Path(args.output) -out.mkdir(parents=True, exist_ok=True) -fig.savefig(out / "fit_plot.png", dpi=150) +ax.scatter(x, y, s=12) +xs = np.linspace(x.min(), x.max(), 2) +ax.plot(xs, fit["slope"] * xs + fit["intercept"], color="C1") +ax.set_xlabel("x") +ax.set_ylabel("y") +ax.set_title(f"slope = {fit['slope']:.3f}") +fig.savefig(Path(args.output) / "fit_plot.png", dpi=150) ``` -Finally, add the dependencies to `requirements.txt`: +Both scripts import from the project's locked environment, so declare +what they need: -```text -scikit-learn -matplotlib +```bash +uv add numpy matplotlib ``` -The Containerfile installs `requirements.txt` into the project image, so -that's all it takes — `lc run` rebuilds the image automatically when the -dependency files change. (If you're running without a container runtime, -install the same packages into `.venv` instead.) +That one command updates `pyproject.toml`, re-locks `uv.lock`, and syncs +`.venv`. It's the only way packages reach a recipe — recipes run +sandboxed in the locked environment, so a stray `pip install` on your +machine changes nothing they can see. That's a feature: the lock *is* +the record of what your results were computed with. -## 4. Add the second universe +## 5. Materialize -`lc init` scaffolded `universes/baseline.yaml`. Point it at our decision's -default: +Commit, then build: -```yaml -id: baseline -description: "Standardized features (the default)." -decisions: - standardize: standardized +```bash +git add -A && git commit -m "Line-fit analysis" +lc materialize ``` -And add the sweep — `universes/raw.yaml`: +The commit isn't ceremony — every output is committed together with the +code that produced it, so a build refuses to start from a tree with +uncommitted edits (it wouldn't be able to say what code ran). Then: -```yaml -id: raw -description: "No preprocessing before the fit." -decisions: - standardize: raw ``` + ✓ made baseline/fit + ✓ made baseline/fit_plot + ! no [project].license in pyproject.toml, so no RO-Crate publication + view is maintained — declare one to enable it -Each universe is one complete selection of decision values; its results -materialize to `results///`. +✓ Made 2 output(s) in /home/you/line-fit-demo +``` -## 5. Run it +(We'll come back to that license line in step 7.) Each output landed in +`results/baseline//` next to a `.lightcone-manifest.json` — +a manifest recording the recipe, the decisions, the input hashes, the +environment, and the commit — and was committed with a run record that +`datalad rerun` can replay. Look at `git log`: the build wrote history, +not just files. + +Check where things stand any time: ```bash -lc run +lc status ``` -`lc run` materializes every universe it finds under `universes/`. To run just -one, or just one output: +``` + mode: direct + sandbox: landlock (fs: declared, network: allowed) -```bash -lc run --universe baseline -lc run r2 + · current baseline/fit a3f1f11 + · current baseline/fit_plot a3f1f11 + +2 current ``` -Then check where things stand: +The commit column is the answer to "which code made this?" — for every +output, current or not. And `lc materialize` is idempotent: run it again +and it reports the project is up to date without executing anything. -```bash -lc status +## 6. Sweep the decision + +Add the second universe — `universes/robust.yaml`: + +```yaml +id: robust +description: "Points beyond 3 sigma of an initial fit are dropped." +decisions: + outliers: clip ``` -Expected output: +Commit and materialize again: + +```bash +git add -A && git commit -m "Add the robust universe" +lc materialize +``` ``` -Universe baseline - ✓ ok r2 - ✓ ok fit_plot + ✓ made robust/fit + ✓ made robust/fit_plot + · up to date baseline/fit + · up to date baseline/fit_plot -Universe raw - ✓ ok r2 - ✓ ok fit_plot +✓ Made 2 output(s) in /home/you/line-fit-demo ``` -Your comparison is on disk: `results/baseline/r2/r2.json` vs -`results/raw/r2/r2.json`, with a figure next to each. +Only the new universe's outputs ran — `baseline` was already exactly +what the spec asks for, so it wasn't touched. Your comparison is on +disk: with this guide's synthetic dataset, clipping drops 4 points and +moves the slope from 2.414 to 2.450 — visibly closer to the true 2.5 +the data was generated with. -If a recipe fails, `lc run` surfaces the error; fix the script or the spec -and rerun — only the affected outputs re-execute. Commit as you go so your -`git log` is a clean record of the build. +If a recipe fails, `lc materialize` reports which output failed and why, +and leaves the tree as clean as it found it; fix the script or the spec, +commit, and rerun — only the affected outputs re-execute. -## 6. Verify integrity +## 7. Publish + +RO-Crate requires a license, so declaring one is how you tell `lc` the +project is meant for the outside world. Add one line under `[project]` +in `pyproject.toml`: + +```toml +license = "CC-BY-4.0" +``` + +then commit and materialize once more: ```bash -lc verify +git add -A && git commit -m "Declare a license" +lc materialize ``` -This recomputes data hashes for every output and walks the input chain back to -declare whether anything has been tampered with since materialization. Useful -pre-publication, when archiving a project, or any time you want a stronger -guarantee than `lc status`. +Nothing is rebuilt — but `ro-crate-metadata.json` appears at the project +root and is committed automatically. From here on, every materialize +keeps it in line with the repository: the project *is* the crate, and +depositing it is just `git archive` (or `datalad export-archive`) on a +repository you already have. ## What just happened -- `astra.yaml` was the only place your analysis was *described* — inputs, - outputs, the decision, and the recipes all live there. +- `astra.yaml` was the only place your analysis was *described* — + inputs, outputs, the decision, and the recipes all live there. - The scripts take decision values as plain command-line arguments, so nothing methodological is hardcoded. -- `lc run` generated `.lightcone/Snakefile` from your spec, dispatched each - rule through Snakemake, and wrote a per-output sidecar manifest recording the - recipe, container image, decisions, input hashes, and output hash. -- `lc status` and `lc verify` rely on those manifests — they don't re-execute - anything; they just check. - -If your laptop dies tomorrow and you `git clone` the repo on a fresh machine -and run `lc run`, you'll get bit-identical results. +- `lc materialize` ran each recipe in the project's locked environment, + sandboxed — free to write its own output directory and nothing else — + and committed every output with a manifest and a re-runnable run + record. +- `lc status` and `lc materialize --check` read those manifests — they + don't re-execute anything; they just classify. An output is remade + when the spec defines it differently than it was made, or when its + declared inputs changed; an output whose *environment* has since + moved is reported as `behind` and deliberately left alone — the + manifest records exactly which environment and commit produced it. + +Clone this repository on a fresh machine, run `lc init` (it rebuilds +the two pieces of local state git doesn't carry — the `.venv` and the +annex), then `lc materialize`: it reports up to date without fetching a +single data byte, because the provenance travels in git. The bytes +themselves follow with `git annex get` whenever you actually need them. ## Where to next - [Running on a Cluster](cluster.md) — take the same project to SLURM. - [Troubleshooting](troubleshooting.md) — when something goes sideways. -- [Glossary](glossary.md) — terms like universe, decision, and manifest in - plain language. +- [Glossary](glossary.md) — terms like universe, decision, and manifest + in plain language. - The [ASTRA docs](https://astra-spec.org/latest/) — the full spec: sub-analyses, prior insights, findings, and evidence. diff --git a/docs/user/index.md b/docs/user/index.md index 7170e4a9..8680753d 100644 --- a/docs/user/index.md +++ b/docs/user/index.md @@ -4,27 +4,28 @@ a reproducible analysis. You describe what you're trying to learn as a precise specification — an `astra.yaml` file following the [**ASTRA**][astra] schema — and the `lc` command line keeps the -resulting code, decisions, and outputs in sync. +resulting code, environments, decisions, and outputs in sync. ASTRA specs are plain YAML, designed to be easy for both humans and AI assistants to write. However the spec gets written, **you stay in charge of the scientific choices** — every methodological decision is declared -in the open, and `lc` records exactly what produced every result. +in the open, and `lc` records exactly what produced every result: the +recipe, the decisions, the input data, the environment, and the commit. ## What this guide covers - [Install](install.md) — get the `lc` command line running on your machine or on a cluster. - [Getting Started](getting-started.md) — create your first project, - run it end-to-end, and understand what each piece does. + build it end-to-end, and understand what each piece does. - [Running on a Cluster](cluster.md) — taking your analysis to a SLURM - HPC system, including Perlmutter-specific notes. + HPC system. - [Troubleshooting](troubleshooting.md) — common issues and how to unstick them. - [Glossary](glossary.md) — the terms that show up everywhere (universe, decision, manifest, …) explained in plain language. -## What you'll do, in three lines +## What you'll do, in a handful of lines !!! tip "Quick start" @@ -32,27 +33,34 @@ in the open, and `lc` records exactly what produced every result. ```bash uv tool install lightcone-cli lc init my-analysis && cd my-analysis - # describe your analysis in astra.yaml, then: - lc run + # describe your analysis in astra.yaml, write your scripts, + # declare what they import (uv add numpy ...), then: + git add -A && git commit -m "First analysis" + lc materialize ``` === "pip" ```bash pip install lightcone-cli lc init my-analysis && cd my-analysis - # describe your analysis in astra.yaml, then: - lc run + # describe your analysis in astra.yaml, write your scripts, + # declare what they import (uv add numpy ...), then: + git add -A && git commit -m "First analysis" + lc materialize ``` -That's the shortest possible path. The rest of the guide is the unhurried version. +That's the shortest possible path. The rest of the guide is the +unhurried version — and the commit is not ceremony: every output is +committed together with the code that produced it, which is why a build +starts from a clean tree. ## What lightcone-cli is *not* - **A statistics package.** It runs your code; it doesn't compute things itself. -- **A workflow language.** Recipes in `astra.yaml` are short shell or - Python commands, not a DSL. There's no learning curve beyond what's - in [Getting Started](getting-started.md). +- **A workflow language.** Recipes in `astra.yaml` are short shell + commands, not a DSL. There's no learning curve beyond what's in + [Getting Started](getting-started.md). - **An IDE.** `lc` is a command-line tool; write `astra.yaml` and your analysis code with whatever editor or tooling you prefer. diff --git a/docs/user/install.md b/docs/user/install.md index 583dda3c..48f13ed0 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -1,78 +1,37 @@ # Install -To get started on a lightcone project, you need two things on your machine: Python 3.11+ and the lightcone command line tool `lc`. -A container runtime is optional but recommended. +To work on a lightcone project you need two things on your machine: +[uv](https://docs.astral.sh/uv/) and git. Everything else — Python +itself included — is installed by uv or ships with `lc`. -## 1. Python +!!! note "Supported platforms" + Linux (glibc 2.34+, x86_64 or aarch64) and macOS (14+ on Apple + silicon, 15+ on Intel). On Windows, use WSL. -If you don't already have a recent Python +## 1. uv and git -=== "macOS" +`lc` uses uv as its only environment substrate — projects are +`pyproject.toml` + `uv.lock`, and uv manages the Python interpreters +too, so there is no separate Python install step. + +=== "macOS / Linux" ```bash - brew install python@3.12 + curl -LsSf https://astral.sh/uv/install.sh | sh ``` -=== "Linux" - Your package manager (`apt install python3.12`, etc.) or - [pyenv](https://github.com/pyenv/pyenv) - -=== "Windows" - [python.org](https://www.python.org/downloads/) or WSL + git is preinstalled on macOS; on Linux use your package manager + (`apt install git`, `dnf install git`, …). === "NERSC Perlmutter" - NERSC doesn't ship `uv`, but it installs into your home dir with a - single curl: + NERSC doesn't ship `uv`, but it installs into your home directory + with a single curl: ```bash curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.12 ``` - Both `uv` and an isolated Python 3.12 land under `~/.local/`. - Make sure `~/.local/bin` is on your `PATH`. - - ??? note "Alternative: NERSC's `python` module" - `module load python` gives you a ready-to-use distribution with - `conda`, `pip`, and many scientific packages already installed: - - ```bash - module load python # NERSC Python (3.11+) - ``` - - Convenient, but the module is shared and read-only. For custom - packages, build a conda env on top: - - ```bash - conda create -n your-env-name python=3.11 -y - conda activate your-env-name - ``` - - This is NERSC's [recommended path for `pip install`](https://docs.nersc.gov/development/languages/python/nersc-python/) - when you need custom packages. - - !!! warning "Storage: 40 GB home quota" - Conda envs land under `~/.conda/envs/` by default. The - Perlmutter home quota is **40 GB**, which gets eaten quickly. - NERSC recommends `/global/common/software//` for - larger envs. If you want them on `$SCRATCH` (note: 12-week - purge), move and symlink: - - ```bash - conda deactivate - mv ~/.conda/envs/your-env-name $SCRATCH/conda-envs/ - ln -s $SCRATCH/conda-envs/your-env-name ~/.conda/envs/your-env-name - ``` - -!!! tip "Recommendation" - We highly recommend the use of [uv](https://docs.astral.sh/uv/) to manage Python installation and virtual environments. - - `uv` can be installed in a single commandline - - curl -LsSf https://astral.sh/uv/install.sh | sh - - and a subsequent version of Python - - uv python install 3.12 + `uv` lands under `~/.local/bin` — make sure it's on your `PATH`. + git is already on the system. ## 2. lightcone-cli @@ -89,74 +48,41 @@ is `lc`. python -m pip install lightcone-cli ``` -=== "NERSC Perlmutter" - With `uv` (recommended — isolates `lc` under `~/.local/share/uv/tools/`): - - ```bash - uv tool install lightcone-cli - ``` - - With pip, the exact command depends on which Python you're using: - - ```bash - # NERSC python module - module load python - python -m pip install --user lightcone-cli # lands in ~/.local/bin/ - - # Conda env - conda activate your-env-name - python -m pip install lightcone-cli - ``` - - `astra-tools` is a transitive dependency — pulled in automatically. - - ??? note "From source (contributors only)" - ```bash - git clone https://github.com/LightconeResearch/lightcone-cli.git - uv pip install -e ./lightcone-cli - ``` - - To also hack on `astra-tools`: - - ```bash - git clone https://github.com/LightconeResearch/ASTRA.git - uv pip install -e ./ASTRA - ``` - Get a confirmation of the proper installation by running - lc --version # → lightcone-cli, version ... + lc --version # → lc, version ... -> **Note** Some people may have already set a personal shell alias `lc='ls --color'`. If that's you, installing lightcone-cli will shadow the alias — make sure to rebind it (e.g. `alias l='ls --color'`). +> **Note** Some people may have already set a personal shell alias +> `lc='ls --color'`. If that's you, installing lightcone-cli will shadow +> the alias — make sure to rebind it (e.g. `alias l='ls --color'`). -## 3. Global configuration +## 3. Tell git who you are -`~/.lightcone/config.yaml` is created automatically the first time you -run any `lc` command. No manual setup step is needed. The file starts -as: +Every output `lc` makes is committed, so git needs an identity before +the first build — `lc materialize` checks up front rather than failing +after your recipes have run: -```yaml -container: - runtime: auto +```bash +git config --global user.name "Ada Lovelace" +git config --global user.email "ada@example.org" ``` -`auto` detects whichever of `podman`, `docker`, or `podman-hpc` is on -your PATH (and skips docker if its daemon isn't running). Feel free to pin the runtime later by editing this file directly. +If you already commit from this machine, you're done. -## 4. (Optional) Docker or Podman +## 4. (Optional) Podman or Docker -If your analysis declares a `container:` (which it usually should — it -makes the result reproducible across machines), you need a container -runtime: +Only *containerized* projects need a container runtime — a project opts +in by declaring `[tool.lightcone.image]` in its `pyproject.toml`, and +until it does, recipes run directly on your machine in the project's +own locked environment. -- Local laptop: install [Podman](https://podman.io/) (rootless, no +- Local machine: install [Podman](https://podman.io/) (rootless, no daemon) or [Docker](https://docs.docker.com/get-docker/). - HPC login node: see [Running on a Cluster](cluster.md). -The `auto` mode picks whichever container runtime you have. If you don't -have either, you can still use `lc` — set `runtime: none` in -`~/.lightcone/config.yaml` and recipes will run on the host without -isolation. +There is nothing to configure: `lc` detects whichever runtime is +available (`podman-hpc`, then `podman`, then `docker` — skipping docker +if its daemon isn't running). ## Sanity check @@ -164,8 +90,7 @@ isolation. lc init --help Both should print help text. If `lc` is shadowed by an `ls` alias, -unset it (`unalias lc`) or use the full path -(`$(which lc) --version`). +unset it (`unalias lc`) or use the full path (`$(which lc) --version`). ## Updating @@ -176,19 +101,12 @@ unset it (`unalias lc`) or use the full path === "pip" ```bash - pip install -U lightcone-cli astra-tools - ``` - -=== "Source" - ```bash - cd path/to/lightcone-cli - git pull - uv pip install -e . # only needed if pyproject.toml changed + pip install -U lightcone-cli ``` - Editable installs auto-follow source edits — switching branches or - pulling new commits is reflected immediately in `lc`. Re-install - only when `pyproject.toml` adds a new dependency. +An upgrade never invalidates your results: the engine's version is +recorded in every output's manifest, but it is not part of any output's +identity, so nothing gets rebuilt just because `lc` moved. ## Uninstalling @@ -202,6 +120,5 @@ unset it (`unalias lc`) or use the full path pip uninstall lightcone-cli ``` -!!! note "Keep your config?" - `~/.lightcone/config.yaml` survives the uninstall. Delete it too - if you want a clean slate. +Your projects are untouched — everything `lc` knows about an analysis +lives in the project's own repository, not in any global state. diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 631ab1da..ed58b020 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -124,8 +124,15 @@ def init(directory: Path, check_only: bool, as_json: bool) -> None: def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: bool) -> None: """Print a convergence report: the items, then the verdict. + + Engine prose and paths are escaped — they are data, not markup, and a + literal ``[project]`` or a bracketed path must reach the terminal as + written. """ + from rich.markup import escape + mark, style = ("·", "yellow") if dry_run else ("✓", "green") + where = escape(str(directory)) lines: list[str] = [] for items, label, item_mark, item_style in ( @@ -133,21 +140,24 @@ def _render_init_output(report: ConvergenceReport, directory: Path, *, dry_run: (report.repaired, "would repair" if dry_run else "repaired", mark, style), (report.blocked, "blocked", "✗", "red"), ): - lines += [f" [{item_style}]{item_mark}[/{item_style}] {label} {item}" for item in items] - lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + lines += [ + f" [{item_style}]{item_mark}[/{item_style}] {label} {escape(item)}" + for item in items + ] + lines += [f" [yellow]![/yellow] {escape(warning)}" for warning in report.warnings] if report.converged: # A dry run over a converged project finds nothing to do because # there is nothing to do — one line serves both moods. - verdict = f"[green]✓[/green] {directory} is already converged — nothing to do" + verdict = f"[green]✓[/green] {where} is already converged — nothing to do" elif report.blocked: # A write run that left an item blocked did not converge the # project either; only a dry run gets to be neutral about it. - verdict = f"[red]✗[/red] {directory} is not converged" + verdict = f"[red]✗[/red] {where} is not converged" elif dry_run: - verdict = f"[yellow]![/yellow] {directory} is not converged" + verdict = f"[yellow]![/yellow] {where} is not converged" else: - verdict = f"[green]✓[/green] Project converged at {directory}" + verdict = f"[green]✓[/green] Project converged at {where}" if lines: lines.append("") # space the verdict off the list @@ -347,6 +357,8 @@ def status(as_json: bool) -> None: click.echo(json.dumps(report.as_dict(), indent=2)) return + from rich.markup import escape + lines = [f" mode: {report.mode}"] if report.image is not None: tag, state = report.image["tag"], report.image["state"] @@ -368,10 +380,10 @@ def status(as_json: bool) -> None: # for machine consumers of `--json`. lines += [ f" {marks[o.status]} {o.status:<8} {o.output:<{width}} " - f"{o.git_sha[:7] or '—':<7}" + (f" [dim]{o.why}[/dim]" if o.why else "") + f"{o.git_sha[:7] or '—':<7}" + (f" [dim]{escape(o.why)}[/dim]" if o.why else "") for o in report.outputs ] - lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + lines += [f" [yellow]![/yellow] {escape(warning)}" for warning in report.warnings] counts = report.counts if not report.outputs: @@ -391,9 +403,16 @@ def status(as_json: bool) -> None: def _render_materialize_output(report: MaterializeReport, root: Path, *, dry_run: bool) -> None: """Print what ran, or what would. + + Engine prose and paths are escaped — they are data, not markup, and a + literal ``[project]`` or a bracketed path must reach the terminal as + written. Task names need no escape: ASTRA ids cannot hold a bracket. """ + from rich.markup import escape + + where = escape(str(root)) lines = [ - f" [yellow]·[/yellow] would run {name} — {why}" + f" [yellow]·[/yellow] would run {name} — {escape(why)}" for name, why in report.planned.items() ] lines += [f" [green]✓[/green] made {name}" for name in report.made] @@ -401,21 +420,22 @@ def _render_materialize_output(report: MaterializeReport, root: Path, *, dry_run # an output came from, and the only line here that tells you something # you could not have worked out from the exit code. lines += [ - f" [cyan]·[/cyan] behind {name} — {why}" for name, why in report.behind.items() + f" [cyan]·[/cyan] behind {name} — {escape(why)}" + for name, why in report.behind.items() ] lines += [f" [dim]·[/dim] up to date {name}" for name in report.current] lines += [f" [red]✗[/red] failed {name}" for name in report.failed] lines += [f" [red]✗[/red] blocked {name}" for name in report.blocked] - lines += [f" [yellow]![/yellow] {warning}" for warning in report.warnings] + lines += [f" [yellow]![/yellow] {escape(warning)}" for warning in report.warnings] if not report.ok: - verdict = f"[red]✗[/red] {root} did not finish" + verdict = f"[red]✗[/red] {where} did not finish" elif report.up_to_date: - verdict = f"[green]✓[/green] {root} is up to date — nothing to do" + verdict = f"[green]✓[/green] {where} is up to date — nothing to do" elif dry_run: verdict = f"[yellow]![/yellow] {len(report.planned)} output(s) would be made" else: - verdict = f"[green]✓[/green] Made {len(report.made)} output(s) in {root}" + verdict = f"[green]✓[/green] Made {len(report.made)} output(s) in {where}" # On the verdict line as well as in the listing: on a large analysis the # listing scrolls away, and this is the one state that reports something # rather than doing it. diff --git a/tests/test_cli.py b/tests/test_cli.py index eec31b5c..03206bf8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -431,6 +431,24 @@ def test_a_failure_exits_nonzero( assert "blocked baseline/report" in result.output +def test_engine_prose_is_rendered_as_written( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A warning naming `[project].license` must reach the terminal with + its brackets — rich would otherwise read them as a style tag and + swallow the word, turning the remedy into "no .license".""" + from lightcone.engine.materialize import MaterializeReport + + _stub( + monkeypatch, + materialize=MaterializeReport(warnings=["no [project].license declared"]), + ) + + result = runner.invoke(main, ["materialize"]) + + assert "no [project].license declared" in result.output + + def test_the_json_report_is_machine_readable( runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 8ee52ee53d33c99262a4f0dadb957f9c49635f70 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:32:17 +0200 Subject: [PATCH 17/23] Docs rewrite 2/5: core concepts, cluster, troubleshooting, glossary (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second pass of the documentation rewrite — the rest of the user guide: a new **Core Concepts** page plus rewrites of **Running on a Cluster**, **Troubleshooting**, and the **Glossary**. Style follows the frozen predecessors. ## What's in it - **`docs/user/concepts.md`** (new, added to nav + user index): the mental model in one page — a project is spec + lock; an output's three recorded facts and why they age differently; the `current`/`behind`/`stale` table with the contradiction-vs-circumstance line; the commit discipline (clean tree in, clean tree out, git log as build log, foreign-write detection); direct vs containerized mode; `lc status` vs `lc materialize --check`; the license ⇒ crate rule. - **`docs/user/cluster.md`**: the venue model replaces the old Snakemake/Dask-Gateway/site-registry content. The allocation *is* the resource declaration (no `--jobs`, no venue config); login-node prep vs in-allocation materialize with interactive/batch tabs; the login guard's refusal quoted verbatim; containers on HPC (podman-hpc detection + migrate, build-on-login-node with the arch gate, the multi-node shared-store refusal); an honest "early days" admonition inviting site reports. - **`docs/user/troubleshooting.md`**: organized around the CLI's actual messages, each captured live against a real project before being quoted — dirty tree (and its commit/discard path split, verified against the source), not-a-project, the sandbox trailer with the three denial remedies (`uv add`, declare the input, write to `{output}`/mkdtemp), behind-after-`uv add` as a non-error, stale-after-spec-edit (including the deliberate "code edits don't cascade" position), unfetched annex content, the NERSC login guard, git identity, and the three containerized refusals. - **`docs/user/glossary.md`**: ASTRA-side entries kept (ASTRA, spec, recipe, decision, universe, sub-analysis, insights/findings); execution-layer half replaced — materialize, manifest, `definition_version` / `env_version` / `data_version` / `input_versions`, the three states, the two modes, image + runtime, sandbox, git-annex, run record, RO-Crate. Snakemake, `code_version`, `ok`/`missing`/`alias`, and the `lc verify` failure kinds are gone with the architecture that defined them. ## Verification - Every quoted refusal/denial was produced by actually triggering it: dirty tree, not-a-project, `NERSC_HOST` login guard, a sandboxed read denial, `ModuleNotFoundError` under the sandbox, and the `behind`-everywhere state after a real `uv add` (plus the `--check` verdict line with its `--refresh` hint). - The dirty-refusal "discard these (lc writes results/)" branch was verified against the source. - `zensical build` passes; the only warnings remain the two frozen pages (`api/snakefile.md`, `hpc/targets.md`) that pass 4 replaces. Next: (3) CLI + JSON reference, (4) developer corner, (5) nav + workflow refresh. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx Co-authored-by: Claude Fable 5 --- docs/user/cluster.md | 333 ++++++++++------------------------- docs/user/concepts.md | 151 ++++++++++++++++ docs/user/getting-started.md | 2 + docs/user/glossary.md | 208 +++++++++++----------- docs/user/index.md | 3 + docs/user/troubleshooting.md | 192 +++++++++++--------- zensical.toml | 1 + 7 files changed, 465 insertions(+), 425 deletions(-) create mode 100644 docs/user/concepts.md diff --git a/docs/user/cluster.md b/docs/user/cluster.md index ca06fb85..f422897c 100644 --- a/docs/user/cluster.md +++ b/docs/user/cluster.md @@ -1,270 +1,127 @@ # Running on a Cluster -When local laptop time isn't enough, you can take the same project to -a SLURM HPC system or a lightcone JupyterHub deployment. There's no -separate configuration to learn — the same `lc run` command works -everywhere, just with more hardware to spread across. +When local laptop time isn't enough, the same project runs on a SLURM +HPC system. There is no separate configuration to learn and no flag to +pass — `lc materialize` detects where it is running, and the allocation +you request *is* the resource declaration. ## The big picture -`lc run` always dispatches through a Dask cluster. Four branches: +`lc materialize` runs its tasks through a scheduler, and picks the venue +by looking at the environment: -1. On your laptop → a `LocalCluster` sized to the machine. -2. **On a JupyterHub deployment** (Dask Gateway detected) → a - run-scoped Gateway cluster created with your project's container - image and shut down when the run finishes. -3. **Inside a SLURM allocation** → an in-process scheduler bound to - the driver's hostname, with one `dask worker` per allocated node - launched via `srun`. -4. With `DASK_SCHEDULER_ADDRESS` set → connect to whatever scheduler - you've pointed at. +1. **Inside a SLURM allocation** (`SLURM_JOB_ID` is set) → the run + spans every node the allocation holds: one worker per node, launched + via `srun`, using every core it was granted. +2. **Anywhere else** → the local machine, using every core. -You don't pick — `lc run` detects which case applies. The only thing -you do differently on a cluster is request the nodes (and on -JupyterHub, not even that). - -## JupyterHub deployments (Kubernetes + Dask Gateway) - -On a lightcone JupyterHub (GKE with Dask Gateway and Cloud Build), -everything is zero-configuration — the deployment injects the whole -contract into your session (`DASK_GATEWAY__*`, `LIGHTCONE_REGISTRY`, -`LIGHTCONE_BUILD_BUCKET`), and `lc` picks it up: - -- The scaffolded project image doubles as the Dask worker pod image - with no hub-specific content: `lc init` pins `lightcone-cli` in - `requirements.txt`, which brings the whole execution stack - (snakemake, dask, distributed, dask-gateway) on top of your own - dependencies — the same image runs anywhere. -- `lc build` builds through the deployment's **GCP Cloud Build** - service (there's no docker in your session) and pushes - `/lc-:` to the hub's Artifact - Registry. Unchanged files never rebuild — freshness is one registry - check. -- `lc run` makes sure the image is up to date, **creates a Dask - Gateway cluster with that image**, runs the pipeline in worker pods - (recipes execute directly in the image — no nested containers), and - **culls the cluster when the run finishes**. Your NFS home is - mounted in every worker pod at the same path, so outputs land in - the project tree exactly as they do locally. - -A cluster's image is fixed at creation, so create-per-run is also what -keeps the environment fresh: edit the Containerfile, `lc run`, and the -next cluster runs the rebuilt image. - -## Pre-flight: pick the right container runtime - -On most HPC sites, docker isn't available on compute nodes. Most -SLURM systems (including NERSC Perlmutter) provide `podman-hpc`. On a -login node: - -```bash -$EDITOR ~/.lightcone/config.yaml -``` - -```yaml -container: - runtime: podman-hpc -``` - -Then build and migrate the images for your project: - -```bash -cd my-analysis -lc build -``` - -`lc build` runs `podman-hpc build` and then `podman-hpc migrate`, -which copies the image into the per-node container cache. Compute -nodes can read it without registry access. - -If your site has only `apptainer` / `singularity`, the Lightcone -toolchain doesn't ship explicit support for those today — you can run -without containers (`runtime: none`) for the moment, with the caveat -that the manifest's `container_image` field will record what was -declared, not what executed. (See [`lc run`](../cli/run.md) for the -provenance warning.) +You already answered every sizing question at `salloc` / `sbatch` — +how many nodes, which constraint, how long — so `lc` asks none of its +own. There is no `--jobs`, no worker count, no venue config file. ## A typical SLURM workflow -### 1. Get an allocation +### 1. Prepare on the login node + +Everything except executing recipes works on a login node — and one +verb is *for* it: ```bash -salloc -N 4 -t 02:00:00 -C gpu # interactive -# or -sbatch run.sbatch # batch +cd $SCRATCH/my-analysis +lc materialize --check # what would run, and why +lc status # where every output stands +lc build # containerized projects: build + commit the image ``` -`run.sbatch` looks like: +### 2. Get an allocation and materialize inside it -=== "Generic" +=== "Interactive" ```bash - #!/bin/bash - #SBATCH -N 4 - #SBATCH -t 02:00:00 - #SBATCH -C gpu - - cd $HOME/my-analysis - source .venv/bin/activate - lc run -j 16 + salloc --nodes=1 --constraint=cpu --qos=interactive --time=02:00:00 + # salloc drops you onto a compute node; from there: + cd $SCRATCH/my-analysis + lc materialize ``` -=== "NERSC Perlmutter" +=== "Batch" ```bash - #!/bin/bash - #SBATCH -A - #SBATCH -q regular - #SBATCH -C gpu - #SBATCH -N 4 - #SBATCH -t 04:00:00 - - cd $SCRATCH/your-analysis - - # make `lc` available — pick the line that matches your install: - export PATH=$HOME/.local/bin:$PATH # uv tool install - # source ~/.conda/envs/your-env-name/bin/activate # conda env - - lc run -j 16 + cd $SCRATCH/my-analysis + sbatch --nodes=1 --constraint=cpu --qos=regular --time=02:00:00 \ + --wrap 'lc materialize' ``` -### 2. `lc run` inside the allocation - -Once `SLURM_JOB_ID` is set in your environment, `lc run` does the rest: + (Make sure `lc` is on `PATH` in the batch environment — with a + `uv tool install`, that's `export PATH=$HOME/.local/bin:$PATH` in + the script if your shell profile doesn't already do it.) -- Starts an in-process Dask scheduler bound to the SLURM node hostname. -- Launches one `dask worker` per node via `srun`. -- Each worker advertises the node's CPU, memory, and GPU resources. -- Snakemake submits each rule via the Dask executor; rules with - per-recipe `resources:` constraints land on workers that can hold - them. +Ask for more nodes and the run uses them — independent outputs and +universes spread across the allocation with nothing else to say. -### 3. Per-recipe resource hints +### 3. Guard rails on known centers -Add resource hints in your `astra.yaml` recipe blocks: +On centers `lc` knows (NERSC today), running `lc materialize` on a +login node refuses with the center's own allocation spellings rather +than quietly hammering a shared node: -```yaml -outputs: - - id: heavy_fit - type: metric - recipe: - command: python scripts/fit.py --output {output[0]} - resources: - cpus_per_task: 32 - mem_mb: 64000 - gpus_per_task: 1 ``` +Error: lc materialize executes recipes on compute nodes, and this is a +NERSC login node (NERSC_HOST is set with no SLURM allocation active). -The Snakemake-via-Dask executor maps these to per-task resource -requests, so a rule that needs a GPU only schedules on nodes that -advertise one. +Get an allocation and run it there: -## Interactive: iterating inside an allocation + interactive: + salloc --nodes=1 --constraint=cpu --qos=interactive --time=02:00:00 + lc materialize -During development you're usually iterating — run something, check the -result, adjust the spec, repeat. For that loop you want an interactive -shell inside a SLURM allocation, so that `lc run` executes on the -compute node rather than the login node. + batch (from the project root): + sbatch --nodes=1 --constraint=cpu --qos=regular --time=02:00:00 \ + --wrap 'lc materialize' -```bash -salloc -A -q interactive -C gpu --nodes=1 -t 00:30:00 -# salloc drops you onto a compute node; from there: -cd /path/to/your-analysis -lc run --universe baseline -lc status +lc materialize --check, lc status and lc run work anywhere. ``` -Everything you launch from that shell (`lc run`, scripts, etc.) -executes on the allocated node. When you're done iterating and want a -hands-off sweep of all universes, submit `lc run` as a batch job -instead (the sbatch template above). - -## What about login-node-only operations? - -Build images, dry-run, look at status — all fine on a login node -without an allocation: - -```bash -lc build # build images (uses podman-hpc on login node) -lc status # offline; reads only manifests -``` - -The actual `lc run` should happen inside an allocation, since that's -where the worker nodes are. - -## External Dask schedulers - -If you have a long-lived Dask cluster (Slurm jobqueue, k8s, etc.) -that you'd rather attach to: - -```bash -export DASK_SCHEDULER_ADDRESS=tcp://my-scheduler:8786 -lc run -``` - -`lc run` notices the env var and connects rather than starting its -own scheduler. It does *not* tear the scheduler down on exit. - -## NERSC Perlmutter: site-specific notes - -!!! note "Setting up on Perlmutter for the first time?" - The [Install](install.md) page has NERSC-specific tabs for Python - (uv vs `module load python`, conda env storage) and lightcone-cli. - Come back here once `lc --version` works. - -### Storage: keep Snakemake state on `$SCRATCH` - -!!! danger "DVS silently ignores `flock()`" - `$HOME` and `/global/cfs/` are mounted on compute nodes via DVS, - which silently ignores `flock()`. Snakemake relies on `flock` for - locking, so its `.snakemake/` directory and Dask spill files - **must** live on Lustre (`$SCRATCH`), which honors `flock`. - Otherwise you get intermittent silent rule-rerun loops or hangs. - -`lc` redirects state automatically when it detects Perlmutter, so -this usually just works. To pin explicitly at project creation: - -```bash -lc init your-analysis --scratch '$SCRATCH' # kept verbatim, expanded at run time -``` - -Or, after the fact, edit `/.lightcone/lightcone.yaml`: - -```yaml -scratch_root: $SCRATCH -``` - -!!! warning "12-week purge on `$SCRATCH`" - Perlmutter purges `$SCRATCH` on a rolling 12-week window. For - outputs you need to keep, copy or symlink to - `/global/cfs/cdirs//`. - -### Further reading - -- [NERSC interactive jobs](https://docs.nersc.gov/jobs/interactive/) - — `salloc` patterns and reservation queues -- [Perlmutter system overview](https://docs.nersc.gov/systems/perlmutter/) - — node types and partitions -- [NERSC queue policy](https://docs.nersc.gov/jobs/policy/) - — QoS options for GPU and CPU partitions - -## Troubleshooting - -- `dask CLI is not on PATH inside the SLURM allocation`. Install - `lightcone-cli` into the venv that your sbatch script activates; - `dask` ships with `distributed`, which is a transitive dep. -- Workers never register. Usually means the SLURM node hostnames - aren't resolvable from each other; check `SLURMD_NODENAME` / - `gethostname()` and confirm the workers can reach the scheduler. -- Image not found on compute nodes. Re-run `lc build` on the login - node — the migrate step is the one that actually publishes the - image to the per-node cache. -- Snakemake locking errors or silent rule-rerun loops on Perlmutter. - `.snakemake/` ended up on DVS-mounted storage — set - `scratch_root: $SCRATCH` in the project's `.lightcone/lightcone.yaml`. -- `pip install` hangs or times out. Compute nodes have no public - internet — always install from a login node. -- `PermissionError` reading another user's symlinked `results/`. - Cross-user scratch path without group ACLs — request access from - the data owner, or copy the manifests into your own scratch. - -For the wiring detail, see -[engine/dask_cluster](../api/dask_cluster.md) in the maintainer docs. +The read-only verbs are exempt on purpose — a login node is exactly +where "where does this project stand?" gets asked. + +## Containers on HPC + +A containerized project (one with `[tool.lightcone.image]` in its +`pyproject.toml`) works the same way, with three site realities to +know: + +- **`podman-hpc` is detected first.** Sites install it precisely + because plain podman's image store is invisible to compute nodes; + where both exist, `lc` prefers the wrapper and runs its extra + `migrate` step automatically, so the image is readable from every + node. +- **Build on a login node, once.** `lc build` builds the image and + commits it into the repository as versioned content — compute nodes + never build and need no registry access; an unfetched image arrives + through the annex like any other data. The archive records the + architecture it was built for, and a mismatched host is refused + before anything runs — so build where the architecture matches the + compute nodes (on NERSC, a login node). +- **Multi-node runs require a shared image store.** With plain podman + or docker the image exists only on the driver's node, so `lc` + refuses a multi-node containerized run unless the runtime is + `podman-hpc`. Single-node allocations work with any runtime. + +## Data on parallel filesystems + +Keep active projects on the filesystem your center recommends for job +I/O (`$SCRATCH` on NERSC), and remember scratch purge policies — the +project is a git repository, so `git push` to a remote (and +`git annex copy --to` for the bytes) is the durable copy. + +!!! warning "Early days" + HPC support is the youngest part of lightcone-cli and has not yet + been broadly validated on production systems. If something refuses, + hangs, or surprises you on your center, please + [open an issue](https://github.com/LightconeResearch/lightcone-cli/issues) + — site reports are exactly what this layer needs right now. + +## Where to next + +- [Core Concepts](concepts.md) — the model all of this rests on. +- [Troubleshooting](troubleshooting.md) — the refusals, quoted, with + remedies. diff --git a/docs/user/concepts.md b/docs/user/concepts.md new file mode 100644 index 00000000..db12a764 --- /dev/null +++ b/docs/user/concepts.md @@ -0,0 +1,151 @@ +# Core Concepts + +The mental model behind `lc`, in one page. Nothing here is required to +follow [Getting Started](getting-started.md) — come back when you want +to know *why* the tool behaves the way it does. + +## A project is three files + +A lightcone project is a directory holding an ASTRA spec and a uv +project: + +- **`astra.yaml`** describes the analysis — inputs, outputs, recipes, + methodological decisions. It is the single source of truth: everything + `lc` does is downstream of it. +- **`pyproject.toml` + `uv.lock`** describe the environment — every + package a recipe may import, resolved to exact versions. The `.venv` + is built *from* the lock and is disposable; the lock is what's real, + and it travels in git. + +There is no global configuration, no registry, no state outside the +project. Clone the repository and you have everything except two pieces +of local machinery (`.venv` and the git-annex initialization), which +`lc init` rebuilds. + +Adding a dependency is a uv operation, not an `lc` one: + +```bash +uv add numpy +``` + +That updates `pyproject.toml`, re-locks, and syncs `.venv` in one step. +Recipes import from the locked environment and nothing else — a stray +`pip install` on your machine changes nothing they can see. + +## An output has an identity, and three facts about it + +Every materialized output records, in its +`.lightcone-manifest.json`: + +1. **What it is** — a hash of its recipe and the decision values that + shaped it (its *definition*). +2. **What it was made from** — a content hash of each declared input. +3. **What it ran under** — a hash of the environment (the lock, the + interpreter, the image declaration if any), plus the git commit the + run started at. + +Those three facts are deliberately not one fact, because they age +differently — and that is what the three states mean: + +| state | means | what `lc materialize` does | +|---|---|---| +| `current` | the output is exactly what the spec asks for, made from these inputs, under this environment | nothing | +| `stale` | the output **contradicts** the project: the spec now defines it differently, or an input's content changed | remakes it | +| `behind` | the output is still exactly what the spec asks for — only the **environment** moved since it was made | reports it, leaves it alone | + +The line between `stale` and `behind` is contradiction versus +circumstance. A stale output is mislabelled — keeping it would be a +lie, so it is remade. A behind output is not wrong in any way: one +`uv add` for a plotting script rewrites the lock for the whole project, +and remaking a week of computation over that buys nothing. Its manifest +records exactly which environment and commit produced it, and that +commit's own `uv.lock` reconstructs the environment if you ever need +it. + +When you *do* want behind outputs remade — before a release, say — +that is one flag: + +```bash +lc materialize --refresh +``` + +`--refresh` only ever widens a run: a `current` output stays current +under it, and there is deliberately no flag in the other direction — +nothing suppresses the rebuild of a stale output. + +One more way an output can be stale: a hand edit. Every output is +committed by the run that made it, so a file changed by hand and +committed shows up in history under a commit that is not a run record — +and the output classifies `stale` everywhere, with `lc status` naming +the foreign commit. + +## Everything is committed, and the tree stays clean + +`lc` versions results in the project's own git repository: git carries +the history and the small files, git-annex carries the data bytes — +transparently, behind the ordinary `git add` / `git commit` you already +type. + +That model has two consequences you'll feel: + +- **A run starts from a clean tree.** Every output is committed + together with the code that produced it; a run that started from + uncommitted edits could not say what that code was. So: commit, then + materialize. +- **A run ends with a clean tree.** Each output is committed as it + lands — with its manifest, in a commit whose message is a *run + record* that `datalad rerun` can replay. A failed recipe's partial + work is rolled back. Your `git log` is the build log. + +`results/` is `lc`'s to write. Don't put files there by hand — a +hand-placed file has no manifest and no run record, and the foreign +write check above exists precisely to catch it. + +## Two modes, derived from the project + +How recipes execute is never configured — it is read off the project: + +- **Direct mode** (the default): recipes run on your machine, in the + project's `.venv`, under an OS sandbox — Landlock on Linux, Seatbelt + on macOS. The project tree is read-only except each recipe's own + output directory; undeclared tools don't execute. +- **Containerized mode**: declaring a `[tool.lightcone.image]` table in + `pyproject.toml` *is* the switch. Recipes then run inside a + content-addressed image built from that declaration — and the image + itself is saved into the repository as versioned content, so a clone + obtains the exact bytes with no registry and no credentials. + `lc status` shows the mode and the image's state. + +Either way, every manifest records what enforcement actually ran +(`hermeticity`) — a host with no sandbox mechanism runs the recipe and +says so, rather than pretending. + +## Reading and gating are different verbs + +- **`lc status`** reports. It always exits 0 — a state is not a + failure — runs nothing, and doesn't mind a dirty tree, because the + moment you most need it is when things aren't clean. It's also the + verb that shows the commit each output was made at. +- **`lc materialize --check`** gates. It classifies everything without + running anything and exits 1 if a run would do work — the thing a + script or CI job branches on. + +Both have `--json`; the first two keys of the check report, `ok` and +`up_to_date`, are the ones to branch on. + +## Publication is a license away + +Declaring a `license` under `[project]` in `pyproject.toml` is +declaring the intent to publish. From then on, every `lc materialize` +maintains `ro-crate-metadata.json` at the project root — an +[RO-Crate](https://www.researchobject.org/ro-crate/) describing the +project, its outputs, and the runs that produced them. The repository +*is* the crate; depositing it is `git archive` on something you already +have. + +## Where to next + +- [Running on a Cluster](cluster.md) — the same model on SLURM. +- [Troubleshooting](troubleshooting.md) — the refusals quoted, with + their remedies. +- [Glossary](glossary.md) — the terms, one at a time. diff --git a/docs/user/getting-started.md b/docs/user/getting-started.md index 3a61d1fd..cb884231 100644 --- a/docs/user/getting-started.md +++ b/docs/user/getting-started.md @@ -377,6 +377,8 @@ themselves follow with `git annex get` whenever you actually need them. ## Where to next +- [Core Concepts](concepts.md) — the model behind what you just did: + the three states, the commit discipline, the two execution modes. - [Running on a Cluster](cluster.md) — take the same project to SLURM. - [Troubleshooting](troubleshooting.md) — when something goes sideways. - [Glossary](glossary.md) — terms like universe, decision, and manifest diff --git a/docs/user/glossary.md b/docs/user/glossary.md index aa0060ea..39fa1d23 100644 --- a/docs/user/glossary.md +++ b/docs/user/glossary.md @@ -9,9 +9,9 @@ plain language. The schema lightcone-cli is built around. ASTRA's job is to capture an analysis's inputs, outputs, and methodological decisions in a single file (`astra.yaml`); lightcone-cli's job is to execute that spec -reproducibly. ASTRA ships separately as the `astra-tools` package and -the `astra` CLI handles the spec itself (validation, paper management, -evidence verification). +reproducibly. ASTRA ships separately as the `astra-tools` package, and +its `astra` CLI handles the spec itself (validation, universe +management, evidence verification). ## astra.yaml @@ -21,20 +21,17 @@ nested via `analyses:` references. ## Recipe -A short shell or Python command that produces an output. Lives inside -an output's `recipe:` block in `astra.yaml`. Outputs declare which -sibling outputs they depend on, and the recipe references them through -placeholders: +A short shell command that produces an output. Lives inside an output's +`recipe:` block in `astra.yaml`. Outputs declare what they depend on, +and the recipe references those dependencies through placeholders: ```yaml outputs: - - id: r2 + - id: fit + inputs: [points] + decisions: [outliers] recipe: - command: python src/fit.py --output {output} - - id: fit_plot - inputs: [r2] - recipe: - command: python src/plot.py --r2_dir {inputs.r2} --output {output} + command: python src/fit.py --points {inputs.points} --outliers {decisions.outliers} --output {output} ``` ## Decision @@ -48,128 +45,139 @@ their `options`, and their `rationale`. One specific selection of decision values. Universes live as YAML files in `universes/` (e.g. `universes/baseline.yaml`, -`universes/permissive.yaml`). Each universe materializes its results +`universes/robust.yaml`). Each universe materializes its results to its own directory: `results///`. -If your spec has no universes, `lc run` materializes against a -universe called `"default"` with all decisions at their declared -defaults. - ## Sub-analysis A nested ASTRA analysis with its own inputs, outputs, and decisions, -referenced from a parent's `analyses:` section. The full tree shares -one set of universes; sub-analyses can reference parent decisions -with `from:` references. Sub-analyses are useful when an analysis has -genuinely different stages (training vs. inference, fit vs. evaluate); -keep things in one analysis when they share the same product. +referenced from a parent's `analyses:` section. A sub-analysis output's +directory uses its qualified id — +`results//./` — so one addressing scheme +spans however deep the spec nests. + +## Materialize + +Making the outputs the spec declares: `lc materialize` runs each recipe +in dependency order and commits every result as it lands. Idempotent — +a second run remakes only what is `stale`, and a run with nothing to do +says so and touches nothing. ## Manifest The per-output sidecar JSON file -(`/.lightcone-manifest.json`) that records what produced -the output and what's inside it. Fields include `code_version`, -`data_version`, `container_image`, `recipe`, `decisions`, -`input_versions`, `git_sha`, `host`, `lc_version`, and a few more. -Manifests are written atomically by `lc run` and read by `lc status` -and `lc verify`. +(`/.lightcone-manifest.json`) recording what produced the +output: the recipe, the decisions, `definition_version`, +`env_version`, `data_version`, `input_versions`, the git commit the +run started at, the engine version, what enforcement actually ran +(`hermeticity`), and — for containerized runs — the image. Written by +the run, read by `lc status` and `lc materialize --check`; kept in +plain git so a clone can classify a whole project without fetching any +data. -## code_version +## definition_version -A SHA-256 over `(recipe + container_image + decisions)`. The -fingerprint of "what does this rule do?" When it drifts, downstream -outputs go `stale` in `lc status`. +A hash of an output's recipe and decision values — the fingerprint of +"what is this output?". When it drifts, the output is `stale` and the +next run remakes it. + +## env_version + +A hash of the environment — the lock file's bytes, the pinned +interpreter, the install settings, and the image declaration if any. +Deliberately *not* part of an output's definition: when it drifts, the +output is `behind`, reported and left alone. ## data_version -A SHA-256 over the contents of an output directory (excluding the -manifest itself). The fingerprint of "what bytes were produced?" -`lc verify` recomputes this and compares to the recorded value to -catch tampering. +A content hash over the files in an output's directory (or of a +declared input). This is what flows downstream: a dependent is remade +when an input's `data_version` changed, and a rebuild that comes out +byte-identical stops the cascade right there. ## input_versions -Inside a manifest, a dict mapping each declared input id to its -version: the upstream output's `data_version` when the input is -another materialized output, or an `mtime-size`/`sha256` -fingerprint when the input is an external file. This is the chain -`lc verify` walks back through. +Inside a manifest, a map from each declared input to the +`data_version` it had when the output was made. Comparing it against +the present is how a change to an input cascades. -## Container +## current / behind / stale -A Docker / Podman / podman-hpc image used to execute a recipe in -isolation. Declared at the analysis level (`container: Containerfile`) -or per-recipe (`recipe: { container: python:3.12-slim }`). Recipe-level -overrides win. +The three states an output can be in: -## Containerfile +- `current` — exactly what the spec asks for, made from these inputs, + under this environment. Nothing to do. +- `behind` — still what the spec asks for; only the environment moved + since it was made. Reported, left alone; `--refresh` remakes. +- `stale` — contradicts the project: the spec defines it differently, + an input changed, or the output was edited by hand. Remade on the + next run. -A Dockerfile by another name (the syntax is identical). lightcone-cli -calls them Containerfiles to make clear they work with podman as well -as docker. +## Direct mode / containerized mode -## Image tag +How recipes execute, derived from the project rather than configured. +Direct mode (the default): the project's `.venv`, under the OS sandbox. +Containerized mode: declaring `[tool.lightcone.image]` in +`pyproject.toml` switches the project over — recipes run inside a +content-addressed image built from that declaration. -The string the runtime uses to identify a built image. lightcone-cli -generates content-addressed tags for Containerfile builds: -`lc--`. The hash covers the Containerfile and -your dependency files, so tags only change when the inputs to the -build change. - -## Runtime +## Image -The OCI tool that actually executes containers: `docker`, `podman`, -or `podman-hpc`. Set in `~/.lightcone/config.yaml` under -`container.runtime`. `auto` picks the first usable; `none` opts out -(runs recipes directly on the host). +Containerized mode's execution world: a base (digest-pinned), optional +apt packages, and the pinned interpreter. Built by `lc build` and saved +**into the repository** as versioned content, so clones obtain the +exact bytes through git-annex with no registry involved. Execution pins +the image's content id, never a tag. -## Snakemake +## Runtime -The workflow engine `lc run` shells out to. You don't need to learn -Snakemake to use lightcone-cli — the Snakefile at `.lightcone/Snakefile` -is auto-generated from your `astra.yaml`. If you're curious, peek at -it; just don't edit it (your changes will get overwritten on the -next `lc run`). +The OCI tool that executes containers. Detected, never configured: +`podman-hpc`, then `podman`, then `docker` (skipped if its daemon is +down). -## Dask +## Sandbox -The distributed scheduler `lc run` dispatches jobs through. On a -laptop it's a `LocalCluster` sized to your machine; inside a SLURM -allocation it's an in-process scheduler with one `dask worker` per -node launched via `srun`. +The isolation every recipe and every `lc run` command executes under — +Landlock on Linux, Seatbelt on macOS, the container boundary in +containerized mode. The project tree is read-only apart from the +output directory being made; undeclared tools don't execute. Each +manifest's `hermeticity` field records what was actually enforced, and +a host with no mechanism says so rather than pretending. -## Prior insight +## git-annex -A piece of evidence from the literature that informs a decision. -Lives in the `prior_insights:` section of `astra.yaml`. Each insight -has a `claim`, one or more `evidence` entries with verbatim quotes, -and a list of decision options it supports. Quotes are -machine-verified against the source PDF. +How the repository carries data: git holds history and small files, +git-annex holds the bytes of `data/` and `results/` behind ordinary +git commands. You never run git-annex yourself except to fetch bytes +on a clone (`git annex get`), and `lc materialize` even does that for +declared inputs it needs. -## Finding +## Run record -A conclusion drawn *from* the analysis (as opposed to a prior -insight, which comes *into* the analysis). Findings live in the -`findings:` section, can cite specific outputs as evidence, and act -as the bridge between materialized results and the eventual paper. +The commit message a materialized output is saved under — a +machine-readable record of the exact command that made it, in a format +`datalad rerun` can replay: it reconstructs the engine, the project +environment, and the sandbox, and remakes the output from its spec. +Your `git log` is the build log. -## Status (`ok`, `stale`, `missing`, `alias`) +## RO-Crate -The four labels `lc status` produces: +The publication view. Declare a `license` under `[project]` in +`pyproject.toml` and every materialize maintains +`ro-crate-metadata.json` — a machine-readable description of the +project, its outputs, and the runs that produced them, following the +Provenance Run Crate profile. The repository is the crate; deposit is +`git archive`. -- `ok` — manifest present, recomputed `code_version` matches. -- `stale` — manifest present but `code_version` drifted. -- `missing` — no manifest at the expected output directory. -- `alias` — output declared without a recipe; just a reference to - another output. +## Prior insight -## Failure kinds (`tampered_data`, `broken_chain`, `missing_manifest`) +A piece of evidence from the literature that informs a decision. +Lives in the `prior_insights:` section of `astra.yaml`, with a `claim` +and verifiable `evidence` (DOI plus exact quote). -The three labels `lc verify` produces when something's wrong: +## Finding -- `tampered_data` — bytes on disk no longer match recorded - `data_version`. -- `broken_chain` — recorded `input_versions` references an upstream - whose `data_version` drifted. -- `missing_manifest` — output directory exists but the manifest is - missing or unparseable. +A conclusion drawn *from* the analysis (as opposed to a prior insight, +which comes *into* it). Findings live in the `findings:` section and +cite specific outputs as evidence — the bridge between materialized +results and the eventual paper. diff --git a/docs/user/index.md b/docs/user/index.md index 8680753d..c2dc24b9 100644 --- a/docs/user/index.md +++ b/docs/user/index.md @@ -18,6 +18,9 @@ recipe, the decisions, the input data, the environment, and the commit. machine or on a cluster. - [Getting Started](getting-started.md) — create your first project, build it end-to-end, and understand what each piece does. +- [Core Concepts](concepts.md) — the model behind the tool: what the + states mean, why everything is committed, and how the two execution + modes differ. - [Running on a Cluster](cluster.md) — taking your analysis to a SLURM HPC system. - [Troubleshooting](troubleshooting.md) — common issues and how to diff --git a/docs/user/troubleshooting.md b/docs/user/troubleshooting.md index b0867502..e13edf62 100644 --- a/docs/user/troubleshooting.md +++ b/docs/user/troubleshooting.md @@ -1,138 +1,156 @@ # Troubleshooting -Common issues and how to unstick them. Roughly ordered by how often -they come up. +Common situations and how to unstick them, roughly ordered by how often +they come up. `lc`'s refusals try to carry their own remedy — this page +adds the context around them. -## "No global configuration found." +## "uncommitted changes in …" -`~/.lightcone/config.yaml` is normally created automatically on first -use, but it may be missing if the home directory was unavailable or if -the file was deleted manually. Re-create it by hand: +``` +Error: uncommitted changes in /home/you/my-analysis — every +materialization is committed with the code that produced it, so a run +cannot start from a tree that does not say what that code is. -```bash -mkdir -p ~/.lightcone -cat > ~/.lightcone/config.yaml <<'EOF' -container: - runtime: auto -EOF + commit these: git add -A . && git commit -m "…" + M src/fit.py ``` -Or just run any `lc` command (e.g. `lc --version`) — the auto-creation -runs before every command. +Not an error in your project — just the order of operations: commit, +then materialize. The refusal sorts the paths it found: work you own +gets the `commit these` line, while leftover files under `results/` +(from an interrupted run of an older `lc`, or a hand write) are listed +as wreckage to discard instead — `results/` is `lc`'s to write, and +committing hand-placed files there defeats the provenance the tool +exists for. -## "No astra.yaml found in current directory or any parent." +## "… is not a Lightcone project" -You're outside an ASTRA project. Either: +You're outside a project. The current directory *is* the project — `lc` +never walks up to find one, by design — so: ```bash cd path/to/your/project ``` -or, if you're starting fresh: - -```bash -lc init my-analysis -cd my-analysis -``` - -`lc init` is idempotent — re-running it inside an existing project is -safe and just fills in anything missing (`lc init --check` tells you -whether it would change anything). +or, starting fresh, `lc init my-analysis && cd my-analysis`. If you're +in a fresh clone, run `lc init` once — it rebuilds the `.venv` and the +annex, the two pieces of local state git doesn't carry. ## "lc: command not found" or `lc` prints a directory listing Two possibilities: -1. The package isn't installed for your current Python. Check - `pip show lightcone-cli` (or `uv pip show lightcone-cli`). +1. The tool isn't on `PATH` — with `uv tool install`, that's + `~/.local/bin`; `uv tool update-shell` fixes the profile. 2. Your shell has a personal alias `lc='ls --color'` shadowing the real command. Run `type lc` to see; `unalias lc` to remove. -## `lc run` warning: "No container runtime found on PATH" +## A recipe fails with "Permission denied" or "No module named …" -You declared a container in `astra.yaml` but `auto` couldn't find any -of `docker`, `podman`, or `podman-hpc`. Two options: +Every sandboxed failure ends with this trailer: -- **Install one.** Podman is the smallest install on Linux and macOS. -- **Opt out explicitly.** Edit `~/.lightcone/config.yaml`: - ```yaml - container: - runtime: none - ``` - This silences the warning, but then your manifests will record an - image that didn't actually run — fine for development, not fine for - archival. +``` +this ran under the lc sandbox (landlock) — a permissions or missing-file +error can mean the command reached for something outside the declared +environment +``` -## `lc run` says "Workflow defines that rule … but no input" +Recipes run in the project's locked environment, with the tree +read-only apart from their own output directory. The common cases: -This is Snakemake speak. It usually means: +- **`ModuleNotFoundError`** — the package isn't in the project's lock. + `uv add `, commit, re-run. (Installing it on the host with + `pip` changes nothing a recipe sees — that's the point.) +- **Reading a file outside the project** — declare it as an ASTRA + input; declared inputs are readable and their content becomes part + of the output's provenance. +- **Writing outside the output directory** — a recipe's product + belongs in `{output}`; for true scratch files, use + `tempfile.mkdtemp()`, which lands in the writable temp area. -- A recipe declares `inputs: [foo]` but no other output produces - `foo`. Either the input is external (in which case it shouldn't be - in the recipe's `inputs:` list — recipes only chain to *sibling* - outputs), or there's a typo. -- Sub-analysis output ids that collide with root output ids — qualify - with `.`. +To probe interactively, `lc run ` runs any command under +exactly the isolation a recipe gets — if it works there, it works as a +recipe. -The fix is in `astra.yaml`. `astra validate astra.yaml` will catch -most typos. +## Everything shows `behind` after a `uv add` -## `lc status` shows everything `stale` after I just ran +Not a problem, and nothing was invalidated. `behind` means: the output +is still exactly what the spec asks for, but the environment has moved +since it was made. Environment changes deliberately don't trigger +rebuilds — the manifest records which environment and commit produced +each output, so nothing is lost by leaving it. When you do want them +remade under the current environment: -Something in the spec changed in a way that affects `code_version`. -That hash covers recipe text, container image identifier, and -decisions. Common causes: +```bash +lc materialize --refresh +``` -- You edited a `Containerfile` or a dependency file (`requirements.txt`, - `pyproject.toml`). The image's content-addressed tag changed → - every recipe that uses it is now `stale`. -- You edited a recipe `command:`. Just rerun. -- You changed the default for a decision. +See [Core Concepts](concepts.md) for the `stale` / `behind` +distinction. -Re-running `lc run` will bring everything back to `ok`. +## Everything shows `stale` after a spec edit -## `lc verify` fails with `tampered_data` +`stale` means the spec now defines the output differently than it was +made — you edited its recipe, a decision, or a declared input's +content changed. That's the invalidation model working; the next +`lc materialize` remakes exactly those outputs. -The bytes in an output directory no longer hash to the recorded -`data_version`. Most innocent cause: someone hand-edited a result -file. Most concerning: results were forged. +One edit that deliberately does *not* invalidate: changing your +analysis code (`src/…`). The recipe *string* is the identity, so if +you want code changes to cascade, declare the source file as an ASTRA +input of the outputs it shapes — that choice is yours to make per +output. -If it was you, regenerate with `lc run --force `. If it -wasn't you, audit your shared filesystem. +## "the content is not in this clone" -## `lc verify` fails with `broken_chain` +``` +data/points.csv: the content is not in this clone — git-annex holds a +reference to it, not the data. Fetch it with `git annex get data/points.csv`. +``` -A downstream output was materialized against an upstream version that -no longer exists. Usually caused by: +The clone has the *pointer* to an annexed file but not its bytes. +`lc materialize` fetches the declared inputs it needs by itself; the +read-only verbs (`lc status`, `--check`) never transfer data, so they +report the fact instead. Fetch by hand only when you want the bytes +for your own inspection. -- The upstream was rerun without rerunning the downstream. -- The upstream's output directory was edited by hand (which would also - trigger `tampered_data` on the upstream itself). +## "… and this is a NERSC login node" -Fix: `lc run` the downstream output. The chain will re-anchor. +`lc materialize` executes recipes, and on centers `lc` recognizes it +refuses to do that on a shared login node. The refusal prints the +center's own `salloc` and `sbatch` spellings — copy one, run the same +command inside the allocation. `lc status`, `lc materialize --check`, +`lc build` and `lc run` work anywhere. See +[Running on a Cluster](cluster.md). -## I want to start the spec over +## git doesn't know who you are -Move `astra.yaml` aside (don't delete it — it's useful context about -what you tried), then write a fresh one: +Every output is committed, so a machine that has never committed needs +an identity before the first run — `lc materialize` checks up front, +before any recipe spends time: ```bash -mv astra.yaml astra.previous.yaml -$EDITOR astra.yaml +git config --global user.name "Ada Lovelace" +git config --global user.email "ada@example.org" ``` -Re-running `lc init` afterwards is safe — it only fills in whatever is -missing and leaves the rest of the layout (`universes/`, `.lightcone/`) -as is. +## Containerized projects + +- **"image absent"** — the declared image hasn't been built and + committed yet: `lc build` (announced by materialize too, which + builds it as a preflight when missing). +- **No runtime found** — install [Podman](https://podman.io/) or + [Docker](https://docs.docker.com/get-docker/); detection is + automatic and there is nothing to configure. +- **Architecture mismatch** — the committed archive records the + architecture it was built for, and a host that can't execute it is + refused before the recipe would have died mid-run. Build on a + matching host (on NERSC, a login node), commit, push, and pull on + the other side. ## Filing a bug Open an issue at [github.com/LightconeResearch/lightcone-cli/issues](https://github.com/LightconeResearch/lightcone-cli/issues). Include the output of `lc --version`, the command you ran, and the -error trace. - -## When all else fails - -Run `lc verify` — it's the fastest way to know whether your problem -is provenance (real problem) or a transient build/run issue (rerun). +full message — the refusals are designed to be pasted. diff --git a/zensical.toml b/zensical.toml index 0b6be05c..705ce74c 100644 --- a/zensical.toml +++ b/zensical.toml @@ -14,6 +14,7 @@ nav = [ {"Welcome" = "user/index.md"}, {"Install" = "user/install.md"}, {"Getting Started" = "user/getting-started.md"}, + {"Core Concepts" = "user/concepts.md"}, {"Running on a Cluster" = "user/cluster.md"}, {"Troubleshooting" = "user/troubleshooting.md"}, {"Glossary" = "user/glossary.md"}, From 86e75f75835e17fcff400de0095227f5719ddecb Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:36:44 +0200 Subject: [PATCH 18/23] Hardening: close the residues recorded through layer 8 (#184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebuild is functionally complete through layer 8; this PR closes the recorded residues that do not need Perlmutter access, one commit per decision. Each choice below was settled explicitly before implementation. ## What lands - **Ambient `UV_*` scrub** (closes #179): `project.child_env` drops every `UV_*` variable outside a closed plumbing allowlist (cache dir, timeouts, TLS, air-gap mode, index credentials, uv's recursion guard), so an exported `UV_NO_BINARY` or `UV_PYTHON` can no longer steer a sync while `env_version` reports nothing moved. The run verbs warn with the names of any non-empty variable dropped, from the same predicate. - **Machine-level uv.toml advisory** (#176, option 2): the scan checks uv's two documented user/system config paths for audited install-settings keys — key presence, not file presence, because list settings concatenate across levels — and reports a hit beside `sdist_built`. Never hashed. - **Network uncontrolled on every mechanism**: the OCI wrap drops `--network none`, its one restriction, so all three mechanisms attest `network: allowed` symmetrically and no consumer can read a promise into "containerized". `denied` stays in the type for a mechanism that genuinely emits a denial flag. Retires the Perlmutter spike item about `--network none` hanging on compute nodes. - **Recipe write scope narrowed to its own output directory** (reverses the recorded "not narrowed" decision): closes the cross-write residue — a concurrent task landing bytes in a sibling's directory before the sibling hashed produced a manifest that was self-consistent and wrong, undetectable by any checksum, so prevention is the only fix. A probe keeps `results/` whole (no output id) — the one probe/recipe asymmetry, and it excludes exactly the accident being prevented. The new enforcement test is mutation-checked through `Unavailable()`. - **`uv_version` manifest attestation**: the engine-closure decision's concrete loss, recorded — probed once per run by the driver and handed down (the HEAD discipline), beside `lc_version`. Attestation, never identity. - **Per-file sha256 in the crate, from annex keys**: every output file is a `File` under its dataset's `hasPart` with `sha256`/`contentSize` parsed from its SHA256E key — verifiable by `sha256sum` after a `git archive` deposit, available bytes-free because keys are repository state (`--include=*` is load-bearing). Non-SHA-256 keys yield size and no digest; git-carried files hash their bytes. Also fixes a latent honesty bug: out-of-tree inputs published lc's *framed* digest under the workflow-run `sha256` term; they now publish none. Validator floor unchanged at five. - **`crate:` line in `lc status`**: the rerun-lag residue made visible by comparing the document's own `datePublished` against the newest manifest `finished_at` the walk already read — no git, no rocrate import on status's path. - **End-of-run edit warning**: an edit made while the graph ran (the `git_dirty` window) is now named in the report; still no manifest field, by decision. - **`[project.scripts]` absence pinned**: the recorded review item — a metadata test asserts the worker and the shim never become console scripts. ## Re-examined and deliberately left (recorded in CLAUDE.md) - The forged-run-record-subject residue: the threat model excludes adversaries, and a copied subject is already deliberate. - The validator RECOMMENDED floor stays pinned at five — no publication-metadata config surface. ## Housekeeping Closed stale pre-rebuild issues #166, #128, #153 with pointers; commented triage on #152 and the advisory landing on #176. ## Verification Full suite: 582 passed (sandbox enforcement real on Linux; crate smoke with the official rocrate-validator, REQUIRED clean and the floor unchanged). ruff and mypy clean. Cross-write denial mutation-checked through `Unavailable()` (rc 0, bytes replaced, unsandboxed). Closes #179. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --------- Co-authored-by: Claude Fable 5 --- .github/workflows/eval.yml | 6 +- CLAUDE.md | 290 ++++++++++++++++++------- src/lightcone/cli/commands.py | 1 + src/lightcone/engine/assets.py | 35 ++- src/lightcone/engine/container.py | 12 +- src/lightcone/engine/crate.py | 130 +++++++---- src/lightcone/engine/dataset.py | 80 ++++++- src/lightcone/engine/identity.py | 60 ++++- src/lightcone/engine/materialize.py | 99 +++++++-- src/lightcone/engine/project.py | 156 ++++++++++++- src/lightcone/engine/run.py | 20 +- src/lightcone/engine/sandbox/denial.py | 8 +- src/lightcone/engine/sandbox/model.py | 8 +- src/lightcone/engine/sandbox/oci.py | 8 +- src/lightcone/engine/sandbox/policy.py | 36 +-- src/lightcone/engine/worker.py | 99 +++++---- tests/conftest.py | 35 +++ tests/test_assets.py | 1 + tests/test_cli.py | 5 +- tests/test_container.py | 12 +- tests/test_container_smoke.py | 18 +- tests/test_crate.py | 115 ++++++++-- tests/test_dataset.py | 91 ++++++++ tests/test_identity.py | 58 ++++- tests/test_materialize.py | 147 ++++++++++++- tests/test_project.py | 79 +++++++ tests/test_run.py | 21 ++ tests/test_sandbox_denial.py | 2 +- tests/test_sandbox_enforcement.py | 24 ++ tests/test_sandbox_oci.py | 8 +- tests/test_sandbox_policy.py | 44 +++- tests/test_worker.py | 33 +-- 32 files changed, 1426 insertions(+), 315 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 5b24e23e..4befd2ba 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -8,7 +8,9 @@ name: Eval on: workflow_dispatch: pull_request: - types: [labeled] + # `ready_for_review` beside the defaults, so flipping a draft to + # ready triggers the run the draft guard below skips. + types: [opened, synchronize, reopened, ready_for_review] # Only one eval per PR at a time — cancel in-progress runs concurrency: @@ -21,7 +23,7 @@ permissions: jobs: eval: - if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-eval' + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false runs-on: ubuntu-latest # Step-level timeout on the agent step (below) is what actually # bounds the run: a job-level timeout would cancel the always() diff --git a/CLAUDE.md b/CLAUDE.md index 6675c45c..f160e694 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -202,8 +202,9 @@ Test, lint and type-check are the whole loop, and they are what `.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers around them plus recipes for the frozen docs and the dormant eval. The -other workflows are `eval.yml` (the agentic eval, on dispatch or PR -label), `pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. +other workflows are `eval.yml` (the agentic eval, on dispatch or any +non-draft PR — flipping a draft to ready triggers it), +`pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. ## Key Invariants (layer 1) @@ -691,18 +692,37 @@ two projects that install the same artifacts are one environment however they spell it. `scan_lock` reads `default-groups` through the same function, because it is asking uv's question too. -What this deliberately cannot reach is **user-level configuration** -(`~/.config/uv/uv.toml`), which uv merges in underneath the project's own -(measured). That is machine state, not project state: hashing it would +What this deliberately cannot reach is **machine-level configuration** +(user `~/.config/uv/uv.toml` and system `/etc/uv/uv.toml`), which uv +merges in underneath the project's own (measured). That is machine +state, not project state: hashing it would make one commit answer differently on two hosts, so a colleague's clone -would report every output as behind. The residue is real and unguarded — -a user-level `no-build` changes what a sync installs and nothing records -it. **There is no flag that closes it**: `--config-file` refuses a +would report every output as behind. **There is no flag that closes +it**: `--config-file` refuses a `pyproject.toml` outright, and both `--config-file ` and `--no-config` drop the project's own `[tool.uv]` along with everything else (measured; `--no-config` also drops `[tool.uv.index]`, which is the -GPU mechanism, so adopting it would break working projects). Tracked as -issue #176 with the options and the measurements; don't re-derive them. +GPU mechanism, so adopting it would break working projects). Since the +hardening pass the hole is *annotated at run time* (issue #176's +advisory option): `identity._machine_config` checks the two documented +paths per platform — those levels can only ever be a `uv.toml`, so the +probe is complete, and it tests which *keys* a file sets because list +settings concatenate across levels — and a hit lands in `scan_lock`'s +advisory tier beside `sdist_built`. Never hashed, still. The env-var +spelling of the same hole (`UV_NO_BINARY` and friends) is *closed*, not +annotated: `project.child_env` scrubs ambient `UV_*` outside a plumbing +allowlist (`_UV_KEPT` — cache dir, link mode, the managed-interpreter +store and its mirror, timeouts, TLS, air-gap, credentials, uv's own +recursion guard), and every uv-acting verb names the non-empty +variables dropped through `project.uv_scrub_warning` — one composer, +one predicate with the scrub: convergence puts it in the report (so +`lc init` says it), materialize in its warnings, and the probe in its +outcome's notes, which is why the CLI never composes it (issue #179). +`lc build` deliberately says nothing: the image's uv runs inside the +container build, where the host environment does not reach. The suite +blinds itself to the host's machine config via +the autouse `machine_uv_config` fixture — `/etc/uv/uv.toml` has no +environment variable to scrub. **The git commit is recorded, never hashed, and never a signal.** It goes in the manifest so the code that produced a result stays recoverable. It @@ -1090,10 +1110,10 @@ design. `lc --help` advertising it would hand people a footgun, and a `uv tool install`. `lightcone/_sandbox_exec.py` is the same shape for the same reason. Keep it cheap to import — **no click, no rich** — it is on the path of every task and every rerun. -`test_the_worker_module_imports_neither_click_nor_rich` pins the imports -and `test_help_does_not_advertise_the_worker` the absence from `--help`; -nothing pins the absence of a `[project.scripts]` entry, so treat that -as a review item. +`test_the_worker_module_imports_neither_click_nor_rich` pins the imports, +`test_help_does_not_advertise_the_worker` the absence from `--help`, and +`test_the_worker_and_the_shim_are_never_console_scripts` the absence of +a `[project.scripts]` entry. **The record's format is datalad's, so it is tested through datalad.** `get_run_info` matches with a regex and returns `(None, None)` on any @@ -1112,25 +1132,22 @@ one thing it does put on disk is the per-run private `$HOME` `rmtree` of that directory has one owner. (`wrap` stays pure; the impurity lives in policy construction, once.) -**There is one policy, `exec_policy`, and a recipe gets exactly what a -probe gets.** The tree is read-only apart from `results/`, for both. So -layer 5's promise is not "in the direction that matters" — it is simply -true: a command that works under `lc run` works as a recipe, with nothing -in between to reason about. - -A recipe is deliberately **not** narrowed to its own output directory. -That would be a second answer to "are these bytes what produced them", -and the manifest's `data_version` is the first — content-addressed, -checked by `lc verify`, and the only one that survives a rebuild on -another machine. Two mechanisms for one guarantee is one more than can be -kept honest, and the sandbox's is the one that cannot travel. - -The residue, recorded rather than argued away: a cross-write that lands -*before* the victim task hashes leaves a manifest that is self-consistent -and wrong, which no checksum can see. It needs concurrent tasks and a -hardcoded sibling path, and the threat model here is accidental leakage -rather than a hostile recipe — but `lc verify` will not catch that one, so -do not describe it as covered. +**There is one policy, `exec_policy`, and it differs between a recipe +and a probe by one keyword.** The tree is read-only apart from the +in-tree write scope: a recipe's is **its own output directory** +(`output_dir=`, handed down from the worker's task), a probe's is +`results/` whole, because a probe has no output id. This narrowing is +the hardening pass reversing an earlier decision (see Recorded +decisions): it exists as leak *prevention*, closing the cross-write +residue — a concurrent task landing bytes in a sibling's directory +before the sibling hashes produced a manifest that was self-consistent +and wrong, which no checksum could ever see, so prevention was the only +possible fix. The probe→recipe promise ("a command that works under +`lc run` works as a recipe") now excludes exactly the commands that +write outside their own output directory — which is the accident being +prevented, not a loophole in the promise. Integrity-answering is still +`data_version`'s job alone; the sandbox prevents the write, it does not +attest the bytes. **`cluster_for_run()` is the seam, and it is two methods wide.** `submit(fn, *args, key=…)` and `completed(handles)`. That is all the @@ -1148,8 +1165,13 @@ provides that, and for what workers do *not* need (git, git-annex). refusal makes it constant. The limitation that comes with that, stated: the check is at start of run while manifests are written per-output much later, so a user who edits `src/fit.py` while a long graph runs gets a - manifest whose `git_sha` no longer describes the code that ran, and - nothing records it. + manifest whose `git_sha` no longer describes the code that ran. Since + the hardening pass the run *says* so — one `dataset.status` call after + the consume loop, warning with the edited paths (the tree started + clean and save/restore keeps `results/` clean, so any dirt appeared + mid-run) — but still records nothing in the manifest: the driver does + not rewrite files the worker owns, and per-output attribution would + need a per-save probe nothing has asked for yet. - **The manifest carries what this layer can honestly fill.** `schema_version`, `output_id`, `universe_id`, `recipe`, `definition_version`, `env_version`, `data_version`, `decisions`, @@ -1161,12 +1183,18 @@ provides that, and for what workers do *not* need (git, git-annex). `started_at` / `finished_at` (ISO 8601 UTC, **millisecond** precision because RO-Crate consumers parse `endTime` with at most three fractional digits; attestation like `lc_version`, defaulted `""`, - never read by `classify`). Spec §3's longer list — - `uv_version`, `platform`, `worker_runtime`, `python_build`, + never read by `classify`), and since the hardening pass `uv_version` + (probed once per run by the driver — `project.uv_version` — and + handed down, the HEAD discipline; the rerun entry point probes its + own; empty on failure, attestation must not fail a run). Spec §3's + remaining list — + `platform`, `worker_runtime`, `python_build`, `dpkg_snapshot_sha256`, `sdist_built`, `env_snapshot`, `gpu_driver` — is attestation nothing here reads; it lands with the verb that reads it (`worker_runtime` is additionally derivable from - `hermeticity.mechanism`, so it may never land at all). + `hermeticity.mechanism`, so it may never land at all — the hardening + pass considered `platform` too and took only `uv_version`, the one + field the engine-closure decision named as its concrete loss). - **`env_version` has four terms.** Layer 6 added the image term — the system layer's identity document, hashed as the literal `null` for a direct project so the formula stays one formula. (The spec's @@ -1328,10 +1356,20 @@ follows POSIX — unlinking a directory needs write on its *parent* — so a recipe granted write on its output directory cannot remove that directory. Seatbelt's `(allow file-write* (subpath …))` covers the directory node, and permits it. Found by CI, which is the point of -running one suite on both. Nothing depends on either answer (the worker -resets the directory anyway, and a recipe that removes it fails to record -its output), so the asymmetry is documented rather than papered over — -but a test that asserts one mechanism's answer will go red on the other. +running one suite on both. Since the hardening pass narrowed a recipe's +write scope to its own output directory, the asymmetry sits exactly on +the node a ported recipe likes to `rm -rf` as a prelude (measured on +real Landlock: `rm -rf "$OUT" && mkdir "$OUT"` is EACCES on Linux and +succeeds on macOS) — so such a recipe is green on a laptop and red on +the Linux venue. Nothing in *lc* depends on either answer: the worker +resets the directory before every execution, so the prelude is +redundant and the fix is deleting it; likewise a temp-then-rename +through a `results/` sibling now fails on both platforms — scratch +belongs in `$TMPDIR` (the private HOME), which is what the write-denial +remedy says. The asymmetry cannot be closed without granting write on +the *parent*, which is precisely the sibling-write hole the narrowing +exists to close; documented rather than papered over, and a test that +asserts one mechanism's answer will go red on the other. **SBPL is last-match-wins; Landlock unions.** This asymmetry decides where a rule can live, and it cuts both ways. A later `(deny …)` can take @@ -1369,9 +1407,10 @@ stays self-describing without one. **The engine never enters the image.** The container is the *recipe's* execution world: driver, git, annex, dask and classification all stay the host's `lc`, and exactly two things ever run in-image — the environment -converge (`container.sync`, network on, project `:rw`, host uv cache -mounted, into `.lightcone/venv`) and each recipe/probe exec -(`--network none`). This deviates from spec v6.1's full-stack rule, +converge (`container.sync`, project `:rw`, host uv cache mounted, into +`.lightcone/venv`) and each recipe/probe exec (mount table only — the +converge differs by its writable mounts, not by network: the network is +uncontrolled on every mechanism). This deviates from spec v6.1's full-stack rule, recorded: v6.1's reason was the host-sync deadlock, which the in-container sync solves, and the spec's own Perlmutter row ("recipe wrap, step 3 only — never the dask worker") is this exact shape. What it @@ -1465,7 +1504,8 @@ attested by the manifest's image id and the archive's bytes. Read from containerized `exec_policy` shape is the same policy minus the host: no OS read baseline, no stdlib root, no exec set — the image *is* those, and everything in it was declared — leaving exactly the paths that -become mounts, project `:ro`, `results/` `:rw`, declared inputs `:ro`, +become mounts, project `:ro`, the write scope `:rw` (a recipe's own +output directory; a probe's `results/`), declared inputs `:ro`, the private HOME `:rw`, `--tmpfs /tmp`, over a **`--read-only` rootfs**: without that flag a write outside the declared set *succeeds* into the container's ephemeral layer and vanishes while the run attests @@ -1479,9 +1519,9 @@ every bind read and `:z` would relabel the user's own files. No in-container Landlock, no seccomp probe, no shim-in-image: the engine container never gets the tree `:rw`, so mounts alone express the whole policy, and the attestation (`mechanism: podman|docker`, -`fs: declared`, `network: denied`) is derived flag-for-flag from the -argv — `--network none` is the codebase's one honest `denied`, loopback -intact. One `OCIBackend`, data-parameterized: podman and docker differ +`fs: declared`, `network: allowed`) is derived flag-for-flag from the +argv — no flag touches the network, the same non-control every +mechanism attests. One `OCIBackend`, data-parameterized: podman and docker differ in spellings (`--userns=keep-id`+`--pull=never` vs `--user uid:gid`), not shape. Runtime is **host capability** — detected podman → docker, the @@ -1521,8 +1561,9 @@ guarantees. discipline, a frozen `container.Runtime` through `worker.materialize` — because resolving per task could answer differently mid-run. The rerun entry point resolves its own, as it does HEAD: it *is* the driver of its -one-task run. `lc status` gained the three header lines (mode, image -state, sandbox) — repository facts only, no runtime and no network +one-task run. `lc status` gained the header lines (mode, image +state, sandbox — and, since the hardening pass, crate) — repository +facts only, no runtime and no network required, which is where the denial note and the runtime-missing refusal point. @@ -1665,11 +1706,10 @@ analogue is a one-time archive→SIF conversion into gitignored `.lightcone/` cache keyed by the same runtime-independent config-blob id (the reason `docker-archive` stays the store format). `container.backend()` stays the single construction point; a future -world-backend is one dataclass in `sandbox/` plus one branch there. A -runtime that cannot deny network must attest `network: allowed` with no -denial flag emitted — no consumer may assume containerized ⇒ denied, -and `_sandbox_line`'s containerized prose is the one place that -assumption lives today. `boundary.run`'s exit-125 note is a +world-backend is one dataclass in `sandbox/` plus one branch there. +Since the hardening pass every mechanism attests `network: allowed` +with no denial flag emitted, so a store-less runtime has nothing to +imitate there. `boundary.run`'s exit-125 note is a podman/docker-family fact, not a `contains_prefix` fact — it becomes mechanism-keyed when a non-OCI backend lands. @@ -1680,8 +1720,7 @@ behavior inside salloc/sbatch steps; `nidXXXXXX` resolution from peer nodes (else `--interface hsn0`); `SLURM_CPUS_ON_NODE` on a CPU node (128 vs 256 hyperthreads); cold-Lustre `distributed` import vs the 120 s worker wait; `podman-hpc migrate` accepting a bare image id and -re-running cheaply; `--network none` on compute nodes (the -`network: denied` attestation hangs on it); podman-hpc `--module` +re-running cheaply; podman-hpc `--module` site-injected mounts vs the honesty of `fs: declared` (the one item that could add a flag); whether Landlock is in the SLES boot LSM list (either answer is handled — a host without it attests `fs: open` with @@ -1701,7 +1740,22 @@ is `git archive` / `datalad export-archive` on a repository that is already a crate; nothing is copied and there is no bundle directory. The rerun entry point deliberately does not regenerate it (it is one task's executor, not the driver), so the crate lags until the next -materialize — recorded residue. +materialize — and since the hardening pass `lc status` says so: its +`crate:` line compares the document's own `datePublished` against the +newest manifest `finished_at` the status walk already read — a content +check, no git and no rocrate import (the recorded constraint that the +crate stays the one materialize-only dependency on status's path). The +`datePublished` pin is therefore load-bearing twice: it keeps the clock +out of the render *and* it is what makes the lag detectable. The line's +claim is scoped to what the proxy can see and worded to it ("up to date +**with the outputs**" / "behind **the outputs**"): a crate-affecting +edit that moves no manifest — the lock's bytes in a `File` entity, a +spec edit — is invisible here, and fine, because the next materialize +converges those anyway; a rerun's lag is the case with no other +surface, and the proxy is exact for it in both directions (a dropped +output regresses the newest stamp just as a rerun advances it). +`license_of` and `CRATE_FILENAME` moved to `project.py` so +status can ask about publication intent without the renderer's stack. **Publication intent is derived, never configured.** A `[project].license` in `pyproject.toml` turns crate maintenance on — @@ -1750,6 +1804,28 @@ JSON-LD drops on expansion — the pre-rebuild exporter's silent failure. The committed archive is one entity, `["File", "ContainerImage"]`, identity (`sha256` = config-blob id) and payload together. +**A published `sha256` is always a raw digest an outsider can verify** — +since the hardening pass, never lc's framed hash. Every file in an +output directory is a `File` under its dataset's `hasPart`, with +`sha256` and `contentSize` parsed from its SHA256E annex key +(`dataset.annex_keys`, one `git annex find --include=*` — the +`--include=*` is load-bearing, bare `find` lists only *present* files +and the crate must answer bytes-free; the key map is injected into +`render` like the writer, so the builder stays git-free). A non-SHA-256 +backend key yields size and no digest, never a wrong one; git-carried +files hash their own working-tree bytes. Out-of-tree declared inputs +publish *no* digest: their recorded `input_versions` value is the framed +hash, which shipped once under the `sha256` term as though `sha256sum` +could check it — the manifests keep that story. The dataset's `version` +stays lc's framed directory digest, deliberately distinct from the +per-file claims. The move re-scoped what an in-tree input's `sha256` +*means*, and the old conflict rule went with it deliberately: a `File` +entity's checksum now describes the deposit — the bytes a `git archive` +carries — never what any particular run consumed, so two manifests +recording different digests for a shared input (a half-rebuilt project) +no longer suppress it; which bytes a *run* consumed is its own +manifest's `input_versions`, in the crate as that manifest `File`. + **The validator floor is pinned as a set, not a count.** `tests/test_crate_smoke.py` materializes a real project and runs the official `rocrate-validator` (dev dependency) against Provenance Run @@ -1846,13 +1922,16 @@ unlinks before writing; a new tampering test should too. own image). One venue cost to remember: the `assets.Versions` memo degrades to once **per worker process**, so a declared input shared by many tasks is re-hashed per process — efficiency, not correctness. - - *The engine's dependency closure left the record entirely, and is not - replaced.* The project lock used to pin what the engine resolved — - most concretely the git-annex build that wrote the bytes. Now - `lc_version` names the engine and nothing pins what it was made of. - Accepted rather than re-provided: the alternative is hashing an + - *The engine's dependency closure left the record entirely, and is + mostly not replaced.* The project lock used to pin what the engine + resolved — most concretely the git-annex build that wrote the bytes. + Now `lc_version` names the engine, the hardening pass added + `uv_version` beside it (the one closure member that decides which + artifacts a lock installs), and nothing pins the rest. Accepted + rather than re-provided: the alternative is hashing an environment lc does not own into artifacts it does, which is the - over-sensitivity the `behind` model exists to avoid. + over-sensitivity the `behind` model exists to avoid — both fields + are attestation, never identity. - *Layer 6 resolved its note the other way:* the image gets **no engine layer at all**. The engine stays on the host in containerized mode too — the container is the recipe's world, never the engine's — @@ -1865,9 +1944,10 @@ unlinks before writing; a new tampering test should too. resolved from PyPI; and recipes no longer find `lc` or `git-annex` on the sandbox PATH — the project `.venv/bin` no longer carries them, which is the boundary telling the truth. The `UV_*` ambient scrub the - launcher would have done is still worth having and is tracked - separately; it protects `env_version`'s install-settings term, not the - delegation that is gone. + launcher would have done landed in the hardening pass, in + `project.child_env` — it protects `env_version`'s install-settings + term, not the delegation that is gone (see the layer-2 residue note + for the allowlist). - **Reads stay restricted, and the OS baseline is ours to maintain.** Codex restricts reads too now, but its Linux read baseline is a *mount table*, not a path list — there is nothing to adopt there. Keeping the @@ -1948,6 +2028,44 @@ unlinks before writing; a new tampering test should too. cross-builds) is layer 7's, with the venue that makes it real. Old archives accumulate in annex history; reclaiming them is the user's `git annex unused`/`drop` — no GC verb. +- **The hardening pass** (2026-08, post-layer-8) closed the residues that + did not need Perlmutter, and re-examined two it then deliberately left: + - *A recipe is narrowed to its own output directory* — reversing the + layer-5 "not narrowed" decision. The old rationale rejected the + sandbox as a second *integrity* mechanism, and that half stands + (`data_version` remains the only answer to "are these bytes what + produced them"); what the narrowing is instead is leak *prevention*, + closing the cross-write residue, whose corruption was undetectable by + construction. A probe keeps `results/` whole (no output id), the one + probe/recipe asymmetry. + - *The network is uncontrolled everywhere*: the OCI wrap dropped + `--network none`, its one restriction, so all three mechanisms attest + `network: allowed` symmetrically and no consumer can read a promise + into "containerized". `denied` stays in the `Attestation` type for a + mechanism that genuinely emits a denial flag. + - *The forged run-record subject stays a recorded residue*, re-examined + and left: the threat model is accidental damage and shortcuts, and a + copied `[DATALAD RUNCMD]` subject is already deliberate — a + body-verifying comparator raises the forgery cost without changing + who it stops, and nothing history-based is adversary-proof anyway. + - *The crate validator floor stays pinned at five*: three entries are + publisher/affiliation metadata lc genuinely does not know, and a + `[tool.lightcone.publication]` surface was considered and rejected — + revisit when a real deposit target demands it. + - *lc's commits are partial commits, and a frozen execution worktree + is deferred, not rejected.* `dataset.save` commits with the same + pathspec it stages, so each save is built from HEAD plus its own + paths alone — work the user staged while a graph ran stays staged + and is named by the end-of-run warning, never swept into an output + or crate commit. The stronger move — executing in a dedicated + `git worktree` at the starting commit, which would also freeze the + *code* a long run reads — was considered and deferred to the venue + era: the branch dance under a live checkout, result propagation + back into it, and a second environment/mid-run-gate story outweigh + a hazard the warning now names precisely, and the case that makes + it genuinely worth it (editing on a login node while an `sbatch` + materialize runs for hours) arrives with the submission-model + venue. ### Recorded deviations from the spec @@ -1970,12 +2088,16 @@ unlinks before writing; a new tampering test should too. invokes the shim on lc's own interpreter, so writer and reader are the same lightcone-cli by construction, and a compatibility field would be backward-compat machinery with no consumer. -- **Network is not controlled, on either platform**, by decision. §7's +- **Network is not controlled, on any mechanism**, by decision. §7's matrix has Seatbelt record `denied`; the generated SBPL explicitly - allows network and both platforms attest `network: allowed`. Symmetric - and honest — nothing pretends to a control it does not apply. (codex - ships a seccomp denylist for this; adding one is a live option, not a - gap we are hiding.) + allows network, and since the hardening pass the OCI backend emits no + `--network` flag either (it briefly shipped `--network none`, dropped + for consistency: three mechanisms, one answer). Every mechanism + attests `network: allowed` — symmetric and honest, nothing pretends + to a control it does not apply. (codex ships a seccomp denylist for + this; adding one is a live option, not a gap we are hiding — and it, + or a runtime that genuinely denies, is what the `denied` literal in + `Attestation` is reserved for.) - **`lc run` has no rename guard and no sandbox flags.** §4's guard against `lc run ` existed only for muscle memory from the pre-rebuild CLI — backward compatibility we do not promise — and §7's @@ -1985,7 +2107,8 @@ unlinks before writing; a new tampering test should too. - **The denial's remedies are only what works today** — `uv add` for a Python package, the system layer (`apt-install` + the containerize note, real since layer 6), the ASTRA input declaration for data, and - "output goes in `results/`" plus `tempfile.mkdtemp()` for a write + "a recipe writes only its own output directory; a probe writes + `results/`" plus `tempfile.mkdtemp()` for a write denial. Nothing in a denial message names a verb, flag, or declaration that does not exist. - **`Attestation` has no serializer of its own.** An earlier draft @@ -1995,17 +2118,20 @@ unlinks before writing; a new tampering test should too. into the manifest's `hermeticity` field. `lc run` is still a probe with no output (§4), so there the attestation is returned and printed, never persisted. -- **`results/` is writable; the rest of the tree is not**, where §4 gives - a probe no output and therefore no in-tree write scope at all. A probe - gets the same write scope a recipe does, so a probe that works means a - recipe will — and the environment a run starts with is the one it - finishes with. +- **The in-tree write scope is writable; the rest of the tree is not**, + where §4 gives a probe no output and therefore no in-tree write scope + at all. A probe writes `results/` whole; a recipe, since the hardening + pass, writes only its own output directory (the cross-write closure — + see the layer-4 policy invariant and Recorded decisions) — and the + environment a run starts with is the one it finishes with. - **The shape was chosen because all three mechanisms express it natively.** A writable directory *nested inside* a read-only tree is the widening direction: Landlock unions rights over ancestors, SBPL restates the write tier after the guard, and podman mounts the - project `:ro` with `results` `:rw` over it — all verified by running - them. The reverse — a writable tree with `.venv` carved out — needs + project `:ro` with the write scope `:rw` over it — all verified by + running them, and the narrowing to one output directory is the same + shape one level deeper. The reverse — a writable tree with `.venv` + carved out — needs rights *subtraction*, which podman and SBPL can do and **Landlock cannot at all**. That asymmetry is the whole argument: the read-only shape is the only one direct mode and containerized mode can both @@ -2059,7 +2185,7 @@ unlinks before writing; a new tampering test should too. | Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | | Change what a run commits | `src/lightcone/engine/materialize.py` + `tests/test_materialize.py` | The driver owns git alone; the tree ends as clean as it started | | Change where a run executes | `src/lightcone/engine/venue.py` + `materialize.cluster_for_run` + `tests/test_venue.py` | One detection ladder, in `cluster_for_run` alone; venues are detected, never configured; test by faking the host (env vars + a stub srun), never the code | -| Change what the crate says | `src/lightcone/engine/crate.py` + `tests/test_crate.py` | Pure builder: sorted iteration, no clock, git injected as `writer`; structure tests, never byte goldens — the one byte-level claim is render-twice-identical. The validator floor lives in `tests/test_crate_smoke.py::_FLOOR` | +| Change what the crate says | `src/lightcone/engine/crate.py` + `tests/test_crate.py` | Pure builder: sorted iteration, no clock, git injected as `writer` and the annex key map as `keys`; structure tests, never byte goldens — the one byte-level claim is render-twice-identical. The validator floor lives in `tests/test_crate_smoke.py::_FLOOR` | | Change how a foreign write is detected | `dataset.last_writer` + `materialize._foreign_write` + `tests/test_dataset.py` | History, never hashing; `datalad_run_subject` is the one spelling of the record's subject; a foreign write classifies `stale` in every verb | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | | Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index ed58b020..75319b24 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -369,6 +369,7 @@ def status(as_json: bool) -> None: }[state] lines.append(f" image: {tag} — {described}") lines.append(f" sandbox: {report.sandbox}") + lines.append(f" crate: {report.crate}") lines.append("") marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} width = max((len(o.output) for o in report.outputs), default=0) diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index 92175ac0..40bdfd37 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -201,11 +201,8 @@ def require_fetched(path: Path) -> None: """ if path.is_symlink() and not path.exists(): unfetched = _ANNEX_OBJECTS in path.readlink().as_posix() - elif path.stat().st_size > _POINTER_MAX_BYTES: - unfetched = False else: - with path.open("rb") as f: - unfetched = f.read(len(_POINTER_PREFIX)) == _POINTER_PREFIX + unfetched = is_pointer(path) if unfetched: raise ContentNotFetchedError( f"{path}: the content is not in this clone — git-annex holds a " @@ -213,6 +210,30 @@ def require_fetched(path: Path) -> None: ) +def is_pointer(path: Path) -> bool: + """Test whether a regular file holds an annex pointer, not content. + + git-annex's own ``isPointerFile`` rule, spelled once: a file no + larger than 32 KiB whose bytes begin ``/annex/objects/``. The locked + shape — a symlink into the object store — is a separate question the + callers ask themselves, because what a symlink means differs by + caller. + + Args: + path: An existing regular file. + + Returns: + Whether it is a pointer. + + Raises: + OSError: If the file cannot be read. + """ + if path.stat().st_size > _POINTER_MAX_BYTES: + return False + with path.open("rb") as f: + return f.read(len(_POINTER_PREFIX)) == _POINTER_PREFIX + + def _feed(h: hashlib._Hash, path: Path) -> None: """Stream *path* into *h* — outputs are not assumed to fit in memory.""" with path.open("rb") as f: @@ -266,6 +287,12 @@ class Manifest: #: existed, not back-compat machinery. started_at: str = "" finished_at: str = "" + #: The uv that resolved and installed the environment the recipe ran + #: in — the one tool between the lock and the installed artifacts. + #: Attestation, like ``lc_version``: outside both hashes, never a + #: rebuild signal, defaulted empty because that is the true value for + #: a manifest written before the field existed. + uv_version: str = "" #: The image the recipe ran in — ``{tag, id, archive, arch}`` — or #: ``None`` on the host. Defaulted, and that is not back-compat #: machinery: ``None`` is the *true* value for every manifest a diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py index 347b44a7..0a7b8540 100644 --- a/src/lightcone/engine/container.py +++ b/src/lightcone/engine/container.py @@ -355,9 +355,9 @@ def sync(root: Path, runtime: Runtime) -> list[str]: """Converge ``.lightcone/venv`` inside the image. The containerized twin of ``project.sync``. - The one container run that gets the network and a writable project - mount — converge once, then execute without writing to the - environment, the same discipline as direct mode. The host's uv cache + The one container run that gets a writable project mount — converge + once, then execute without writing to the environment, the same + discipline as direct mode. The host's uv cache is mounted at its identical path, so a complete environment materializes from cache hits in about a second; the cache location is ``uv cache dir``'s answer, never a guess, because that is uv's own @@ -414,7 +414,9 @@ def converge(runtime: Runtime) -> list[str]: return sync(runtime.root, runtime) -def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: +def policy_for( + runtime: Runtime, read_paths: list[Path], *, output_dir: Path | None = None +) -> sandbox.Policy: """Build the exec policy for a resolved runtime. The one place the ``env_dir``/``containerized`` pair is assembled — @@ -426,6 +428,7 @@ def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: Args: runtime: The resolved runtime. read_paths: Declared inputs, as :func:`sandbox.exec_policy` takes. + output_dir: A recipe's own output directory; absent for a probe. Returns: The policy for this world. @@ -435,6 +438,7 @@ def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: read_paths=read_paths, env_dir=runtime.env_dir, containerized=runtime.mode == "containerized", + output_dir=output_dir, ) diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py index a3ead986..78c10c9b 100644 --- a/src/lightcone/engine/crate.py +++ b/src/lightcone/engine/crate.py @@ -26,10 +26,12 @@ from __future__ import annotations +import bisect +import hashlib import json -import tomllib +import re import uuid -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import Any @@ -42,8 +44,6 @@ from lightcone.engine.plan import Graph, Key from lightcone.engine.project import SPEC_FILENAME -CRATE_FILENAME = "ro-crate-metadata.json" - #: The vocabulary the run-level facts come from. Without it in the #: ``@context``, terms like ``containerImage`` and ``sha256`` are #: undefined and silently dropped on JSON-LD expansion — typed but inert. @@ -63,32 +63,15 @@ #: data entities and every action's ``object`` list cannot drift apart. _ENVIRONMENT = ("pyproject.toml", "uv.lock", ".python-version") +#: A SHA-256-backed annex key: ``SHA256E-s--<64 hex>`` (the E +#: backend keeps the extension). The hex *is* the raw sha256 of the +#: content, which is what makes a checksum publishable with none of the +#: bytes fetched. Any other backend yields a size and no digest — never +#: a wrong one. +_SHA256_KEY = re.compile(r"^SHA256E?-s(\d+)--([0-9a-f]{64})(?:\..*)?$") -def license_of(root: Path) -> str: - """Read the project's declared license out of ``pyproject.toml``. - - Presence is what turns crate maintenance on: RO-Crate requires a - license, a run must not refuse over a missing key, and inventing one - would assert terms over someone's data — so declaring - ``[project].license`` is declaring the intent to publish. - - Args: - root: The project root. - - Returns: - The license as declared — an SPDX expression, a URL, free text, - or a file path for the table forms — or empty when undeclared. - """ - try: - data = tomllib.loads((root / "pyproject.toml").read_text()) - except (OSError, tomllib.TOMLDecodeError): - return "" - declared = data.get("project", {}).get("license") - if isinstance(declared, str): - return declared - if isinstance(declared, dict): - return str(declared.get("text") or declared.get("file") or "") - return "" +#: Any backend key's size field, for ``contentSize`` alone. +_KEY_SIZE = re.compile(r"^\w+-s(\d+)") def render( @@ -98,6 +81,7 @@ def render( license: str, dsid: str, writer: Callable[[Path], LastWrite], + keys: Mapping[str, str], ) -> str: """Build the crate document for the project as it stands. @@ -109,17 +93,21 @@ def render( Args: root: The project root. graph: The full task graph — every universe, every output. - license: The declared license, from :func:`license_of`. + license: The declared license, from :func:`project.license_of`. dsid: The dataset UUID, the namespace absolute entity ids are minted under so they are stable across clones. writer: Answers "which commit last touched this path" — :func:`dataset.last_writer` bound to the root, injected so the builder stays free of git. + keys: Each annexed file's key, repository-relative — + :func:`dataset.annex_keys`'s answer, injected for the same + reason as *writer*. SHA-256-backed keys become per-file + checksums an archive can verify with ``sha256sum``. Returns: The ``ro-crate-metadata.json`` text, trailing newline included. """ - build = _Builder(root, graph, license, dsid, writer) + build = _Builder(root, graph, license, dsid, writer, keys) return build.document() @@ -137,6 +125,7 @@ def __init__( license: str, dsid: str, writer: Callable[[Path], LastWrite], + keys: Mapping[str, str], ) -> None: from astra.helpers import load_yaml @@ -144,6 +133,11 @@ def __init__( self.graph = graph self.license = license self.writer = writer + self.keys = dict(keys) + #: Sorted once: each output selects its files by bisecting this, + #: not by rescanning the whole map — the map holds every annexed + #: file in the repository, data/ included. + self.sorted_keys = sorted(self.keys) self.crate = ROCrate() self.crate.metadata.extra_contexts.append(_WORKFLOW_RUN_CONTEXT) #: Every materialized task, sorted: the one iteration order. @@ -311,11 +305,47 @@ def _environment_files(self) -> None: readme["about"] = {"@id": "./"} def _file(self, name: str) -> Any: + # Idempotent by id, so the second asker (the license file is + # asked for by the workflow and the root) does not hash again. + if (existing := self.crate.dereference(name)) is not None: + return existing properties: dict[str, Any] = {} if fmt := _format_of(name): properties["encodingFormat"] = fmt + properties.update(self._integrity(name)) return self.crate.add_file(self.root / name, name, properties=properties) + def _integrity(self, name: str) -> dict[str, str]: + """``sha256`` and ``contentSize`` for one repository file. + + An annexed file answers from its key — the working tree may hold + only a pointer, and hashing that would publish a digest of the + wrong bytes — and a git-carried file from the bytes themselves. + Both are repository state, so the render stays pure. A file + neither annexed nor readable carries no claim at all. + + The byte path re-checks the pointer shape rather than trusting + the key map's absence: ``annex_keys`` answers empty for a whole + repository whenever git-annex cannot answer at all, and a + pointer file reads perfectly well — so without the guard, one + failed ``git annex find`` would publish a well-formed digest of + the pointer text for every annexed file, silently. + """ + if key := self.keys.get(name): + if digest := _SHA256_KEY.match(key): + return {"contentSize": digest.group(1), "sha256": digest.group(2)} + if size := _KEY_SIZE.match(key): + return {"contentSize": size.group(1)} + return {} + path = self.root / name + try: + if path.is_symlink() or assets.is_pointer(path): + return {} + data = path.read_bytes() + except OSError: + return {} + return {"contentSize": str(len(data)), "sha256": hashlib.sha256(data).hexdigest()} + def _dataset_id(self, key: Key) -> str: """One output directory's crate id — :func:`plan.declared_path`'s answer, never a second spelling of the results layout.""" @@ -331,6 +361,15 @@ def _dataset(self, key: Key, manifest: assets.Manifest) -> None: manifest_file = self._file(f"{dataset_id}{assets.MANIFEST_FILENAME}") manifest_file["about"] = {"@id": dataset_id} entity["subjectOf"] = {"@id": manifest_file.id} + # Every file the directory holds, each with the checksum its + # annex key already carries — the claim `sha256sum` can check + # after a `git archive` deposit, where `version` above is lc's + # own framed directory digest and deliberately is not that. + parts = [manifest_file] + lo = bisect.bisect_left(self.sorted_keys, dataset_id) + hi = bisect.bisect_left(self.sorted_keys, dataset_id + "\uffff") + parts += [self._file(name) for name in self.sorted_keys[lo:hi]] + entity["hasPart"] = [{"@id": part.id} for part in parts] # ----- the runs ----- @@ -374,7 +413,7 @@ def _objects(self, key: Key, manifest: assets.Manifest) -> list[dict[str, str]]: if upstream in self.made_keys: refs.append({"@id": self._dataset_id(upstream)}) continue - refs.append({"@id": self._external(name, task.inputs[name], manifest)}) + refs.append({"@id": self._external(name, task.inputs[name])}) # The *recorded* decisions: the values the recipe actually ran # under, whatever the spec says today. A decision the workflow no # longer declares gets no exampleOfWork — there is no parameter @@ -394,33 +433,36 @@ def _objects(self, key: Key, manifest: assets.Manifest) -> list[dict[str, str]]: refs.append({"@id": value.id}) return refs - def _external(self, name: str, path: Path, manifest: assets.Manifest) -> str: + def _external(self, name: str, path: Path) -> str: """A declared input the spec points at, in or out of the tree. In-or-out is :func:`plan.declared_path`'s answer — relative inside the tree, absolute outside it — never a second spelling of that rule here: two copies of one path rule is how the first one shipped a bug. + + An in-tree input's checksum comes from its annex key, like every + other file: a ``File`` entity's ``sha256`` describes the + *deposit* — the bytes a ``git archive`` carries — never what any + particular run consumed, so manifests disagreeing about a shared + input (a half-rebuilt project) do not suppress it; which bytes a + run consumed is its own manifest's ``input_versions``. An + out-of-tree input carries none: its recorded digest is lc's + *framed* hash, not a raw sha256, so publishing it under the + workflow-run ``sha256`` term would be a checksum nothing can + verify — the manifests keep the full story, which is the layer's + stated weaker promise. """ declared = plan.declared_path(self.root, path) in_tree = not Path(declared).is_absolute() entity_id = declared if in_tree else Path(declared).as_uri() - recorded = manifest.input_versions.get(name, "") - digest = recorded.removeprefix("sha256:") if recorded.startswith("sha256:") else "" - if (existing := self.crate.dereference(entity_id)) is not None: - # Two manifests can testify to different bytes for one input - # — a half-rebuilt project. Publishing either digest would - # contradict a manifest in the same crate, so publish - # neither; the manifests themselves keep the full story. - if digest and existing.get("sha256") not in (None, digest): - existing.pop("sha256") + if self.crate.dereference(entity_id) is not None: return entity_id properties: dict[str, Any] = {"@type": "File", "name": declared} if fmt := _format_of(declared): properties["encodingFormat"] = fmt - if digest: - properties["sha256"] = digest if in_tree: + properties.update(self._integrity(declared)) self.crate.add_file(path, declared, properties=properties) else: # Outside the repository: recorded by content, not stored diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index a59f73ad..def066fb 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -231,16 +231,42 @@ def last_writer(directory: Path, path: Path) -> LastWrite: git cannot answer at all. """ argv = ["log", "-1", "--format=%H%x00%s%x00%an%x00%ae%x00%as", "--", _rel(directory, path)] - try: - proc = project._run(["git", *argv], cwd=directory) - except OSError: - return LastWrite() - out = str(proc.stdout or "").strip("\n") - if proc.returncode != 0 or not out: + out = _ask(argv, cwd=directory) + if not (out := (out or "").strip("\n")): return LastWrite() return LastWrite(*out.split("\0")) +def annex_keys(directory: Path) -> dict[str, str]: + """Map every annexed file to its key, repository-relative. + + One process for the whole tree. ``--include=*`` is load-bearing: a + bare ``find`` lists only files whose *content* is present, and the + crate must answer on a clone that holds none of the bytes — the keys + are repository state, which is what keeps the render pure. + + "Cannot say" — no annex, no git, an unborn repository — is the empty + answer, never an error, the :func:`last_writer` discipline. + + Args: + directory: The project root. + + Returns: + ``{relative path: key}`` for every annexed file. + """ + out = _ask(["annex", "find", "--include=*", "--format=${file}\\t${key}\\n"], cwd=directory) + keys: dict[str, str] = {} + for line in (out or "").splitlines(): + # From the *last* tab: git-annex emits ${file} unescaped, so a + # tab in a filename would otherwise split inside the path and + # hand back a truncated file with a corrupted key. Keys never + # contain tabs, so the rightmost split is always the real one. + file, sep, key = line.rpartition("\t") + if sep and file and key: + keys[file] = key + return keys + + def save(directory: Path, paths: Iterable[Path], message: str) -> bool: """Commit *paths*. @@ -270,6 +296,14 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: Per-add and never written to the repository's config, so a user's own ``git add`` keeps git-annex's stock behavior. + The commit carries the same pathspec as the add, making it a + *partial* commit: git builds it from HEAD plus these paths alone and + leaves anything else in the index staged and untouched. Without the + pathspec, ``git commit`` commits the whole index — so a file the + user staged while a graph was running would be swept, silently, + into whichever save landed next. The end-of-run warning names such + edits; this is what keeps lc's commits from eating them. + Args: directory: The repository root. paths: What to stage, absolute or repository-relative. @@ -279,13 +313,16 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: False if there was nothing to commit. """ relative = [_rel(directory, p) for p in paths] - _git( - ["-c", "annex.thin=true", "-c", "annex.dotfiles=true", "add", "-A", "--", *relative], - cwd=directory, - ) - if _git_ok(["diff", "--cached", "--quiet"], cwd=directory): + annex = ["-c", "annex.thin=true", "-c", "annex.dotfiles=true"] + _git([*annex, "add", "-A", "--", *relative], cwd=directory) + # Scoped like the commit: foreign staged content must neither count + # as "something to commit" here nor be committed below. + if _git_ok(["diff", "--cached", "--quiet", "--", *relative], cwd=directory): return False - _git(["commit", "-q", "-m", message], cwd=directory) + # The annex config rides on the commit too: a partial commit takes + # the paths' content through the clean filter again, and without the + # flags that pass would route the bytes by stock rules. + _git([*annex, "commit", "-q", "-m", message, "--", *relative], cwd=directory) return True @@ -313,6 +350,25 @@ def restore(directory: Path, paths: Iterable[Path]) -> None: # ============================================================================= +def _ask(argv: list[str], *, cwd: Path) -> str | None: + """Run git where "cannot say" must be an answer, never an error. + + The read-only-verbs discipline, as one seam: an unborn HEAD, a + stripped ``.git``, a host without git — states a project can really + be in — come back as ``None``, and the caller renders its own empty. + + Returns: + git's stdout, or ``None`` when git cannot answer. + """ + try: + proc = project._run(["git", *argv], cwd=cwd) + except OSError: + return None + if proc.returncode != 0: + return None + return str(proc.stdout or "") + + def _git(argv: list[str], *, cwd: Path) -> str: """Run git in *cwd*, returning its stdout; a nonzero exit raises.""" proc = project._run(["git", *argv], cwd=cwd) diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py index 505444c8..68567443 100644 --- a/src/lightcone/engine/identity.py +++ b/src/lightcone/engine/identity.py @@ -41,6 +41,7 @@ import hashlib import json +import os import re import tomllib from collections.abc import Mapping @@ -174,6 +175,10 @@ class LockScan: #: Groups outside uv's default set. Advisory: they are installable #: states `env_version` does not distinguish. non_default_groups: tuple[str, ...] + #: Machine-level uv config files setting audited install settings. + #: Advisory: they steer the sync underneath the project's own + #: settings, and ``env_version`` deliberately cannot see them. + machine_config: tuple[str, ...] def scan_lock(root: Path) -> LockScan: @@ -225,9 +230,54 @@ def scan_lock(root: Path) -> LockScan: refusals=tuple(sorted(refusals)), sdist_built=tuple(sorted(sdist_built)), non_default_groups=tuple(sorted(non_default)), + machine_config=_machine_config(), ) +def _machine_config_paths() -> tuple[Path, ...]: + """uv's user- and system-level config files, per its documented rule. + + These levels can only ever be a ``uv.toml`` — never a + ``pyproject.toml`` — so two known paths per platform make the probe + complete rather than a heuristic. + """ + if os.name == "nt": + return tuple( + Path(os.environ[var]) / "uv" / "uv.toml" + for var in ("APPDATA", "PROGRAMDATA") + if var in os.environ + ) + config_home = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") + config_dirs = os.environ.get("XDG_CONFIG_DIRS") or "/etc/xdg" + return ( + config_home / "uv" / "uv.toml", + *(Path(d) / "uv" / "uv.toml" for d in config_dirs.split(":") if d), + Path("/etc/uv/uv.toml"), + ) + + +def _machine_config() -> tuple[str, ...]: + """Name the machine-level uv config files that steer install settings. + + uv merges user- and system-level configuration *underneath* the + project's, and list settings concatenate across levels — a user-level + ``no-binary-package`` adds to the project's — so the test is which + keys a file sets, never whether the file exists. Deliberately never + hashed: machine state in ``env_version`` would make one commit answer + differently on two hosts. A file uv itself cannot parse is skipped — + the sync fails loudly on it without our help. + """ + findings = [] + for path in _machine_config_paths(): + try: + data = tomllib.loads(path.read_text()) + except (OSError, tomllib.TOMLDecodeError): + continue + if keys := sorted(set(data) & set(_INSTALL_SETTINGS)): + findings.append(f"{path} sets {', '.join(keys)}") + return tuple(findings) + + # ============================================================================= # Reading the project's files # ============================================================================= @@ -262,11 +312,11 @@ def _uv_config(root: Path) -> dict[str, Any]: differ when uv installs the same thing in each. uv warns about the pair itself, and convergence already lifts its warnings. - What this cannot reach is user-level configuration - (``~/.config/uv/uv.toml``), which uv *does* merge in underneath. That - is machine state rather than project state: hashing it would make one - commit answer differently on two hosts, reporting every output as - behind on a colleague's clone. + What this cannot reach is user- and system-level configuration, which + uv *does* merge in underneath. That is machine state rather than + project state: hashing it would make one commit answer differently on + two hosts, reporting every output as behind on a colleague's clone. + :func:`_machine_config` reports it instead. """ config = root / "uv.toml" if not config.is_file(): diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 12fea41a..331d7b37 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -317,6 +317,10 @@ class StatusReport: image: dict[str, str] | None = None #: One line naming the enforcement a run here would get. sandbox: str = "" + #: Where the publication view stands — maintained, and if so whether + #: it still reflects the outputs. Repository facts only, like the + #: rest of the header. + crate: str = "" @property def counts(self) -> dict[str, int]: @@ -336,6 +340,7 @@ def as_dict(self) -> dict[str, Any]: "mode": self.mode, "image": self.image, "sandbox": self.sandbox, + "crate": self.crate, "counts": self.counts, "outputs": [output.as_dict() for output in self.outputs], "warnings": self.warnings, @@ -367,7 +372,10 @@ def status(root: Path) -> StatusReport: if state != "direct": result.image = {"tag": tag, "state": state, "archive": archive} result.sandbox = _sandbox_line(result.mode) + stamps = [] for key, verdict, manifest, foreign in _classified(root, [], report, refresh=False): + if manifest and manifest.finished_at: + stamps.append(manifest.finished_at) result.outputs.append( OutputStatus( output=_name(key), @@ -378,10 +386,46 @@ def status(root: Path) -> StatusReport: foreign_write=foreign.sha if foreign else "", ) ) + result.crate = _crate_line(root, max(stamps, default="")) result.warnings = report.warnings return result +def _crate_line(root: Path, newest: str) -> str: + """One line placing the publication view, from repository facts alone. + + Lag is read off the document itself, not history: the render pins + ``datePublished`` to the newest manifest ``finished_at``, so a date + that no longer matches the manifests — in either direction, a rerun + adds an output the view predates and a dropped output regresses the + newest — means the view no longer describes the outputs. That is the + line's whole claim, and it is worded to it: a crate-affecting edit + that moves no manifest (the lock's bytes, the spec) is invisible + here, and fine — the next materialize converges those anyway, where + a rerun's lag has no other surface. No rocrate import (the crate is + the one materialize-only dependency on status's path) and no git: + the manifests were already read by the walk. + """ + spdx = project.license_of(root) + path = root / project.CRATE_FILENAME + if not spdx: + if path.is_file(): + return "no longer maintained — pyproject.toml declares no [project].license" + return "not maintained — declare [project].license to enable it" + if not path.is_file(): + return "will be created by the next `lc materialize`" + try: + entities = json.loads(path.read_text()).get("@graph", []) + published = next( + (str(e.get("datePublished", "")) for e in entities if e.get("@id") == "./"), "" + ) + except (OSError, ValueError, AttributeError): + return "unreadable — the next `lc materialize` rewrites it" + if newest and published != newest: + return "behind the outputs — `lc materialize` refreshes it" + return "up to date with the outputs" + + def _foreign_write(root: Path, key: Key) -> dataset.LastWrite | None: """Find the commit that last touched *key*'s directory, unless it is the output's own run record — then ``None``, the clean answer. The @@ -403,7 +447,7 @@ def _sandbox_line(mode: str) -> str: """ if mode == "containerized": if runtime := container.runtime_hint(): - return f"{runtime} (fs: declared, network: denied)" + return f"{runtime} (fs: declared, network: allowed)" return "no container runtime — install podman (or docker)" from lightcone.engine import sandbox @@ -449,6 +493,8 @@ def materialize( project.require_git_annex() dataset.require_committer(root) report = MaterializeReport() + if warning := project.uv_scrub_warning(): + report.warnings.append(warning) # The dirty check comes before anything that writes: the image # converge below *commits*, and `dataset.save` stages scoped but # commits the whole index — on a dirty tree the user's staged edits @@ -496,15 +542,19 @@ def materialize( # rerun does not come through here; its entry point converges too.) report.warnings.extend(f"uv: {w}" for w in container.converge(runtime)) - # Read once, for every task: the driver commits each output as it lands, - # so HEAD moves during the run, and a per-task read would give later - # manifests a commit this run created — nondeterministically, depending - # on whether a recipe finished before or after the previous save. - head = dataset.head(root) - # One memo for the run, for the same reason as one HEAD read: a - # declared input shared by several outputs — or by one output across - # several universes — is the same bytes every time it is asked for. - versions = assets.Versions() + # The run's driver-resolved facts, each read once: HEAD because the + # driver commits as outputs land and a per-task read would stamp + # later manifests with a commit this run created; the uv probe + # because attestation is a fact about the run (and empty is an + # answer, not a failure); one content-hash memo because a declared + # input shared by several outputs is the same bytes every time. + context = worker.RunContext( + env_version=env_version, + head=dataset.head(root), + versions=assets.Versions(), + runtime=runtime, + uv_version=project.uv_version(root), + ) # The history question is the driver's to answer — workers have no # git, by design — so each task is told up front whether its # directory was last written by something other than its own run @@ -532,12 +582,9 @@ def materialize( worker.materialize, root, task, - env_version, - head, - versions, + context, refresh, foreign[key], - runtime, *[pending[dep] for dep in task.depends_on], key=_name(key), ) @@ -551,6 +598,17 @@ def materialize( # survive. for task in outstanding.values(): dataset.restore(root, [task.output_dir]) + # The tree was clean at the start-of-run refusal and save/restore + # keeps `results/` clean, so anything dirty *now* was edited while + # the graph ran — and every manifest records the starting commit, + # which no longer describes that code. A warning, never a manifest + # field: the driver does not rewrite files the worker owns. + if edited := dataset.status(root): + names = ", ".join(sorted(path for _, path in edited)) + report.warnings.append( + f"edited while the run was in flight: {names} — the manifests " + "record the starting commit, which no longer describes this code" + ) _converge_crate(root, report, full, dsid) return report @@ -751,12 +809,12 @@ def _converge_crate(root: Path, report: MaterializeReport, full: Graph, dsid: st # crate is the one materialize-only dependency on their shared path. from lightcone.engine import crate - spdx = crate.license_of(root) - path = root / crate.CRATE_FILENAME + spdx = project.license_of(root) + path = root / project.CRATE_FILENAME if not spdx: report.warnings.append( f"pyproject.toml no longer declares [project].license, so " - f"{crate.CRATE_FILENAME} is no longer maintained" + f"{project.CRATE_FILENAME} is no longer maintained" if path.exists() else "no [project].license in pyproject.toml, so no RO-Crate " "publication view is maintained — declare one to enable it" @@ -776,6 +834,7 @@ def _converge_crate(root: Path, report: MaterializeReport, full: Graph, dsid: st license=spdx, dsid=dsid, writer=functools.partial(dataset.last_writer, root), + keys=dataset.annex_keys(root), ) if not (path.is_file() and path.read_text() == document): path.write_text(document) @@ -939,6 +998,12 @@ def _graph( "dependency groups outside uv's default set are installable states " f"the environment's identity does not distinguish: {', '.join(scan.non_default_groups)}" ) + if scan.machine_config: + report.warnings.append( + "machine-level uv configuration steers install settings underneath " + "the project's own, and env_version cannot see it: " + + "; ".join(scan.machine_config) + ) env_version = identity.env_version(root) full = plan.build(root) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index a139f245..836ca691 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -20,6 +20,9 @@ SPEC_FILENAME = "astra.yaml" +#: The publication view `lc materialize` converges at the project root. +CRATE_FILENAME = "ro-crate-metadata.json" + class ProjectError(Exception): """A project cannot be read or converged.""" @@ -195,6 +198,8 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: require_git_annex() c = _Converger(write=write) + if warning := uv_scrub_warning(): + c.warn(warning) if write: directory.mkdir(parents=True, exist_ok=True) @@ -515,6 +520,37 @@ def mode(directory: Path) -> Literal["direct", "containerized"]: return "direct" if image._table(directory) is None else "containerized" +def license_of(directory: Path) -> str: + """Read the project's declared license out of ``pyproject.toml``. + + Presence is what turns crate maintenance on: RO-Crate requires a + license, a run must not refuse over a missing key, and inventing one + would assert terms over someone's data — so declaring + ``[project].license`` is declaring the intent to publish. The same + derived-never-configured shape as :func:`mode`, and it lives here so + ``lc status`` can ask without importing the crate renderer's stack. + + Args: + directory: The project root. + + Returns: + The license as declared — an SPDX expression, a URL, free text, + or a file path for the table forms — or empty when undeclared. + """ + import tomllib + + try: + data = tomllib.loads((directory / "pyproject.toml").read_text()) + except (OSError, tomllib.TOMLDecodeError): + return "" + declared = data.get("project", {}).get("license") + if isinstance(declared, str): + return declared + if isinstance(declared, dict): + return str(declared.get("text") or declared.get("file") or "") + return "" + + def env_dir(directory: Path) -> Path: """Locate the project environment for this project's mode. @@ -652,18 +688,126 @@ def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: ) +#: The ambient ``UV_*`` variables :func:`child_env` keeps. Plumbing only — +#: where bytes come from and how fast, never *what* a sync installs: the +#: cache location (shared-filesystem hosts point it at scratch), network +#: timeouts and concurrency, TLS trust, air-gap mode, and index +#: credentials. Anything with install semantics (``UV_NO_BINARY``, +#: ``UV_PYTHON``, ``UV_INDEX_URL``, …) is dropped: the same settings are +#: hashed into ``env_version`` when a project declares them, so an ambient +#: spelling would steer a sync while every hash agrees nothing changed. +_UV_KEPT = frozenset( + { + "UV_CACHE_DIR", + "UV_HTTP_TIMEOUT", + "UV_REQUEST_TIMEOUT", + "UV_CONCURRENT_BUILDS", + "UV_CONCURRENT_DOWNLOADS", + "UV_CONCURRENT_INSTALLS", + "UV_NATIVE_TLS", + "UV_INSECURE_HOST", + "UV_OFFLINE", + # How package content lands (hardlink/copy/symlink) — the same + # line the install-settings hash draws: `link-mode` is not an + # audited setting either. + "UV_LINK_MODE", + # The managed-interpreter store — the shared-filesystem story + # `UV_CACHE_DIR` is kept for, and *which* interpreter is pinned + # by `.python-version`, not by where its bytes live. There is no + # project-level spelling for this one, so scrubbing it would + # come with a remedy that does not exist. + "UV_PYTHON_INSTALL_DIR", + # Where the pinned interpreter downloads from, not which one. + "UV_PYTHON_INSTALL_MIRROR", + # Auth plumbing, the credentials family. + "UV_KEYRING_PROVIDER", + # uv's own recursion guard, set on every `uv run` child — lc + # itself frequently *is* one. Dropping it disables the guard and + # makes the scrub report uv's variable as the user's. + "UV_RUN_RECURSION_DEPTH", + } +) + + +def _uv_scrubbed(name: str) -> bool: + """Decide whether one ambient variable is dropped by the UV scrub.""" + if not name.startswith("UV_"): + return False + # UV_INTERNAL__* is uv talking to its own children, never a setting. + if name in _UV_KEPT or name.startswith("UV_INTERNAL__"): + return False + # Index credentials (UV_INDEX__USERNAME / _PASSWORD) are how a + # private registry authenticates; the registry itself is the + # project's `[tool.uv.index]` to declare. + return not ( + name.startswith("UV_INDEX_") and name.endswith(("_USERNAME", "_PASSWORD")) + ) + + def child_env() -> dict[str, str]: """Build the environment external tools run in. - Ours, minus ``VIRTUAL_ENV``. Every uv invocation names its project - explicitly, so an activated environment elsewhere is never what we - mean — and uv warns once per invocation when it ignores one, which - would otherwise land in the report and in ``--json``. + Ours, minus ``VIRTUAL_ENV`` and minus every ``UV_*`` variable outside + the :data:`_UV_KEPT` plumbing allowlist. Every uv invocation names its + project explicitly, so an activated environment elsewhere is never + what we mean — and an ambient install setting would change what a + sync installs without moving ``env_version``, which is the identity + hole the scrub closes. + + Returns: + The current environment without ``VIRTUAL_ENV`` or scrubbed ``UV_*``. + """ + return { + k: v + for k, v in os.environ.items() + if k != "VIRTUAL_ENV" and not _uv_scrubbed(k) + } + + +def uv_scrub_warning() -> str: + """Compose the dropped-ambient-variables warning, once for every verb. + + A user whose ``UV_PYTHON`` or ``UV_INDEX_URL`` stopped steering uv + deserves a pointer to why on *whichever* verb they hit first — + ``lc init`` resolving against the wrong index fails with uv's raw + error otherwise. One spelling here, and one predicate with + :func:`child_env`, so the verbs cannot drift from each other or the + report from the scrub. Empty variables steer nothing and are not + reported. + + Returns: + The warning, or ``""`` when nothing non-empty was dropped. + """ + if dropped := sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)): + return ( + f"ignored ambient {', '.join(dropped)} — an install setting is " + "the project's to declare (pyproject.toml), and an ambient one " + "would steer uv without moving env_version" + ) + return "" + + +def uv_version(directory: Path) -> str: + """Ask uv its version, for the manifest's attestation. + + Read once per run by whoever owns the run — the driver, or the rerun + entry point — and handed down, the HEAD discipline. Empty on any + failure: attestation must never fail a run. + + Args: + directory: Where to run the probe. Returns: - The current environment without ``VIRTUAL_ENV``. + The version token (``0.12.5``), or ``""``. """ - return {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} + try: + proc = _run(["uv", "--version"], cwd=directory) + except OSError: + return "" + words = str(proc.stdout or "").split() + if proc.returncode != 0 or len(words) < 2 or words[0] != "uv": + return "" + return words[1] def _check_call(argv: list[str], *, cwd: Path) -> list[str]: diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index e406909d..766ec59e 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -15,11 +15,18 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import replace from pathlib import Path from typing import Any from lightcone.engine import container, sandbox -from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv, uv_prefix +from lightcone.engine.project import ( + SPEC_FILENAME, + child_env, + require_uv, + uv_prefix, + uv_scrub_warning, +) def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: @@ -53,14 +60,14 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: built = container.policy_for(runtime, input_paths(project, spec)) with sandbox.scope(built) as policy: - return sandbox.run( + outcome = sandbox.run( container.backend(runtime), policy, list(command), cwd=project, # The direct hop converges; the containerized one must not — - # the converge above already did, and the exec runs with the - # network denied, where a sync cannot. + # the converge above already did, into the in-image + # environment the hop is about to enter. prefix=uv_prefix(project, sync=runtime.mode == "direct"), # Same reason as convergence: this uv invocation names its # project explicitly, so an environment activated elsewhere @@ -68,6 +75,11 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: # the middle of the probe's own output. env=child_env(), ) + # The probe is what called `child_env`, so the probe's outcome is + # where the scrub's fact belongs — the caller prints notes verbatim. + if warning := uv_scrub_warning(): + outcome = replace(outcome, notes=(warning, *outcome.notes)) + return outcome def read_spec(project: Path) -> dict[str, Any]: diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py index 713a8b3e..c19325ec 100644 --- a/src/lightcone/engine/sandbox/denial.py +++ b/src/lightcone/engine/sandbox/denial.py @@ -184,9 +184,11 @@ def _render_write(path: Path) -> list[str]: return _message( f"cannot write {path}", [ - " output goes in results/ — the rest of the tree is read-only, so", - " the environment a run starts with is the one it ends with. For", - " anything that is not output, write somewhere scratch:", + " a recipe writes only its own output directory ({output} in the", + " recipe); a probe writes results/. The rest of the tree is", + " read-only, so the environment a run starts with is the one it", + " ends with. For anything that is not output, write somewhere", + " scratch:", " import tempfile; tempfile.mkdtemp() # or $TMPDIR", ], ) diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index ea4fdb39..b8ad809b 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -87,10 +87,10 @@ class Attestation: """The hermeticity record for one exec. Derived from the flags actually applied, never from the mechanism - matrix's expectations. ``network`` is ``allowed`` wherever lc applied - no restriction — the direct-mode mechanisms — and ``denied`` only - where a flag actually denied it, which today is the OCI backend's - ``--network none``. + matrix's expectations. ``network`` is ``allowed`` everywhere today — + lc controls the filesystem, not the network, and every mechanism says + so identically. ``denied`` stays in the type for a mechanism that + genuinely emits a denial flag; nothing may attest it without one. """ mechanism: Literal["landlock", "seatbelt", "podman", "docker", "podman-hpc", "none"] diff --git a/src/lightcone/engine/sandbox/oci.py b/src/lightcone/engine/sandbox/oci.py index 3cd07f5c..0acbf4e9 100644 --- a/src/lightcone/engine/sandbox/oci.py +++ b/src/lightcone/engine/sandbox/oci.py @@ -84,7 +84,6 @@ def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: overlay = [f"--env={k}={v}" for k, v in sorted(policy.env.items())] return [ self.runtime, "run", "--rm", - "--network", "none", "--entrypoint", "", # The rootfs is read-only so a write outside the declared set # is a loud denial rather than bytes vanishing with the @@ -113,9 +112,9 @@ def attest(self, policy: Policy) -> Attestation: Every value is a flag in :meth:`wrap`'s output: the mounts plus the read-only rootfs bound the filesystem to the declared set, - and ``--network none`` is a real denial — the one place in the - codebase that honestly says ``denied`` (loopback stays intact, - which is the meaning of it). + and no flag touches the network — ``allowed``, the same answer + every mechanism gives, because lc does not control the network + anywhere and the attestation says only what was enforced. Args: policy: The policy being wrapped. @@ -126,5 +125,4 @@ def attest(self, policy: Policy) -> Attestation: return Attestation( mechanism=self.runtime, fs="declared", - network="denied", ) diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py index d3e7fbf1..feaf59d9 100644 --- a/src/lightcone/engine/sandbox/policy.py +++ b/src/lightcone/engine/sandbox/policy.py @@ -6,7 +6,8 @@ The shape it encodes is what a container gives the command, minus the container: the project and the declared inputs and the OS baseline -readable, ``results/`` and a private scratch scope writable, and only +readable, the in-tree write scope (a recipe's own output directory; a +probe's ``results/``) and a private scratch scope writable, and only the project's own environment plus a versioned utility allowlist runnable. What it catches is a command reaching *outside* that set — a tool, a library, or a data file that is on this machine and would not be @@ -180,19 +181,18 @@ def exec_policy( read_paths: Sequence[Path] = (), env_dir: Path | None = None, containerized: bool = False, + output_dir: Path | None = None, ) -> Policy: """Build what a sandboxed command may touch. - The tree is read-only apart from ``results/``, so ``lc run`` and a - recipe get the same scope: a command that works under one works under - the other. - - A recipe is not narrowed to its own output directory. Whether an - output's bytes are its own is what the manifest's ``data_version`` - answers, and that answer travels; a second mechanism for it would not. - The residue: a cross-write landing before the victim task hashes - leaves a manifest that is self-consistent and wrong, which no checksum - can see. + The tree is read-only apart from the write scope: a recipe writes its + own output directory and nothing else in the tree, so a concurrent + task cannot land bytes in a sibling's directory before that sibling + hashes — the one corruption ``data_version`` could never see, because + the manifest it produces is self-consistent and wrong. A probe has no + output, so ``lc run`` gets ``results/`` whole; the probe→recipe + promise therefore excludes exactly the commands that write outside + their own output directory, which is the accident being prevented. The containerized shape is the same policy with the host stripped out: the *image* is the OS baseline and the exec set — everything @@ -207,11 +207,14 @@ def exec_policy( ``.venv``. containerized: Build the mount-shaped policy instead of the host one. + output_dir: The one in-tree directory a recipe may write; absent + for a probe, which gets ``results/`` whole. Returns: - The policy. ``results/`` is granted only if it exists — a policy - describes, it does not prepare. Creates the per-run HOME on disk; - the caller owns removing it (see + The policy. The in-tree write scope is granted only if it exists — + a policy describes, it does not prepare (the worker resets the + output directory before building one). Creates the per-run HOME on + disk; the caller owns removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). """ env_dir = env_dir if env_dir is not None else project / ".venv" @@ -228,6 +231,7 @@ def exec_policy( for sub in _HOME_LAYOUT.values(): (tmp_home / sub).mkdir(parents=True, exist_ok=True) + in_tree_write = output_dir if output_dir is not None else project / "results" if containerized: # Declared spellings, not realpaths — the one shape that keeps # its paths unresolved. These become mount *destinations*, and a @@ -237,7 +241,7 @@ def exec_policy( # *source* side itself.) return Policy( read=_declared([project, *read_paths]), - write=_declared([tmp_home, project / "results"]), + write=_declared([tmp_home, in_tree_write]), execute=(), tmp_home=tmp_home, env=home_overlay(tmp_home, env_dir, containerized=True), @@ -247,7 +251,7 @@ def exec_policy( # EXECUTE on the interpreter *file*; READ on the install root beside # it, for the stdlib. See :func:`_venv_python` and :func:`_stdlib_root`. stdlib = _stdlib_root(python) - write = _existing([tmp_home, project / "results", *_write_roots(project)]) + write = _existing([tmp_home, in_tree_write, *_write_roots(project)]) read = _existing([project, *read_paths, *stdlib, *(Path(p) for p in _OS_READ_BASELINE)]) return Policy( diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index bbed7208..5d019224 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -36,7 +36,7 @@ from pathlib import Path from typing import Literal -from lightcone.engine import assets, container, dataset, identity, plan, sandbox, venue +from lightcone.engine import assets, container, dataset, identity, plan, project, sandbox, venue from lightcone.engine.plan import Key, Task from lightcone.engine.project import ( ProjectError, @@ -85,6 +85,32 @@ def usable(self) -> bool: return self.status in ("ok", "current", "behind") +@dataclass(frozen=True) +class RunContext: + """The driver-resolved facts of one run, handed to every task. + + Each field is read or resolved exactly once, by whoever owns the run + — the driver, or the rerun entry point — because a per-task read + could answer differently mid-run: HEAD moves as the driver commits, + a runtime resolved twice could disagree, and a provenance field that + depends on task timing is worse than either answer. Frozen and + picklable, so it crosses to workers by value; one object, so the + next attestation field is one line here rather than an edit to five + signatures. + """ + + #: The run's environment identity, checked either side of each recipe. + env_version: str + #: The run's ``(commit sha, origin URL)``. + head: Head + #: The run's content-hash memo for declared inputs. + versions: assets.Versions + #: The execution world — the host mechanism, or the project image. + runtime: container.Runtime + #: The uv that converges environments this run. Attestation only. + uv_version: str + + # ============================================================================= # The Dask unit: decide, then execute # ============================================================================= @@ -93,12 +119,9 @@ def usable(self) -> bool: def materialize( root: Path, task: Task, - env_version: str, - head: Head, - versions: assets.Versions, + context: RunContext, refresh: bool, foreign: dataset.LastWrite | None, - runtime: container.Runtime, *upstream: TaskResult, ) -> TaskResult: """Make *task* if it needs making. What Dask submits, once per task. @@ -111,19 +134,12 @@ def materialize( Args: root: The project root. task: The output to make. - env_version: The run's environment identity, checked either side - of the recipe. - head: The run's ``(commit sha, origin URL)``, read once by the - driver because it commits as outputs land and HEAD moves. - versions: The run's content-hash memo for declared inputs. + context: The run's driver-resolved facts. refresh: Whether to remake an output that is merely behind. foreign: The commit that last wrote the output's directory in place of its own run record, or ``None`` — answered by the driver, because history is git's and workers have no git; handed to the one classification rule, where it is `stale`. - runtime: The execution world, resolved once by the driver — the - same discipline as *head*, because resolving per task could - answer differently mid-run. *upstream: The results of this task's dependencies, arriving as the futures it was given — which is what makes Dask the scheduler rather than a loop here. @@ -132,9 +148,7 @@ def materialize( What happened. Never raises. """ try: - return _materialize( - root, task, env_version, head, versions, refresh, foreign, runtime, upstream - ) + return _materialize(root, task, context, refresh, foreign, upstream) except Exception as e: # the contract is that this function returns return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") @@ -142,12 +156,9 @@ def materialize( def _materialize( root: Path, task: Task, - env_version: str, - head: Head, - versions: assets.Versions, + context: RunContext, refresh: bool, foreign: dataset.LastWrite | None, - runtime: container.Runtime, upstream: tuple[TaskResult, ...], ) -> TaskResult: reported = {u.key: u for u in upstream if u.usable} @@ -157,19 +168,19 @@ def _materialize( live = {key: u.data_version for key, u in reported.items()} inputs = { - name: live[key] if (key := task.produced_by.get(name)) else versions.of(path) + name: live[key] if (key := task.produced_by.get(name)) else context.versions.of(path) for name, path in task.inputs.items() } manifest = assets.read(task.output_dir) verdict = assets.classify( definition_version=task.definition_version, - env_version=env_version, + env_version=context.env_version, manifest=manifest, inputs=inputs, foreign=foreign, ) if verdict.calls_for_a_remake(refresh=refresh): - return execute(root, task, env_version, inputs, head=head, runtime=runtime) + return execute(root, task, inputs, context) # Left alone, so the bytes on disk stand. Their *recorded* digest, # never a recomputed one: on a clone that has fetched no annex content @@ -189,36 +200,29 @@ def _materialize( def execute( root: Path, task: Task, - env_version: str, input_versions: Mapping[str, str], - *, - head: Head, - runtime: container.Runtime, + context: RunContext, ) -> TaskResult: """Run *task*'s recipe and record what it produced. The output directory is reset first: the recipe owns it, and a file left from a previous run would otherwise enter the content hash and be - committed as part of an output that never produced it. + committed as part of an output that never produced it. The context's + ``env_version`` is checked either side of the recipe, so a mid-run + lock edit cannot be recorded as if it had been in force. Args: root: The project root. task: The output to make. - env_version: The run's environment identity, checked either side - of the recipe so a mid-run lock edit cannot be recorded as if - it had been in force. input_versions: Each declared input's content identity, recorded in the manifest as the chain. - head: The run's ``(commit sha, origin URL)``. - runtime: The execution world the recipe enters — the host under - the platform's mechanism, or the project image behind its - mount table. + context: The run's driver-resolved facts. Returns: ``ok`` with the output's ``data_version``, or ``failed``. Commits nothing and never touches git beyond reading HEAD. """ - if moved := _gate(root, env_version): + if moved := _gate(root, context.env_version): return TaskResult(task.key, "failed", reason=moved) # The whole directory, not a list of expected files: a recipe declares @@ -232,11 +236,11 @@ def execute( task.output_dir.mkdir(parents=True) read_paths = [p for p in task.inputs.values() if p.exists()] - policy = container.policy_for(runtime, read_paths) + policy = container.policy_for(context.runtime, read_paths, output_dir=task.output_dir) started_at = _now() with sandbox.scope(policy): outcome = sandbox.run( - container.backend(runtime), + container.backend(context.runtime), policy, [_SHELL, "-c", task.recipe], cwd=root, @@ -252,14 +256,14 @@ def execute( reason=f"the recipe exited {outcome.returncode}", notes=outcome.notes, ) - if moved := _gate(root, env_version): + if moved := _gate(root, context.env_version): return TaskResult(task.key, "failed", reason=moved, notes=outcome.notes) # Guarded separately from the boundary catch above it, because these # two failures deserve different words: "your recipe failed" and "your # recipe worked and we could not record it" are different problems. try: - sha, remote = head + sha, remote = context.head data_version = assets.data_version(task.output_dir) assets.write( task.output_dir, @@ -268,17 +272,18 @@ def execute( universe_id=task.universe_id, recipe=task.recipe, definition_version=task.definition_version, - env_version=env_version, + env_version=context.env_version, data_version=data_version, decisions=dict(task.decisions), input_versions=dict(input_versions), git_sha=sha, git_remote=remote, lc_version=lc_version(), + uv_version=context.uv_version, hermeticity=asdict(outcome.attestation), started_at=started_at, finished_at=finished_at, - image=runtime.manifest_image(), + image=context.runtime.manifest_image(), ), ) except (OSError, ProjectError) as e: @@ -389,10 +394,14 @@ def main(argv: list[str]) -> int: result = execute( root, task, - identity.env_version(root), _from_disk(task), - head=dataset.head(root), - runtime=runtime, + RunContext( + env_version=identity.env_version(root), + head=dataset.head(root), + versions=assets.Versions(), + runtime=runtime, + uv_version=project.uv_version(root), + ), ) except ProjectError as e: print(f"error: {e}", file=sys.stderr) diff --git a/tests/conftest.py b/tests/conftest.py index 9be56af2..51536f3c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,41 @@ def venue_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv(name, raising=False) +@pytest.fixture(autouse=True) +def ambient_uv(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip scrubbable ``UV_*`` out of the suite's environment. + + CI pins its matrix interpreter through an ambient ``UV_PYTHON``, + which the scrub correctly drops and reports — so without this every + converge in the suite carries the warning and every ``warnings == + []`` assertion depends on the host. Derived from the scrub's own + predicate, so a variable the allowlist later admits stops being + stripped here for free; the scrub tests set their own variables + back deliberately. + """ + import os + + from lightcone.engine.project import _uv_scrubbed + + for name in [k for k in os.environ if _uv_scrubbed(k)]: + monkeypatch.delenv(name) + + +@pytest.fixture(autouse=True) +def machine_uv_config(monkeypatch: pytest.MonkeyPatch) -> None: + """Blind the suite to the host's machine-level uv configuration. + + The advisory probe reads ``~/.config/uv/uv.toml`` and + ``/etc/uv/uv.toml`` — host state a fixture cannot scrub through the + environment (``/etc`` has no variable), so a developer's own config + would add a warning to every scan. The probe's own tests monkeypatch + the paths back to fixtures deliberately. + """ + from lightcone.engine import identity + + monkeypatch.setattr(identity, "_machine_config_paths", tuple) + + @pytest.fixture(autouse=True) def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: """Fake every external tool convergence shells out to, so the suite is diff --git a/tests/test_assets.py b/tests/test_assets.py index ad6a07cd..e2ef5fed 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -40,6 +40,7 @@ def _manifest(**overrides: object) -> Manifest: "git_sha": "abc123", "git_remote": "https://example/demo.git", "lc_version": "0.4.2", + "uv_version": "0.12.5", "hermeticity": {"mechanism": "landlock", "fs": "declared", "network": "allowed"}, } return Manifest(**{**base, **overrides}) # type: ignore[arg-type] diff --git a/tests/test_cli.py b/tests/test_cli.py index 03206bf8..8970165c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -548,7 +548,8 @@ def test_status_headers_answer_mode_image_and_sandbox( "state": "absent", "archive": ".datalad/environments/lc-env-0123456789abcdef/image", } - report.sandbox = "podman (fs: declared, network: denied)" + report.sandbox = "podman (fs: declared, network: allowed)" + report.crate = "up to date with the outputs" _status_stub(monkeypatch, report) output = runner.invoke(main, ["status"]).output @@ -557,6 +558,7 @@ def test_status_headers_answer_mode_image_and_sandbox( assert "lc-env-0123456789abcdef" in output assert "needs build" in output and "lc build" in output assert "podman" in output + assert "crate: up to date with the outputs" in output def test_build_on_a_direct_project_is_an_explanatory_no_op( @@ -627,6 +629,7 @@ def test_status_json_is_machine_readable( "mode": "direct", "image": None, "sandbox": "", + "crate": "", "counts": {"current": 1, "behind": 1, "stale": 1}, "outputs": [ { diff --git a/tests/test_container.py b/tests/test_container.py index ad7a8a1b..dc7a8f6a 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -123,6 +123,12 @@ def _argvs(calls: list[list[str]], *head: str) -> list[list[str]]: return [c for c in calls if c[: len(head)] == list(head)] +def _git_calls(calls: list[list[str]], sub: str) -> list[list[str]]: + """git calls carrying *sub* anywhere — `-c key=val` pairs may precede + the subcommand, so a prefix match misses them.""" + return [c for c in calls if c[0] == "git" and sub in c] + + # ---- runtime detection ------------------------------------------------------ @@ -178,7 +184,7 @@ def test_a_missing_archive_refuses_unless_the_caller_may_build( with pytest.raises(ProjectError, match="lc build"): container.runtime_for_run(root, build=False) assert _argvs(fake, "podman", "build") == [] - assert _argvs(fake, "git", "commit") == [] + assert _git_calls(fake, "commit") == [] runtime = container.runtime_for_run(root, build=True) @@ -264,11 +270,11 @@ def test_the_build_saves_and_commits_the_archive(root: Path, fake: list[list[str configured = {c[-2] for c in _argvs(fake, "git", "config", "-f", ".datalad/config")} assert f"datalad.containers.{runtime.image_tag}.image" in configured assert f"datalad.containers.{runtime.image_tag}.cmdexec" in configured - (add,) = [c for c in fake if c[0] == "git" and "add" in c] + (add,) = _git_calls(fake, "add") assert f".datalad/environments/{runtime.image_tag}" in " ".join(add) # The dot-path routing: without it the archive is a full blob in git. assert "annex.dotfiles=true" in add - assert len(_argvs(fake, "git", "commit")) == 1 + assert len(_git_calls(fake, "commit")) == 1 def test_an_unrouted_archive_refuses_before_building( diff --git a/tests/test_container_smoke.py b/tests/test_container_smoke.py index 76b61d4c..5010aed9 100644 --- a/tests/test_container_smoke.py +++ b/tests/test_container_smoke.py @@ -169,8 +169,8 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: `lc run` never builds: the first probe refuses naming `lc build`, and succeeding after one is the mutation check. The mounts are the mechanism: an undeclared host file simply is not there (while the - host itself reads it fine). And `--network none` is a real denial - with loopback intact — the meaning of `network: denied`.""" + host itself reads it fine). The network is not controlled here any + more than on the host mechanisms — `allowed`, symmetrically.""" with pytest.raises(ProjectError, match="lc build"): engine_run.probe(cproject, ["bc", "--version"]) @@ -178,7 +178,7 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: outcome = engine_run.probe(cproject, ["bc", "--version"]) assert outcome.returncode == 0 assert outcome.attestation.mechanism == runtime - assert outcome.attestation.network == "denied" + assert outcome.attestation.network == "allowed" assert outcome.attestation.fs == "declared" outside = Path.home() / ".lc-smoke-outside.txt" @@ -205,16 +205,6 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: ) assert loopback.returncode == 0 - egress = engine_run.probe( - cproject, - [ - "python", "-c", - "import socket, urllib.request; socket.setdefaulttimeout(3); " - 'urllib.request.urlopen("http://1.1.1.1")', - ], # fmt: skip - ) - assert egress.returncode != 0 - # ---- lc materialize --------------------------------------------------------- @@ -234,7 +224,7 @@ def test_materialize_end_to_end_in_the_image(runtime: str, cproject: Path) -> No manifest = assets.read(cproject / "results/baseline/sums") assert manifest is not None assert manifest.hermeticity["mechanism"] == runtime - assert manifest.hermeticity["network"] == "denied" + assert manifest.hermeticity["network"] == "allowed" assert manifest.hermeticity["fs"] == "declared" assert manifest.image is not None assert manifest.image["id"] == _inspect_id(runtime, manifest.image["id"]) diff --git a/tests/test_crate.py b/tests/test_crate.py index e25679db..99a4872c 100644 --- a/tests/test_crate.py +++ b/tests/test_crate.py @@ -107,8 +107,15 @@ def _writer(path: Path) -> LastWrite: return LastWrite("a" * 40, "irrelevant", "Ada Lovelace", "ada@example.org", "2026-08-19") -def _render(root: Path, graph: Graph, writer: Writer = _writer) -> dict[str, Any]: - text = crate.render(root, graph, license="MIT", dsid=_DSID, writer=writer) +def _render( + root: Path, + graph: Graph, + writer: Writer = _writer, + keys: dict[str, str] | None = None, +) -> dict[str, Any]: + text = crate.render( + root, graph, license="MIT", dsid=_DSID, writer=writer, keys=keys or {} + ) loaded = json.loads(text) assert isinstance(loaded, dict) return loaded @@ -126,8 +133,9 @@ def test_rendering_twice_at_the_same_state_is_byte_identical(project: Path) -> N _made(project, "baseline", "second", git_sha="aaa111") graph = _graph(project) - first = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) - second = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) + keys = {"results/baseline/first/out.txt": "SHA256E-s24--" + "c" * 64 + ".txt"} + first = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer, keys=keys) + second = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer, keys=keys) assert first == second @@ -221,7 +229,6 @@ def test_an_action_chains_its_inputs_and_its_environment(project: Path) -> None: second = actions["run of `second` in universe `baseline`"] first_objects = {ref["@id"] for ref in first["object"]} assert {"uv.lock", ".python-version", "pyproject.toml", "data/catalog.csv"} <= first_objects - assert entities["data/catalog.csv"]["sha256"] == "cafe" assert "results/baseline/first/" in {ref["@id"] for ref in second["object"]} assert second["result"] == [{"@id": "results/baseline/second/"}] assert second["description"] == "make second" @@ -312,16 +319,96 @@ def test_the_license_is_a_local_entity_never_a_minted_url(project: Path) -> None } -def test_license_of_reads_every_spelling(tmp_path: Path) -> None: - cases = { - 'license = "MIT"': "MIT", - 'license = { text = "BSD-3-Clause" }': "BSD-3-Clause", - 'license = { file = "LICENSE" }': "LICENSE", - "": "", +# ---- per-file integrity ---------------------------------------------------- + + +def test_output_files_carry_checksums_from_their_annex_keys(project: Path) -> None: + """The keys are repository state, so a bytes-free clone renders the + same claims — and the hex in a SHA256E key is the raw sha256 an + archive can verify with `sha256sum` after a `git archive` deposit, + where the dataset's `version` is lc's framed digest and cannot be.""" + _made(project, "baseline", "first", git_sha="aaa111") + digest = "d" * 64 + keys = {"results/baseline/first/out.txt": f"SHA256E-s21--{digest}.txt"} + entities = _entities(_render(project, _graph(project), keys=keys)) + + part = entities["results/baseline/first/out.txt"] + assert part["sha256"] == digest + assert part["contentSize"] == "21" + parts = {ref["@id"] for ref in entities["results/baseline/first/"]["hasPart"]} + assert parts == { + "results/baseline/first/.lightcone-manifest.json", + "results/baseline/first/out.txt", } - for spelling, expected in cases.items(): - (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\n{spelling}\n') - assert crate.license_of(tmp_path) == expected, spelling + + +def test_a_non_sha256_key_yields_size_and_no_digest(project: Path) -> None: + """`annex.backend` is the researcher's to set, and a wrong checksum + is worse than none — the publish-neither discipline.""" + _made(project, "baseline", "first", git_sha="aaa111") + keys = {"results/baseline/first/out.txt": "MD5E-s21--" + "e" * 32 + ".txt"} + entities = _entities(_render(project, _graph(project), keys=keys)) + + part = entities["results/baseline/first/out.txt"] + assert part["contentSize"] == "21" + assert "sha256" not in part + + +def test_pointer_shaped_bytes_never_become_a_checksum(project: Path) -> None: + """`annex_keys` answers empty for the whole repository whenever + git-annex cannot answer at all, and an unlocked pointer file reads + perfectly well — so the byte fallback re-checks the pointer shape + instead of publishing a well-formed digest of the pointer text.""" + _made(project, "baseline", "first", git_sha="aaa111") + pointer = "/annex/objects/SHA256E-s300--" + "a" * 64 + ".csv\n" + (project / "data" / "catalog.csv").write_text(pointer) + + entities = _entities(_render(project, _graph(project), keys={})) + + catalog = entities["data/catalog.csv"] + assert "sha256" not in catalog + assert "contentSize" not in catalog + + +def test_git_carried_files_are_hashed_by_their_bytes(project: Path) -> None: + """The lock and its companions are in git, so their working-tree + bytes are the content — repository state, and the render stays + pure.""" + import hashlib + + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + body = (project / "uv.lock").read_bytes() + assert entities["uv.lock"]["sha256"] == hashlib.sha256(body).hexdigest() + assert entities["uv.lock"]["contentSize"] == str(len(body)) + + +def test_an_out_of_tree_input_publishes_no_checksum( + project: Path, tmp_path: Path +) -> None: + """Its recorded input_versions digest is lc's *framed* hash, not a + raw sha256 — publishing it under the workflow-run term would be a + checksum nothing can verify. The manifests keep the full story.""" + catalog = tmp_path / "shared" / "catalog.csv" + catalog.parent.mkdir() + catalog.write_text("a,b\n") + task = Task( + "baseline", + "first", + project / "results/baseline/first", + "make first", + {"catalog": catalog}, + {}, + {}, + "sha256:def", + ) + _made(project, "baseline", "first", git_sha="aaa111", inputs={"catalog": "sha256:cafe"}) + entities = _entities(_render(project, Graph({("baseline", "first"): task}))) + + external = entities[catalog.as_uri()] + assert "sha256" not in external + assert "contentSize" not in external def _as_list(value: Any) -> list[Any]: diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 9356c057..c21e6491 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -231,6 +231,35 @@ def test_a_locked_file_without_its_content_is_refused_too(repo: Path) -> None: # ---- committing ------------------------------------------------------------ +def test_save_leaves_foreign_staged_work_staged_and_uncommitted(repo: Path) -> None: + """The user can `git add` while a graph runs; the next save must not + sweep it. The commit is a partial commit — built from HEAD plus the + saved paths alone — so their work stays exactly where they left it: + staged, and in no commit of lc's.""" + (repo / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=repo) + out = repo / "results" / "fit" + out.mkdir() + (out / "value.txt").write_text("42\n") + + assert dataset.save(repo, [out], "make fit") + + committed = dataset._git(["show", "--name-only", "--format=", "HEAD"], cwd=repo).split() + assert "notes.py" not in committed + assert sorted(committed) == ["results/fit/value.txt"] + staged = dataset._git(["diff", "--cached", "--name-only"], cwd=repo).split() + assert staged == ["notes.py"], "still staged, exactly as the user left it" + + +def test_save_sees_nothing_to_commit_past_foreign_staged_work(repo: Path) -> None: + """The nothing-to-commit probe is scoped like the commit, or foreign + staged content would make an empty save attempt a commit and fail.""" + (repo / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=repo) + + assert not dataset.save(repo, [repo / "results"], "nothing here") + + def test_save_reports_when_there_was_nothing_to_commit(repo: Path) -> None: """`lc materialize` may not leave an empty commit behind for an output that produced nothing new.""" @@ -454,11 +483,73 @@ def test_the_annex_executables_are_ours_to_install() -> None: assert ours.get(name) == value, f"{name} is not re-declared as {value}" +def test_the_worker_and_the_shim_are_never_console_scripts() -> None: + """`python -m lightcone.engine.worker` makes an output unconditionally, + commits nothing, and leaves the tree dirty by design; the shim is the + sandbox's own plumbing. A `[project.scripts]` entry would put either + on `$PATH` through `uv tool install` — a footgun `lc --help` already + refuses to advertise. Every entry point is either the CLI or a + mirrored git-annex executable, and nothing else.""" + from importlib.metadata import distribution + + theirs = {e.value for e in distribution("git-annex").entry_points} + for entry in distribution("lightcone-cli").entry_points: + assert "lightcone.engine" not in entry.value + assert "_sandbox_exec" not in entry.value + assert entry.value.startswith("lightcone.cli") or entry.value in theirs, entry + + # ---- who last wrote a path ------------------------------------------------- +def test_annex_keys_maps_every_annexed_file_content_present_or_not(repo: Path) -> None: + """The crate's per-file checksums come from here, and they must + answer on a clone that holds none of the bytes — `--include=*` is + what turns `find` from "present files" into "annexed files".""" + out = repo / "results" / "fit" + out.mkdir() + (out / "value.dat").write_bytes(b"x" * 300) + dataset.save(repo, [out], "make fit") + + keys = dataset.annex_keys(repo) + key = keys["results/fit/value.dat"] + assert key.startswith("SHA256E-s300--"), key + assert ".gitattributes" not in keys, "git-carried files have no key" + + clone = repo.parent / "clone" + dataset._git(["clone", "-q", str(repo), str(clone)], cwd=repo.parent) + # A clone inherits no local config, and annex init in a *clone* has + # remote git-annex branch state to commit — identity required, where + # a fresh repo's init tolerates its absence. + for key_, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key_, value], cwd=clone) + dataset.init_annex(clone) + assert dataset.annex_keys(clone)["results/fit/value.dat"] == key, ( + "keys are repository state, bytes not required" + ) + + +def test_annex_keys_survives_a_tab_in_a_filename(repo: Path) -> None: + """git-annex emits ${file} unescaped, so the parse splits from the + *last* tab — keys never contain one, filenames legally can.""" + out = repo / "results" / "fit" + out.mkdir() + (out / "run\t1.dat").write_bytes(b"x" * 300) + dataset.save(repo, [out], "make fit") + + keys = dataset.annex_keys(repo) + assert keys["results/fit/run\t1.dat"].startswith("SHA256E-s300--") + + +def test_annex_keys_of_a_plain_directory_is_empty(tmp_path: Path, real_tools: None) -> None: + """Cannot say is empty, never an error — the last_writer discipline.""" + bare = tmp_path / "bare" + bare.mkdir() + assert dataset.annex_keys(bare) == {} + + def test_last_writer_names_the_commit_that_last_touched_a_path(repo: Path) -> None: """The foreign-write fact's whole mechanism: every output is committed, so a hand edit needs a commit, and history names it.""" diff --git a/tests/test_identity.py b/tests/test_identity.py index e263bd77..594703c9 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -201,7 +201,9 @@ def test_the_environment_is_not_part_of_what_an_output_is(root: Path) -> None: def test_a_clean_lock_scans_clean(root: Path) -> None: - assert scan_lock(root) == LockScan(refusals=(), sdist_built=(), non_default_groups=()) + assert scan_lock(root) == LockScan( + refusals=(), sdist_built=(), non_default_groups=(), machine_config=() + ) def test_a_path_dependency_is_refused(root: Path) -> None: @@ -285,3 +287,57 @@ def test_the_default_group_set_is_read_from_uv_toml_too(root: Path) -> None: ) (root / "uv.toml").write_text('default-groups = ["dev", "plots"]\n') assert scan_lock(root).non_default_groups == () + + +# ---- machine-level uv configuration ---------------------------------------- + + +def test_machine_config_setting_an_audited_key_is_advisory( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """uv merges user- and system-level config underneath the project's, + and list settings concatenate across levels — so a machine-level file + steers the sync while env_version sees nothing. Reported, never + hashed: machine state in the hash would make one commit answer + differently on two hosts.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text("no-build = true\n") + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + scan = scan_lock(root) + + assert scan.machine_config == (f"{user} sets no-build",) + + +def test_machine_config_without_audited_keys_stays_silent( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """cache-dir, link-mode and friends decide where bytes come from, not + what gets installed — the same line the install-settings hash draws.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text('cache-dir = "/scratch/uv"\nlink-mode = "copy"\n') + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + assert scan_lock(root).machine_config == () + + +def test_absent_or_unreadable_machine_config_stays_silent( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A file uv cannot parse fails the sync loudly on its own; the + advisory adds nothing by refusing first.""" + from lightcone.engine import identity + + broken = tmp_path / "broken-uv.toml" + broken.write_text("no-build = [unclosed\n") + monkeypatch.setattr( + identity, + "_machine_config_paths", + lambda: (tmp_path / "missing" / "uv.toml", broken), + ) + + assert scan_lock(root).machine_config == () diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 13582add..5ff96e8d 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -71,6 +71,16 @@ def _commits(root: Path) -> int: return len(dataset._git(["log", "--oneline"], cwd=root).splitlines()) +def _cluster(monkeypatch: pytest.MonkeyPatch, scheduler: _Inline) -> None: + """Point the run at a custom scheduler — the one monkeypatch point.""" + + @contextmanager + def fake() -> Iterator[_Inline]: + yield scheduler + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + # ---- a run, end to end ----------------------------------------------------- @@ -179,6 +189,21 @@ def test_refresh_remakes_what_is_behind_and_commits_it(root: Path, inline: None) assert manifest.env_version == identity.env_version(root) +def test_the_manifest_records_the_uv_that_converged_the_environment( + root: Path, inline: None +) -> None: + """Probed once by the driver and handed to every task — attestation + beside lc_version, never a rebuild signal.""" + from lightcone.engine import project + + engine.materialize(root, ["first"]) + + manifest = assets.read(root / "results/baseline/first") + assert manifest is not None + assert manifest.uv_version == project.uv_version(root) + assert manifest.uv_version.count(".") >= 1, "a real version token, not prose" + + def test_check_reports_behind_without_planning_it(root: Path, inline: None) -> None: """`--check` is a gate, and `behind` must not close it — a project of curated results would never pass again.""" @@ -450,6 +475,90 @@ def test_a_lock_that_builds_from_source_is_a_warning_not_a_refusal(root: Path) - assert any("oldlib" in w for w in report.warnings) +def test_machine_level_uv_config_is_reported( + root: Path, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The config-file half of the same hole the ambient scrub closes: + env_version cannot see it, so the run says so.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text("no-binary = true\n") + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + report = engine.check(root, []) + + assert any("env_version cannot see it" in w and str(user) in w for w in report.warnings) + + +def test_ambient_uv_settings_are_scrubbed_and_reported( + root: Path, inline: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """The scrub protects env_version's install-settings term; the warning + is what tells a user why their variable stopped steering the sync.""" + monkeypatch.setenv("UV_NO_BINARY", "1") + + report = engine.materialize(root, []) + + assert report.ok + assert any("UV_NO_BINARY" in w for w in report.warnings) + + +def test_an_edit_while_the_graph_runs_is_reported( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The dirty check runs at start of run and manifests are written + per-output later, so an edit in between leaves manifests whose + git_sha no longer describes the code that ran. The run ends with one + status call and says so — the honest floor under the unwritten + `git_dirty` field.""" + + class Editing(_Inline): + def completed(self, handles: list[object]) -> Iterator[object]: + (root / "notes.md").write_text("scribbled while the graph ran\n") + yield from handles + + _cluster(monkeypatch, Editing()) + + report = engine.materialize(root, []) + + assert report.ok + assert any("notes.md" in w and "in flight" in w for w in report.warnings) + + +def test_a_mid_run_stage_is_not_swept_into_lcs_commits( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`dataset.save` stages scoped and commits scoped — a partial + commit — so work the user staged while the graph ran ends the run + exactly where they left it: staged, warned about, and in none of + lc's commits (the per-output saves and the trailing crate commit + alike).""" + _declare_license(root) + + class Staging(_Inline): + def completed(self, handles: list[object]) -> Iterator[object]: + (root / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=root) + yield from handles + + _cluster(monkeypatch, Staging()) + + report = engine.materialize(root, []) + + assert report.ok + assert any("notes.py" in w and "in flight" in w for w in report.warnings) + staged = dataset._git(["diff", "--cached", "--name-only"], cwd=root).split() + assert staged == ["notes.py"] + ever_committed = dataset._git(["log", "--name-only", "--format="], cwd=root).split() + assert "notes.py" not in ever_committed + + +def test_a_clean_run_reports_no_in_flight_edit(root: Path, inline: None) -> None: + report = engine.materialize(root, []) + assert not any("in flight" in w for w in report.warnings) + + # ---- leaving the tree as clean as it was found ----------------------------- @@ -522,11 +631,7 @@ def completed(self, handles: list[Any]) -> Iterator[TaskResult]: yield handles[0] raise KeyboardInterrupt - @contextmanager - def fake() -> Iterator[_Interrupted]: - yield _Interrupted() - - monkeypatch.setattr(engine, "cluster_for_run", fake) + _cluster(monkeypatch, _Interrupted()) with pytest.raises(KeyboardInterrupt): engine.materialize(root, []) @@ -1047,6 +1152,38 @@ def test_a_removed_license_stops_maintenance_but_keeps_the_file( assert any("no longer maintained" in w for w in report.warnings) +def test_status_places_the_publication_view(root: Path, inline: None) -> None: + """The `crate:` header line, through its three plain states.""" + assert engine.status(root).crate == "not maintained — declare [project].license to enable it" + + _declare_license(root) + assert engine.status(root).crate == "will be created by the next `lc materialize`" + + engine.materialize(root, []) + assert engine.status(root).crate == "up to date with the outputs" + + +def test_status_sees_the_crate_lag_a_rerun_leaves(root: Path, inline: None) -> None: + """The recorded residue made visible: a rerun rewrites a manifest but + never regenerates the view. Status reads the mismatch off the + document's own datePublished against the manifests it already read — + no git, no rocrate import.""" + from dataclasses import replace + + _declare_license(root) + engine.materialize(root, []) + directory = root / "results/baseline/second" + manifest = assets.read(directory) + assert manifest is not None + assets.write(directory, replace(manifest, finished_at="2027-01-01T00:00:00.000+00:00")) + dataset.save(root, [directory], "a rerun-shaped manifest rewrite") + + assert engine.status(root).crate.startswith("behind") + + engine.materialize(root, []) + assert engine.status(root).crate == "up to date with the outputs" + + def test_an_output_the_spec_dropped_is_excluded_and_named(root: Path, inline: None) -> None: _declare_license(root) engine.materialize(root, []) diff --git a/tests/test_project.py b/tests/test_project.py index 6630393a..463af2eb 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -674,6 +674,69 @@ def test_ambient_virtualenv_is_not_passed_to_tools( assert env["LC_TEST_CANARY"] == "kept", "the rest of the environment is untouched" +def test_ambient_uv_install_settings_are_scrubbed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ambient install setting would steer what a sync installs without + moving env_version — the identity hole the scrub closes. Plumbing (the + cache, timeouts, index credentials) survives: it decides where bytes + come from and how fast, never what gets installed.""" + import os + + from lightcone.engine.project import child_env, uv_scrub_warning + + for name in [k for k in os.environ if k.startswith("UV_")]: + monkeypatch.delenv(name) # the suite itself may run under `uv run` + monkeypatch.setenv("UV_NO_BINARY", "1") + monkeypatch.setenv("UV_PYTHON", "3.10") + monkeypatch.setenv("UV_INDEX_URL", "https://elsewhere.invalid/simple") + monkeypatch.setenv("UV_CACHE_DIR", "/scratch/uv") + monkeypatch.setenv("UV_INDEX_INTERNAL_PASSWORD", "hunter2") + monkeypatch.setenv("UV_OFFLINE", "1") + monkeypatch.setenv("UV_PYTHON_INSTALL_DIR", "/scratch/uv/python") + monkeypatch.setenv("UV_LINK_MODE", "copy") + monkeypatch.setenv("LC_TEST_CANARY", "kept") + + env = child_env() + assert "UV_NO_BINARY" not in env + assert "UV_PYTHON" not in env + assert "UV_INDEX_URL" not in env + assert env["UV_CACHE_DIR"] == "/scratch/uv", "shared-cache plumbing survives" + assert env["UV_INDEX_INTERNAL_PASSWORD"] == "hunter2", "credentials survive" + assert env["UV_OFFLINE"] == "1", "air-gap mode survives" + assert env["UV_PYTHON_INSTALL_DIR"] == "/scratch/uv/python", ( + "the interpreter store is plumbing, and it has no project-level spelling" + ) + assert env["UV_LINK_MODE"] == "copy", "link-mode is not an audited setting either" + assert env["LC_TEST_CANARY"] == "kept" + assert "UV_INDEX_URL, UV_NO_BINARY, UV_PYTHON" in uv_scrub_warning(), ( + "the warning names exactly what the scrub dropped" + ) + + +def test_converge_reports_the_uv_scrub( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`lc init` resolves and syncs, so a user whose ambient UV_INDEX_URL + was dropped must hear it here — not only on the verbs they have not + reached when resolution fails with uv's raw error.""" + monkeypatch.setenv("UV_INDEX_URL", "https://mirror.invalid/simple") + + report = converge(tmp_path / "proj") + + assert any("UV_INDEX_URL" in w for w in report.warnings) + + +def test_an_empty_scrubbed_variable_is_not_reported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An empty variable steers nothing, so warning about it is noise.""" + from lightcone.engine.project import uv_scrub_warning + + monkeypatch.setenv("UV_NO_BUILD", "") + assert "UV_NO_BUILD" not in uv_scrub_warning() + + def test_relays_uv_warnings_into_the_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -747,3 +810,19 @@ def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) with pytest.raises(ProjectError, match="no solution found"): converge(tmp_path / "proj") + + +def test_license_of_reads_every_spelling(tmp_path: Path) -> None: + """Publication intent, derived never configured — the crate is + maintained iff [project].license is declared, in any of its forms.""" + from lightcone.engine.project import license_of + + cases = { + 'license = "MIT"': "MIT", + 'license = { text = "BSD-3-Clause" }': "BSD-3-Clause", + 'license = { file = "LICENSE" }': "LICENSE", + "": "", + } + for spelling, expected in cases.items(): + (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\n{spelling}\n') + assert license_of(tmp_path) == expected, spelling diff --git a/tests/test_run.py b/tests/test_run.py index 8334c2d6..d80d6377 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -196,3 +196,24 @@ def test_a_recipe_does_not_sync_where_a_probe_does(project: Path) -> None: concurrent worker writes the same `.venv`.""" assert "--no-sync" in uv_prefix(project, sync=False) assert "--exact" not in uv_prefix(project, sync=False) + + +def test_the_probe_reports_the_uv_scrub_in_its_notes( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The probe is what builds the child environment, so the scrub's + fact rides its outcome — the caller prints notes verbatim, and no + verb has to remember to ask.""" + from lightcone.engine import run as engine_run + from lightcone.engine import sandbox + from lightcone.engine.sandbox.model import Attestation + + monkeypatch.setenv("UV_NO_BINARY", "1") + outcome = sandbox.Outcome( + returncode=0, attestation=Attestation(mechanism="none", fs="open") + ) + monkeypatch.setattr(sandbox, "run", lambda *a, **k: outcome) + + outcome = engine_run.probe(project, ["true"]) + + assert any("UV_NO_BINARY" in note for note in outcome.notes) diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py index f38dfbbf..02ef15b2 100644 --- a/tests/test_sandbox_denial.py +++ b/tests/test_sandbox_denial.py @@ -97,7 +97,7 @@ def test_an_in_tree_write_is_its_own_kind_of_denial(policy: Policy, project: Pat stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" joined = "\n".join(denial.explain(stderr, policy, cwd=project)) assert "cannot write" in joined - assert "output goes in results/" in joined + assert "its own output directory" in joined assert "inputs:" not in joined diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py index f673ab25..93c86c67 100644 --- a/tests/test_sandbox_enforcement.py +++ b/tests/test_sandbox_enforcement.py @@ -382,6 +382,30 @@ def test_results_can_be_written(backend: sandbox.Backend, project: Path) -> None assert (project / "results" / "out.csv").read_text() == "out" +def test_a_recipe_cannot_write_a_sibling_output_directory( + backend: sandbox.Backend, project: Path +) -> None: + """The cross-write closure, at the kernel: a recipe granted its own + output directory cannot land bytes in a sibling's — the corruption + that would otherwise enter the sibling's digest as though its recipe + wrote it. Both writes target user-owned paths, so only the boundary + can refuse the first; the second is the mutation check in-place.""" + own = project / "results" / "baseline" / "first" + sibling = project / "results" / "baseline" / "second" + own.mkdir(parents=True) + sibling.mkdir(parents=True) + (sibling / "value.txt").write_text("theirs\n") + with sandbox.scope(sandbox.exec_policy(project, output_dir=own)) as policy: + crossed = shell( + backend, policy, f"printf forged > {sibling / 'value.txt'}", cwd=project + ) + owned = shell(backend, policy, f"printf mine > {own / 'value.txt'}", cwd=project) + assert crossed.returncode != 0 + assert (sibling / "value.txt").read_text() == "theirs\n", "the file changed anyway" + assert owned.returncode == 0, owned.stderr + assert (own / "value.txt").read_text() == "mine" + + def test_a_declared_input_is_read_only( backend: sandbox.Backend, project: Path, outside: Path ) -> None: diff --git a/tests/test_sandbox_oci.py b/tests/test_sandbox_oci.py index c39bda95..cf8cdccd 100644 --- a/tests/test_sandbox_oci.py +++ b/tests/test_sandbox_oci.py @@ -160,9 +160,11 @@ def test_execution_pins_the_image_by_id_never_a_tag(root: Path, policy: Policy) assert not any("lc-env-" in part for part in argv) -def test_the_network_is_denied_by_flag(root: Path, policy: Policy) -> None: +def test_no_flag_touches_the_network(root: Path, policy: Policy) -> None: + """lc does not control the network on any mechanism, and the argv is + where that has to be true for the `allowed` attestation to be honest.""" argv = _backend(root).wrap(policy, ["true"]) - assert "--network" in argv and argv[argv.index("--network") + 1] == "none" + assert "--network" not in argv def test_runtimes_differ_only_in_their_spellings(root: Path, policy: Policy) -> None: @@ -211,7 +213,7 @@ def test_the_attestation_is_derived_from_the_flags(root: Path, policy: Policy) - attested = _backend(root, runtime).attest(policy) assert attested.mechanism == runtime assert attested.fs == "declared" - assert attested.network == "denied" + assert attested.network == "allowed" assert attested.landlock_abi is None diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py index bdb3b08d..e05a492b 100644 --- a/tests/test_sandbox_policy.py +++ b/tests/test_sandbox_policy.py @@ -46,20 +46,56 @@ def test_the_tree_is_read_only_apart_from_results( assert not built.grants(project / ".venv" / "bin" / "python", built.write) -def test_results_is_writable(tmp_path: Path) -> None: - """Output goes here, and it is the same scope a recipe gets — which - is what makes a probe a probe of the real thing. +def test_results_is_writable_for_a_probe(tmp_path: Path) -> None: + """A probe has no output id, so its write scope is `results/` whole. A writable directory nested inside a read-only tree is the shape all three mechanisms express natively: Landlock unions rights so a nested grant only widens, SBPL restates the write tier after the guard, and - podman mounts `results` `:rw` over a `:ro` project.""" + podman mounts the scope `:rw` over a `:ro` project.""" project = tmp_path / "proj" (project / "results").mkdir(parents=True) with scope(policy_module.exec_policy(project)) as built: assert built.grants(project / "results" / "out.csv", built.write) +def test_a_recipe_is_narrowed_to_its_own_output_directory(tmp_path: Path) -> None: + """The cross-write closure: a concurrent task landing bytes in a + sibling's directory before the sibling hashes produces a manifest + that is self-consistent and wrong — no checksum can ever see it, so + prevention is the only fix. Same nested-writable shape, one level + deeper.""" + project = tmp_path / "proj" + own = project / "results" / "baseline" / "first" + sibling = project / "results" / "baseline" / "second" + own.mkdir(parents=True) + sibling.mkdir(parents=True) + with scope(policy_module.exec_policy(project, output_dir=own)) as built: + assert built.grants(own / "out.csv", built.write) + assert not built.grants(sibling / "out.csv", built.write) + assert not built.grants(project / "results", built.write) + assert built.grants(sibling / "out.csv", built.read), ( + "an upstream output is still a readable input" + ) + + +def test_the_containerized_recipe_mounts_only_its_own_output_directory( + tmp_path: Path, +) -> None: + """The mount table derives from the write set, so the narrowing must + survive into the containerized shape untranslated.""" + project = tmp_path / "proj" + own = project / "results" / "baseline" / "first" + own.mkdir(parents=True) + with scope( + policy_module.exec_policy( + project, containerized=True, env_dir=project / ".lightcone/venv", output_dir=own + ) + ) as built: + assert own in built.write + assert project / "results" not in built.write + + def test_results_is_granted_only_if_it_exists(tmp_path: Path) -> None: """Convergence makes it. A policy that made directories would be a side effect nobody asked a probe for.""" diff --git a/tests/test_worker.py b/tests/test_worker.py index 014b882f..ad715c86 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -67,21 +67,23 @@ def _runtime(root: Path) -> container.Runtime: return container.runtime_for_run(root, build=False) +def _context(root: Path, env_version: str | None = None) -> worker.RunContext: + """The driver-resolved facts a real run would hand down.""" + return worker.RunContext( + env_version=env_version if env_version is not None else identity.env_version(root), + head=_HEAD, + versions=assets.Versions(), + runtime=_runtime(root), + uv_version="0.0.0-test", + ) + + def _make( root: Path, output_id: str, *upstream: TaskResult, refresh: bool = False ) -> TaskResult: """Run one task the way Dask would, handed its upstream results.""" - task = _task(root, output_id) return worker.materialize( - root, - task, - identity.env_version(root), - _HEAD, - assets.Versions(), - refresh, - None, - _runtime(root), - *upstream, + root, _task(root, output_id), _context(root), refresh, None, *upstream ) @@ -127,6 +129,9 @@ def test_the_manifest_is_complete_before_anything_is_saved(root: Path) -> None: # The engine's version is attestation, not identity: with lc outside # the project's lock, this field is the record of which engine ran. assert manifest.lc_version == worker.lc_version() + # And so is the uv that converged the environment: probed once by the + # driver, handed down, never a rebuild signal. + assert manifest.uv_version == "0.0.0-test" def test_the_recipe_runs_under_the_boundary(root: Path) -> None: @@ -249,10 +254,7 @@ def test_a_stale_file_does_not_survive_a_rebuild(root: Path) -> None: output = root / "results/baseline/first" (output / "leftover.txt").write_text("from a previous run\n") - worker.execute( - root, _task(root, "first"), identity.env_version(root), {}, head=_HEAD, - runtime=_runtime(root), - ) + worker.execute(root, _task(root, "first"), {}, _context(root)) assert not (output / "leftover.txt").exists() assert (output / "value.txt").exists() @@ -311,8 +313,7 @@ def test_an_environment_that_moved_under_the_run_is_refused(root: Path) -> None: """A manifest may not claim an environment that had already been edited by the time the recipe ran.""" result = worker.execute( - root, _task(root, "first"), "sha256:from-another-run", {}, head=_HEAD, - runtime=_runtime(root), + root, _task(root, "first"), {}, _context(root, env_version="sha256:from-another-run") ) assert result.status == "failed" From bbe6d8517f8bd873069ba4484738ed38bb8ca303 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:42:21 +0200 Subject: [PATCH 19/23] Docs rewrite 3/5: the CLI reference (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third pass — `docs/cli/` rebuilt around the five verbs that exist, in the frozen pages' structure (Synopsis / semantics / options table / JSON / examples). ## What's in it - **Overview** (`cli/index.md`): the three global behaviors (current directory is the project — no walk-up, no global config; nothing waits on a human; refusals carry their remedy) and the exit-code contract, including `lc run`'s proxying and the `--check` gate semantics. - **`lc init`**: the four convergence classes (created / repaired / **blocked** / warned — with blocked counting against convergence and warnings never affecting exit), the real scaffold tree, what it deliberately doesn't create (`src/`, dependencies), and the deliberately bare flag surface. - **`lc materialize`** (new page): target grammar (`fit` vs `robust/fit`, matching-nothing is an error), the remake rules (stale vs behind vs current under `--refresh`), the run's contract (clean in/clean out, fetches declared inputs, commits as it goes, reports every independent failure, converges the crate), check mode's dirty-tree exemption, the absent-flags rationale (`--jobs`, `--force`), and the JSON report with `ok`/`up_to_date` called out as the branching keys. - **`lc status`**: the header facts, the commit column as the verb's reason to exist, the three states, the report-vs-gate split, and the JSON shape including `foreign_write`. - **`lc run`**: the probe — same environment and sandbox as a recipe, environment converge first, exit-code proxying (`128+N`), denial notes, works on any uv project, no sandbox opt-out. - **`lc build`**: the system-layer-only image, the closed declaration key set, the committed `docker-archive` as the store (id-pinned, no registry), the arch refusal, the clean-tree requirement, and the JSON result (`action: built | present`). - `verify.md` and `export.md` deleted with their verbs; nav updated (materialize added, order follows the workflow). ## Verification - All `--help` texts, the init/materialize/status JSON shapes, and the console blocks were captured from the current CLI against a real project. - The build JSON keys and `action` values were read from `commands.py`/`container.py`, and the tag format corrected against `image.py` (`lc-env-<16 hex>` — my first draft had the pre-rebuild `lc--` spelling). - `zensical build`: no new warnings; the two remaining are in frozen pages PR 4 replaces. Next: (4) developer corner, (5) nav + workflow refresh + final reconciliation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx Co-authored-by: Claude Fable 5 --- docs/cli/build.md | 134 ++++++++++++++++++++-------------------- docs/cli/export.md | 106 ------------------------------- docs/cli/index.md | 40 ++++++++---- docs/cli/init.md | 119 +++++++++++++++++------------------ docs/cli/materialize.md | 113 +++++++++++++++++++++++++++++++++ docs/cli/run.md | 121 ++++++++++++------------------------ docs/cli/status.md | 111 ++++++++++++++++++++------------- docs/cli/verify.md | 67 -------------------- zensical.toml | 5 +- 9 files changed, 375 insertions(+), 441 deletions(-) delete mode 100644 docs/cli/export.md create mode 100644 docs/cli/materialize.md delete mode 100644 docs/cli/verify.md diff --git a/docs/cli/build.md b/docs/cli/build.md index d53445cf..f4603a20 100644 --- a/docs/cli/build.md +++ b/docs/cli/build.md @@ -1,7 +1,9 @@ # lc build -Build container images declared in `astra.yaml` (or pre-pull registry -images so `lc run` can use `--pull=never`). +Build the project's system-layer image, and commit it. Containerized +projects only — a project containerizes by declaring a +`[tool.lightcone.image]` table in `pyproject.toml`, and on a direct +project this verb just says so and exits. ## Synopsis @@ -9,82 +11,80 @@ images so `lc run` can use `--pull=never`). lc build [OPTIONS] ``` -## Options +Idempotent: an image that is already built and committed is left +alone. -| Option | Default | Effect | -|--------|---------|--------| -| `--force` | off | Rebuild / re-pull even if the tag already exists locally. | -| `--runtime {docker,podman,podman-hpc,kubernetes}` | resolved from `~/.lightcone/config.yaml` | Override the runtime for this build. | - -## What it does - -For every distinct `container:` value found in the project (root, -sub-analysis, or recipe-level): - -- **Path to a Containerfile** → compute the content-addressed tag - `lc--`, build the image, and (for `podman-hpc`) - migrate it into the per-node container cache. -- **Anything else** (e.g. `python:3.12-slim`, `ghcr.io/foo/bar:tag`) → - pull it into the local image store. This is what lets `lc run` pass - `--pull=never` to the runtime, sidestepping `unqualified-search-registries` - resolution issues with content-addressed tags. - -On the `kubernetes` runtime (a lightcone JupyterHub deployment, where -no local OCI runtime exists) the same command builds through the -deployment's **GCP Cloud Build** service instead: the staged build -context is uploaded to the deployment's build bucket and the resulting -image is pushed as `$LIGHTCONE_REGISTRY/lc-:` — -the same content-addressed identity, so an unchanged environment is a -single registry check and no build at all. Pre-built registry images -are left alone (worker pods pull them directly). Auth is the pod's -Workload Identity; nothing to configure. - -If the runtime is `none` (either by config or because `auto` couldn't -find one), `lc build` prints a friendly note and exits 0. There is -nothing to build. - -## Tag computation +## What the image is -```text -lc-- +The image is the *system layer* only: the declared base (digest-pinned, +or the default), the declared apt packages, and the pinned Python +interpreter. Your analysis environment is not in it — recipes' Python +packages come from the project's lock, synced into the container at run +time — and neither is `lc` itself. That is what makes "editing code +never rebuilds the image" structural: no project file enters the build +context at all. + +The declaration is a closed set of keys, hashed into the image's +identity: + +```toml +[tool.lightcone.image] +base = "docker.io/library/debian@sha256:..." # optional; default pinned by lc +apt-install = ["libfftw3-dev"] # optional +run-commands = ["curl -L ... | tar xz"] # optional, the bounded escape +env = { OMP_NUM_THREADS = "1" } # optional ``` -The hash covers the Containerfile contents plus any of these dependency -files at the project root: +## The archive is the store -- `requirements.txt` -- `requirements-dev.txt` -- `requirements-test.txt` -- `pyproject.toml` -- `setup.py` -- `setup.cfg` -- `poetry.lock` -- `Pipfile.lock` +`lc build` saves the built image into the repository — +`.datalad/environments//image`, a `docker-archive` committed +through git-annex — so the exact bytes travel with the project: a +clone obtains them with a fetch, no registry and no credentials +involved. Execution always pins the image's content *id*, never a tag, +so nothing can substitute a different image under the same name. -Edit any one of those and the tag changes. That, in turn, changes -`code_version` in every recipe that uses the image, which marks all -downstream outputs `stale` in `lc status`. +The archive records the architecture it was built for, and a host that +can't execute that architecture is refused up front — build where the +architecture matches the machines that will run recipes (on NERSC, a +login node). -## Examples +## Requirements -```bash -lc build # build / pull whatever's missing -lc build --force # rebuild / re-pull everything -lc build --runtime podman-hpc # force the HPC runtime -``` +- A clean tree — the image commit must not sweep your staged edits in, + and the tag derives from the committed declaration. +- A build-capable runtime: `podman-hpc`, `podman`, or `docker` + (detected in that order; nothing to configure). -## Pre-staging for HPC +`lc materialize` also builds as a preflight when the committed +declaration has no image yet, announcing it first — `lc build` exists +so you can pay the minutes when *you* choose to. -On a login node: +## Options -```bash -$EDITOR ~/.lightcone/config.yaml # container.runtime: podman-hpc -lc build # builds + migrates everything +| Option | Default | Effect | +|--------|---------|--------| +| `--json` | off | Emit the result as JSON on stdout. | + +## The JSON result + +```json +{ + "mode": "containerized", + "tag": "lc-env-1a2b3c4d5e6f7a8b", + "id": "sha256:...", + "archive": ".datalad/environments/lc-env-1a2b3c4d5e6f7a8b/image", + "action": "built" +} ``` -Then submit a SLURM job for `lc run`. The compute nodes will find every -image already cached. +`action` is `"built"` when this invocation built and committed the +image, `"present"` when it was already there. On a direct project the +result is just `{"mode": "direct"}`. -See [api/container](../api/container.md) for the implementation and -[Architecture](../architecture.md) for why we wrap recipes ourselves -instead of using Snakemake's `container:` directive. +## Examples + +```bash +lc build # build + commit, or confirm it's already there +lc build --json # the machine-readable form +``` diff --git a/docs/cli/export.md b/docs/cli/export.md deleted file mode 100644 index 49ecc00e..00000000 --- a/docs/cli/export.md +++ /dev/null @@ -1,106 +0,0 @@ -# lc export - -Export project artifacts in interoperable formats. Currently the only -exporter is `wrroc` (Workflow Run RO-Crate); the group is shaped to host -future formats without breaking the CLI surface. - -## Synopsis - -```text -lc export wrroc [OPTIONS] -``` - -## lc export wrroc - -Walk the project's per-output `.lightcone-manifest.json` sidecars and -emit a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) -bundle — a JSON-LD package readable by WorkflowHub, Zenodo's RO-Crate -plugin, and any RO-Crate-aware archive. The on-disk manifest format is -unchanged; the bundle is the *publication view*, generated on demand. - -### Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--output`, `-o PATH` | `./wrroc` | Bundle directory, or `.zip` path when `--zip` is set. | -| `--universe`, `-u NAME` | every universe with materialized outputs | Restrict to specific universes. Repeatable. | -| `--author "NAME "` | git `user.name` / `user.email`, then `LIGHTCONE_AUTHOR` env | Override the author recorded in the bundle. | -| `--license URL` | `https://creativecommons.org/licenses/by/4.0/` | License URL or SPDX identifier for the bundle. Required by the WRROC profile. | -| `--zip` / `--no-zip` | `--no-zip` | Package the bundle as a single `.zip` after building. | -| `--metadata-only` | off | Skip data files; bundle manifests + `astra.yaml` + universe files only. | - -### What gets bundled - -- `astra.yaml` → `ComputationalWorkflow` (`programmingLanguage: snakemake`). -- Each materialized output → `Dataset` with `version = data_version`. -- Each recipe execution → `CreateAction` with `object` (inputs, both upstream datasets via stable `@id` and external files), `result` (the produced dataset), and `instrument` (the recipe `SoftwareApplication`). -- Each container → `SoftwareApplication` referenced via `softwareRequirements`. -- Each decision → `FormalParameter` + per-run `PropertyValue`. -- Author → `Person`. - -Provenance metadata (`code_version`, `data_version`, `git_sha`, `lc_version`, host) lands as `PropertyValue` entries on the relevant entities. - -### Examples - -```bash -lc export wrroc # ./wrroc/ directory -lc export wrroc -o run.zip --zip # zip bundle for upload -lc export wrroc --metadata-only # provenance graph only, no data files -lc export wrroc -u baseline -u alt_method # restrict to specific universes -lc export wrroc --author "Ada Lovelace " -lc export wrroc --license https://opensource.org/licenses/MIT -``` - -### Output - -```text -✓ Wrote WRROC directory: ./wrroc - Captured 7 runs across universes: baseline, alt_method -``` - -If no materialized outputs are found, the bundle still writes — but only -contains the workflow definition, and a warning is printed: - -```text -✓ Wrote WRROC directory: ./wrroc -Warning: no materialized outputs were found — the bundle contains only - the workflow definition. - This usually means recipes haven't been run yet (try lc run) or the - .lightcone-manifest.json sidecars are missing. - Workflow-only bundles will not pass strict Provenance Run Crate - validation; that profile requires at least one materialized run. -``` - -### Failure modes - -| Error | Cause | -|---|---| -| `No astra.yaml at ; cannot export.` | The cwd is not inside an ASTRA project. | -| ` is non-empty; refuse to clobber.` | The target directory already has contents. Pass a fresh path or remove the existing one. | -| ` is an existing directory; cannot overwrite with a zip.` | `--zip` was requested but the output path resolves to a directory. Use a file path like `bundle.zip`. | - -Manifests that exist but are unreadable (e.g. cross-user symlinks under -`results/` with permission denied) are warned about and skipped — they -do not abort the export. - -### Validation - -The bundle conforms to the [Provenance Run Crate 0.5](https://w3id.org/ro/wfrun/provenance/0.5) -profile. To validate locally: - -```bash -pip install git+https://github.com/crs4/rocrate-validator.git -rocrate-validator -y validate ./wrroc/ -``` - -### When to run - -- Before submitting a paper or depositing artifacts in Zenodo / WorkflowHub. -- After a clean run (`lc verify` clean) on the final commit you intend to publish. -- For external collaborators who don't have `lc` installed but need to inspect provenance. - -### Related - -- [`lc verify`](verify.md) — confirm the manifest chain is intact before exporting. -- [`lc status`](status.md) — see which outputs will be captured by the export. -- [api/manifest](../api/manifest.md) — the on-disk format the export reads from. diff --git a/docs/cli/index.md b/docs/cli/index.md index 889ebd36..eda2f2ec 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -7,21 +7,27 @@ and audit it. ## Global behavior -- `~/.lightcone/config.yaml` is created automatically on first use of - any `lc` command. You do not need to create it manually. -- All commands except `init` walk up from the cwd looking for - `astra.yaml`. If none is found, the command errors out. +- **The current directory is the project.** Every command except + `init` assumes it is invoked from the project root; there is no + walk-up and no global configuration. Outside a project, a command + errors cleanly. +- **Nothing waits on a human.** No command prompts or opens an + interactive shell — every verb runs to completion on its arguments + alone, which is what makes the CLI safe to drive from scripts and + agents. +- **Refusals carry their remedy.** When a command refuses (a dirty + tree, a login node, a missing image), the message names the exact + command that fixes it. ## Commands | Command | Purpose | |---------|---------| -| [`lc init`](init.md) | Scaffold a new ASTRA project (`astra.yaml`, `Containerfile`, `.lightcone/`, MyST report template, optional venv & git). | -| [`lc run`](run.md) | Generate the Snakefile and dispatch through Snakemake + Dask. | -| [`lc build`](build.md) | Build container images declared in `astra.yaml`. | -| [`lc status`](status.md) | Manifest-driven status report. No Snakemake import needed. | -| [`lc verify`](verify.md) | Recompute hashes, walk the input chain, surface tampering. | -| [`lc export`](export.md) | Emit interoperable bundles (Workflow Run RO-Crate) for publication. | +| [`lc init`](init.md) | Converge a directory into a Lightcone project (idempotent). | +| [`lc materialize`](materialize.md) | Make the analysis's outputs; commit each one as it lands. | +| [`lc status`](status.md) | Report the state of every output. Reads only; always exits 0. | +| [`lc run`](run.md) | Run an ad-hoc command in the project environment, under isolation. | +| [`lc build`](build.md) | Containerized projects: build the image and commit it. | ## Global options @@ -33,7 +39,15 @@ Options: --help Show this message and exit. ``` -## Removed commands +## Exit codes -For historical context: `lc dev`, `lc setup`, `lc target`, and `lc update` no -longer exist as explicit commands. See the removal pages for details. +- `0` — the command did what it says. +- `1` — a refusal or a failure, with the reason on stderr. For + `lc materialize --check` and `lc init --check`, exit 1 means "work + would be done" — the gate form scripts branch on. +- `lc run` is a proxy: it exits with the command's own code + (`128 + N` for a signal), so pipelines read it exactly as they would + the bare command. + +Every verb with a report takes `--json` for the machine-readable form; +each verb's page shows its shape. diff --git a/docs/cli/init.md b/docs/cli/init.md index 58feff94..e6429b73 100644 --- a/docs/cli/init.md +++ b/docs/cli/init.md @@ -1,8 +1,8 @@ # lc init -Converge a directory into an ASTRA project. Idempotent — safe to run -at any time, on an empty directory, a half-scaffolded one, or an -existing project. +Converge a directory into a Lightcone project. Idempotent — safe to run +at any time, on an empty directory, a half-scaffolded one, an existing +project, or a fresh clone. ## Synopsis @@ -19,65 +19,66 @@ manages, and never overwrites files you own: - **Created if missing** — every item in the tree below. A directory that already holds an `astra.yaml` is *adopted*: the spec is left - untouched and only the missing lightcone pieces are added. -- **Repaired** — the managed `.gitignore` block (appended exactly once, - keyed on its `# lightcone-cli` marker), and the stored scratch root - when `--scratch` is passed and differs from the project config. -- **Warned about** — problems `lc` can see but must not fix, reported - in the `warnings` list: an unsupported directory `COPY` in your - Containerfile, an unparseable `.lightcone/lightcone.yaml`. Warnings - don't affect the exit code. -- **Never touched** — anything you authored. - -`--check` reports what a run *would* create or repair, writes nothing, -and exits `1` when the project is not converged. `--json` prints the -report as machine-readable JSON: + untouched and only the missing lightcone pieces are added. A + directory inside an existing git repository adopts that repository + rather than nesting a new one. +- **Repaired** — derived artifacts that have drifted: a `uv.lock` that + no longer matches `pyproject.toml`, a `.venv` that no longer matches + the lock, a managed `.gitignore` or `.gitattributes` entry that a + newer `lc` added. Repairs only ever append or rebuild derived state; + hand-written lines are never reordered or removed. +- **Blocked** — something convergence can see but must not fix by + appending: a `.gitignore` rule that would silently swallow + `results/`, a `.gitattributes` whose ordering would misroute storage. + A blocked item names the file and line at fault, counts against + convergence, and is yours to resolve. +- **Warned about** — advisory facts (e.g. uv falling back to file + copies across filesystems). Warnings never affect the exit code. + +`--check` computes the same report without writing anything and exits +`1` when the project is not converged. `--json` prints it +machine-readable: ```json { - "converged": false, - "created": ["Containerfile"], - "repaired": [".gitignore"], - "unchanged": ["astra.yaml", "..."], + "converged": true, + "created": [], + "repaired": [], + "unchanged": ["astra.yaml", "pyproject.toml", "..."], + "blocked": [], "warnings": [] } ``` -Agents driving a project should run `lc init` (or `lc init --check ---json`) at the start of a session to make sure the directory is -workable. +Agents driving a project should run `lc init --check --json` at the +start of a session to make sure the directory is workable. ## What it creates -The spec scaffold follows the `astra init` boilerplate -(`astra.yaml`, `universes/baseline.yaml`), with the -lightcone-specific pieces layered on top. Inside `DIRECTORY` -(creating it if needed): +Inside `DIRECTORY` (creating it if needed): ```text -astra.yaml # tiny boilerplate spec with one example output +astra.yaml # boilerplate spec with one example output universes/ baseline.yaml # the default universe -Containerfile # project image; referenced by `container:` in astra.yaml -requirements.txt # analysis dependencies (numpy, pandas to start) -.gitignore # Python + lightcone state + MyST build output -.lightcone/ - lightcone.yaml # project config: { target: local } (+ scratch_root if --scratch) -results/ - README.md # the materialization contract; outputs land here via `lc run` -myst.yml # MyST report configuration (MySTRA plugin) -index.md # template report referencing astra.yaml elements -.venv/ # Python venv with the analysis dependencies (skipped with --no-venv) +pyproject.toml # the uv project — the environment's source of truth +.python-version # the exact interpreter, pinned +uv.lock # derived: converged by correctness, not existence +.venv/ # derived: built from the lock (local, never committed) +.gitignore # managed entries, converged line-wise +.git/ # a git repository, with git-annex initialized +.gitattributes # the storage policy: what the annex carries +.datalad/config # dataset identity (a DataLad dataset from birth) +data/ + README.md # declared input data lives here +results/ + README.md # outputs land here — lc's to write +myst.yml # MyST report configuration +index.md # template report referencing astra.yaml ``` -The boilerplate `container: python:3.12-slim` from the astra -boilerplate is rewritten to `container: Containerfile`, so the project -builds its own content-addressed image and dependencies can evolve -under `lc build`. - -On a known site (NERSC Perlmutter, a lightcone JupyterHub), `lc init` -also prints the detected site and the scratch root that `lc run` will -use for its operational state. +Two things it deliberately does *not* create: a `src/` directory +(where analysis code lives is your layout, and git doesn't track empty +directories), and any dependency in `pyproject.toml` — the lock +carries only what *your* analysis imports, added with `uv add`. ## Options @@ -85,21 +86,17 @@ use for its operational state. |--------|---------|--------| | `--check` | off | Report drift without writing; exit 1 if not converged. | | `--json` | off | Emit the convergence report as JSON on stdout. | -| `--no-git` | off | Skip `git init`. | -| `--no-venv` | off | Skip venv creation (`uv venv` if available, else `python -m venv`). | -| `--scratch ` | site default | Scratch root for snakemake state, dask spill, and run locks. Shell expressions like `$SCRATCH` are kept verbatim and expanded at run time. | -> The historical `--target`, `--existing-project`, `--sub-analysis`, -> and `--permissions` flags have been removed. +There is deliberately nothing else — no `--no-git`, no template +selection. The project layout is the contract the other verbs rely on. ## Examples ```bash -lc init # converge cwd -lc init my-analysis # scaffold/converge ./my-analysis -lc init my-analysis --no-git --no-venv # bare bones -lc init . --scratch '$SCRATCH' # pin the scratch root explicitly -lc init --check --json # is this directory workable? (for scripts/agents) +lc init # converge cwd +lc init my-analysis # scaffold/converge ./my-analysis +lc init --check --json # is this directory workable? (for scripts/agents) +lc init # in a fresh clone: rebuild .venv + the annex ``` ## Next steps @@ -107,9 +104,9 @@ lc init --check --json # is this directory workable? (for script ```bash cd my-analysis # Describe your analysis in astra.yaml — inputs, outputs, recipes, -# decisions. ASTRA specs are plain YAML; write them by hand or draft -# them with your AI coding assistant of choice. -lc run # materialize the outputs -lc status # check what's ok / stale / missing -myst start # preview the report (requires: npm i -g mystmd) +# decisions — and write the scripts the recipes name. +uv add numpy # declare what the scripts import +git add -A && git commit -m "First analysis" +lc materialize # make the outputs +lc status # see where everything stands ``` diff --git a/docs/cli/materialize.md b/docs/cli/materialize.md new file mode 100644 index 00000000..6a7283c7 --- /dev/null +++ b/docs/cli/materialize.md @@ -0,0 +1,113 @@ +# lc materialize + +Make the analysis's outputs, and commit each one as it lands. This is +the build verb: it validates the spec, converges the environment, runs +every recipe that needs running — in dependency order, in parallel +where the graph allows — and commits each result together with its +manifest, in a commit whose message is a replayable run record. + +## Synopsis + +```text +lc materialize [OPTIONS] [TARGETS]... +``` + +With no targets, everything the spec declares, across every universe. +A target narrows the run to an output and whatever it depends on: + +- `fit` — the output `fit` in every universe that has it. +- `robust/fit` — exactly one universe's output. + +A target that matches nothing is an error listing what exists — +quietly making nothing is the least useful thing a build tool can do. + +## What gets remade + +An output is remade when it is `stale` — the analysis defines it +differently than it was made (a changed recipe or decision), one of +its declared inputs changed content, or it was edited by hand since. +Inputs are compared by content, so a rebuild that comes out +byte-identical stops the cascade there. + +An output that is `behind` — still exactly what the spec asks for, +but made under an earlier environment — is reported and left alone; +`--refresh` widens the run to remake those too. A `current` output is +never touched, under any flag. + +## The run's contract + +- **Starts clean, ends clean.** A dirty tree is a refusal (the message + sorts your uncommitted work from stray files under `results/`); a + failed or interrupted recipe's partial work is rolled back. +- **Fetches what it needs.** Declared inputs whose annexed content is + not in this clone are fetched before anything hashes. +- **Commits as it goes.** Each output lands in its own commit, written + by the driver in one thread while other recipes keep running. +- **Reports every independent failure.** One failing recipe doesn't + abort the rest; its dependents report `blocked` and the run exits 1 + with all of it listed. +- **Maintains the publication view.** With a `[project].license` + declared, the run converges `ro-crate-metadata.json` in a trailing + commit. + +On a containerized project, the run resolves the committed image first +(building it as a preflight if the declaration is committed but the +image never built). Inside a SLURM allocation, the run spans every +allocated node — see [Running on a Cluster](../user/cluster.md). + +## Check mode + +`--check` classifies every output without executing, committing, or +fetching anything, and exits `1` if a run would do work — the gate a +script or CI job branches on. It is exempt from the dirty-tree +refusal: reading the state of a project before deciding what to commit +is what it is for. + +## Options + +| Option | Default | Effect | +|--------|---------|--------| +| `--check` | off | Report what would run and why; exit 1 if anything is out of date. | +| `--refresh` | off | Also remake `behind` outputs. Never touches `current` ones. | +| `--json` | off | Emit the report as JSON on stdout. | + +There is deliberately no `--jobs` (a run takes every core; sizing +belongs to the allocation you run it in), no `--force`, and no flag to +*skip* a stale output — deleting its directory is your own file +operation, and stronger consent than a flag. + +## The JSON report + +```json +{ + "ok": true, + "up_to_date": true, + "made": [], + "current": ["baseline/fit", "robust/fit", "baseline/fit_plot", "robust/fit_plot"], + "behind": {}, + "failed": [], + "blocked": [], + "planned": {}, + "warnings": [], + "notes": [] +} +``` + +The first two keys are the ones to branch on: `ok` — everything +attempted finished; `up_to_date` — nothing needed doing (a failed run +is never up to date, and `behind` outputs don't count against it). +`planned` is check mode's answer, mapping each would-run output to why; +`behind` maps each left-alone output to the commit that can rebuild its +environment. `notes` carries sandbox messages verbatim — denial +remedies are built to be pasted. + +## Examples + +```bash +lc materialize # everything, all universes +lc materialize fit # one output (and upstreams), every universe +lc materialize robust/fit # one universe's output +lc materialize --check # would anything run? (exit 1 = yes) +lc materialize --refresh # also remake behind outputs +lc materialize --check --json # the machine-readable gate +``` diff --git a/docs/cli/run.md b/docs/cli/run.md index 3e311be9..594a18a8 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -1,97 +1,58 @@ # lc run -Materialize outputs declared in `astra.yaml`. Generates a Snakefile -and dispatches through Snakemake on a Dask cluster. +Run an ad-hoc command in the project environment, under isolation. +This is the probe verb: it executes exactly one command the way a +recipe would be executed — same environment, same sandbox — so "does +it work under `lc run`?" and "will it work as a recipe?" are the same +question. ## Synopsis ```text -lc run [OPTIONS] [OUTPUTS]... +lc run COMMAND... ``` -`OUTPUTS` is zero or more output ids. With no arguments, materializes -everything (Snakemake's `rule all`). - -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | all universes in `universes/*.yaml` (or `["default"]` if none exist) | Restrict to one universe. | -| `--jobs`, `-j N` | `os.cpu_count()` | Parallel jobs / Dask submission concurrency. Passed as both `--cores` and `--jobs` to Snakemake. | -| `--rerun-triggers TRIGGERS` | `code,input,mtime,params` | Comma-separated rerun triggers (forwarded to Snakemake). | -| `--force`, `-f` | off | `--force` when targets are named, `--forceall` otherwise. | -| `--verbose`, `-v` | off | Show the underlying Snakemake / executor chatter and the spawned `snakemake` invocation. | - -## What happens, step by step - -1. Find the project (walk up looking for `astra.yaml`). -2. Discover universes from `universes/*.yaml` (default to `["default"]`). -3. Resolve the container runtime via - `lightcone.engine.container.load_runtime`. If `auto` falls back to - `none` while the spec declares containers, print a loud provenance - warning. -4. Generate `.lightcone/Snakefile` and - `.lightcone/snakefile-config.json` for the selected universes. -5. Translate any explicit `OUTPUTS` into Snakemake target paths - (`/.lightcone-manifest.json`) — this is what tells - Snakemake "build that specific output." -6. Open a Dask cluster context (`local`, `srun`-backed inside - `SLURM_JOB_ID`, or external if `DASK_SCHEDULER_ADDRESS` is set). -7. Spawn `snakemake -s … -d … --cores N --jobs N --executor dask - --rerun-triggers …` with `DASK_SCHEDULER_ADDRESS` in the environment. -8. In the default (non-verbose) path, filter the executor's banner - chatter so the output reads as lightcone's, not Snakemake's. Real - error content always passes through. - -## Output qualification - -When the same `output_id` appears in multiple sub-analyses, you must -qualify it as `.`: +Everything after `run` is the command, verbatim — flags included. +`lc run` takes no options of its own, so nothing needs escaping: ```bash -lc run inference # error if 'inference' is ambiguous -lc run hod_fitting.inference # disambiguated +lc run python -c "import numpy; print(numpy.__version__)" +lc run python src/fit.py --points data/points.csv --outliers keep --output /tmp/probe ``` -Each rule's body wraps the recipe in a ` run --rm --pull=never --v "$PWD":"$PWD" -w "$PWD" bash -c ''` shell when a -container is configured. After the recipe shell exits, the Snakefile -calls `write_manifest()` host-side and the validation snippet emits -warnings for empty / all-NaN / wrong-extension outputs. +## What it does + +- **Converges the environment first.** The probe syncs `.venv` to the + lock before executing, so what you probe is what a recipe gets. +- **Applies the recipe policy.** The project tree is read-only apart + from `results/`, declared inputs are readable, undeclared tools + don't execute. On a containerized project, the command runs inside + the committed image (which must already be built — the probe never + builds). +- **Proxies the exit code.** `lc run` exits with the command's own + code — `128 + N` when a signal killed it — so scripts and pipelines + read it exactly as they would the bare command. +- **Explains denials.** On a nonzero exit, a note on stderr says the + command ran sandboxed; when the failure looks like a denial, the + note names the path and the remedy (`uv add` for a missing package, + an ASTRA input declaration for data, `results/` or + `tempfile.mkdtemp()` for writes). + +A probe has no output and writes no manifest: nothing it does is +recorded anywhere. Any uv project works — `lc run` doesn't require an +`astra.yaml`, only `pyproject.toml`, `uv.lock` and `.venv` in the +current directory. + +## What it is not + +There is no sandbox opt-out and no flag surface — a command that needs +more than the policy grants is a command that would fail as a recipe, +and the fix (declare the dependency) is the same in both places. ## Examples ```bash -lc run # all outputs, all universes -lc run --universe baseline # one universe -lc run accuracy # one output -lc run accuracy precision --universe baseline # several -lc run --jobs 4 --verbose # parallel, with stack noise -lc run --force --universe baseline # rebuild everything -lc run --rerun-triggers params,input # tighter staleness -``` - -## Inside SLURM - -```bash -salloc -N 4 ... -lc run --universe baseline -j 16 +lc run python -c "import scipy" # is the package in the lock? +lc run bash -c 'echo $HOME' # see the private HOME a recipe gets +lc run python src/fit.py --help # exercise a script exactly as a recipe would ``` - -`lc run` detects `SLURM_JOB_ID`, binds the Dask scheduler to the -driver's hostname, and launches one `dask worker` per node via `srun`. -Workers advertise `cpus`, `memory`, and `gpus` resources. Per-rule -resource hints (`cpus_per_task`, `mem_mb`, `gpus_per_task`) constrain -which workers can pick up which jobs. - -## Provenance gotcha - -If `~/.lightcone/config.yaml` says `runtime: auto` and no runtime is -on PATH, `lc run` falls back to running recipes on the host. Because -each manifest still records the *declared* `container_image`, this is a -provenance lie. `lc run` prints a yellow warning telling you to either -install a runtime or set `container.runtime: none` explicitly. - -See [api/dask_cluster](../api/dask_cluster.md) for the cluster-shape -decision and [Architecture](../architecture.md) for the full execution -flow. diff --git a/docs/cli/status.md b/docs/cli/status.md index 7985c3a5..fae5fdb9 100644 --- a/docs/cli/status.md +++ b/docs/cli/status.md @@ -1,7 +1,10 @@ # lc status -Manifest-driven status report for every output declared in -`astra.yaml`. +Report what state each of the analysis's outputs is in. Reads only: it +runs nothing, commits nothing, transfers no data, does not mind an +unclean tree, and always exits `0` — a state is not a failure. The +moment you most need to know where a project stands is when it isn't +clean, so this verb works there. ## Synopsis @@ -9,59 +12,79 @@ Manifest-driven status report for every output declared in lc status [OPTIONS] ``` -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | every universe in `universes/*.yaml` | Restrict to one universe. | -| `--json` | off | Emit machine-readable JSON instead of a styled table. | - ## Output -Per universe, one line per declared output: +```text + mode: direct + sandbox: landlock (fs: declared, network: allowed) + + · current baseline/fit a3f1f11 + · current baseline/fit_plot a3f1f11 + · behind robust/fit 00cc14e made under an earlier environment + ! stale robust/fit_plot — no manifest — it has never been materialized -``` -Universe baseline - ✓ ok accuracy - ✸ stale precision - ✗ miss recall - → alias inference +2 current · 1 behind · 1 stale ``` -Statuses (defined in `lightcone.engine.status.StatusLiteral`): +The header is repository facts: which mode the project executes in +(and, for a containerized project, the image's tag and state), and +what enforcement a run on this host would get. No runtime and no +network is needed to answer either. -| Status | Meaning | When you see it | -|--------|---------|-----------------| -| `ok` | Manifest present, recomputed `code_version` matches what the manifest recorded. | The output is up to date. | -| `stale` | Manifest present, but `code_version` drifted. | You changed the recipe, image, or a decision since the last run. `lc run` will re-execute. | -| `missing` | No manifest at the expected output path. | Never built, or the directory was deleted. | -| `alias` | The output has no `recipe:` of its own — it's just a name pointing at a sibling output (typical for ASTRA "promoted" outputs from sub-analyses). | Status is implicitly determined by the upstream. | +Then one line per output the spec declares, in dependency order: its +state, **the commit it was made at**, and — for anything not current — +why. The commit column is the verb's reason to exist: "which code made +this?" has an answer for a current output too, and for a `behind` +output that commit is where the environment that produced it can be +read back. -## Why it doesn't import Snakemake +## States -`lc status` reads only the per-output `.lightcone-manifest.json` files -and recomputes `code_version` against the current spec. It never -imports Snakemake or touches `.snakemake/`. That makes it usable on: +- `current` — exactly what the spec asks for. Nothing to do. +- `behind` — still what the spec asks for; the environment moved since. + Left alone by runs; `--refresh` remakes. +- `stale` — contradicts the project: definition changed, an input's + content changed, or the output was edited by hand since it was made + (a *foreign write* — the offending commit is named). -- A fresh clone before any `lc run`. -- A frozen archive copied off a cluster. -- A read-only workspace. +## Report vs gate -If a manifest is missing, the output reports `missing`. If a manifest is -unparseable, `read_manifest` returns `None` and you also see `missing` -— that is the agent-forged-file scenario; investigate with `lc verify`. +`lc status` reports; **`lc materialize --check` gates.** Two verbs +answering the same question with different exit codes is how a script +comes to depend on the wrong one, so the split is sharp: use status for +eyes, check for exit codes. -## Examples +## Options -```bash -lc status # every output, every universe -lc status --universe baseline # just baseline -lc status --json # machine-readable JSON output +| Option | Default | Effect | +|--------|---------|--------| +| `--json` | off | Emit the report as JSON on stdout. | + +## The JSON report + +```json +{ + "mode": "direct", + "image": null, + "sandbox": "landlock (fs: declared, network: allowed)", + "counts": {"current": 4, "behind": 0, "stale": 0}, + "outputs": [ + { + "output": "baseline/fit", + "status": "current", + "why": "", + "git_sha": "a3f1f11791430d1becbe5548477b5910ab59a94a", + "data_version": "sha256:939e9a55...", + "foreign_write": "" + } + ], + "warnings": [] +} ``` -## Related - -- [`lc verify`](verify.md) — recomputes data hashes too (slower; catches - tampering and broken chains). -- [api/status](../api/status.md) — the Python API. -- [api/manifest](../api/manifest.md) — the manifest schema. +Per output: the state, the reason (empty for `current`), the commit it +was materialized at and its content identity (both empty if it never +was), and `foreign_write` — the sha of a hand-edit's commit when one +was detected, which the prose `why` cannot carry for a machine +consumer. For a containerized project, `image` is +`{"tag": ..., "state": "present" | "absent" | "unfetched"}`. diff --git a/docs/cli/verify.md b/docs/cli/verify.md deleted file mode 100644 index 6e1ae216..00000000 --- a/docs/cli/verify.md +++ /dev/null @@ -1,67 +0,0 @@ -# lc verify - -Recompute hashes for every materialized output and walk the recorded -input chain. Catches tampering, drift, and forged manifests. - -## Synopsis - -```text -lc verify [OPTIONS] -``` - -## Options - -| Option | Default | Effect | -|--------|---------|--------| -| `--universe`, `-u NAME` | every universe | Restrict to one universe. | - -## Output - -``` -Universe baseline - ✓ ok accuracy - ✗ tampered_data precision recorded 'sha256:abc…' != actual 'sha256:def…' - ✗ broken_chain recall upstream 'features' data_version drifted - ✗ missing_manifest f1 No manifest found at output directory -``` - -Exit code is non-zero if any output failed. - -## Failure modes - -| Failure | What it means | -|---------|----------------| -| `missing_manifest` | The output directory exists but `.lightcone-manifest.json` is missing or unparseable. Most innocent cause: someone deleted the manifest. Most concerning: the directory was created by something other than `lc run`. | -| `tampered_data` | The bytes inside the output directory no longer hash to the `data_version` recorded in the manifest. Files were edited, regenerated outside the harness, or the directory contents differ from what was originally written. | -| `broken_chain` | The manifest records a specific upstream `data_version`, but the upstream's current `data_version` doesn't match. Usually means the upstream was rerun without rebuilding the downstream. Fix: `lc run` the downstream. | - -## Outputs without recipes - -Alias outputs (declared in `astra.yaml` without their own `recipe:`) -are skipped — there's no manifest to verify. They are checked -implicitly via the upstream output they reference. - -## Outputs that aren't materialized - -If an output's directory doesn't exist at all, `lc verify` skips it -(no failure to report). Use [`lc status`](status.md) to know what's -missing in the first place. - -## Examples - -```bash -lc verify # every output, every universe — non-zero exit on any failure -lc verify --universe baseline # just baseline -``` - -## When to run - -- Before publishing a result. -- After moving a project between machines. -- Periodically on shared archives. -- Whenever `lc status` shows `ok` but the data feels suspicious. - -## Related - -- [api/verify](../api/verify.md) — implementation and `VerifyResult`. -- [api/manifest](../api/manifest.md) — the manifest schema and what's hashed. diff --git a/zensical.toml b/zensical.toml index 705ce74c..8178b33a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -25,11 +25,10 @@ nav = [ {"CLI Reference" = [ {"Overview" = "cli/index.md"}, {"lc init" = "cli/init.md"}, + {"lc materialize" = "cli/materialize.md"}, + {"lc status" = "cli/status.md"}, {"lc run" = "cli/run.md"}, {"lc build" = "cli/build.md"}, - {"lc status" = "cli/status.md"}, - {"lc verify" = "cli/verify.md"}, - {"lc export" = "cli/export.md"}, ]}, {"Python API" = [ {"Overview" = "api/index.md"}, From 8099d85a47924142986769be1e81a42ffdf89970 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:55:15 +0200 Subject: [PATCH 20/23] Docs rewrite 4/5: the developer corner (#188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth pass — the maintainer half of the site: `maintainer.md`, `architecture.md`, an **Engine Internals** section replacing the old Python API pages, and the contributing trio. ## What's in it - **`architecture.md`**: the CLI/engine/ASTRA split; the run pipeline as one annotated diagram (guards → converge → plan → fetch → venue → workers → driver loop → crate); identity (two hashes, three states, the one-classify-two-callers discipline); storage (DataLad model, run records, thin commits); the exec boundary (policy → argv rewrite → attestation); the container hatch; venues; the crate. This is CLAUDE.md's invariants translated for humans who didn't live the rebuild. - **`maintainer.md`**: the welcome page in its old shape — what the corner covers, the three-command dev loop, and the house rules (no dead code/foreshadowing, no escape hatches, literal behavior, one implementation per rule, honest reporting). - **Engine Internals** (`docs/api/`, 12 pages): an overview map plus one hand-written tour per module — project, dataset, identity, plan, assets, worker, materialize, venue, sandbox, image & container, crate. Each page: responsibility, key-symbols table, "what must stay true", and where its tests live. Hand-written rather than autodoc, deliberately: the engine is not a public API (projects don't depend on lightcone-cli), so contract beats signatures. The 17 old pages documented modules that no longer exist; all deleted. - **Contributing**: `setup.md` (uv-only loop, the gated-suite table, docs/wheel builds, pre-PR checklist), `testing.md` (the `tools`/`real_tools` seam, a where-does-this-question-belong table, the enforcement suite's four properties, the mutation-check rule and its two traps), and a new `extending.md` (the change-map table plus the everywhere-rules and conventions). - **Deleted**: `docs/hpc/` (site-registry model), `contributing/backends.md`, `contributing/hpc-sites.md`. Nav updated (Python API → Engine Internals; Extending added). ## Verification - `zensical build` now reports **zero issues** — the two warnings that survived passes 1–3 lived in pages this one deletes. - Module pages were anchored on the modules' own docstrings and CLAUDE.md's invariants; symbol names spot-checked against the source (e.g. `runtime_for_run`'s `(Runtime, "built"|"present")` return, `datalad_run_subject`, `_SITES`). - No remaining references to any deleted page anywhere in `docs/`. Next and last: (5) final polish — `check-docs.yml` refresh and CLAUDE.md reconciliation (the docs-frozen rules, the design-doc pointer). 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx Co-authored-by: Claude Fable 5 --- docs/api/assets.md | 55 +++- docs/api/cli.md | 88 ------ docs/api/cloudbuild.md | 63 ----- docs/api/container.md | 259 +++++------------- docs/api/crate.md | 61 +++++ docs/api/dask_cluster.md | 104 -------- docs/api/dask_executor.md | 118 -------- docs/api/dataset.md | 58 ++++ docs/api/identity.md | 53 ++++ docs/api/index.md | 99 +++---- docs/api/io_manager.md | 8 - docs/api/manifest.md | 112 -------- docs/api/materialize.md | 72 +++++ docs/api/plan.md | 57 ++++ docs/api/project.md | 54 ++++ docs/api/runner.md | 14 - docs/api/sandbox.md | 70 +++++ docs/api/site_registry.md | 54 ---- docs/api/snakefile.md | 106 -------- docs/api/status.md | 63 ----- docs/api/targets.md | 11 - docs/api/tree.md | 104 -------- docs/api/validation.md | 50 ---- docs/api/venue.md | 63 +++++ docs/api/verify.md | 66 ----- docs/api/worker.md | 59 ++++ docs/architecture.md | 473 ++++++++++++--------------------- docs/contributing/backends.md | 42 --- docs/contributing/extending.md | 55 ++++ docs/contributing/hpc-sites.md | 24 -- docs/contributing/setup.md | 107 ++++---- docs/contributing/testing.md | 158 ++++++----- docs/hpc/containers.md | 52 ---- docs/hpc/index.md | 17 -- docs/hpc/site-registry.md | 6 - docs/hpc/targets.md | 3 - docs/maintainer.md | 66 +++-- zensical.toml | 25 +- 38 files changed, 1110 insertions(+), 1839 deletions(-) delete mode 100644 docs/api/cli.md delete mode 100644 docs/api/cloudbuild.md create mode 100644 docs/api/crate.md delete mode 100644 docs/api/dask_cluster.md delete mode 100644 docs/api/dask_executor.md create mode 100644 docs/api/dataset.md create mode 100644 docs/api/identity.md delete mode 100644 docs/api/io_manager.md delete mode 100644 docs/api/manifest.md create mode 100644 docs/api/materialize.md create mode 100644 docs/api/plan.md create mode 100644 docs/api/project.md delete mode 100644 docs/api/runner.md create mode 100644 docs/api/sandbox.md delete mode 100644 docs/api/site_registry.md delete mode 100644 docs/api/snakefile.md delete mode 100644 docs/api/status.md delete mode 100644 docs/api/targets.md delete mode 100644 docs/api/tree.md delete mode 100644 docs/api/validation.md create mode 100644 docs/api/venue.md delete mode 100644 docs/api/verify.md create mode 100644 docs/api/worker.md delete mode 100644 docs/contributing/backends.md create mode 100644 docs/contributing/extending.md delete mode 100644 docs/contributing/hpc-sites.md delete mode 100644 docs/hpc/containers.md delete mode 100644 docs/hpc/index.md delete mode 100644 docs/hpc/site-registry.md delete mode 100644 docs/hpc/targets.md diff --git a/docs/api/assets.md b/docs/api/assets.md index 39815f8a..77dce6ef 100644 --- a/docs/api/assets.md +++ b/docs/api/assets.md @@ -1,5 +1,52 @@ -# lightcone.engine.assets (removed) +# lightcone.engine.assets -This module was the Dagster asset factory. It no longer exists. The Snakemake -generator that replaced it lives at -[engine/snakefile](snakefile.md). +One output: its directory, its manifest, and whether it is still +current. The classification rule lives here, next to the manifest it +reads and the hashes it compares — and it is the one place in the +engine where a bug is quiet rather than loud, which is why it may not +have two implementations. + +Source: `src/lightcone/engine/assets.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `classify(...)` | The one rule: `current` / `behind` / `stale`, with the why. Two callers — the worker and the read-only walk. | +| `Verdict.calls_for_a_remake(refresh=)` | The one place a state becomes an action: `stale` always, `behind` only when asked. | +| `data_version(path)` | Content hash of a directory or file — computed in the worker, before anything is annexed. | +| `Versions` | Per-run memo so a shared declared input hashes once, not once per dependent. | +| `read(dir)` / `write(...)` | The manifest, `.lightcone-manifest.json`. | +| `output_dir(root, u, o)` | The path, guarded: an id that is not one path component is refused — this guard is what lets the worker's reset stay a whole-directory delete. | +| `ContentNotFetchedError` | An annexed file whose content is not in this clone, in either shape it takes. | + +## What must stay true + +- **One `classify`, two callers, one differing value.** The worker + hands live input digests; check mode hands `None` for anything + upstream that will run ("this is going to change"). That value is + the entire difference — never a second body of logic. History (the + foreign-write fact) enters the same way: computed by whoever has + git, handed in as a value. +- **The comparison is fourfold**: `definition_version`, the declared + input *set* (separate on purpose — a dropped dependency moves + neither hash), each recorded input digest, then `env_version`. + `stale` wins over `behind`; `behind` does not propagate and a behind + upstream still feeds its dependents. +- **A skip returns the *recorded* digest, never a recomputed one** — + on a bytes-free clone, rehashing dangling symlinks would quietly + report a different output. +- **Unfetched content refuses loudly, in both shapes.** A pointer file + hashes to a well-formed digest of the wrong thing; a dangling + symlink drops out of an `is_file()` walk without a word. Both raise + `ContentNotFetchedError` naming `git annex get`; only dangling + symlinks are added back to the directory walk. +- **`calls_for_a_remake` has three callers** (worker, check, the + cascade walk) and no inline re-spellings — the third copy is where + they start to disagree. + +## Tests + +`tests/test_assets.py` — pure; nothing on disk beyond `tmp_path`. +The pointer-file and dangling-symlink traps are pinned against real +annex shapes in `tests/test_dataset.py`. diff --git a/docs/api/cli.md b/docs/api/cli.md deleted file mode 100644 index df679953..00000000 --- a/docs/api/cli.md +++ /dev/null @@ -1,88 +0,0 @@ -# lightcone.cli.commands - -The Click surface. Defined in `src/lightcone/cli/commands.py`. Six -public commands: `init`, `run`, `status`, `verify`, `build`, `export`. - -The user-facing reference is in [CLI Overview](../cli/index.md). This -page is a tour of the module internals. - -## Entry point - -```python -@click.group() -@click.version_option(package_name="lightcone-cli") -@click.pass_context -def main(ctx: click.Context) -> None: - ctx.ensure_object(dict) - _ensure_global_config() # auto-create ~/.lightcone/config.yaml with defaults -``` - -`main` is exposed as `lightcone.cli.main` (re-exported from -`lightcone.cli.__init__`) and is the entry point declared in -`pyproject.toml::project.scripts`: - -```toml -[project.scripts] -lc = "lightcone.cli:main" -``` - -## Helpers - -### `_config_path() → Path` - -Returns `~/.lightcone/config.yaml`. Used by `_ensure_global_config()`, -which the `main` group calls to create the file with defaults -(`container: {runtime: auto}`) on first invocation. - -### `_project_root(start: Path | None = None) → Path` - -Walks up from `start` (or `cwd`) looking for `astra.yaml`. Raises -`click.ClickException` if none found. Used by `run`, `status`, `verify`, -`build`. - -### `_target_for(project: Path, output_id: str, universe: str) → str` - -Translate an `output_id` (or qualified `.`) into -the Snakemake target path that materializes it — specifically the -manifest file `results///.lightcone-manifest.json`. -Raises `click.ClickException` if the id is unknown or ambiguous. - -### `_run_snakemake(cmd, *, env, scratch_root, verbose)` - -Spawn `snakemake` and forward the run's narrative output: lines the -executor plugin prefixes with the sentinel -(`lightcone.engine.runner.SENTINEL`) stream to the terminal with the -prefix stripped; everything else (DAG chatter, job stats) is dropped -unless `verbose`. stderr is tailed into a bounded ring buffer and, on -failure, dumped to `snakemake-stderr-.log` under the scratch -root. Returns the exit code. - -### `_status_label(s: str) → str` - -Map a status literal to the Rich-formatted display label: - -| Status | Display | -|--------|---------| -| `ok` | `[green]✓ ok[/green]` | -| `stale` | `[yellow]✸ stale[/yellow]` | -| `missing` | `[red]✗ miss[/red]` | -| `alias` | `[dim]→ alias[/dim]` | - -## Boilerplate text - -`_CONTAINERFILE_TEMPLATE`, `_REQUIREMENTS`, `_GITIGNORE_BASE`, -`_GITIGNORE_APPEND`, `_MYST_YML`, and `_INDEX_MD_BODY` are multi-line -strings written at `lc init` time (the spec boilerplate itself comes -from astra's boilerplate helper). Edit them to change what new -projects look like. - -`init` is a convergence loop, not a one-shot scaffolder: each managed -item is created if missing, offered to an optional -`repair(text) -> str | None` hook otherwise (today only -`_repair_gitignore`, which appends the managed block once), and left -alone when the hook returns `None`. `--check` computes the same report -without writing (exit 1 when not converged); `--json` prints it as -`{converged, created, repaired, unchanged, warnings}`. Warnings carry -problems init can see but must not fix (e.g. a directory `COPY` in the -Containerfile, detected via -`lightcone.engine.container.directory_copy_sources`). diff --git a/docs/api/cloudbuild.md b/docs/api/cloudbuild.md deleted file mode 100644 index 17f0f6a3..00000000 --- a/docs/api/cloudbuild.md +++ /dev/null @@ -1,63 +0,0 @@ -# lightcone.engine.cloudbuild - -Remote image builds through GCP Cloud Build — the build backend for -deployments with no OCI runtime on the host (a JupyterHub user pod on -GKE). Pure `urllib` REST against the metadata server, GCS, the Cloud -Build API, and the Docker Registry v2 API; no SDK dependency, no -stored credentials, no git remote required. - -Source: `src/lightcone/engine/cloudbuild.py`. - -## Deployment contract - -Env vars injected into every user pod by the deployment (see the -hub-deploy `lightcone` hub config): - -| Env var | Meaning | -|---------|---------| -| `LIGHTCONE_REGISTRY` | Artifact Registry prefix (`-docker.pkg.dev//`); also names the GCP project builds run in. Declared in `engine.container` (`REGISTRY_ENV`). | -| `LIGHTCONE_BUILD_BUCKET` | GCS bucket for build sources and logs. Its presence (with the registry) selects this backend — `cloudbuild_available()`. | -| `LIGHTCONE_BUILD_SERVICE_ACCOUNT` | Optional dedicated build SA; the deployment grants it registry-writer rights only. | - -Auth is the pod's Workload Identity, spoken to the GCE metadata server -(`_metadata_access_token()`). The pod's identity needs -`cloudbuild.builds.editor`, `iam.serviceAccountUser` on the build SA, -object create/view on the bucket, and `artifactregistry.reader` for -the freshness probe. - -## `ensure_image(project, containerfile_spec, *, project_name, force=False, on_progress=None) → str` - -Make sure the project's image is in the registry; return its ref. -Content-addressed and git-free: - -1. Compute the ref `$LIGHTCONE_REGISTRY/lc-:` — the - same `image_identity()` digest as the local `lc--` - tag, spelled for a registry. -2. `registry_image_exists(ref)` — one HEAD on the Docker Registry v2 - manifest endpoint. Present → done (no build, no upload). `force` - skips this probe. -3. Tar the **staged build context** (`_populate_build_context` — the - exact file set the tag hashes) and upload it to the bucket under a - content-addressed object name. -4. Submit the build (docker builder step, image push, logs to - `gs:///logs` with `GCS_ONLY` — required for custom SAs and - the source of the failure tail), poll to a terminal status. -5. Non-`SUCCESS` → `CloudBuildError` carrying the build-log tail. - -`on_progress(phase, detail)` phases: `cached`, `staging`, then Cloud -Build statuses lowercased (`queued`, `working`, `success`, …). - -## `registry_image_exists(ref) → bool | None` - -`None` — not `False` — when unknowable (no metadata credentials, -registry unreachable), so callers can distinguish "absent, build it" -from "can't tell". Artifact Registry accepts the OAuth2 access token -directly as a Bearer on `/v2/` endpoints. - -## Tests - -`tests/test_cloudbuild.py` mocks the two HTTP seams -(`_metadata_access_token`, `_request`) and exercises the real control -flow: backend selection, freshness probe, staging, submission -(including the custom-SA payload), polling, failure-tail reporting, -and the staged-tarball ↔ hashed-context equivalence. diff --git a/docs/api/container.md b/docs/api/container.md index dad10864..79d99d46 100644 --- a/docs/api/container.md +++ b/docs/api/container.md @@ -1,192 +1,73 @@ -# lightcone.engine.container - -The container layer. Two surfaces: build-time (`compute_image_tag`, -`build_image`, `pull_image`) and run-time wrap (`wrap_recipe`, -`make_image_tag_resolver`). - -Source: `src/lightcone/engine/container.py`. - -## Constants - -| Constant | Value | -|----------|-------| -| `RUNTIMES` | `("podman", "docker", "podman-hpc")` — detection priority order | -| `DEPENDENCY_FILES` | `("requirements.txt", "requirements-dev.txt", "requirements-test.txt", "pyproject.toml", "setup.py", "setup.cfg", "poetry.lock", "Pipfile.lock")` | - -Detection priority is podman before docker for two reasons: it's -rootless (less surprising on shared machines), and the docker probe -includes `docker info` so a stopped daemon doesn't silently win over a -healthy podman. - -## Runtime detection - -### `detect_runtime() → str | None` - -Returns the first usable runtime in `RUNTIMES`. "Usable" means the -binary is on PATH and (for docker) `docker info` succeeds. Returns -`None` if nothing's available. - -### `load_runtime(*, project_path=None) → RuntimeChoice` - -Resolve the runtime to use. Reads `container.runtime` from -`~/.lightcone/config.yaml`: - -- `auto` (default) → first available, else `"none"` with `explicit=False`. - On a site declaring `container_runtime: kubernetes` (a Dask Gateway - deployment), auto resolves to `kubernetes` with no PATH probing. -- `docker | podman | podman-hpc` → explicit; binary must exist or - raises `ContainerBuildError`. -- `kubernetes` → explicit; no binary involved (the worker pod is the - container). -- `none` → explicit opt-out. -- Anything else → `ContainerBuildError`. - -`project_path` is accepted for future per-project overrides but is not -consulted today. - -### `RuntimeChoice` (dataclass) - -```python -@dataclass(frozen=True) -class RuntimeChoice: - runtime: str # docker | podman | podman-hpc | none - explicit: bool # True if pinned, False if `auto` produced this -``` - -`explicit=False` + `runtime="none"` means auto fell back silently. Callers -should warn — that case mismatches the manifest's recorded -`container_image` against what actually executed. - -## Image tag computation - -### `compute_image_tag(project_name, containerfile, project_path) → str` - -Returns `lc--`. The hash covers the -Containerfile contents plus every dependency file from `DEPENDENCY_FILES` -that exists at the project root. - -Sanitization: lowercase + spaces → hyphens. - -### `find_dependency_files(project_path) → list[Path]` - -Sorted list of dependency files actually present. Used by -`compute_image_tag`. - -### `hash_file_contents(files) → str` - -Concatenated SHA-256 hex digest of the listed files. Internal helper. - -### `is_containerfile(spec, project_path) → bool` - -True if `spec` resolves to an existing file (i.e. it's a Containerfile, -not a registry image). - -## Build - -### `build_image(tag, containerfile, context, *, runtime, build_args=None) → ContainerBuildResult` - -Run ` build -t -f [--build-arg …] `. -For `podman-hpc`, also runs `podman-hpc migrate ` so compute nodes -can read the image. Raises `ContainerBuildError` on any failure. - -### `pull_image(image, *, runtime) → None` - -Run ` pull `, then (for podman-hpc) `migrate`. Used by -`lc build` to pre-stage registry images so `lc run` can pass -`--pull=never`. - -### `image_exists_locally(tag, *, runtime) → bool` - -Check the local image store. Routes to `image_exists_podman_hpc(tag)` -for `podman-hpc`, otherwise runs ` image inspect `. - -### `_podman_hpc_migrate(tag)` (private) - -Wraps `podman-hpc migrate`. Raises `ContainerBuildError` on failure. - -## Run-time wrap - -### `wrap_recipe(recipe, *, image, runtime) → str` - -Wrap `recipe` so it executes inside `image` under `runtime`. Returns a -shell-command string for Snakemake's `shell()`. - -No-op cases (`recipe` returned unchanged): - -- `image is None` -- `runtime == "none"` -- `runtime == "kubernetes"` — the Dask worker pod executing the recipe - was started from `image`; wrapping would containerize twice. The - image still flows into `code_version` and the manifest. - -Otherwise produces: - -```bash - run --rm --pull=never \ - -v "$PWD":"$PWD" -w "$PWD" \ - bash -c '' -``` - -`--pull=never` is critical: it sidesteps podman's -`unqualified-search-registries` resolution, which fails for our -content-addressed `lc--` tags. The cost: registry images -have to be pre-pulled by `lc build`. - -The bind mount and `-w "$PWD"` ensure recipes that write to relative -paths land in the project tree. Snakemake invokes us with `cwd=project`, -so `$PWD` is the project root. - -Snakemake placeholders inside `recipe` (`{output[0]}`, `{input.X}`, -`{wildcards.universe}`) are preserved — they substitute through Python's -`str.format` at execution time, after wrapping. - -### `make_image_tag_resolver(project_path, project_name) → Callable` - -Returns a memoizing wrapper around `resolve_image_for_run`. Multiple -outputs typically share a Containerfile; resolving re-hashes the file -plus all dependency files (lockfiles can be megabytes), so caching by -spec string for the lifetime of the caller's loop matters. - -### `resolve_image_for_run(spec, *, project_path, project_name, registry=None) → str | None` - -Translate an `astra.yaml` `container:` value into the image tag the -runtime will execute: - -- `None` / empty → `None` -- Containerfile path → `lc--` (the tag `lc build` would - produce), or `/lc-:` when `registry` is given - (a deployment with a remote builder — same content-addressed - identity, spelled for a registry) -- Anything else → returned as-is - -## Status - -### `get_container_status(spec, project_path, project_name, *, runtime) → ContainerStatus` - -Without building or pulling, return a `ContainerStatus` describing what -would happen. - -### `ContainerStatus` (dataclass) - -```python -@dataclass -class ContainerStatus: - type: str # "none" | "prebuilt" | "build" - image: str | None = None # the tag (always set for "prebuilt"/"build") - exists: bool | None = None # local-store presence (None for "none" runtime) - containerfile: str | None = None # the spec, only set for "build" -``` - -## Exceptions - -### `ContainerBuildError` - -Raised by `build_image`, `pull_image`, `_podman_hpc_migrate`, and -`load_runtime` (configuration errors). Message carries the failing -runtime and stderr. +# lightcone.engine.image & container + +The container hatch, split down the pure/impure line. `image.py` is +what a containerized project *declares* and how that becomes an +identity — pure, no subprocess anywhere. `container.py` is building, +storing and entering images — impure, every command through +`project._run`. The exec side (the mount table) lives with the other +backends in `sandbox/oci.py`. + +Sources: `src/lightcone/engine/image.py`, +`src/lightcone/engine/container.py`, `src/lightcone/engine/sandbox/oci.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `image.declaration(root)` | The `[tool.lightcone.image]` table, validated — a closed key set (`base`, `apt-install`, `run-commands`, `env`), because every key is hashed. | +| `image.tag(root)` | `lc-env-<16 hex>` over the rendered Containerfile *and* the identity document. | +| `image.archive_path(root, tag)` | `.datalad/environments//image` — the `datalad containers-add` layout. | +| `container.build(root)` | Build + save + commit, idempotent; returns `(Runtime, "built" \| "present")`. | +| `container.runtime_for_run(root, *, build)` | One function, two strictnesses: `lc build`/materialize-preflight may build and commit; the probe and worker only ever find, fetch, and load. | +| `container.backend(...)` | The single construction point for the exec backend — the only mode branch. | +| `container.sync(...)` | The in-container environment converge: network on, project `:rw`, host uv cache mounted, into `.lightcone/venv`. | +| `Runtime` | Facts only — root/mode/name/tag/id/arch — never mechanism. | + +## What must stay true + +- **The user never sees a Containerfile.** The render exists only in a + transient build context; the image's `LABEL` carries the identity + document so the archive stays self-describing. There is deliberately + no `pip-install` key — the Python environment is the lock's + business, never the image's. +- **The engine never enters the image.** The container is the + *recipe's* world: driver, git, annex, and classification stay the + host's `lc`; exactly two things run in-image (the sync and each + exec, `--network none`). +- **No project file enters the build context** — that is what makes + "code edits never rebuild" structural rather than incidental. +- **The dataset is the store; runtime stores are caches.** Execution + pins the archive's config-blob **id** (readable with no runtime), + never a tag; a dropped archive never substitutes — a rebuild is a + new archive under a new id. +- **Builds and archive commits happen only on a clean tree, and only + after the graph resolves** — a refusal over a typo must not cost a + minutes-long build, and `dataset.save` commits the whole index. +- **The mount table is the mechanism** (`sandbox/oci.py`): project + `:ro`, `results/` `:rw`, declared inputs `:ro`, private HOME, + `--tmpfs /tmp`, over a `--read-only` rootfs — without that flag a + stray write *succeeds* into the ephemeral layer and vanishes while + the attestation claims `fs: declared`. Mounts are resolved source, + **declared** destination — the one policy shape that keeps its paths + unresolved, because they are addresses the recipe uses. +- **Runtime differences are spellings, never shapes.** One + `OCIBackend`, data-parameterized; the podman family is stated once + (`_PODMAN_FAMILY`) and asked positively, so a new runtime falls + outside it by default. podman-hpc adds exactly one step (`migrate`, + outside the load branch) and joins `_SHARED_STORE_RUNTIMES`. + Detection order podman-hpc → podman → docker; docker's daemon is + probed at detection. +- **The architecture gate refuses before the load** — a wrong-arch + `load` succeeds and then dies as `exec format error` deep inside a + recipe. Ignorance passes; a recorded mismatch refuses, naming the + fix. ## Tests -`tests/test_container.py` covers detection, image tag computation, -build invocation, recipe wrapping, and the `RuntimeChoice` resolution -matrix. +`tests/test_image.py` (pure: structure and ordering, tag sensitivity +both ways, the `env_version` frame), `tests/test_container.py` +(lifecycle against the stubbed `_run` — every refusal on recorded +argv), `tests/test_sandbox_oci.py` (the mount table, pure), and +`tests/test_container_smoke.py` — the runtime's answer, gated by +`LC_CONTAINER_TESTS_REQUIRED=1` in CI, building a real image and +proving the record on a bytes-free clone with a real `datalad rerun`. diff --git a/docs/api/crate.md b/docs/api/crate.md new file mode 100644 index 00000000..af2bf0d3 --- /dev/null +++ b/docs/api/crate.md @@ -0,0 +1,61 @@ +# lightcone.engine.crate + +The publication view: the repository described as a Workflow Run +RO-Crate. The project *is* the crate — `ro-crate-metadata.json` sits at +the root, describes what the repository already holds, and a deposit is +`git archive`, not an export step. lc's manifests stay the canonical +record; the crate is the same facts in schema.org vocabulary for +archives and viewers that will never run `lc`. + +Source: `src/lightcone/engine/crate.py` (converged by +`materialize._converge_crate`). + +## Key symbols + +| Symbol | Role | +|---|---| +| `render(root, graph, *, license, dsid, writer)` | The document, as bytes. A pure function of repository state — git comes in as the `writer` callable, the dataset id as a value. | +| `license_of(root)` | `[project].license` from `pyproject.toml`; empty means no crate is maintained. Presence is publication intent. | +| `CRATE_FILENAME` | `ro-crate-metadata.json`. | + +## What must stay true + +- **The clock never enters the render.** `datePublished` is the newest + manifest `finished_at` (the spec file's last-commit date for a + never-materialized project) and must override rocrate's + construction-time default. Entities build in sorted order, + serialization is `sort_keys` — render-twice-identical is the one + byte-level claim, and it is what makes convergence sound. +- **Maintenance is derived, never configured.** RO-Crate requires a + license; materialize must not refuse to run science over a missing + key, and inventing one asserts terms over someone's data. Absent ⇒ + one report line; removed later ⇒ the file is left, and the line says + it is no longer maintained. +- **Run identity comes free from `git_sha`** — the driver reads HEAD + once per run, so grouping manifests by it *is* grouping by run: one + `OrganizeAction` per materialize, a `ControlAction` per execution, a + `HowToStep` per output id (deduped across universes — a step is spec + structure, an action is one execution). +- **The `Person` is the author of the output's *saving* commit** (via + `writer`), never the manifest's `git_sha` — that is the commit the + run *started* at and can be someone else's. +- **The manifest is not transliterated.** `env_version`, + `definition_version` and `hermeticity` get no invented schema.org + spelling — the manifest itself is in the crate as a `File`, + `subjectOf` its output. Real vocabulary comes from the workflow-run + `@context`, without which `containerImage` and `sha256` are + undefined terms JSON-LD silently drops — the pre-rebuild exporter's + failure mode. +- **The rerun entry point does not regenerate the crate** — it is one + task's executor, so the crate lags until the next materialize. + Recorded residue, not a bug. + +## Tests + +`tests/test_crate.py` — pure: fixture manifests, a hand-built graph, a +stub writer, no git anywhere; structure and ordering assertions plus +the single render-twice byte check. `tests/test_crate_smoke.py` — the +official `rocrate-validator` against Provenance Run Crate 0.5: +REQUIRED clean, RECOMMENDED pinned to the recorded `_FLOOR` set (a new +failure is a regression, a disappearing one is the floor to shrink), +required in CI via `LC_CRATE_TESTS_REQUIRED=1`. diff --git a/docs/api/dask_cluster.md b/docs/api/dask_cluster.md deleted file mode 100644 index be2aebd2..00000000 --- a/docs/api/dask_cluster.md +++ /dev/null @@ -1,104 +0,0 @@ -# lightcone.engine.dask_cluster - -Cluster lifecycle for `lc run`. One context manager (`cluster_for_run`), -four branches, no service to manage. - -Source: `src/lightcone/engine/dask_cluster.py`. - -## `cluster_for_run(*, verbose=False, worker_image=None, max_workers=None) → Iterator[dict[str, str]]` - -Yields the env overlay the child snakemake needs to reach the cluster -(the executor plugin lives in a different process, so connection info -travels via environment variables). Four branches in priority order: - -1. **`DASK_SCHEDULER_ADDRESS` already set** → yield it as-is. We don't - own the cluster, so we don't tear it down. -2. **`DASK_GATEWAY__ADDRESS` set** (a JupyterHub deployment) → - **create** a run-scoped Dask Gateway cluster with `worker_image` as - its `image` cluster option, scale it adaptively `1..max_workers`, - wait (bounded by `LIGHTCONE_GATEWAY_WORKER_TIMEOUT`, default 600 s) - for the first worker, and shut the cluster down on exit — success - or failure. Yields `{LIGHTCONE_GATEWAY_CLUSTER: }`: Gateway - schedulers speak a `gateway://` comm scheme a bare `Client` cannot - dial, so the executor rejoins by name through the Gateway API. -3. **`SLURM_JOB_ID` set** → start an in-process scheduler bound to the - driver's SLURM hostname (`SLURMD_NODENAME` or `gethostname()`), - then `srun` one `dask worker` per node across the allocation. -4. **None of the above** → `LocalCluster()` sized to the local machine. - -Outside the Gateway branch the scheduler is always in-process, so its -lifetime equals the run's lifetime: no orphaned schedulers if the -driver crashes. On the Gateway branch the same contract is enforced -server-side — create per run, cull on exit (the deployment's idle -timeout is the backstop). Create-per-run is also what makes image -updates seamless: a Gateway cluster's image is fixed at creation. - -The Gateway branch self-provisions the worker environment through the -deployment's **standard `environment` cluster option** (no -lightcone-specific injection needed server-side): the -`DASK_DISTRIBUTED__WORKER__RESOURCES__*` scheduling contract mirrored -from the declared `worker_cores`/`worker_memory` option values, the -driver's `HOME`/`USER`/`LOGNAME` (passwd-less uid-1000 images crash -`getpass.getuser()` without them), and `LIGHTCONE_WORKER_IMAGE` as -manifest ground truth. It also fails fast on two silent-hang failure -modes: zero workers within the timeout (unpullable image, -unschedulable pool), and workers that don't advertise the -`cpus`/`memory` resource contract (a deployment that doesn't expose -the `environment` option). - -## Resource keys - -These string constants form a contract with the executor plugin: - -```python -RESOURCE_CPUS = "cpus" -RESOURCE_MEMORY = "memory" -RESOURCE_GPUS = "gpus" -``` - -Workers must advertise every key the executor may request — Dask -matches by exact key presence. The local-cluster path includes all -three even when the executor doesn't ask, so per-rule -`mem_mb`/`gpus_per_task` rules still schedule on a workstation. - -## Node-shape detection - -`_detect_node_shape()` reads SLURM env vars with sane fallbacks: - -| Resource | Env var | Fallback | -|----------|---------|----------| -| CPUs | `SLURM_CPUS_ON_NODE` | `os.cpu_count()` | -| Memory | `SLURM_MEM_PER_NODE` (MB) | `psutil.virtual_memory().total` if installed; otherwise 0 (advisory; workers won't enforce caps) | -| GPUs | `SLURM_GPUS_ON_NODE` | `0` | - -## SLURM-backed cluster details - -```python -srun --ntasks=$SLURM_NNODES --ntasks-per-node=1 \ - dask worker --nthreads $cpus --nworkers 1 \ - --resources "cpus=N memory=B gpus=G" --no-dashboard -``` - -The `--ntasks-per-node=1` is important: we want one worker per node, -not per CPU. The worker uses `--nthreads` to advertise its parallelism -within the node. - -After spawning workers, the manager opens a temporary `Client(addr)` to -`wait_for_workers(n_workers=nnodes, timeout=120)`. If the workers -haven't connected within two minutes, raise. - -On exit, the manager `terminate()`s the worker subprocess group, waits -up to 10s, then `kill()`s anything still alive. - -## Why no `dask-jobqueue`? - -`dask-jobqueue` would `sbatch` workers from inside an existing job — -fine, but adds dependency and indirection. Since we already require the -user to be inside an allocation (`salloc` / `sbatch`), `srun` is enough -and keeps everything in one process tree. - -## Tests - -`tests/test_dask_cluster.py` covers the three branches and the -resource-advertising contract. The SLURM branch is tested with mocked -`subprocess.Popen` plus a stubbed `Client.wait_for_workers`. diff --git a/docs/api/dask_executor.md b/docs/api/dask_executor.md deleted file mode 100644 index 28f2ae5d..00000000 --- a/docs/api/dask_executor.md +++ /dev/null @@ -1,118 +0,0 @@ -# snakemake_executor_plugin_dask - -Snakemake executor plugin that submits each rule as a `client.submit()` -on a `dask.distributed` cluster. Lives at the top of `src/` because -Snakemake discovers executor plugins through the -`snakemake_executor_plugin_*` package-naming convention. - -Source: `src/snakemake_executor_plugin_dask/`. - -## Module shape - -``` -snakemake_executor_plugin_dask/ -├── __init__.py # plugin metadata + Executor re-export -└── executor.py # DaskExecutor class -``` - -## Plugin metadata - -```python -common_settings = CommonSettings( - job_deploy_sources=True, # send Snakefile + sources to workers - non_local_exec=True, # workers may live elsewhere - implies_no_shared_fs=False, # we *do* assume a shared FS -) -``` - -We assume a shared filesystem because all our workers (local threads, -SLURM nodes) see the project tree the same way. If you change this, -you also need to teach `wrap_recipe()` not to use `$PWD` bind mounts. - -## `DaskExecutor` - -Inherits from `snakemake_interface_executor_plugins.executors.remote.RemoteExecutor`. - -### `__init__(workflow, logger)` - -Imports `dask.distributed` lazily; raises `WorkflowError` if missing -("`pip install distributed`"). Connects via `_connect_client()`: if -`LIGHTCONE_GATEWAY_CLUSTER` is set, rejoins that Dask Gateway cluster -through `Gateway().connect(name)` (with `shutdown_on_close=False` — -the executor is a guest; `lc run` owns the cluster lifecycle); -otherwise dials `DASK_SCHEDULER_ADDRESS` with a bare `Client`. Raises -`WorkflowError` if neither is set — `lc run` is responsible for -setting one. - -### `run_job(job)` - -Translate a Snakemake job to a Dask submission: - -```python -client.submit( - _run_shell, job.format_job_exec(), - resources=_build_resources(job), - pure=False, - key=f"snakejob-{job.name}-{job.jobid}", -) -``` - -`_run_shell` runs `subprocess.run(cmd, shell=True, check=False)` on -the worker and returns `(exit_code, output_block)`. The block is the -child snakemake's sentinel-prefixed lines (see -`lightcone.engine.runner.SENTINEL`), kept verbatim; on a failure that -produced no sentinel line at all (the child died before the rule body -— import error, missing package in the worker image) a bounded raw -tail is sentinel-framed instead so bootstrap failures don't vanish -into worker logs. Returning output through the task result is the only -channel that works uniformly across LocalCluster threads, srun-launched -workers, and Gateway worker pods (whose stdout goes to pod logs). The -recipe is already container-wrapped at Snakefile generation time, so -the worker has no runtime logic of its own. - -### `check_active_jobs(active_jobs)` - -Async generator: for each submitted job, check `future.done()`. Yield -back jobs that are still in flight. For finished jobs, unpack the -result via `_unpack_result` (which also accepts the bare-int result of -a worker running an older lightcone-cli release), write the output -block to stdout — `lc run` filters and forwards it — then: - -- `future.exception() is not None` → `report_job_error(...)` -- exit code `!= 0` → `report_job_error(...)` -- otherwise → `report_job_success(...)` - -### `cancel_jobs(active_jobs)` - -Best-effort `future.cancel()` on each in-flight job. **Does not** close -the `Client` — Snakemake calls `cancel_jobs` for partial cancellations -as well as at shutdown, so closing here would break subsequent -submissions. The client is closed in `shutdown()` exclusively. - -### `shutdown()` - -`self._client.close()` then `super().shutdown()`. - -## Resource translation - -```python -def _build_resources(job) -> dict[str, float]: - res = {} - cpus = job.resources.get("cpus_per_task") or job.threads - if cpus: res["cpus"] = float(cpus) - mem_mb = job.resources.get("mem_mb") - if mem_mb: res["memory"] = float(mem_mb) * 1e6 # MB → bytes - gpus = job.resources.get("gpus_per_task") or job.resources.get("gpus") - if gpus: res["gpus"] = float(gpus) - return res -``` - -Returns `None` if the resulting dict is empty (Dask's "no constraints" -sentinel). Resource keys must match those advertised by workers -(`cpus`, `memory`, `gpus` — see [`engine.dask_cluster`](dask_cluster.md)). - -## Tests - -`tests/test_dask_plugin.py` exercises the `_build_resources` mapping -and the executor's lifecycle (init / run / check / shutdown) against a -local `LocalCluster` fixture. diff --git a/docs/api/dataset.md b/docs/api/dataset.md new file mode 100644 index 00000000..89c3dd1e --- /dev/null +++ b/docs/api/dataset.md @@ -0,0 +1,58 @@ +# lightcone.engine.dataset + +The git + git-annex seam: how a project stores what it produced. +Storage follows the DataLad model — git carries the pointers and the +history, git-annex carries the bytes — reached through ordinary `git` +commands. Every command goes through `project._run`, so there is one +monkeypatch point and every invocation is inspectable. + +Source: `src/lightcone/engine/dataset.py` (+ +`templates/files/gitattributes.tmpl` for the routing policy). + +## Key symbols + +| Symbol | Role | +|---|---| +| `save(root, paths, message)` | Stage scoped, commit — with `-c annex.thin=true` and `-c annex.dotfiles=true`, per-add and never written to config. | +| `restore(root, paths)` | `git clean` always; `git checkout HEAD --` only when HEAD has the path. Never `-- .`. | +| `status(root)` | The dirty question, scoped to the project (`-- .`, prefix-stripped) so a project inside a larger repository works. | +| `head(root)` | The commit a run started at — read once per run, by the driver. | +| `last_writer(root, dir)` | Who last touched an output's directory — the foreign-write question. Answers "cannot say" as empty, never an error. | +| `require_committer(root)` | Refuses a repository with no git identity, before any recipe spends time. Asked as `git var`, the question a commit itself asks. | +| `dataset_id(root)` | The DataLad dataset UUID, read via `git config -f`. | + +## What must stay true + +- **Nobody is ever asked to run a git-annex command.** `filter=annex` + plus the `.gitattributes` policy make an ordinary `git add` do the + right thing; `annex.largefiles=nothing` comes first and outputs and + data opt out — last match wins, and + `test_analysis_code_stays_in_git_and_stays_writable` pins it against + a real annex. +- **Manifests stay in git**, exempted back out of the annex, so a + bytes-free clone can classify a whole project. +- **An unfetched file exists, in two shapes** — an unlocked pointer + file (readable, hashes to the wrong thing) and a locked dangling + symlink (drops out of naive walks silently). `assets.data_version` + refuses both with `ContentNotFetchedError`; detection handles both + regardless of which shape lc writes, because `annex.thin` and + `git annex lock` are the researcher's to set. +- **Thin is per-add and only where lc writes.** Thin's hazard is an + in-place write rewriting the annex object under its own key; lc + always resets output directories rather than writing in place, but + `data/` is the researcher's, and their tools (`h5py`, astropy + `mode='update'`) do open files for update — so the flag never + reaches repository config. +- **`restore` is asymmetric on purpose:** a first materialization has + no HEAD version to go back to, and a failed task must not discard + edits made elsewhere while the graph ran. +- **Committing an archive or dot-named file needs `annex.dotfiles`** — + git-annex routes dotfiles to git whatever `largefiles` says, and + without the flag an image archive lands as a git blob, silently. + +## Tests + +`tests/test_dataset.py`, deliberately against **real tools** +(`real_tools` fixture): whether bytes land in the annex or as a blob +in git is not a question a stub can answer, and every bug this seam +has had was invisible to one. diff --git a/docs/api/identity.md b/docs/api/identity.md new file mode 100644 index 00000000..e7f865ca --- /dev/null +++ b/docs/api/identity.md @@ -0,0 +1,53 @@ +# lightcone.engine.identity + +What a materialized output is identified by: two hashes that answer +different questions, and the lock scan that decides whether an +environment can be audited at all. + +Source: `src/lightcone/engine/identity.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `definition_version(recipe, decisions)` | What the spec says an output *is* — the rebuild trigger. | +| `env_version(root)` | What it ran under: lock bytes ‖ interpreter pin ‖ install settings ‖ image document. The `behind` trigger. | +| `scan_lock(root)` | Refusals, reports and advisories about what the lock pins. | + +## What must stay true + +- **`env_version` is not part of `definition_version`.** That is the + whole shape of the invalidation model: an environment edit stales + nothing, it makes outputs *behind*. (The original design nested + them; staling every output in every project on an engine upgrade was + the bug, not the cost.) +- **Both hashes are length-framed** — label, length, bytes per field — + so a boundary shift between adjacent fields cannot yield the same + digest from different inputs. Mutation-checked in the suite. +- **The lock is hashed as raw bytes, never parsed.** A comment reflow + moves `env_version`, deliberately: over-invalidation costs a report + line, while a parse of our own can silently disagree with uv. +- **The install-settings list is closed** (`_INSTALL_SETTINGS`), every + key hashed whether or not the project sets it — a setting outside + the list must not move the hash, one merely *matching* today's + default must. Settings are read where uv reads them (`uv.toml` + **replaces** `[tool.uv]`, measured); only values are hashed, never + which file supplied them. User-level uv config is deliberately out + of reach — machine state, not project state — and the residue is + tracked as issue #176. +- **The git commit is recorded, never hashed, and never a signal** — + one sha covers the whole tree, so hashing it stales everything on a + README edit. The honest consequence: editing `src/fit.py` remakes + nothing unless the file is declared as an ASTRA input. Do not add a + heuristic that scans recipes for repo paths. +- **The lock scan refuses only what cannot be audited** — path, + directory, and editable dependencies (two syncs of one lock can + install different code). A registry package with no wheel is a + report; a non-default group is advisory; the project's own package + is exempt. Names compare in PEP 503 form, or a project named + `my_project` fails to recognise itself. + +## Tests + +`tests/test_identity.py` — pure, and written as sensitivity tests in +both directions: what must move each hash, and what must not. diff --git a/docs/api/index.md b/docs/api/index.md index 491843b1..f3126ea8 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,61 +1,38 @@ -# Python API Reference - -The interesting public surface lives in `lightcone.engine.*`. The CLI -is a thin Click wrapper around these modules. - -## Module map - -| Module | Role | -|--------|------| -| [`lightcone.cli.commands`](cli.md) | Click CLI: `init`, `run`, `build`, `status`, `verify`, `setup`. | -| [`lightcone.engine.manifest`](manifest.md) | Per-output `.lightcone-manifest.json` write/read; `code_version`, `sha256_dir`. The integrity layer. | -| [`lightcone.engine.snakefile`](snakefile.md) | Generate `.lightcone/Snakefile` and `snakefile-config.json` from `astra.yaml`. | -| [`lightcone.engine.container`](container.md) | Runtime detection, content-addressed image tags, `wrap_recipe`. | -| [`lightcone.engine.dask_cluster`](dask_cluster.md) | Cluster lifecycle for `lc run` (local / SLURM / external). | -| [`lightcone.engine.status`](status.md) | Manifest-driven status walker. | -| [`lightcone.engine.verify`](verify.md) | Recompute hashes; walk the input chain. | -| [`lightcone.engine.tree`](tree.md) | Sub-analysis tree helpers — outputs, decisions, `from:` resolution. | -| [`lightcone.engine.validation`](validation.md) | Post-recipe sanity checks (empty dir, all-NaN columns, …). | -| [`snakemake_executor_plugin_dask`](dask_executor.md) | Snakemake executor plugin → `dask.distributed`. | -| `lightcone.engine.site_registry` | Vestigial — no active code path imports it. See [api/site_registry](site_registry.md). | - -## Common entry points - -```python -from pathlib import Path -from lightcone.engine.snakefile import generate, discover_universes -from lightcone.engine.container import load_runtime - -project = Path("my-analysis") -runtime = load_runtime(project_path=project).runtime -universes = discover_universes(project) # ['baseline', 'experiment'] -snakefile, cfg = generate(project, universes=universes, runtime=runtime) -# Now invoke `snakemake -s snakefile -d project --executor dask ...` -``` - -```python -from lightcone.engine.status import get_output_status - -for s in get_output_status(project, universe_id="baseline"): - print(s.status, s.output_id) # 'ok', 'stale', 'missing', or 'alias' -``` - -```python -from lightcone.engine.verify import verify_outputs - -failed = [r for r in verify_outputs(project, universe_id="baseline") if not r.passed] -for r in failed: - print(r.failure, r.output_id, r.detail) -``` - -```python -from lightcone.engine.container import ( - detect_runtime, - compute_image_tag, - build_image, -) - -runtime = detect_runtime() # 'podman' / 'docker' / 'podman-hpc' / None -tag = compute_image_tag("my-project", Path("Containerfile"), Path(".")) -build_image(tag, Path("Containerfile"), Path("."), runtime=runtime) -``` +# Engine Internals + +The `lightcone.engine.*` modules, one page each: what the module owns, +its key symbols, and the invariants a change must keep. These are +hand-written tours, not generated API dumps — the engine is not a +public API (projects don't depend on lightcone-cli), so what matters +is responsibility and contract, not every signature. + +## The map + +| Module | Owns | Character | +|---|---|---| +| [`project`](project.md) | What a project is: convergence, discovery, mode, the `_run` seam | impure | +| [`dataset`](dataset.md) | How a project stores: git + git-annex, run records, restore | impure | +| [`identity`](identity.md) | `env_version`, `definition_version`, the lock scan | pure | +| [`plan`](plan.md) | The spec, read as a graph of tasks (through ASTRA) | pure | +| [`assets`](assets.md) | One output: its directory, manifest, and state | pure | +| [`worker`](worker.md) | Making one output; the rerun entry point | impure | +| [`materialize`](materialize.md) | The driver: gates, scheduling, the save/restore loop, status | impure | +| [`venue`](venue.md) | Where a run executes: SLURM detection, the login guard | impure | +| [`sandbox`](sandbox.md) | The exec boundary: policy, backends, attestation, denials | mixed | +| [`image` & `container`](container.md) | The container hatch: declaration → image → archive → runtime | pure / impure | +| [`crate`](crate.md) | The publication view: the repo as an RO-Crate | pure | + +"Pure" here is a testing fact: pure modules are tested with nothing on +disk beyond `tmp_path` and nothing spawned; impure ones go through the +one subprocess seam (`project._run`) that the suite stubs — see +[Testing](../contributing/testing.md). + +Two files sit outside the engine on purpose: + +- **`lightcone/_sandbox_exec.py`** — the Landlock shim. Stdlib-only, + zero lightcone imports; it runs on every sandboxed exec, and an + engine import there would put click and the astra stack on that + path. Pinned by tests. +- **`lightcone/cli/commands.py`** — the CLI: flags, rendering, exit + codes. Imports the engine inside callbacks so `lc --help` stays + cheap; never contains logic worth testing beyond rendering. diff --git a/docs/api/io_manager.md b/docs/api/io_manager.md deleted file mode 100644 index ebfe3099..00000000 --- a/docs/api/io_manager.md +++ /dev/null @@ -1,8 +0,0 @@ -# lightcone.engine.io_manager (removed) - -The Dagster IO manager was retired. Output paths are now baked into the -generated Snakefile by [engine/snakefile](snakefile.md), and the canonical -location of every output directory is computed by -[`resolve_output_path`](tree.md) — root outputs land at -`results///`, and path-rooted sub-analyses land at -`/results///`. diff --git a/docs/api/manifest.md b/docs/api/manifest.md deleted file mode 100644 index 345bc451..00000000 --- a/docs/api/manifest.md +++ /dev/null @@ -1,112 +0,0 @@ -# lightcone.engine.manifest - -The integrity layer. Every materialized output gets a sidecar -`.lightcone-manifest.json` written by this module on the host -immediately after the recipe shell exits. - -Source: `src/lightcone/engine/manifest.py`. Schema version: `1` -(`SCHEMA_VERSION = 1` — bump if you change the manifest shape). - -## Public surface - -```python -__all__ = [ - "MANIFEST_FILENAME", # ".lightcone-manifest.json" - "SCHEMA_VERSION", # 1 - "code_version", - "fingerprint_external", - "read_manifest", - "sha256_dir", - "write_manifest", -] -``` - -## `code_version(*, recipe, container_image, decisions) → str` - -Deterministic content hash of everything that defines what the recipe -*does*: the recipe text, the resolved container image identifier, and -the canonicalized decision dict. Returns `"sha256:"`. - -The runtime used to invoke the container (docker / podman / podman-hpc) -is intentionally excluded — the same image produces the same data -regardless of which OCI tool launched it. - -`code_version` is embedded in each rule's `params.cfg` so Snakemake's -`params` rerun-trigger detects drift automatically. - -## `sha256_dir(path) → str` - -Deterministic content hash of a directory tree. Walks recursively, -hashes each file along with its relative path (so renames change the -hash), and excludes: - -- `.lightcone-manifest.json` (chicken-and-egg) -- `.snakemake_timestamp` (touched by Snakemake *after* the rule body - completes — including it would make every hash unstable) - -Raises `FileNotFoundError` if `path` does not exist. - -## `fingerprint_external(path, *, strict=False) → str` - -External (non-manifested) input fingerprint: - -- File: `mtime-size:-` by default; `sha256:` when - `strict=True`. -- Directory: always `sha256:` (via `sha256_dir`). -- Missing path: literal string `"missing"`. - -## `read_manifest(output_dir) → dict | None` - -Read `/.lightcone-manifest.json`. Returns `None` if the -file is missing or unparseable. **Does not** catch `OSError` — a -permission-denied or I/O error is surfaced rather than silently -masquerading as "missing", because it would otherwise hide real -problems from `lc verify` / `lc status`. - -## `write_manifest(*, output_dir, inputs, cfg) → Path` - -Atomically write the manifest for an already-materialized output. -Called from each rule's `run:` block. - -Required keys in `cfg`: - -- `output_id`, `universe_id` -- `recipe`, `container_image`, `decisions` -- `code_version` -- `git_sha`, `lc_version` - -`inputs` is a `dict[str, Path]` mapping declared input id → filesystem -path. For each input, the function reads the upstream manifest if -present and records its `data_version`; otherwise falls back to -`fingerprint_external`. - -Atomicity: writes to `.tmp`, then `os.replace()` rename. -Either both data and manifest exist at the end, or Snakemake reruns -the rule. - -## Manifest shape - -```jsonc -{ - "schema_version": 1, - "output_id": "accuracy", - "universe_id": "baseline", - "code_version": "sha256:…", - "data_version": "sha256:…", - "container_image": "lc-myproject-abc123", - "recipe": "python scripts/eval.py", - "decisions": { "scaling": "standard", "use_pca": "no" }, - "input_versions": { "features": "sha256:…", "labels": "mtime-size:…-…" }, - "git_sha": "...", - "lc_version": "0.4.0", - "host": "saul01", - "slurm_job_id": "1234567", - "finished_at": 1717000000.0 -} -``` - -## Tests - -`tests/test_manifest.py` covers `code_version` determinism, `sha256_dir` -exclusions, `fingerprint_external` modes, and `write_manifest` end-to-end -including the atomic rename. diff --git a/docs/api/materialize.md b/docs/api/materialize.md new file mode 100644 index 00000000..54b570de --- /dev/null +++ b/docs/api/materialize.md @@ -0,0 +1,72 @@ +# lightcone.engine.materialize + +Making a whole analysis: what runs, in what order, and what gets +committed. The driver refuses dirt, hands the graph to Dask, and owns +git alone — plus the read-only halves (`check`, `status`) that share +its classification walk. + +Source: `src/lightcone/engine/materialize.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `materialize(root, targets, *, refresh)` | The run: guards → converge → plan → fetch → schedule → save/restore loop → crate converge. | +| `check(root, targets, *, refresh)` | The same classification without executing, committing, or fetching. Exempt from the dirty refusal. | +| `status(root)` | The report: every output's state and provenance commit, plus the mode/image/sandbox header facts. | +| `MaterializeReport` / `StatusReport` | The JSON surfaces; `ok` and `up_to_date` first. | +| `cluster_for_run()` | The venue ladder, and the two-method scheduler seam (`submit`, `completed`). | +| `run_record(...)` / `datalad_run_subject(...)` | The commit message `datalad rerun` replays, and the one spelling of its subject line — shared with the foreign-write comparator, because two strings here would drift. | +| `_engine_requirement()` | How a record pins its engine: by version for a release, by source commit (hatch-vcs) for a dev build. | + +## The run's order, and why + +1. **Login guard first** — the allocation is the remedy with queue + latency, so the user submits it before fixing anything else. +2. **Dirty refusal before the environment converge** — in + containerized mode the converge can commit an image archive, and + `dataset.save` commits the whole index; on a dirty tree the user's + staged edits would be swept in. +3. **Converge before the graph runs** — `uv run --locked --no-sync` + in workers would otherwise execute recipes against a drifted + `.venv` while manifests record the new lock (measured; the state + is made impossible rather than detected). +4. **Graph (validation, lock scan) before the image** — a refusal + over a typo must not cost a minutes-long build. +5. **HEAD, runtime, and foreign-write facts read once, handed down** + — the driver commits as results arrive, so any per-task read could + answer differently mid-run. Nondeterminism in a provenance field is + worse than either answer. +6. **Save on `ok`, restore otherwise, `try/finally` around the loop** + — an interrupt restores whatever is still outstanding; the tree + ends as clean as it started. + +## What must stay true + +- **The driver owns git, alone** — one thread, as results arrive. + A dependent may start while its upstream is being annexed; that is + measured-safe (the clean filter renames over the path, which never + stops existing) and must not be "fixed" by moving the save into the + task. +- **`up_to_date` is `ok and not made and not planned`** — a run where + every recipe failed must not report "nothing to do", and `behind` + never counts against it. +- **A read-only verb never tracebacks.** Anything `check`/`status` + cannot read classifies as "will be remade" and the real error + belongs to the recipe that follows. +- **The run record is genuinely re-runnable**: engine pinned by + requirement, project environment rebuilt by the worker from the + rerun commit's own lock, format tested *through datalad's parser* + and a real `datalad rerun` — a golden test over our own JSON stays + green through a silent break. +- **The crate converge is contained**: it runs after the loop, on the + full graph, and a failure there is a warning — the outputs are + already committed, and the crate is the publication view, not the + run. + +## Tests + +`tests/test_materialize.py` — real repositories, real recipes, a real +`LocalCluster` through the seam exactly once, real `datalad rerun` for +the record's whole claim. `cluster_for_run` is the one monkeypatch +point for venue-free tests. diff --git a/docs/api/plan.md b/docs/api/plan.md new file mode 100644 index 00000000..980eef4a --- /dev/null +++ b/docs/api/plan.md @@ -0,0 +1,57 @@ +# lightcone.engine.plan + +The spec, read as a graph of tasks. `astra.yaml` × `universes/*.yaml` +gives one task per `(universe, output)` pair that has a recipe; a task +carries everything executing it needs — the rendered command, where its +bytes go, what it reads, its decisions, its `definition_version` — and +nothing about *how* it will be executed. + +Source: `src/lightcone/engine/plan.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `build(root)` | Validate the spec with ASTRA's own validators, resolve every universe, return the `Graph`. | +| `Graph` | Tasks keyed on `(universe_id, output_id)`; `order()` for the read-only topological walk, `resolve(targets)` for what a user typed, `closure(keys)` to narrow a run. | +| `Task` | One output in one universe, frozen. | +| `declared_path(root, path)` | The one rule that names a path: project-relative inside the tree, absolute outside, never resolved. | + +## What must stay true + +- **What the spec *means* is ASTRA's to say.** `astra.resolve` settles + decisions, resolves inputs, drops `when:`-excluded outputs, and + renders the placeholder grammar. This module holds only what + *execution* adds. A prior in-house interpretation diverged three + ways (couldn't build ASTRA's own nested example, ignored `when:`, + invented an input spelling `astra validate` rejects) — that history + is why re-derivation is banned. Missing semantics → PR to + astra-tools. +- **A spec ASTRA rejects never reaches a recipe.** `build` runs the + schema, file, and universe validators before resolving anything — + resolution answers what a *valid* spec means and does not re-check + that it is one. +- **The layout is flat and path-addressed.** + `results///`, and the path in a + rendered recipe *is* the path on disk — no staging, no relocation. +- **`declared_path` is lexical, never `resolve()`d.** A declared input + under `data/` is an annex symlink; resolving it writes + `.git/annex/objects/…` into the run record — the storage instead of + the input. This shipped once. +- **Two universes cannot share an id** (the id names a directory; + `build` refuses, naming both files), and an out-of-tree absolute + input is **reported, not refused** — its bytes still hash and + cascade, but the repository cannot bring it back, and saying so is + the whole obligation. +- **A target that matches nothing is an error** listing what exists — + quietly making nothing is the least useful thing a build tool can + do. + +## Tests + +`tests/test_plan.py` — pure; tests what lc *adds* (directories, edges, +versions, the validation gate), never what a spec means — that +coverage lives in astra-tools' own suite, and re-asserting it here +would recreate the second implementation this module deleted. Every +fixture must be a spec `astra validate` accepts; the gate enforces it +for free. diff --git a/docs/api/project.md b/docs/api/project.md new file mode 100644 index 00000000..a8c9f027 --- /dev/null +++ b/docs/api/project.md @@ -0,0 +1,54 @@ +# lightcone.engine.project + +What a project is: the convergence engine behind `lc init`, project +discovery, mode detection, and the one subprocess seam the whole +engine shares. + +Source: `src/lightcone/engine/project.py` (+ +`engine/templates/` for the scaffold's file content). + +## Key symbols + +| Symbol | Role | +|---|---| +| `converge(dir, *, write)` | The whole scaffold operation. `write=False` is check mode — the *same* decision path with side effects off. | +| `ConvergenceReport` | `created` / `repaired` / `unchanged` / `blocked` / `warnings`, plus `.converged` and `.as_dict()`. | +| `current_project()` | The cwd as a project: requires `pyproject.toml`, `uv.lock`, `.venv`. | +| `declared_project()` | The weaker question — what the repository carries, without `.venv`. One caller: the worker entry point, which builds the venv a moment later. | +| `mode(root)` | `"direct"` or `"containerized"` — presence of `[tool.lightcone.image]`, nothing else. | +| `uv_prefix(root, *, sync)` | The one spelling of the project uv hop. Callers differ only in `sync`: a probe converges the environment, a recipe must not. | +| `project_name(dir)` | PEP 503-ish name from the directory name. | +| `_run` / `_check_call` | Every external tool invocation, and the suite's one monkeypatch point. | +| `ProjectError` | The engine's one exception; the CLI translates it once. | + +## What must stay true + +- **Everything routes through the converger.** Every scaffold item + goes through `_Converger.item` / `.file` / `.blocked`; nothing + writes or records outside that mechanism. `.file` takes a *thunk*, + so check mode renders no template at all. +- **Derived artifacts converge by correctness, not existence.** + `uv.lock` and `.venv` are probed with uv's own no-write checks + (`uv lock --check`, `uv sync --locked --exact --check`); drift + reports as `repaired`. Check mode may probe but never mutates — + pinned by `test_check_mode_only_probes`. +- **A warning is advisory; a blocked item counts.** Convergence never + claims a project is converged while something it owns is absent or + unfixable — and repairs only ever append (`.gitignore` / + `.gitattributes` are converged entry-wise, order judged against the + template). +- **Only what git can carry is converged.** No `src/`, no empty + directories — a clone must need nothing but `.venv` and + `git annex init`, and + `test_a_clone_of_a_converged_project_is_converged` pins it. +- **There is no discovery.** The invoked directory is the project or + it is a clean error; every uv call carries an explicit `--project`. +- **Templates are files** (`templates/files/*.tmpl`, `string.Template` + with strict substitution), and a template gets a function only when + there is a value to decide or a merge policy to hold. + +## Tests + +`tests/test_project.py` (semantics, against the stubbed `_run`), +`tests/test_templates.py` (content, substitution, repair logic), +`tests/test_cli.py` (the `lc init` surface). diff --git a/docs/api/runner.md b/docs/api/runner.md deleted file mode 100644 index 8efc1cc6..00000000 --- a/docs/api/runner.md +++ /dev/null @@ -1,14 +0,0 @@ -# lightcone.engine.runner (removed) - -The pluggable runner (`docker`, `venv`, `local`, `slurm`) was replaced by -two thinner pieces: - -- The Snakefile generator at [engine/snakefile](snakefile.md) wraps each - recipe in a ` run --rm ...` invocation at generation time - (or leaves it bare when no container is configured). -- The Dask cluster manager at [engine/dask_cluster](dask_cluster.md) - decides whether the run is local, SLURM-backed via `srun`, or attached - to an external scheduler. - -There is no longer a single "backend" abstraction — those two -modules together cover what the runner used to do. diff --git a/docs/api/sandbox.md b/docs/api/sandbox.md new file mode 100644 index 00000000..d1175a35 --- /dev/null +++ b/docs/api/sandbox.md @@ -0,0 +1,70 @@ +# lightcone.engine.sandbox + +The exec boundary: what a command may touch, and how that is enforced. +A `Policy` says *what* in mechanism-free path sets; a `Backend` turns +it into **a different argv that sandboxes itself**; `boundary` picks +one, runs it, and reports what was actually enforced. `run.py` (the +`lc run` engine) and the worker are the two consumers. + +Source: `src/lightcone/engine/sandbox/` — `model.py`, `policy.py`, +`boundary.py`, `landlock.py`, `seatbelt.py`, `oci.py`, `denial.py` — +plus `lightcone/_sandbox_exec.py`, the Landlock shim. + +## Key symbols + +| Symbol | Role | +|---|---| +| `Policy` | What we will enforce: path sets, env overlay, exec allowlist. No mechanism ever appears in it. | +| `Capability` | What this host can do — `detect()`'s answer, the only `sys.platform` branch. | +| `Attestation` | What was actually enforced, derived from the flags applied — never from what the matrix says should have happened. | +| `Backend.wrap(policy, argv)` | The pure rewrite. `contains_prefix` declares whether the uv hop rides inside (a container is a world; a host mechanism trusts host plumbing). | +| `exec_policy(...)` | The one policy: probe and recipe get the same thing. Building it is where the impurity lives (the per-run private `$HOME`); `scope()` owns its cleanup. | +| `Unavailable` | A real backend that wraps to the same argv and attests `fs: open`. Saying so is the caller's job; pretending is nobody's. | +| `denial.explain()` / `denial.trailer()` | Best-guess remedies (allowed to return nothing) and the unconditional trailer on every nonzero sandboxed exit. | + +## What must stay true + +- **`wrap` stays pure** — no temp files, no FDs, no global state + (pinned by `test_wrap_is_pure`). That is what makes every backend + testable on a host that cannot run it, and it is why the Landlock + policy travels as JSON on argv rather than an inherited ruleset FD. +- **The shim stays alone**: stdlib only, zero lightcone imports, setup + failures exit the reserved 97, and it never falls through to running + the command unsandboxed. +- **Never grant EXECUTE on a directory that could be a system + prefix.** Landlock unions rights over ancestors, so one EXECUTE on + `/usr` outranks the whole per-file allowlist — with every test still + green, because the allowlisted binaries are exactly the ones that + were going to work. This shipped once (a venv on a system python); + the rule and its test are the fix. +- **SBPL is last-match-wins; Landlock unions.** The asymmetry decides + where a rule can live: the macOS guard takes back writes the + vendored defaults hand out, and the write tier is restated *after* + the guard — get the order wrong and layer 4 materializes on Linux + and refuses on macOS with the golden test still green. +- **Anything every backend must do belongs to the seam** — the env + overlay is composed in `boundary.env_argv()` once, for every + mechanism, so a mechanism added later cannot forget what it never + had to remember. (While each backend applied its own, `Unavailable` + applied none.) +- **The macOS profiles are vendored, not authored** (codex-derived, + provenance header, single delta) — the read baseline is a list of + things that break, found one production failure at a time. Put our + rules in the generator, keep `diff` against upstream as the re-sync + tool. +- **A denial is never invisible**: `explain()` may find nothing, so + the trailer fires on every nonzero exit, unconditionally. Remedies + name only what exists today. + +## Tests + +The suite splits along the seam: +`test_sandbox_policy/wrap/denial.py` (pure, every OS), +`test_sandbox_shim.py` (the shim as a real subprocess), +`test_sandbox_oci.py` (the mount table, pure), and +`test_sandbox_enforcement.py` — **the kernel's answer**, one suite for +both mechanisms, run against the *real* `exec_policy`, with +`LC_SANDBOX_TESTS_REQUIRED=1` turning "no mechanism, skip" into a hard +failure in CI. Every denial test is mutation-checked through +`Unavailable()` — a denial test that would pass unsandboxed is testing +nothing, silently. diff --git a/docs/api/site_registry.md b/docs/api/site_registry.md deleted file mode 100644 index 6efca3d7..00000000 --- a/docs/api/site_registry.md +++ /dev/null @@ -1,54 +0,0 @@ -# lightcone.engine.site_registry - -Known-site defaults. When lightcone-cli runs on a recognized site -(NERSC Perlmutter, a lightcone JupyterHub deployment), the matching -entry here supplies site-specific defaults — most importantly the -scratch root and the preferred container runtime. - -Source: `src/lightcone/engine/site_registry.py`. - -## What the module exposes - -- `SITE_DEFAULTS` — a dict mapping site keys (`"perlmutter"`, - `"jupyterhub"`, `"local"`) to a structured defaults dict (display - name, hostname patterns or env markers, backend, container runtime, - `scratch_root`, suggested QoS / constraint / time-limit options). -- `detect_current_site() → HostSite` — the high-level entry point. - Single source of truth for "which site are we on?": environment - markers win over hostname patterns (a pod's hostname is noise; the - injected env is the signal). Returns a falsy `HostSite` when nothing - matches. -- `HostSite` — frozen dataclass bundling the matched site key with its - defaults; `site.get("scratch_root")` etc. -- Lower-level pieces: `detect_site(hostname_or_name)`, - `detect_site_from_env()`, `get_site_defaults(site_key)`, - `list_known_sites()`, `get_site_scratch_deny_rules(site_key)`. - -## Who calls it - -- `lc init` (`lightcone.cli.commands`) — detects the site to surface - the resolved scratch root the run layer will use. -- `lightcone.engine.scratch` — `resolve_scratch_root()` falls back to - the site's declared `scratch_root` (e.g. `$SCRATCH` on Perlmutter, - `$HOME` on a JupyterHub pod) when the project config doesn't pin one. -- `lightcone.engine.container` — `auto` runtime resolution prefers the - site's declared `container_runtime` (`podman-hpc` on Perlmutter, - `kubernetes` on a hub). - -Everything should go through `detect_current_site()` rather than -re-deriving `socket.gethostname() + detect_site + get_site_defaults`. - -## Vestigial pieces - -`get_site_scratch_deny_rules()` and `list_known_sites()` currently -have no callers — they are residue from the removed target system. -The `suggested_options` blocks (QoS/constraint/time-limit guidance) -are likewise declared but not consumed yet. - -## Adding a site - -Append an entry to `SITE_DEFAULTS`. HPC sites match by -`hostname_patterns`; deployment-style sites (pods with arbitrary -hostnames) match by `env_markers`. Declare `scratch_root` for any site -where the default tempdir is wrong — see the `jupyterhub` entry's -comment for why a shared filesystem matters there. diff --git a/docs/api/snakefile.md b/docs/api/snakefile.md deleted file mode 100644 index ed8f3b43..00000000 --- a/docs/api/snakefile.md +++ /dev/null @@ -1,106 +0,0 @@ -# lightcone.engine.snakefile - -Generate `.lightcone/Snakefile` and `.lightcone/snakefile-config.json` -from `astra.yaml`. Both are auto-generated on every `lc run` — never -edit by hand. - -Source: `src/lightcone/engine/snakefile.py`. - -## Public surface - -```python -__all__ = ["generate", "discover_universes", "LIGHTCONE_DIR"] -``` - -## `generate(project_path, *, universes, runtime="none") → (Path, Path)` - -Reads `astra.yaml`, resolves the analysis tree, and writes: - -- `.lightcone/Snakefile` — the workflow. -- `.lightcone/snakefile-config.json` — per-`(rule_key, universe)` config. - -Returns the two paths. - -`runtime` is one of `docker | podman | podman-hpc | none` and is used to -wrap each recipe at generation time (see -[engine.container.wrap_recipe](container.md#wrap_recipe)). Resolution is -done once here, not per rule, so all rules use a consistent runtime. - -## `discover_universes(project_path) → list[str]` - -Sorted list of universe ids from `universes/*.yaml`, or `["default"]` if -the directory is empty / missing. - -## Generated Snakefile shape - -For each output with a `recipe:` block: - -```python -rule : - input: - ="/...", # only for sibling outputs - output: - data=directory(""), - manifest="/.lightcone-manifest.json", - params: - cfg=lambda wc: CFG[""][wc.universe], - run: - shell('printf "▶ [%s]\\n" "{wildcards.universe}" >&2') - shell(params.cfg["shell_command"]) - write_manifest( - output_dir=Path(output.data), - inputs={"": Path(input.), ...}, - cfg=params.cfg, - ) - for _w in validate_output(Path(output.data), params.cfg.get("output_type"), params.cfg["output_id"]): - print(f"\033[33m⚠\033[0m {_w}", file=sys.stderr) -``` - -## `cfg` content - -Per-`(rule_key, universe)` entry written into -`snakefile-config.json`: - -| Key | Source | Used by | -|-----|--------|---------| -| `output_id` | `tree_out.output_id` | `write_manifest` | -| `output_type` | `output_def["type"]` | `validate_output` | -| `universe_id` | universe name | `write_manifest` | -| `recipe` | `recipe.command` | `write_manifest` | -| `shell_command` | `wrap_recipe(recipe, image, runtime)` prefixed with `: lc_code_version=…;` | the rule body | -| `container_image` | raw `container:` spec from astra.yaml | `write_manifest` (for provenance) | -| `decisions` | merged universe decisions | `write_manifest`, `code_version` | -| `code_version` | `code_version(recipe, image_tag, decisions)` | drift detection via Snakemake `params` trigger | -| `git_sha`, `lc_version` | runtime metadata | `write_manifest` | -| `inputs` | resolved input paths (with `{universe}` substituted) | informational | - -## Why we own the container wrap - -Snakemake supports container directives (`container:` and -`--sdm apptainer`), but we deliberately don't use them. Two reasons, -both pragmatic: - -- `--sdm apptainer` adds an extra container layer that defeats - podman-hpc's migrate workflow. -- Default registry resolution on podman fails for - `lc--` tags because they trip - `unqualified-search-registries`. We pass `--pull=never` to skip the - lookup; Snakemake's machinery doesn't make this easy to thread - through. - -## Naming details - -- **`_rule_key(tree_out)`** — `output_id` for root outputs, - `.` for sub-analysis outputs. This is the - user-visible name and the cfg key. -- **`_rule_name(tree_out)`** — same as `_rule_key` but with `.` → - `__` because Snakemake rule names must be Python identifiers. -- **`_output_dir_pattern(tree_out)`** — wildcard path. Root and inline - sub-analyses: `results/{universe}/`. Path-rooted - sub-analyses: `/results/{universe}/`. - -## Tests - -`tests/test_snakefile.py` covers rule generation across root + sub-analyses, -input wiring, container wrapping, `code_version` embedding, and (last -test) parses the generated Snakefile via `snakemake -n`. diff --git a/docs/api/status.md b/docs/api/status.md deleted file mode 100644 index 85f4f31c..00000000 --- a/docs/api/status.md +++ /dev/null @@ -1,63 +0,0 @@ -# lightcone.engine.status - -Manifest-driven status walker. Reads only the per-output -`.lightcone-manifest.json` files; does **not** import Snakemake. - -Source: `src/lightcone/engine/status.py`. - -## Public surface - -### `get_output_status(project_path, *, universe_id) → Iterator[OutputStatus]` - -Yield an `OutputStatus` for every declared output in `project_path`'s -`astra.yaml`, against the named universe. Used by `lc status` and by -external tooling that wants a structured view. - -The function: - -1. Loads and resolves the analysis tree. -2. Loads merged universe decisions (tolerates a missing universe file — - returns an empty dict). -3. For each tree output: - - If it has no `recipe:` → `alias`. - - If no manifest at the output dir → `missing`. - - Otherwise recomputes `code_version` against the current spec and - compares to the manifest's recorded value. Match → `ok`, - mismatch → `stale`. - -### `OutputStatus` (dataclass) - -```python -@dataclass -class OutputStatus: - output_id: str - universe_id: str - analysis_id: str | None # None for root-level outputs - output_dir: Path - status: StatusLiteral # "ok" | "stale" | "missing" | "alias" - manifest: dict | None # None for missing/alias -``` - -### `StatusLiteral` - -```python -StatusLiteral = Literal["ok", "stale", "missing", "alias"] -``` - -## Why `code_version` is the staleness signal - -The manifest records the `code_version` that produced the data. Drift -detection just recomputes the current `code_version` from the live -spec and compares. Anything that touches recipe text, container image -tag, or decisions changes `code_version`; everything else is irrelevant -for staleness. - -For staleness against external inputs (e.g. someone edited a CSV under -`inputs/`), `lc status` doesn't catch it — that's outside `code_version`'s -scope. Use `lc verify` or rely on Snakemake's `mtime`/`input` rerun -triggers in `lc run`. - -## Tests - -`tests/test_status.py` covers the four status branches end-to-end -against tmp projects, including the alias and decision-drift paths. diff --git a/docs/api/targets.md b/docs/api/targets.md deleted file mode 100644 index 4f29d61b..00000000 --- a/docs/api/targets.md +++ /dev/null @@ -1,11 +0,0 @@ -# lightcone.engine.targets (removed) - -The target configuration module is gone. The only remaining global config -is `~/.lightcone/config.yaml`, which today carries one key: - -```yaml -container: - runtime: auto # auto | docker | podman | podman-hpc | none -``` - -It is read by [`lightcone.engine.container.load_runtime`](container.md). diff --git a/docs/api/tree.md b/docs/api/tree.md deleted file mode 100644 index 5e5301b3..00000000 --- a/docs/api/tree.md +++ /dev/null @@ -1,104 +0,0 @@ -# lightcone.engine.tree - -Walk the resolved analysis tree. Used by the Snakefile generator, -`status`, and `verify` to enumerate outputs, resolve `from:` -references, and merge universe decisions across nested sub-analyses. - -Source: `src/lightcone/engine/tree.py`. - -## `TreeOutput` (dataclass) - -```python -@dataclass -class TreeOutput: - output_id: str - output_def: dict # the raw dict from astra.yaml - analysis_id: str | None # None for root-level outputs - analysis_path: str | None # e.g. "./analyses/hod_fitting" - analysis_spec: dict # the sub-analysis spec dict (root spec for root outputs) -``` - -## Public functions - -### `collect_tree_outputs(spec) → list[TreeOutput]` - -Walk the resolved tree (root-level outputs first, then each -sub-analysis under `analyses:`) and return one `TreeOutput` per output -declaration. - -### `collect_tree_inputs(spec) → dict[str, dict]` - -Return `{qualified_id: input_def}` where `qualified_id` is `input_id` -for root inputs and `analysis_id.input_id` for sub-analysis inputs. - -### `resolve_universe_decisions(project_path, spec, universe_id) → dict` - -Load and merge universe decisions from root and sub-analyses. Returns -a flat dict using qualified keys for sub-analysis decisions -(`analysis_id.decision_id`) to avoid collisions. - -Behavior: - -- Loads root universe from `universes/.yaml`. -- For each sub-analysis with `path:`, looks for the corresponding - universe file at `/universes/.yaml`. The - sub-universe id comes from `root_universe.analyses..universe` - if present, otherwise the root universe id. -- `from:` references on sub-analysis decisions (`from: ../parent_decision`) - are resolved to the corresponding root decision value. -- Local sub-decisions not referenced via `from:` are still added. - -### `get_decisions_for_analysis(merged_decisions, analysis_id) → dict` - -Extract the decisions relevant to a specific analysis. Root analysis -(`analysis_id=None`): returns all unqualified keys. Sub-analysis: -returns decisions with the matching `analysis_id.` prefix, stripped to -local names. - -### `resolve_output_path(project_path, tree_output, universe_id) → Path` - -Returns the *parent* directory of the output dir (i.e. the -`results//` directory). The actual output dir is this path -joined with `tree_output.output_id`. - -- Root + inline sub-analyses: `/results//` -- Path-rooted sub-analyses: `//results//` - -### `resolve_container_spec(tree_output, root_spec) → str | None` - -Pick the container declaration in priority order: - -``` -recipe-level > sub-analysis-level > root-level -``` - -Returns the raw spec string (Containerfile path or registry image), or -`None` when no container is declared anywhere. - -### `find_upstream_output(consumer, inp_id, all_outputs) → TreeOutput | None` - -Resolve a recipe input id to the producing `TreeOutput`. Mirrors the -lookup the Snakefile generator does for `rule.input`. Handles: - -- Dotted `.` → match by qualified key. -- Inside a sub-analysis, bare `inp_id` → first try - `.`, then bare. -- `inp_id` referencing an analysis-level input with `from:` pointing - at a sibling output → resolved through that. - -Returns `None` for inputs that refer to external files (no producer -rule). - -### `resolve_input_path(project_path, spec, from_ref, universe_id) → str | None` - -Resolve a `from:` reference on an input to a concrete filesystem path. -Handles: - -- `../parent_input` → root input's `source` (only if absolute). -- `../sibling.output_id` → sibling sub-analysis's results path. -- `sibling.output_id` (no `../`) → same as above (root convenience). - -## Tests - -`tests/test_tree.py` covers all four resolvers, including the `from:` -edge cases for sub-analysis decisions and inputs. diff --git a/docs/api/validation.md b/docs/api/validation.md deleted file mode 100644 index ce020198..00000000 --- a/docs/api/validation.md +++ /dev/null @@ -1,50 +0,0 @@ -# lightcone.engine.validation - -Post-recipe sanity checks. Called by every rule's body after -`write_manifest`. **Never raises** — all problems are returned as -warning strings and printed to stderr. - -Source: `src/lightcone/engine/validation.py`. - -## `validate_output(output_dir, output_type, output_id) → list[str]` - -Inspect the output directory after a successful recipe run. Empty list -means no problems. Returned strings are human-readable and prefixed by -the rule body with a `⚠ ` marker. - -The check fires unconditionally on a few "this is almost certainly -wrong" situations: - -- Output directory does not exist after a successful run. -- Output directory exists but is a file rather than a directory. -- Output directory is empty after a successful run. - -Beyond that, behavior depends on the declared `type:` in `astra.yaml`: - -| `output_type` | Check | -|---------------|-------| -| `metric` | At least one `*.json` file present, parseable, not all-null/all-NaN. | -| `table` | At least one `*.csv` file present; parseable; warns on individual all-NaN numeric columns and on tables where every numeric column is all-NaN. | -| `figure` | At least one `*.png/jpg/jpeg/svg/pdf/eps` file present; warns on zero-byte files. | -| anything else | Empty list (no specific check). | - -## What it does *not* do - -This is a smoke test, not a validator. It does not: - -- Check the schema or shape of metric JSON beyond null-detection. -- Compare against expected values. -- Catch silent computational errors. -- Block the run — warnings are printed but the manifest is still - written. - -For deeper validation, layer your own checks in the recipe (`assert`, -`pydantic`, …). The point of `validate_output` is to flag the cheap -common silent failures: empty directories, NaN-only columns, missing -files. - -## Tests - -`tests/test_validation.py` covers each output-type branch including -the malformed-input edge cases (unparseable JSON, missing CSV, zero-byte -figures, …). diff --git a/docs/api/venue.md b/docs/api/venue.md new file mode 100644 index 00000000..d86735dc --- /dev/null +++ b/docs/api/venue.md @@ -0,0 +1,63 @@ +# lightcone.engine.venue + +Where a run executes. A venue is host state, never project state — +nothing here reads the project or enters any identity. The one venue +beyond the local machine is a SLURM allocation, detected rather than +configured: the user already answered every resource question at +`salloc`, so the allocation *is* the declaration and lc's job is to +span it. + +Source: `src/lightcone/engine/venue.py` (consumed by +`materialize.cluster_for_run`). + +## Key symbols + +| Symbol | Role | +|---|---| +| `slurm_client()` | The allocation branch: a scheduler in the driver process bound to `SLURMD_NODENAME`, one `srun --overlap` launching a worker per node on `sys.executable`. | +| `require_compute_node(command)` | The login guard: refuses iff a known center's marker is set and `SLURM_JOB_ID` is not, printing that center's own `salloc`/`sbatch` spellings. | +| `allocation_nodes()` | How many nodes the allocation holds; 0 outside one. | +| `_SITES` | One row per known center — name, marker, remedies, **verified against the center's documentation, never guessed**. NERSC is the seeded row. | + +## What must stay true + +- **The detection ladder lives in `cluster_for_run()` alone.** Nothing + else asks where a run executes; a future submission-model venue is + one more branch there plus only the config it genuinely needs. +- **Workers run the driver's own interpreter** (`sys.executable -m + distributed.cli.dask_worker`) — on HPC that is the tool env on the + shared filesystem, so driver and workers are the identical + installation and version skew is structurally out. Workers need no + git and no annex. +- **The worker flags are each load-bearing**: `--nthreads=` + (tasks block in `subprocess.wait()` with the GIL released), + `--no-nanny` (srun won't relaunch either), `--memory-limit 0` (the + real work is behind the exec boundary; Dask would pause workers over + phantom numbers), `--death-timeout 60` (a worker whose driver died + exits instead of holding the node), `--local-directory /tmp` + **literal** (a site prolog can scope `TMPDIR` per node or step, so a + driver-resolved path can be absent elsewhere). +- **The srun child is the one documented exception to `project._run`** + — it lives as long as the run and its stderr must reach the terminal + live. Teardown retires workers first, then wait → terminate → kill, + bounded; connection is a poll loop so a dead srun reports *its exit + code* now, not a timeout later. +- **A leak refuses loudly, never falls back silently**: `SLURM_JOB_ID` + with no srun on PATH, a non-integer count variable, an unresolvable + `SLURMD_NODENAME` — each is a named refusal. +- **The guard is materialize-scoped** (plus the rerun entry point — + the record's `cmd` is how recipes reach login nodes without `lc` in + the command line). `check`, `status` and `lc run` never call it: a + login node is exactly where "where does this stand" gets asked. +- **A containerized multi-node run requires a shared image store** — + `_SHARED_STORE_RUNTIMES` (podman-hpc), asked positively, checked in + `materialize()` before the runtime resolves so the refusal costs no + build. + +## Tests + +`tests/test_venue.py` — fakes the *host*, never the code: SLURM +variables set deliberately, a bash stub standing in for srun, and the +end-to-end tests run a real graph through the real bind/launch/teardown +on any machine. The `venue_env` autouse fixture scrubs venue variables +suite-wide (derived from `_SITES`, so a new center is one row). diff --git a/docs/api/verify.md b/docs/api/verify.md deleted file mode 100644 index 491f5c7d..00000000 --- a/docs/api/verify.md +++ /dev/null @@ -1,66 +0,0 @@ -# lightcone.engine.verify - -Recompute on-disk hashes and walk the recorded input chain. Catches -tampering, drift, and forged manifests. Like `status`, this module -never imports Snakemake. - -Source: `src/lightcone/engine/verify.py`. - -## Public surface - -### `verify_outputs(project_path, *, universe_id) → Iterator[VerifyResult]` - -Yield a `VerifyResult` for every materialized output (i.e. every output -with a recipe whose directory exists on disk) in the named universe. - -Outputs that aren't materialized at all are silently skipped — that's a -question for `lc status`, not `lc verify`. - -For each materialized output: - -1. Read its manifest. If missing or unparseable → `missing_manifest`. -2. Recompute `sha256_dir(output_dir)`. If it doesn't match the recorded - `data_version` → `tampered_data` (with a `recorded … != actual …` - detail message). -3. Walk recorded `input_versions`: - - For each declared recipe input, look up the upstream output via - `find_upstream_output`. - - If upstream is external (no producer rule) → nothing to chain to. - - If upstream's current manifest is missing → `broken_chain` - ("upstream … missing manifest"). - - If upstream's current `data_version` ≠ recorded → `broken_chain` - ("upstream … data_version drifted"). - - If the input is missing from the manifest entirely → - `broken_chain` ("input … missing from manifest"). -4. Otherwise `passed=True`. - -### `VerifyResult` (dataclass) - -```python -@dataclass -class VerifyResult: - output_id: str - universe_id: str - output_dir: Path - passed: bool - failure: FailureKind | None - detail: str | None = None -``` - -### `FailureKind` - -```python -FailureKind = Literal["missing_manifest", "tampered_data", "broken_chain"] -``` - -## Performance notes - -`sha256_dir` is the dominant cost. Hashing 10 GB of float arrays takes -real wall time. `lc status` is the cheap version that recomputes -`code_version` only — use that for the day-to-day "is this stale?" -question, and `lc verify` for periodic / pre-publication audits. - -## Tests - -`tests/test_verify.py` covers each failure kind end-to-end against tmp -projects, plus the chain-walking through nested sub-analyses. diff --git a/docs/api/worker.md b/docs/api/worker.md new file mode 100644 index 00000000..2dbe05b0 --- /dev/null +++ b/docs/api/worker.md @@ -0,0 +1,59 @@ +# lightcone.engine.worker + +Making one output — the unit of work, and the only thing that runs a +recipe. Also an entry point: + +```text +python -m lightcone.engine.worker / +``` + +which is what the `[DATALAD RUNCMD]` record in every materialization +commit names, behind an engine-pinning `uv run --no-project --with …`. +It is a module rather than an `lc` verb on purpose: it makes the +output unconditionally, commits nothing, and leaves the tree dirty by +design — precisely the state `lc materialize` refuses to start from — +so advertising it would hand people a footgun. + +Source: `src/lightcone/engine/worker.py`. + +## Key symbols + +| Symbol | Role | +|---|---| +| `materialize(task, versions, ...)` | The unit: classify → reset the directory → sandbox → recipe → hash → manifest. Returns a `TaskResult`, always. | +| `TaskResult` | `ok` / `current` / `behind` / `failed` / `blocked`, the output's `data_version`, and the attestation. `.usable` is what dependents check. | +| `main(argv)` | The rerun entry point: guards, converges the project environment from the commit's own lock, resolves its own HEAD and runtime, executes. | +| `lc_version()` | The engine version every manifest records. | + +## What must stay true + +- **The worker never raises** — enforced at the unit boundary, so the + contract holds for failure modes nobody enumerated. Raising would + make Dask abort every task in flight; reporting all independent + failures in one run is most of what owning the loop buys. +- **`data_version` is computed here, before anything is staged** — the + dependent's argument *is* this return value, so the digest must + exist while the files are still unannexed. Deriving it from + `git annex find` records `sha256([])` for everything, silently, with + green tests — and couples the digest to the annex backend, which is + deliberately not pinned. +- **The reset takes the whole directory** — a crashed previous run can + have left anything there, and there is no "expected file list" to + delete by. The `output_dir` guard bounds the blast radius, not a + narrower delete. +- **No git in here.** The driver commits; a worker that asked git + would race the index lock and could read a HEAD this same run moved. +- **`main`'s "no output ``" message covers the task lookup only.** + It once wrapped the whole body, and a `KeyError` from anywhere + inside astra surfaced as "bad target" — a rerun misdiagnosing itself + at the one place nobody is watching. +- **Keep it cheap to import — no click, no rich.** It is on the path + of every task and every rerun; two tests pin the imports and the + absence from `--help`. (Nothing pins the absence of a + `[project.scripts]` entry — treat that as a review item.) + +## Tests + +`tests/test_worker.py` — real recipes through the real boundary +against a real repository (the `analysis` fixture): whether gates +hold and bytes land are not questions a stub can answer. diff --git a/docs/architecture.md b/docs/architecture.md index f8461629..6d28ba8b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,313 +1,190 @@ # Architecture -The whole story in one sentence: **lightcone-cli is a thin shim over -Snakemake that owns provenance.** This page expands that sentence. +How lightcone-cli is put together, for someone about to change it. The +[user-guide concepts page](user/concepts.md) covers what the tool +promises; this page covers how the promises are kept. -## Three subsystems +## The split that everything else follows -1. **Snakefile generation** — translate `astra.yaml` into a - `.lightcone/Snakefile` and a sidecar `snakefile-config.json` keyed by - `(rule, universe)`. Snakemake handles the rest of execution. -2. **Manifest layer** — a per-output sidecar JSON written *by us* on the - host immediately after each rule's recipe shell exits. The integrity - contract lives here. -3. **Cluster management** — `lc run` always dispatches through a Dask - scheduler whose lifetime equals the run's lifetime. The cluster - manager picks the right shape (local / SLURM / external) on the fly. - -Everything the user touches is the `lc` CLI on top of these: the engine -(Snakefile generation + cluster management) and the integrity layer -(manifests, `lc status`, `lc verify`). - ---- - -## 1. Snakefile generation - -Generator: [`lightcone.engine.snakefile.generate`](api/snakefile.md). - -For each output in the resolved analysis tree (root + sub-analyses, -expanded by `astra.helpers.resolve_analysis_tree`), the generator emits -one Snakemake rule per output. The rule body is a `run:` block: - -```python -rule : - input: ... # from upstream outputs (sibling rules) - output: - data=directory("results/{universe}/"), - manifest="results/{universe}//.lightcone-manifest.json", - params: - cfg=lambda wc: CFG[""][wc.universe], - run: - shell('printf "▶ [%s]\\n" "{wildcards.universe}" >&2') - shell(params.cfg["shell_command"]) # the recipe (already container-wrapped) - write_manifest(output_dir=Path(output.data), inputs={...}, cfg=params.cfg) - for w in validate_output(...): print(f"⚠ {w}", file=sys.stderr) -``` - -### What goes in `cfg` - -`snakefile-config.json` is keyed by ` → → cfg` where -the inner dict carries: - -- `shell_command` — the recipe pre-wrapped at generation time. When - containers are configured, this looks like - ` run --rm --pull=never -v "$PWD":"$PWD" -w "$PWD" bash -c ''`. - Snakemake's own `container:` directive and `--sdm apptainer` are - intentionally *not* used — we own the runtime end-to-end. -- `code_version` — `sha256(recipe + container_image + decisions)`. - Embedded as a `: lc_code_version=…;` no-op prefix on the shell command - so it lands in any shell trace. -- `recipe`, `container_image`, `decisions`, `output_id`, `output_type`, - `universe_id`, `git_sha`, `lc_version`, resolved input paths. - -### Why pre-wrap, not Snakemake's `container:`? - -Two reasons. First, `--sdm apptainer` adds an extra container layer that -defeats podman-hpc's migrate workflow. Second, registry image resolution -on podman fails for our content-addressed `lc--` tags -because they trip `unqualified-search-registries` in `registries.conf`. -We pass `--pull=never` to skip the lookup entirely; that requires -images to be present locally, which is what `lc build` does. - -### Staleness detection - -The generator does *not* override Snakemake's rerun logic — it just -makes sure drift is visible to it. We default to -`--rerun-triggers code,input,mtime,params`. The `params` trigger is the -one that fires today: `cfg` is per-universe and contains -`code_version`, so any change to recipe / container image / decisions -flows through. - ---- - -## 2. The manifest layer - -Module: [`lightcone.engine.manifest`](api/manifest.md). Filename: -`.lightcone-manifest.json` (constant; `SCHEMA_VERSION = 1`). - -Every successful rule writes a manifest to its output directory. The -write is atomic (`os.replace` rename); a missing or unparseable manifest -re-runs the rule on the next `lc run`. - -### Fields - -```json -{ - "schema_version": 1, - "output_id": "...", - "universe_id": "baseline", - "code_version": "sha256:…", - "data_version": "sha256:…", - "container_image": "lc-myproject-abc123" , - "recipe": "python scripts/compute.py", - "decisions": {...}, - "input_versions": { "": "sha256:…" }, - "git_sha": "...", - "lc_version": "...", - "host": "...", - "slurm_job_id": "...", - "finished_at": 1700000000.0 -} -``` - -### `data_version` exclusions - -`sha256_dir()` skips two filenames: `.lightcone-manifest.json` (chicken -and egg) and `.snakemake_timestamp` (Snakemake touches the directory -*after* the rule body completes — including it would make every hash -unreproducible). - -### `input_versions` semantics - -For each declared recipe input: -- If the input is a sibling output (has its own manifest) → - `data_version` from that manifest. -- Otherwise treated as external → - `mtime-size:-` for files, `sha256_dir(...)` for - directories, `"missing"` for absent paths. - -### What `lc verify` checks - -- **`tampered_data`** — `sha256_dir()` of the on-disk output no longer - matches the recorded `data_version`. -- **`broken_chain`** — a recorded `input_versions[id]` no longer matches - the upstream output's current `data_version`. -- **`missing_manifest`** — the output directory exists but has no - manifest, or the manifest fails to parse. - -### What `lc status` checks - -- **`ok`** — manifest present, recomputed `code_version` matches. -- **`stale`** — manifest present but `code_version` drifted (recipe, - image, or decisions changed). -- **`missing`** — no manifest. -- **`alias`** — output declared without a recipe; materialized only as a - side effect of an upstream. - -`status` reads only manifests. No Snakemake import, no `.snakemake/` -directory required, works on a fresh clone or frozen archive. - ---- - -## 3. Cluster management - -Module: [`lightcone.engine.dask_cluster`](api/dask_cluster.md). - -`cluster_for_run()` is the only entry point. It is a context manager -that yields a Dask scheduler address valid for the duration of the run, -across three branches: - -1. `DASK_SCHEDULER_ADDRESS` already set → yield as-is. We don't own the - cluster, we don't tear it down. -2. `SLURM_JOB_ID` set → start an in-process scheduler bound to the - driver hostname (`SLURMD_NODENAME` or `gethostname()`), then `srun` - one `dask worker` per node across the allocation. Workers advertise - the node's resources via Dask abstract resources (`cpus`, `memory`, - `gpus`). The Snakemake executor plugin maps per-rule - `cpus_per_task` / `mem_mb` / `gpus_per_task` to per-task constraints. -3. Neither → `LocalCluster()` sized to the local machine. - -The scheduler is always in-process so its lifetime equals the run's -lifetime: no service to manage, no orphaned schedulers. - -### The Snakemake executor - -Module: [`snakemake_executor_plugin_dask`](api/dask_executor.md). - -Snakemake calls `run_job(job)`, we translate it to: - -```python -client.submit( - _run_shell, cmd, - resources=_build_resources(job), - pure=False, - key=f"snakejob-{job.name}-{job.jobid}", -) +```text +lc (CLI) engine ASTRA +───────────── ───────────────────── ───────────── +flags, rendering, ──► what a project is, ──► what a spec +exit codes how outputs are made *means* ``` -The worker shells out to the (already container-wrapped) command. There -is no per-rule "executor logic" to write — recipes are wrapped at -generation time, so the worker just runs them. - ---- - -## Container layer - -Module: [`lightcone.engine.container`](api/container.md). - -Two surfaces: - -- **Build** — `compute_image_tag()` + `build_image()`. Tags are - `lc--` over the Containerfile and dependency - files (`requirements.txt`, `pyproject.toml`, `poetry.lock`, - `Pipfile.lock`, …). Rebuilds happen only when the hash changes. -- **Run-time wrap** — `wrap_recipe()` produces the command string that - the Snakefile generator embeds into each rule. - -Runtime resolution: `~/.lightcone/config.yaml` carries -`container.runtime` (`auto | docker | podman | podman-hpc | none`). -`auto` picks the first usable in `(podman, docker, podman-hpc)`, -skipping docker if its daemon is unreachable. `none` is an explicit -opt-out — recipes run on the host. When `auto` falls back to `none` -silently, `lc run` warns that the manifest's `container_image` field -will misrepresent what actually executed. - -For `podman-hpc`, the build path also runs `podman-hpc migrate ` -so compute nodes can read the image without a registry. - ---- - -## Sub-analysis tree - -`astra.yaml` can declare nested `analyses:` pointing to sub-directories -each with their own `astra.yaml`. The full tree is resolved by -`astra.helpers.resolve_analysis_tree()` before any operation. - -Output paths follow the analysis layout: - -- Root + inline sub-analyses: `results///` -- Path-rooted sub-analyses: `/results///` - -`from:` references on inputs and decisions are resolved by helpers in -[`engine.tree`](api/tree.md). When an output id is ambiguous (the same -name appears in multiple sub-analyses), `lc run` errors and asks for -the qualified `.` form. - ---- - -## Repository at a glance +- **`cli/commands.py`** owns flags, console rendering, and exit codes — + nothing else. It imports the engine *inside* command callbacks, so + `lc --help` stays cheap. The engine never imports click and never + prints. +- **The engine** owns everything about what a project is and how + outputs get made. It raises `ProjectError`; the CLI's group class + translates that into a clean error message, once, for every verb. +- **ASTRA** owns what a spec means. Scoping, `from:` references, + conditional outputs, universe resolution, and the recipe placeholder + grammar are all answered by `astra.resolve` and checked by + `astra.validation` — never re-implemented here. When the spec's + *meaning* looks wrong, the fix is a PR to astra-tools. + +The engine ships as the `lightcone.*` PEP 420 namespace — +`src/lightcone/` has **no `__init__.py`**, so sibling distributions can +share the namespace. The engine is the host's `uv tool`, never a +project dependency: a project's lock carries only what the analysis +imports. + +## One run, end to end ```text -src/lightcone/ # PEP 420 namespace package — NO __init__.py -├── cli/ # Click surface -│ ├── __init__.py # exposes main() -│ └── commands.py # init, run, status, verify, build, export -├── engine/ # execution substrate -│ ├── manifest.py # write_manifest, sha256_dir, code_version -│ ├── snakefile.py # generate .lightcone/Snakefile from astra.yaml -│ ├── container.py # docker/podman/podman-hpc build + recipe wrap -│ ├── cloudbuild.py # GCP Cloud Build backend (kubernetes runtime) -│ ├── dask_cluster.py # cluster lifecycle (local/SLURM/Gateway/external) -│ ├── scratch.py # scratch-root resolution, run dirs, run lock -│ ├── status.py # manifest-driven status walker (no Snakemake) -│ ├── verify.py # recompute hashes, walk the chain -│ ├── tree.py # sub-analysis tree helpers -│ ├── validation.py # post-recipe output sanity checks -│ ├── wrroc.py # Workflow Run RO-Crate export -│ └── site_registry.py # known-site defaults (scratch root, runtime) - -src/snakemake_executor_plugin_dask/ # Snakemake executor → dask.distributed - -tests/ # pytest, mirrors src/ -pyproject.toml # hatchling + hatch-vcs; ASTRA + Snakemake as deps +lc materialize + │ guard: compute node? tools? git identity? + │ refuse: dirty tree + │ converge: uv.lock ⇄ .venv (and the image, containerized) + │ plan: astra validate + resolve → Graph of Tasks + │ fetch: git annex get (declared inputs not in this clone) + │ venue: SLURM allocation? → srun workers · else LocalCluster + ├─► workers: reset output dir → sandbox → recipe → hash → manifest + │ (never raise; return ok/current/behind/failed/blocked) + └─ driver: consume results in one thread + ok → dataset.save (commit + run record) + failed → dataset.restore (tree as clean as it started) + finally → converge ro-crate-metadata.json (if licensed) ``` -The `lightcone.*` namespace is a PEP 420 implicit namespace package. -**Do not add `src/lightcone/__init__.py`** — that would turn it into a -regular package and break coexistence with future sibling distributions -(`lightcone-ui`, etc.). Any new `lightcone-*` package must live under -`src/lightcone//` and ship only its own subpackage. - ---- +The division of labor is strict and load-bearing: + +- **The driver owns git, alone.** Workers execute and return a + `TaskResult`; the driver commits as results arrive, in one thread. + Concurrent git operations race on the index lock — this split is not + a preference. +- **Dask owns the ordering.** Every task is submitted with its + upstream futures as arguments; there is no ready-set loop or + hand-rolled topological sort on the execution path. +- **The worker never raises.** A recipe failure, a gate failure, an + unreadable manifest — all come back as a state, so one failure + doesn't abort every task in flight, and a run reports *all* its + independent failures. +- **Values are resolved once and handed down.** HEAD, the container + runtime, and the foreign-write facts are read by the driver and + passed to workers as values — a worker that asked git itself could + get a different answer mid-run, and workers have no git anyway. + +## Identity: two hashes, three states + +`identity.py` computes two digests that deliberately answer different +questions: + +- **`definition_version`** = hash(rendered recipe ‖ decisions) — what + the spec says the output *is*. When it moves, the artifact + contradicts the spec: **stale**, remade. +- **`env_version`** = hash(lock bytes ‖ interpreter pin ‖ install + settings ‖ image document) — what the output *ran under*. When it + moves, the artifact is merely from another time: **behind**, + reported, left alone. + +`assets.classify` is the one implementation of the rule, with two +callers: the worker (live input digests) and the read-only walk +(`None` for anything upstream that will run — "this is going to +change"). That single value is the entire difference between run and +check, which is what keeps `--check` honest. `behind` does not +propagate; `stale` wins when both apply; and a foreign write (an +output's directory last touched by a commit that is not its own run +record) classifies stale through the same rule, as one more input +value. + +Both hashes are length-framed (label, length, bytes per field), so a +boundary shift between concatenated fields cannot produce a collision. +The lock is hashed as raw bytes, never parsed — over-invalidation +costs a report line; a parse that disagrees with uv costs correctness. + +## Storage: the repository is the record + +`dataset.py` is the whole git + git-annex seam. The model is DataLad's: +git carries pointers and history, the annex carries bytes, and +`.gitattributes` routes content (`annex.largefiles=nothing` by +default; `data/` and `results/` opt out). A researcher only ever types +ordinary `git add` / `git commit`. + +Each output is committed with a **run record** — a `[DATALAD RUNCMD]` +commit message whose `cmd` reconstructs the engine +(`uv run --no-project --with lightcone-cli==`) and re-executes the +worker entry point, so `datalad rerun` replays the making of an output +with the gates, the sandbox, and the manifest intact. Results are +committed *thin* (hard-linked to their annex object), which is safe +precisely because lc never writes an output in place — the worker +resets the directory first. + +## The exec boundary + +Every recipe and every `lc run` command goes through +`engine/sandbox/`: a `Policy` (mechanism-free path sets) is turned +into *a different argv that sandboxes itself* by a `Backend` — +Landlock via the stdlib-only shim `lightcone/_sandbox_exec.py`, +Seatbelt via `sandbox-exec`, the OCI mount table in containerized +mode, and `Unavailable` (wrap = identity) where no mechanism exists. +Because every backend is a pure argv rewrite, all of them are testable +on a host that can't run them, and the manifest's `hermeticity` field +records what was *actually* enforced — never what should have been. + +There is one policy, `exec_policy`: probe and recipe get exactly the +same thing (tree read-only apart from `results/`), so "works under +`lc run`" and "works as a recipe" stay the same fact. + +## The container hatch + +Containerized mode changes the recipe's world and nothing else. +`image.py` (pure) turns the `[tool.lightcone.image]` declaration into +a rendered Containerfile, an identity document, and a content tag; +`container.py` (impure) builds it, saves it as a `docker-archive` +inside the repository (`.datalad/environments//image`, annexed), +and enters it. The engine never enters the image — driver, git, and +classification stay on the host; exactly two things run in-image: the +environment sync and each recipe exec, over a read-only rootfs with +the mount table as the whole policy. Execution pins the archive's +config-blob id, never a tag. + +## Venues + +`materialize.cluster_for_run()` is the one place that decides where a +run executes, and the seam it returns is two methods wide — +`submit(fn, *args, key=…)` and `completed(handles)`. A SLURM +allocation (detected by `SLURM_JOB_ID`) gets one worker per node via a +single `srun`, running the driver's own interpreter so driver and +workers are the identical installation. Anything else is the local +machine. Venues are detected, never configured; the only venue config +that exists is the allocation the user already requested. + +## The publication view + +`crate.py` renders the repository as a Provenance Run Crate — a pure +function of repository state (sorted iteration, no clock, git injected +as a callable), which is what lets `materialize` converge +`ro-crate-metadata.json` byte-for-byte and commit only differences. +Run identity comes free from the manifests' `git_sha` (the driver +reads HEAD once per run), so one materialize maps onto one +`OrganizeAction` with no new manifest field. -## Execution flow +## Repository at a glance ```text -astra.yaml ── snakefile.generate() ──► .lightcone/Snakefile + .lightcone/snakefile-config.json - │ - snakemake -s … -d … --executor dask - │ - ┌──────────────────────┼──────────────────────┐ - │ │ │ - DAG resolution per-rule run: dask scheduler - (Snakemake) shell(recipe) (LocalCluster / - + write_manifest() SLURM-srun / - external) - │ - └─► results///data - results///.lightcone-manifest.json +src/lightcone/ # namespace — NO __init__.py +├── _sandbox_exec.py # the Landlock shim — stdlib only, zero lightcone imports +├── cli/commands.py # flags, rendering, exit codes — nothing else +└── engine/ + ├── project.py # what a project is: convergence, discovery, mode + ├── dataset.py # the git + git-annex seam + ├── identity.py # env_version, definition_version, the lock scan + ├── image.py # the system layer, declared → rendered — pure + ├── container.py # runtimes, the build, the archived image — impure + ├── crate.py # the publication view — pure + ├── assets.py # an output: directory, manifest, state + ├── plan.py # the spec, read as a graph of tasks + ├── worker.py # making one output; the rerun entry point + ├── materialize.py # the driver: gates, Dask, the save/restore loop + ├── run.py # what `lc run` is + ├── venue.py # where a run executes + ├── sandbox/ # the exec boundary + └── templates/ # the scaffold's file content, as real files ``` -What Snakemake owns (we don't write it): DAG construction, topological -execution, parallelism, dry-run, locking, retry, log capture, -per-rule resources, `--rerun-triggers` for staleness detection. - -What we own: a Snakefile generator, the manifest layer (write/read/verify), -a status walker, a verify routine, the Dask cluster manager, the -container-runtime layer, and a Snakemake executor plugin that submits -each rule to a Dask scheduler. - ---- - -## Configuration files - -| File | Scope | Purpose | -|------|-------|---------| -| `astra.yaml` | Project | The spec. Inputs, outputs, recipes, decisions, sub-analyses. | -| `.lightcone/Snakefile` | Project (generated) | Auto-generated by `lc run`. Don't edit. | -| `.lightcone/snakefile-config.json` | Project (generated) | Per-`(rule, universe)` config. | -| `.lightcone/lightcone.yaml` | Project | Tiny scratchpad — currently writes only `target: local`. Not consumed by today's code. | -| `~/.lightcone/config.yaml` | User | `container.runtime`. | - -The `dagster.yaml` and `~/.lightcone/targets/*.yaml` files referenced in -older docs are no longer used — historical residue. +Each module's page in [Engine Internals](api/index.md) carries its +public surface and the invariants that bind it. diff --git a/docs/contributing/backends.md b/docs/contributing/backends.md deleted file mode 100644 index 5c651e2d..00000000 --- a/docs/contributing/backends.md +++ /dev/null @@ -1,42 +0,0 @@ -# Adding an Execution Backend (rewritten) - -The `ASTRAContainerRunner` plugin point is gone. Execution is structured -quite differently now, and "adding a backend" decomposes into one or both -of these: - -## Adding a container runtime - -The supported runtimes are `docker`, `podman`, and `podman-hpc` (plus the -`none` no-op). They are listed in -`src/lightcone/engine/container.py::RUNTIMES`. To add a new one: - -1. Append the binary name to `RUNTIMES` (detection priority is the tuple - order). -2. If detection needs a probe (like the docker-daemon ping), extend - `detect_runtime()`. -3. If `wrap_recipe()` needs different flags for the runtime, branch on - `runtime` there. -4. If post-build migration is required (the `podman-hpc migrate` model), - add a `__migrate(tag)` and call it from `build_image()` / - `pull_image()`. -5. Add tests in `tests/test_container.py`. - -## Adding a Dask cluster shape - -Today the cluster manager has three branches: existing scheduler, SLURM -allocation, local. To add a fourth (for example, a custom GPU farm): - -1. Add a branch to `cluster_for_run()` in - `src/lightcone/engine/dask_cluster.py`. -2. Make sure it advertises the same resource keys (`cpus`, `memory`, - `gpus`) so the [Snakemake executor plugin](../api/dask_executor.md) - can match. -3. Add tests in `tests/test_dask_cluster.py`. - -## Adding a non-Snakemake executor - -In principle Snakemake supports multiple executors and we ship one -(`snakemake_executor_plugin_dask`). If you need a different scheduler, -you can write another Snakemake executor plugin and pass it through -`lc run --executor ` — but that flag does not exist today and would -need to be added to `src/lightcone/cli/commands.py::run`. diff --git a/docs/contributing/extending.md b/docs/contributing/extending.md new file mode 100644 index 00000000..ae3312f9 --- /dev/null +++ b/docs/contributing/extending.md @@ -0,0 +1,55 @@ +# Extending the Codebase + +Where each kind of change belongs, what to read first, and the +invariant it must keep. The engine has one implementation per rule — +most review feedback is some form of "that spelling already exists; +use it". + +## The map + +| To change… | Edit | Keep true | +|---|---|---| +| What a scaffolded file contains | `engine/templates/files/*.tmpl` (+ `test_templates.py`) | A template gets a function only when a value must be decided or a merge policy held. | +| What gets converged | `engine/project.py` (+ `test_project.py`) | Everything through `_Converger.item`/`.file`/`.blocked`; repairs only append; only what git can carry. | +| How a project stores bytes | `engine/dataset.py` + `gitattributes.tmpl` (+ `test_dataset.py`, real annex) | Every command through `project._run`; nobody is asked to run git-annex. | +| How an output is identified | `engine/identity.py` (+ `test_identity.py`) | Sensitivity both ways: what must move the hash, what must not. Length-framing stays. | +| When an output is remade | `engine/assets.py` (+ `test_assets.py`) | One `classify`; callers differ by one input value, never by logic. Ask first: does the change *contradict* the project (stale) or is it *circumstance* (behind)? | +| How the spec becomes a graph | `engine/plan.py` (+ `test_plan.py`) | Ask `astra.resolve`; a missing answer is a PR to astra-tools; ambiguity is a `ProjectError`, never a guess. | +| How a recipe runs | `engine/worker.py` (+ `test_worker.py`) | Never raises; no git; mutation-check every denial test. | +| What a run commits | `engine/materialize.py` (+ `test_materialize.py`) | The driver owns git alone; the tree ends as clean as it started. | +| Where a run executes | `engine/venue.py` + `cluster_for_run` (+ `test_venue.py`) | One detection ladder; venues detected, never configured; test by faking the host. | +| Supporting a new HPC center | `venue._SITES` | One row — marker + the center's own `salloc`/`sbatch` spellings, verified against its documentation, never guessed. | +| What a sandboxed command may touch | `sandbox/policy.py` (+ `test_sandbox_policy.py`) | Path sets only — no mechanism leaks in. | +| Adding a sandbox mechanism | one module in `sandbox/` + one line in `detect()` | `wrap` pure, `attest` honest, `contains_prefix` answered. Nothing above the seam changes. | +| A denial message | `sandbox/denial.py` (+ `test_sandbox_denial.py`) | Remedies copy-pasteable and real *today*; the trailer stays unconditional. | +| What the image is made of | `engine/image.py` (+ `test_image.py`) | Pure; every declaration key hashed; structure tests, never byte goldens. | +| How images are built/stored/entered | `engine/container.py` + `sandbox/oci.py` (+ `test_container.py`) | `runtime_for_run`'s two strictnesses; runtime differences are spellings inside `OCIBackend`, never new shapes. | +| What the crate says | `engine/crate.py` (+ `test_crate.py`) | Pure builder: sorted, no clock, git injected; render-twice-identical. The validator floor lives in `test_crate_smoke._FLOOR`. | +| How a foreign write is detected | `dataset.last_writer` + `materialize._foreign_write` | History, never hashing; `datalad_run_subject` is the one spelling of the record's subject. | +| A CLI verb | `cli/commands.py` (+ `test_cli.py`) | Logic in the engine; raise `ProjectError`; render here; engine imports stay inside callbacks. | + +## Rules that apply everywhere + +- **Land code, tests, and dependencies together.** A dependency enters + `pyproject.toml` with the change that needs it, never speculatively. +- **No dead code, no foreshadowing.** Nothing references a verb, flag, + or feature that doesn't exist yet; `lc --help` advertises only what + works. +- **No escape hatches.** Enforcement ships without a flag to turn it + off; there is deliberately no `--no-sandbox`, no `--force`, no + rebuild-the-world flag. +- **Nothing waits on a human.** No prompt, no interactive shell — + either is a hang for the agents that run these verbs most. +- **Refusals carry remedies, and remedies are verified.** A message + that tells someone to run a command has been run; a center's + spellings come from its documentation. +- **Docstrings are Google-style, comments carry *why*.** A design + decision gets a sentence; its history belongs in the design record, + not the code. + +## Conventions + +Ruff (E, F, I, N, W, UP; line length 100), mypy strict with +`namespace_packages = true`. `src/lightcone/` must never gain an +`__init__.py` — the namespace is shared with future sibling +distributions, and a real package there breaks the contract. diff --git a/docs/contributing/hpc-sites.md b/docs/contributing/hpc-sites.md deleted file mode 100644 index 6e30e06f..00000000 --- a/docs/contributing/hpc-sites.md +++ /dev/null @@ -1,24 +0,0 @@ -# Adding an HPC Site - -The old target system is gone; what remains is the lightweight -[`site_registry`](../api/site_registry.md) module, which supplies -per-site defaults (scratch root, preferred container runtime) via -`detect_current_site()`. - -If you want lightcone-cli to behave well on a new cluster, what you -actually need is: - -1. **A container runtime that works on compute nodes.** `podman-hpc` is - the supported case. Wire it up via `~/.lightcone/config.yaml`, or - declare it as the site's `container_runtime` in `SITE_DEFAULTS`. -2. **Dask workers reachable from the scheduler.** `lc run` already does - the right thing inside an `salloc`/`sbatch` allocation — the cluster - manager binds the scheduler to the SLURM canonical hostname and - launches one worker per node via `srun`. See - [api/dask_cluster](../api/dask_cluster.md). -3. **A sane scratch root.** `lc run` keeps its operational state - (snakemake metadata, dask spill, cross-node run locks) under a - scratch root that must honour `flock` — on Perlmutter that means - `$SCRATCH` (Lustre), not DVS-mounted home/CFS. Declare - `scratch_root` in the site's `SITE_DEFAULTS` entry; users can - override it per-project with `lc init --scratch`. diff --git a/docs/contributing/setup.md b/docs/contributing/setup.md index ed11c9c4..c2348966 100644 --- a/docs/contributing/setup.md +++ b/docs/contributing/setup.md @@ -1,92 +1,77 @@ # Development Setup -You'll need: - -- Python 3.11+ -- [uv](https://docs.astral.sh/uv/) — `curl -LsSf https://astral.sh/uv/install.sh | sh` -- [just](https://github.com/casey/just) — `brew install just` or `cargo install just` -- Git -- One of: docker, podman, podman-hpc (optional — only needed for - container tests and for projects that declare `container:`) +Everything runs through [uv](https://docs.astral.sh/uv/); there is no +task runner and no other build tooling. ## Clone & install ```bash git clone https://github.com/LightconeResearch/lightcone-cli.git cd lightcone-cli -just install # uv sync --all-groups (dev + docs) +uv sync --group dev ``` -`just` (alone, with no recipe) lists everything available — the -recipes that follow are the ones you'll touch most. +That resolves the engine and the dev tools (pytest, ruff, mypy, +datalad, the rocrate validator) into `.venv`. `uv run lc --version` +runs the checkout's `lc`. + +You also need `git` on `PATH` (the one tool uv cannot install); +git-annex arrives as a wheel with the sync. -## Running the test suite +## The loop ```bash -just test # uv run pytest -just test-cov # with coverage report +uv run pytest # the suite +uv run ruff check src/ tests/ # lint (--fix to apply) +uv run mypy src/ # strict mode ``` -The opt-in `slow` marker covers tests that spin up real subsystems -(local Dask cluster, etc.). They are excluded by default; run with -`uv run pytest -m slow` to include them. +These three are exactly what CI runs (`tests.yml`, `lint.yml`) — green +locally means green there, modulo the gated suites below. -## Linting & types +Most of the suite is hermetic: an autouse fixture stubs the engine's +one subprocess seam, so tests spawn nothing and touch no network. The +exceptions opt in explicitly — see [Testing](testing.md). -```bash -just lint # ruff + mypy -just fix # ruff --fix -just fmt # ruff format -``` +### The gated suites -Ruff rules: `E, F, I, N, W, UP`. Line length: 100. Target: Python 3.11. -Mypy is strict, with `namespace_packages = true` and -`explicit_package_bases = true` (we ship a PEP 420 namespace package). +Three suites answer questions only a real mechanism can, and each +skips where its mechanism is missing — with an environment variable CI +sets to turn the skip into a hard failure: -## Building the docs locally +| Variable | Suite | Needs | +|---|---|---| +| `LC_SANDBOX_TESTS_REQUIRED=1` | `test_sandbox_enforcement.py` | Landlock (Linux) or Seatbelt (macOS) | +| `LC_CONTAINER_TESTS_REQUIRED=1` | `test_container_smoke.py` | podman or docker | +| `LC_CRATE_TESTS_REQUIRED=1` | `test_crate_smoke.py` | nothing beyond dev deps | + +## Building the docs ```bash -just docs-serve # syncs docs group + live preview at http://127.0.0.1:8000 -just docs-strict # build with --strict -just docs # one-shot build into site/ +uv sync --group docs +uv run zensical build # renders into site/ +uv run zensical serve # live preview ``` -The docs use [zensical](https://zensical.org). The nav lives in -`zensical.toml`. +The site deploys on release (`docs-deploy.yml`), so docs track the +released CLI, not `main`. ## Building the wheel ```bash -just build # uv build -just version # current version (from git tags via hatch-vcs) -``` - -The wheel ships two packages: - -```toml -[tool.hatch.build.targets.wheel] -packages = ["src/lightcone", "src/snakemake_executor_plugin_dask"] -``` - -## Repo layout - -```text -src/lightcone/ # main namespace (PEP 420; no __init__.py at the package root) -src/snakemake_executor_plugin_dask/ # Snakemake → Dask executor plugin -tests/ # pytest tree, mirrors src/ -evals/ # agentic eval: prompt.md + task seeds (tasks/snae/) -docs/ # docs site +uv build ``` -## Pre-commit checklist +CI runs this only to publish. The version comes from hatch-vcs — the +git tag for a release, tag-plus-commit for a dev build — which is also +what lets a run record pin a dev engine by its source commit. -Quick sequence before pushing a PR: - -```bash -just lint # ruff + mypy -just test # full pytest run -just docs-strict # docs still build cleanly -``` +## Pre-PR checklist -Each line maps to one CI check. CI runs them serially; running locally -catches everything before the PR machinery starts. +1. `uv run pytest` — including, if your change touches the sandbox, + containers, or the crate, the relevant gated suite on a host that + can run it. +2. `uv run ruff check src/ tests/` and `uv run mypy src/`. +3. New behavior lands with its tests, in the same PR. +4. Read [Extending](extending.md) — it says where each kind of change + belongs, and the invariants it must keep. diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index e25b165a..8143fac4 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -1,85 +1,77 @@ # Testing -## Test layout - -```text -tests/ -├── conftest.py # shared fixtures -├── test_cli.py # Click CliRunner integration tests -├── test_container.py # detection, image tag, build_image, wrap_recipe, RuntimeChoice -├── test_dask_cluster.py # cluster_for_run branches & resource keys -├── test_dask_plugin.py # snakemake_executor_plugin_dask -├── test_eval_tasks.py # eval task seed specs validate against astra -├── test_manifest.py # write_manifest, sha256_dir, code_version -├── test_snakefile.py # generator + final `snakemake -n` parse test -├── test_status.py # OutputStatus across ok/stale/missing/alias -├── test_tree.py # collect_tree_outputs, find_upstream_output, … -├── test_validation.py # validate_output across metric/table/figure types -└── test_verify.py # verify_outputs across all three failure kinds -``` - -Tests mirror `src/` 1:1 — when you add a module, add a test file at the -matching path. - -## Common patterns - -### CLI tests (Click `CliRunner`) - -```python -from click.testing import CliRunner -from lightcone.cli.commands import main - -def test_init_creates_structure(tmp_path): - runner = CliRunner() - result = runner.invoke(main, ["init", str(tmp_path / "myproject"), "--no-git", "--no-venv"]) - assert result.exit_code == 0 - assert (tmp_path / "myproject" / "astra.yaml").exists() -``` - -### End-to-end against a tmp project - -`test_status.py`, `test_verify.py`, and `test_snakefile.py` build a -minimal ASTRA project under `tmp_path` (one `astra.yaml`, one -`universes/baseline.yaml`, optional sub-analyses), then run the -function under test. Helpers: - -- `astra.helpers.load_yaml` / `resolve_analysis_tree` mirror what - production code does. -- `lightcone.engine.snakefile.generate(project, universes=[...], runtime="none")` - for tests that need an actual Snakefile. - -### Snakefile parsing - -`tests/test_snakefile.py` ends with a parse test that runs -`snakemake -n -s ` to confirm the generator -produces a Snakefile the upstream tool actually accepts. Add a similar -assertion when changing rule shape. - -### Slow tests - -```bash -uv run pytest -m slow # opt in to the slow tests -``` - -The `slow` marker is reserved for tests that start a real Dask cluster. -Do not use it for things that are merely a bit chatty — prefer trimming -test scope. - -## Eval harness (separate) - -The agentic eval is a plain GitHub Actions workflow — -`.github/workflows/eval.yml` — with no Python harness behind it. On -each PR it scaffolds a project with `lc init`, overlays the seed files -from `evals/tasks/snae/` (`astra.yaml`, `data/`), runs Claude Code -headlessly with `evals/prompt.md` (the astra skill is installed from -the `LightconeResearch/agent-skills` plugin marketplace), and then -checks the outcome with `astra validate` and `lc status --json` — the -job fails unless every declared output is materialized. Run metrics -(turns, tool calls, cost, wall time) are extracted from the transcript -by `.github/scripts/trace_digest.py` and posted as a sticky PR comment -and job summary. Two artifacts are uploaded: `agent-trace` (the raw -stream-json transcript plus a human-readable markdown digest) and -`eval-project` (the built project with its provenance manifests). - -To reproduce locally, run the same commands the workflow does with -`claude`, `lc`, and `astra` on PATH. +The suite's shape follows the engine's: pure modules get pure tests, +the subprocess seam gets a stub, and the questions only a kernel, a +runtime, or a validator can answer get real ones — gated so they can't +pass by not running. + +## The one seam + +`tests/conftest.py`'s autouse `tools` fixture stubs +`engine.project._run` — the single choke point every external command +goes through — emulating each tool's observable effect (`uv lock` +writes `uv.lock`, `git init` makes `.git`, …) and recording every +argv. Under the stub the suite is hermetic: no network, no resolution, +no subprocesses. + +The `real_tools` fixture opts back out, putting the real `_run` back. +Everything built on it (the `analysis` fixture, the rerun tests) does +spawn and may touch the network — that is the deliberate price of +testing execution. + +## Where a question belongs + +| Question | File | Character | +|---|---|---| +| Convergence semantics | `test_project.py` | stubbed | +| Template content & repair | `test_templates.py` | pure | +| Do bytes land in the annex? | `test_dataset.py` | **real tools** — every bug this seam had was invisible to a stub | +| Identity sensitivity | `test_identity.py` | pure, both directions | +| The graph, the gate | `test_plan.py` | pure — tests what lc *adds*, never what a spec means (that's astra-tools' suite) | +| Classification | `test_assets.py` | pure | +| One output, real recipe | `test_worker.py` | real boundary, real repo | +| The run, the record | `test_materialize.py` | real repos; one real `LocalCluster`; real `datalad rerun` | +| Venue detection & launch | `test_venue.py` | fakes the *host* (env vars, a stub srun), never the code | +| Policy / wrap / denial | `test_sandbox_*.py` | pure, run on every OS | +| The kernel's answer | `test_sandbox_enforcement.py` | gated | +| Image identity | `test_image.py` | pure — structure and ordering, never byte goldens | +| Runtime lifecycle | `test_container.py` | stubbed; refusals asserted on recorded argv | +| The runtime's answer | `test_container_smoke.py` | gated | +| The crate | `test_crate.py` | pure; the one byte claim is render-twice-identical | +| The validator's answer | `test_crate_smoke.py` | gated | +| CLI surface | `test_cli.py` | `CliRunner`; assert short unwrappable fragments | + +## The enforcement suite + +`test_sandbox_enforcement.py` is the only file that can tell you the +sandbox works, and four properties keep it honest: + +1. **One suite, both mechanisms** — parameterized by `detect()` alone; + a leak only Linux catches is a leak, and macOS CI is the sole place + the generated SBPL ever executes. +2. **The real policy** — always `exec_policy`, never one hand-built to + make the point. (`/usr` once sat in the exec set through a fully + green suite built the other way.) +3. **Real leaks, tried literally** — undeclared tools executed, + undeclared libraries `dlopen`ed, undeclared data read. +4. **It cannot pass by not running** — `LC_SANDBOX_TESTS_REQUIRED=1` + in CI turns the skip into a failure, and two tests cover the guard + itself. + +**Mutation-check every denial test**: run the same command through +`Unavailable()` and confirm it *succeeds*. A denial test that would +pass unsandboxed is testing nothing, and the failure mode is silent. +Two related traps: a write-denial must target a path the OS would let +you write (a `/etc` write pins nothing), and enforcement fixtures must +not live under `/tmp`, which is inside the write baseline — the +`outside` fixture roots at `$HOME` for exactly this reason. + +## Conventions + +- Don't add a flag whose only user is a test — stub `project._run` + instead. +- A forged-output test must break the annex hard link before writing + (`test_materialize._forge` shows how) — results are committed thin, + so an in-place write dirties every byte-identical sibling. +- Record formats are tested through their consumer (datalad's parser, + the rocrate validator), never as golden files of our own JSON. diff --git a/docs/hpc/containers.md b/docs/hpc/containers.md deleted file mode 100644 index 786dc86d..00000000 --- a/docs/hpc/containers.md +++ /dev/null @@ -1,52 +0,0 @@ -# Container Builds for HPC - -HPC nodes generally cannot reach a docker daemon, so lightcone-cli ships -support for `podman-hpc` (NERSC Perlmutter and friends). The build/migrate -workflow is owned by `lightcone.engine.container`. - -## podman-hpc workflow - -`podman-hpc` is rootless and HPC-aware. After a build, the image must be -*migrated* into the per-node container cache so compute nodes can read it -without a registry. - -```bash -# On a login node, with podman-hpc on PATH: -lc setup # writes ~/.lightcone/config.yaml -$EDITOR ~/.lightcone/config.yaml # set container.runtime: podman-hpc -lc build # builds + migrates each image -``` - -`lc build` checks for cached tags and skips rebuilds. Use `--force` to -rebuild everything. - -## Tag computation - -Tags are content-addressed: - -``` -lc-- -``` - -The hash covers the Containerfile contents plus any of these dependency -files found at the project root: -`requirements.txt`, `requirements-dev.txt`, `requirements-test.txt`, -`pyproject.toml`, `setup.py`, `setup.cfg`, `poetry.lock`, `Pipfile.lock`. - -## At run time - -`lc run` does **not** re-shell into Snakemake's `container:` directive or -`--sdm apptainer`. The Snakefile generator wraps each rule's recipe in: - -```bash -podman-hpc run --rm --pull=never -v "$PWD":"$PWD" -w "$PWD" \ - bash -c '' -``` - -`--pull=never` is critical: short-name resolution would otherwise try -`unqualified-search-registries` for tags like `lc-myproject-abc123` and -fail. Pre-pulling registry images via `lc build` (or pre-staging -Containerfile images via `lc build`) is therefore mandatory. - -See also: [api/container](../api/container.md) for the implementation, -and [`lc build`](../cli/build.md) for the user-facing command. diff --git a/docs/hpc/index.md b/docs/hpc/index.md deleted file mode 100644 index 672e3fc1..00000000 --- a/docs/hpc/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# HPC & SLURM (consolidated) - -The standalone HPC subsystem (target files, site registry, sbatch -generation) is gone. SLURM execution is now handled by Dask: when `lc run` -is invoked inside an existing SLURM allocation, the cluster manager -launches one `dask worker` per allocated node via `srun` and Snakemake -dispatches each rule across them. - -For the user-facing flow, see [Running on a Cluster](../user/cluster.md). - -For maintainer detail: - -- [api/dask_cluster](../api/dask_cluster.md) — the three-branch decision - (existing scheduler / SLURM allocation / local). -- [api/dask_executor](../api/dask_executor.md) — the Snakemake executor - plugin that turns each rule into a `client.submit(...)` call. -- [api/container](../api/container.md) — `podman-hpc` build & migrate. diff --git a/docs/hpc/site-registry.md b/docs/hpc/site-registry.md deleted file mode 100644 index 49dd39e7..00000000 --- a/docs/hpc/site-registry.md +++ /dev/null @@ -1,6 +0,0 @@ -# Site Registry (orphaned) - -The `lightcone.engine.site_registry` module still exists but is not imported -by any active code path. It carries Perlmutter scheduler defaults that used -to feed the wizard for the (now removed) target system. See -[api/site_registry](../api/site_registry.md) for the current state. diff --git a/docs/hpc/targets.md b/docs/hpc/targets.md deleted file mode 100644 index 2db4548c..00000000 --- a/docs/hpc/targets.md +++ /dev/null @@ -1,3 +0,0 @@ -# Target Configuration (removed) - -The per-machine target system is gone. See [`lc target`](../cli/target.md). diff --git a/docs/maintainer.md b/docs/maintainer.md index 0cbdd3c6..fdfac4a5 100644 --- a/docs/maintainer.md +++ b/docs/maintainer.md @@ -1,24 +1,27 @@ # Developer corner -`lightcone-cli` is a thin shim over Snakemake that owns provenance. This guide -covers everything below the user surface: how the execution and integrity layers -work, what each engine module does, and how to get a working dev loop. +`lightcone-cli` is a small engine with strong opinions: one way to +identify an output, one way to store it, one boundary to execute it +behind. This guide covers everything below the user surface — how the +engine is put together, what each module owns, and how to get a +working dev loop. If you're looking for the user-facing docs, the [user guide](user/index.md) is the other half of this site. ## What this covers -- [Architecture](architecture.md) — the three subsystems (Snakefile generation, - manifest layer, cluster management) and the invariants that hold them together. -- [CLI Reference](cli/index.md) — every `lc` command: flags, options, and the - exact Snakemake invocation each one triggers. -- [Python API](api/index.md) — the `lightcone.engine.*` modules: public - signatures, common entry points, and module responsibilities. -- [HPC & SLURM](hpc/index.md) — how the Dask cluster manager adapts to local, - SLURM, and external schedulers. -- [Contributing](contributing/setup.md) — clone, install, run the test suite, - lint, and build the docs locally. +- [Architecture](architecture.md) — the CLI/engine/ASTRA split, the + run pipeline, identity, storage, the exec boundary, and the + invariants that hold them together. +- [CLI Reference](cli/index.md) — every `lc` command: flags, JSON + report shapes, exit codes. +- [Engine Internals](api/index.md) — the `lightcone.engine.*` + modules: what each owns, its key symbols, and what must stay true + of it. +- [Contributing](contributing/setup.md) — clone, install, run the + test suite; [how the suite is shaped](contributing/testing.md); and + [where a change belongs](contributing/extending.md). ## Get started in three commands @@ -27,18 +30,29 @@ If you're looking for the user-facing docs, the ```bash git clone https://github.com/LightconeResearch/lightcone-cli.git cd lightcone-cli - just install # uv sync --all-groups - just test # pytest + uv sync --group dev && uv run pytest ``` - Run `just` with no arguments to see all available recipes. - -## What lightcone-cli *owns* - -The codebase is intentionally small. Snakemake handles DAG construction, -parallelism, cluster submission, staleness detection, locking, and log capture — -we do not replicate any of that. The parts that are ours: - -- **Snakefile generator** — translates `astra.yaml` into `.lightcone/Snakefile`. -- **Manifest layer** — writes and verifies `.lightcone-manifest.json` per output. -- **Cluster manager** — picks local / SLURM / external Dask shape at runtime. +Test, lint (`uv run ruff check src/ tests/`) and type-check +(`uv run mypy src/`) are the whole loop — there is deliberately no +task runner in between. + +## The house rules + +A few conventions run through every module; changes are reviewed +against them: + +- **No dead code, no foreshadowing.** Nothing lands before the layer + that calls it, and no message names a verb or flag that doesn't + exist yet. `lc --help` advertises only what works. +- **No escape hatches around guarantees.** A feature that enforces + something ships without a flag to turn the enforcement off. +- **Literal behavior over invented convenience.** The current + directory is the project; erroring beats walking up or guessing. + Nothing prompts — a verb is run by an agent more often than a + person, and a prompt is a hang. +- **One implementation per rule.** Classification, path naming, the + run-record subject, tool resolution — each has exactly one spelling, + and a second copy is where the two start to disagree. +- **Honest reporting.** What was enforced, what was skipped, and what + a clone can't see are all recorded or said — never assumed. diff --git a/zensical.toml b/zensical.toml index 8178b33a..cea1338e 100644 --- a/zensical.toml +++ b/zensical.toml @@ -30,23 +30,24 @@ nav = [ {"lc run" = "cli/run.md"}, {"lc build" = "cli/build.md"}, ]}, - {"Python API" = [ + {"Engine Internals" = [ {"Overview" = "api/index.md"}, - {"cli/commands" = "api/cli.md"}, - {"engine/manifest" = "api/manifest.md"}, - {"engine/snakefile" = "api/snakefile.md"}, - {"engine/container" = "api/container.md"}, - {"engine/cloudbuild" = "api/cloudbuild.md"}, - {"engine/status" = "api/status.md"}, - {"engine/verify" = "api/verify.md"}, - {"engine/tree" = "api/tree.md"}, - {"engine/validation" = "api/validation.md"}, - {"engine/dask_cluster" = "api/dask_cluster.md"}, - {"snakemake_executor_plugin_dask" = "api/dask_executor.md"}, + {"project" = "api/project.md"}, + {"dataset" = "api/dataset.md"}, + {"identity" = "api/identity.md"}, + {"plan" = "api/plan.md"}, + {"assets" = "api/assets.md"}, + {"worker" = "api/worker.md"}, + {"materialize" = "api/materialize.md"}, + {"venue" = "api/venue.md"}, + {"sandbox" = "api/sandbox.md"}, + {"image & container" = "api/container.md"}, + {"crate" = "api/crate.md"}, ]}, {"Contributing" = [ {"Development Setup" = "contributing/setup.md"}, {"Testing" = "contributing/testing.md"}, + {"Extending" = "contributing/extending.md"}, ]}, ]}, {"ASTRA docs" = "https://astra-spec.org/latest/"}, From 5d474857d511f00fa9bea7f23bd1f67fa10c0f56 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 11:01:49 +0200 Subject: [PATCH 21/23] Docs rewrite 5/5: reconcile CLAUDE.md and the doc-drift check (#189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final pass of the documentation rewrite — no doc pages this time, just bringing the two things that *govern* the docs in line with their new reality. ## CLAUDE.md - **The freeze rule is replaced** by its successor: docs are live (PRs #185–#188), two tracks, and a change lands with its docs — a verb change updates its `docs/cli/` page, an engine change its `docs/api/` module page, user-visible behavior the user guide. The docs inherit the codebase's own discipline: document only what exists, quote refusals from real runs, verify command blocks by executing them. - **The design-doc pointer** now records the decision taken during this rewrite: the design records stay in the sibling checkout and are **dropped when the rebuild completes** — never imported into `docs/` (reverses the paragraph's earlier plan). - **The workflow inventory** catches up: eval runs on dispatch or the `run-eval` label (re-trigger by re-adding it), `check-docs.yml` reviews merged PRs, `docs-deploy.yml` tracks releases; the docs build command is stated. - The extending table's "add the next layer" row loses "Docs are deliberately deferred". Deliberately **not** done: slimming CLAUDE.md's invariant sections now that the developer corner overlaps them. CLAUDE.md is still the working memory for the remaining rebuild (Perlmutter spike, deferred layers), and gutting it mid-rebuild trades a known-good reference for dedup. Worth revisiting when the rebuild closes and the design docs drop. ## check-docs.yml The post-merge doc-drift check was written before the freeze and knew nothing about the structure. Its prompt now carries: - a **map from change kind to doc home** (verb → `docs/cli/`, module → `docs/api/` + architecture, user-visible → user guide + README, workflow/conventions → contributing); - the two failure modes that matter most for these docs: **quoted console output drifting** from what the CLI prints, and **documenting anything the code no longer delivers** (the no-foreshadowing rule applied to docs); - an updated skip list (CLAUDE.md and `evals/` are maintained separately) replacing the stale `skills/` rule. ## Verification - `zensical build`: zero issues. - check-docs.yml validates as YAML. - Grep for `frozen`/stale workflow descriptions in CLAUDE.md comes back clean. This closes the 5-PR docs rewrite. The site deploys with the next release (or a manual `docs-deploy` dispatch). 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx Co-authored-by: Claude Fable 5 --- .github/workflows/check-docs.yml | 26 +++++++++++++++++++++----- CLAUDE.md | 31 ++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index a8d68c73..fe9d23eb 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -34,14 +34,30 @@ jobs: Your task: check whether the project documentation is still accurate and complete given the code changes introduced by this PR. + The documentation is structured, and each kind of change has a home: + - A verb's flags, output, JSON shape, or exit codes → its page in docs/cli/ + (one page per verb, plus the overview's exit-code contract). + - An engine module's responsibility, key symbols, or invariants → its page + in docs/api/ (hand-written module tours) and, for cross-cutting shifts, + docs/architecture.md. + - User-visible behavior (scaffold contents, states, refusal messages, + environment model, SLURM, publication) → docs/user/ (getting-started + quotes real console output; troubleshooting quotes real refusals) and + README.md's quick start. + - Test structure, dev workflow, or conventions → docs/contributing/. + Steps to follow: 1. Run: git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} to get the list of changed files. - 2. Read the changed source files (focus on .py, .ts, .js, .go, .rs files). - 3. Read existing documentation files (README*, docs/**/*.md, CHANGELOG*, CONTRIBUTING*). - 4. SKIP any files under skills/ or named SKILL.md — these are not user-facing docs. - 5. Identify any documentation that is now inaccurate, incomplete, or missing - given the code changes. + 2. Read the changed source files (focus on src/**/*.py and the workflows). + 3. Read the documentation pages the map above points at for those changes. + 4. SKIP CLAUDE.md and evals/ — agent instructions and the eval harness are + maintained separately, not user-facing docs. + 5. Identify documentation that is now inaccurate, incomplete, or missing. + Two failure modes matter most here: a quoted console output or refusal + message that no longer matches what the CLI prints, and a documented + flag, verb, state, or file that no longer exists (the docs must never + describe more than the code delivers — no foreshadowing). Then: - Post a comment on PR #${{ github.event.pull_request.number }} summarising diff --git a/CLAUDE.md b/CLAUDE.md index f160e694..bea1b08f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,8 +34,10 @@ spec: > spec, the decisions win. It lives in the sibling checkout for now > (branch `redesign_prototype`), alongside its decision records > (rationale, substrate tradeoffs, hermeticity enforcement, the v6 -> review); it moves into this repo's `docs/design/` when the docs are -> rewritten at the end of the rebuild. +> review). It stays in the sibling checkout and is **dropped when the +> rebuild completes** (decision, 2026-08): the design records are not +> imported into this repo's docs — the rewritten `docs/` carries the +> current design, and this file carries the decisions. The pre-rebuild codebase (Snakemake shim, authored Containerfiles, `container:` in `astra.yaml`, vendored dask executor plugin, WRROC export) @@ -104,10 +106,15 @@ Each of these has been asked for in review at least once; none is optional. - **No dead code.** If nothing in the current layer calls it, it doesn't land yet. `lc --help` advertises only verbs that work. -- **`docs/` is frozen at its pre-rebuild state, by decision.** It still - describes the old Snakemake architecture and the full command set. - Don't patch it layer by layer — it gets rewritten in one pass once the - rebuild is complete. Same for `README.md` and `zensical.toml`. +- **`docs/` is live again** (rewritten 2026-08, PRs #185–#188; the + freeze is over). The site is two tracks — user guide + developer + corner — and a change now lands with its docs: a new or changed verb + updates its `docs/cli/` page, an engine change updates its + `docs/api/` module page, and user-visible behavior updates the user + guide. The docs' own rules match this file's: document only what + exists, quote refusals from real runs, and verify every command + block by executing it. `check-docs.yml` reviews each merged PR for + drift. - **Port with intent.** Prior implementations (this repo's git history, and the `redesign_prototype` branch of the sibling `lightcone-cli` checkout) are references, not sources of truth. Neither is the spec by @@ -201,10 +208,12 @@ uv build # wheel + sdist (CI runs this only to publish) Test, lint and type-check are the whole loop, and they are what `.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers -around them plus recipes for the frozen docs and the dormant eval. The -other workflows are `eval.yml` (the agentic eval, on dispatch or any -non-draft PR — flipping a draft to ready triggers it), -`pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. +around them. The docs build with `uv sync --group docs && uv run +zensical build`. The other workflows are `eval.yml` (the agentic eval, +on dispatch or the `run-eval` PR label; re-trigger by re-adding the +label), `check-docs.yml` (doc-drift review on merged PRs), +`pypi-publish.yaml`, and `docs-deploy.yml` (deploys on release, so the +site tracks the released CLI). ## Key Invariants (layer 1) @@ -2172,7 +2181,7 @@ unlinks before writing; a new tampering test should too. | To... | Read | Key patterns | |---|---|---| -| Add the next layer | the spec (§11 = the layer ordering) | Land code + tests + deps together; update the layer table above. Docs are deliberately deferred | +| Add the next layer | the spec (§11 = the layer ordering) | Land code + tests + deps together; update the layer table above and the docs pages the layer touches | | Change what a scaffolded file contains | `src/lightcone/engine/templates/files/` | Edit the `.tmpl`; add new ones to `TEMPLATE_NAMES`, and a renderer only if the file needs a substituted value or a merge policy | | Add a value to the scaffold | `src/lightcone/engine/templates/__init__.py` | Derive it from the environment or our own metadata before introducing a constant | | Change what gets converged | `src/lightcone/engine/project.py` + `tests/test_project.py` | `_Converger.item` / `.file`; repairs only ever append | From 960de67e085809a8a202bd2de2bbe2ceb121d840 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 11:05:16 +0200 Subject: [PATCH 22/23] Catch the docs up to the hardening pass: network is uncontrolled everywhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #184 dropped the OCI wrap's --network none so all three mechanisms attest network: allowed symmetrically; the container internals page still said recipes exec network-denied. Found by the drift check the new check-docs prompt describes — a quoted flag the code no longer delivers. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx --- docs/api/container.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/api/container.md b/docs/api/container.md index 79d99d46..899f5164 100644 --- a/docs/api/container.md +++ b/docs/api/container.md @@ -32,8 +32,10 @@ Sources: `src/lightcone/engine/image.py`, business, never the image's. - **The engine never enters the image.** The container is the *recipe's* world: driver, git, annex, and classification stay the - host's `lc`; exactly two things run in-image (the sync and each - exec, `--network none`). + host's `lc`; exactly two things run in-image — the sync and each + exec. Network is uncontrolled on every mechanism, symmetrically, and + the attestation says so — no consumer may read a promise into + "containerized". - **No project file enters the build context** — that is what makes "code edits never rebuild" structural rather than incidental. - **The dataset is the store; runtime stores are caches.** Execution From 13495f99665a72d69b37ef8c3c7e750434fb7ad1 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 12:24:44 +0200 Subject: [PATCH 23/23] Eval trace follow-up: lc run syntax note, clean harness env (#191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the PR #190 eval trace, which surfaced three harness-attributable frictions (the other findings were agent-side noise). ## The `lc run` convention question, settled Checked the ecosystem before touching anything: `uv run "python -V"` fails with ``Failed to spawn: `python -V` `` (verified live), `docker run img "echo hi"` fails with `exec: "echo hi": executable file not found` — argv with no shell parsing is the convention across `docker run`, `uv run`, `kubectl exec`, `pipx run`, `poetry run`, `srun`. **`lc run` already matches it**, so the behavior stands unchanged and the syntax gets one line in the two places it was missing: - `evals/prompt.md`: "Argv style, like `docker run` or `uv run`: `lc run python scripts/fit.py --output /tmp/x`, never a single quoted shell string; for shell syntax use `lc run bash -c '...'`." - `docs/cli/run.md`: same sentence, in the synopsis. In the trace, the missing line cost one round trip through a genuinely cryptic failure (`/usr/bin/env: '': No such file or directory` + the sandbox trailer insinuating a denial). ## Harness environment - **`UV_PYTHON` noise**: `setup-uv`'s `python-version` input exports `UV_PYTHON`, and the #184 install-settings scrub then (correctly) warned `ignored ambient UV_PYTHON — …` on *every* `lc run`/`lc materialize` — a dozen-plus unactionable lines the agent ended up `grep -v`'ing to read its own output. The input is dropped; determinism is kept by pinning `--python 3.12` on both tool installs. Same category as the `VIRTUAL_ENV` cleanup earlier. - **Truthful dev versions**: the shallow checkout had no tags, so hatch-vcs stamped `0.1.dev1+g` into the installed engine — and into every manifest's `lc_version`. `fetch-depth: 0` fixes it. Validation: label this PR `run-eval` to see the updated prompt and clean environment in action. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01DJzmp2MUhwiNHR94cB91dx --------- Co-authored-by: Claude Fable 5 --- .github/workflows/eval.yml | 14 +++++++++++--- docs/cli/run.md | 5 ++++- evals/prompt.md | 5 ++++- src/lightcone/cli/commands.py | 4 ++-- tests/test_cli.py | 17 +++++++++++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 4befd2ba..949205c9 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -31,11 +31,19 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + with: + # Full history so hatch-vcs sees the tags and stamps a truthful + # dev version into the installed engine (and every manifest's + # lc_version) instead of a 0.1.dev fallback + fetch-depth: 0 - name: Set up uv + # No python-version input: it would export an ambient UV_PYTHON, + # which lc's install-settings scrub then rightly warns about on + # every single invocation the agent makes. The interpreter is + # pinned per tool install below instead. uses: astral-sh/setup-uv@v6 with: - python-version: "3.12" enable-cache: true - name: Install lightcone-cli + astra (uv tools) @@ -49,8 +57,8 @@ jobs: # venv is activated: the agent's shell sees the tools exactly as # an end user's would. run: | - uv tool install "$GITHUB_WORKSPACE" - uv tool install "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)" + uv tool install --python 3.12 "$GITHUB_WORKSPACE" + uv tool install --python 3.12 "astra-tools==$(grep -oP 'astra-tools==\K[0-9][0-9.]*' pyproject.toml)" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - name: Install Claude Code + astra plugin diff --git a/docs/cli/run.md b/docs/cli/run.md index 594a18a8..2171b98c 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -13,7 +13,10 @@ lc run COMMAND... ``` Everything after `run` is the command, verbatim — flags included. -`lc run` takes no options of its own, so nothing needs escaping: +Argv, the `docker run` / `uv run` convention: a single quoted string +would be exec'd as one filename, so probe shell syntax through +`bash -c` instead. `lc run` takes no options of its own, so nothing +else needs escaping: ```bash lc run python -c "import numpy; print(numpy.__version__)" diff --git a/evals/prompt.md b/evals/prompt.md index c0f948f4..aa05abcf 100644 --- a/evals/prompt.md +++ b/evals/prompt.md @@ -26,7 +26,10 @@ This project is driven by two CLIs — use them rather than improvising: making. - `lc run ` runs an ad-hoc command in the project environment under the same isolation a recipe gets — useful for - probing why a recipe would fail. + probing why a recipe would fail. Argv style, like `docker run` or + `uv run`: `lc run python scripts/fit.py --output /tmp/x`, never a + single quoted shell string; for shell syntax use + `lc run bash -c '...'`. - Outputs land in `results/baseline//`, each with a `.lightcone-manifest.json` manifest written and committed by the engine. Never write into `results/` yourself: a hand-placed file diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 75319b24..8fbf4ca5 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -368,8 +368,8 @@ def status(as_json: bool) -> None: "unfetched": "content not in this clone — the next build or run fetches it", }[state] lines.append(f" image: {tag} — {described}") - lines.append(f" sandbox: {report.sandbox}") - lines.append(f" crate: {report.crate}") + lines.append(f" sandbox: {escape(report.sandbox)}") + lines.append(f" crate: {escape(report.crate)}") lines.append("") marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} width = max((len(o.output) for o in report.outputs), default=0) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8970165c..1f5bd9e2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -561,6 +561,23 @@ def test_status_headers_answer_mode_image_and_sandbox( assert "crate: up to date with the outputs" in output +def test_status_header_prose_is_not_markup( + runner: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The crate line names `[project].license`, and rich would read the + brackets as a style tag and swallow the one word that names the fix.""" + from lightcone.engine.materialize import StatusReport + + report = _report() + assert isinstance(report, StatusReport) + report.crate = "not maintained — declare [project].license to enable it" + _status_stub(monkeypatch, report) + + output = runner.invoke(main, ["status"]).output + + assert "declare [project].license to enable it" in output + + def test_build_on_a_direct_project_is_an_explanatory_no_op( runner: CliRunner, project: Path ) -> None: