From 5ddd380d1f9890c0e5a94483b0d311f2535a6c96 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:28:40 +0200 Subject: [PATCH 01/12] Close the uv identity holes: scrub ambient UV_*, report machine-level uv.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ambient half (#179): child_env() drops every UV_* variable outside a closed plumbing allowlist — cache location, timeouts, TLS, air-gap mode, index credentials, uv's own recursion guard — so an exported UV_NO_BINARY or UV_PYTHON can no longer steer what a sync installs while env_version reports nothing moved. The run verbs (materialize, lc run) name any non-empty variable the scrub dropped, from the same predicate, so the report cannot disagree with the scrub. The config-file half (#176, the advisory option): uv merges user- and system-level uv.toml underneath the project's settings, and list settings concatenate across levels — so the scan now checks the two documented paths per platform for audited install-settings keys and reports a hit beside sdist_built. Reported, never hashed: machine state in env_version would make one commit answer differently on two hosts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- src/lightcone/cli/commands.py | 11 ++++- src/lightcone/engine/identity.py | 55 ++++++++++++++++++++-- src/lightcone/engine/materialize.py | 12 +++++ src/lightcone/engine/project.py | 73 ++++++++++++++++++++++++++--- tests/conftest.py | 15 ++++++ tests/test_identity.py | 54 +++++++++++++++++++++ tests/test_materialize.py | 29 ++++++++++++ tests/test_project.py | 44 +++++++++++++++++ 8 files changed, 280 insertions(+), 13 deletions(-) diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 631ab1da..5f918509 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -165,9 +165,16 @@ def run(command: tuple[str, ...]) -> None: """Run COMMAND in the project environment, under isolation. """ from lightcone.engine import run as engine_run - from lightcone.engine.project import current_project + from lightcone.engine.project import current_project, scrubbed_uv_vars - outcome = engine_run.probe(current_project(), command) + directory = current_project() + if dropped := scrubbed_uv_vars(): + click.echo( + f"ignored ambient {', '.join(dropped)} — an install setting is " + "the project's to declare (pyproject.toml)", + err=True, + ) + outcome = engine_run.probe(directory, command) if outcome.notes: click.echo("\n".join(["", *outcome.notes]), err=True) # `Popen.returncode` is negative for a signal, and `sys.exit(-9)` diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py index 505444c8..3bba9d00 100644 --- a/src/lightcone/engine/identity.py +++ b/src/lightcone/engine/identity.py @@ -41,6 +41,7 @@ import hashlib import json +import os import re import tomllib from collections.abc import Mapping @@ -174,6 +175,10 @@ class LockScan: #: Groups outside uv's default set. Advisory: they are installable #: states `env_version` does not distinguish. non_default_groups: tuple[str, ...] + #: Machine-level uv config files setting audited install settings. + #: Advisory: they steer the sync underneath the project's own + #: settings, and ``env_version`` deliberately cannot see them. + machine_config: tuple[str, ...] = () def scan_lock(root: Path) -> LockScan: @@ -225,9 +230,49 @@ def scan_lock(root: Path) -> LockScan: refusals=tuple(sorted(refusals)), sdist_built=tuple(sorted(sdist_built)), non_default_groups=tuple(sorted(non_default)), + machine_config=_machine_config(), ) +def _machine_config_paths() -> tuple[Path, ...]: + """uv's user- and system-level config files, per its documented rule. + + These levels can only ever be a ``uv.toml`` — never a + ``pyproject.toml`` — so two known paths per platform make the probe + complete rather than a heuristic. + """ + if os.name == "nt": + return tuple( + Path(os.environ[var]) / "uv" / "uv.toml" + for var in ("APPDATA", "PROGRAMDATA") + if var in os.environ + ) + config_home = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") + return (config_home / "uv" / "uv.toml", Path("/etc/uv/uv.toml")) + + +def _machine_config() -> tuple[str, ...]: + """Name the machine-level uv config files that steer install settings. + + uv merges user- and system-level configuration *underneath* the + project's, and list settings concatenate across levels — a user-level + ``no-binary-package`` adds to the project's — so the test is which + keys a file sets, never whether the file exists. Deliberately never + hashed: machine state in ``env_version`` would make one commit answer + differently on two hosts. A file uv itself cannot parse is skipped — + the sync fails loudly on it without our help. + """ + findings = [] + for path in _machine_config_paths(): + try: + data = tomllib.loads(path.read_text()) + except (OSError, tomllib.TOMLDecodeError): + continue + if keys := sorted(set(data) & set(_INSTALL_SETTINGS)): + findings.append(f"{path} sets {', '.join(keys)}") + return tuple(findings) + + # ============================================================================= # Reading the project's files # ============================================================================= @@ -262,11 +307,11 @@ def _uv_config(root: Path) -> dict[str, Any]: differ when uv installs the same thing in each. uv warns about the pair itself, and convergence already lifts its warnings. - What this cannot reach is user-level configuration - (``~/.config/uv/uv.toml``), which uv *does* merge in underneath. That - is machine state rather than project state: hashing it would make one - commit answer differently on two hosts, reporting every output as - behind on a colleague's clone. + What this cannot reach is user- and system-level configuration, which + uv *does* merge in underneath. That is machine state rather than + project state: hashing it would make one commit answer differently on + two hosts, reporting every output as behind on a colleague's clone. + :func:`_machine_config` reports it instead. """ config = root / "uv.toml" if not config.is_file(): diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 12fea41a..ff43ef99 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -449,6 +449,12 @@ def materialize( project.require_git_annex() dataset.require_committer(root) report = MaterializeReport() + if dropped := project.scrubbed_uv_vars(): + report.warnings.append( + f"ignored ambient {', '.join(dropped)} — an install setting is " + "the project's to declare (pyproject.toml), and an ambient one " + "would steer the sync without moving env_version" + ) # The dirty check comes before anything that writes: the image # converge below *commits*, and `dataset.save` stages scoped but # commits the whole index — on a dirty tree the user's staged edits @@ -939,6 +945,12 @@ def _graph( "dependency groups outside uv's default set are installable states " f"the environment's identity does not distinguish: {', '.join(scan.non_default_groups)}" ) + if scan.machine_config: + report.warnings.append( + "machine-level uv configuration steers install settings underneath " + "the project's own, and env_version cannot see it: " + + "; ".join(scan.machine_config) + ) env_version = identity.env_version(root) full = plan.build(root) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index a139f245..4f838c99 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -652,18 +652,79 @@ def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: ) +#: The ambient ``UV_*`` variables :func:`child_env` keeps. Plumbing only — +#: where bytes come from and how fast, never *what* a sync installs: the +#: cache location (shared-filesystem hosts point it at scratch), network +#: timeouts and concurrency, TLS trust, air-gap mode, and index +#: credentials. Anything with install semantics (``UV_NO_BINARY``, +#: ``UV_PYTHON``, ``UV_INDEX_URL``, …) is dropped: the same settings are +#: hashed into ``env_version`` when a project declares them, so an ambient +#: spelling would steer a sync while every hash agrees nothing changed. +_UV_KEPT = frozenset( + { + "UV_CACHE_DIR", + "UV_HTTP_TIMEOUT", + "UV_REQUEST_TIMEOUT", + "UV_CONCURRENT_BUILDS", + "UV_CONCURRENT_DOWNLOADS", + "UV_CONCURRENT_INSTALLS", + "UV_NATIVE_TLS", + "UV_INSECURE_HOST", + "UV_OFFLINE", + # uv's own recursion guard, set on every `uv run` child — lc + # itself frequently *is* one. Dropping it disables the guard and + # makes the scrub report uv's variable as the user's. + "UV_RUN_RECURSION_DEPTH", + } +) + + +def _uv_scrubbed(name: str) -> bool: + """Decide whether one ambient variable is dropped by the UV scrub.""" + if not name.startswith("UV_"): + return False + # UV_INTERNAL__* is uv talking to its own children, never a setting. + if name in _UV_KEPT or name.startswith("UV_INTERNAL__"): + return False + # Index credentials (UV_INDEX__USERNAME / _PASSWORD) are how a + # private registry authenticates; the registry itself is the + # project's `[tool.uv.index]` to declare. + return not ( + name.startswith("UV_INDEX_") and name.endswith(("_USERNAME", "_PASSWORD")) + ) + + def child_env() -> dict[str, str]: """Build the environment external tools run in. - Ours, minus ``VIRTUAL_ENV``. Every uv invocation names its project - explicitly, so an activated environment elsewhere is never what we - mean — and uv warns once per invocation when it ignores one, which - would otherwise land in the report and in ``--json``. + Ours, minus ``VIRTUAL_ENV`` and minus every ``UV_*`` variable outside + the :data:`_UV_KEPT` plumbing allowlist. Every uv invocation names its + project explicitly, so an activated environment elsewhere is never + what we mean — and an ambient install setting would change what a + sync installs without moving ``env_version``, which is the identity + hole the scrub closes. + + Returns: + The current environment without ``VIRTUAL_ENV`` or scrubbed ``UV_*``. + """ + return { + k: v + for k, v in os.environ.items() + if k != "VIRTUAL_ENV" and not _uv_scrubbed(k) + } + + +def scrubbed_uv_vars() -> list[str]: + """Name the ambient non-empty ``UV_*`` variables the scrub drops. + + The run verbs surface these as a warning: a user whose ``UV_PYTHON`` + stopped steering a sync deserves a pointer to why. One predicate with + :func:`child_env`, so the report can never disagree with the scrub. Returns: - The current environment without ``VIRTUAL_ENV``. + Sorted variable names, set and non-empty in this process. """ - return {k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"} + return sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)) def _check_call(argv: list[str], *, cwd: Path) -> list[str]: diff --git a/tests/conftest.py b/tests/conftest.py index 9be56af2..c3fa42f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,21 @@ def venue_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv(name, raising=False) +@pytest.fixture(autouse=True) +def machine_uv_config(monkeypatch: pytest.MonkeyPatch) -> None: + """Blind the suite to the host's machine-level uv configuration. + + The advisory probe reads ``~/.config/uv/uv.toml`` and + ``/etc/uv/uv.toml`` — host state a fixture cannot scrub through the + environment (``/etc`` has no variable), so a developer's own config + would add a warning to every scan. The probe's own tests monkeypatch + the paths back to fixtures deliberately. + """ + from lightcone.engine import identity + + monkeypatch.setattr(identity, "_machine_config_paths", tuple) + + @pytest.fixture(autouse=True) def tools(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: """Fake every external tool convergence shells out to, so the suite is diff --git a/tests/test_identity.py b/tests/test_identity.py index e263bd77..88cc0ac6 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -285,3 +285,57 @@ def test_the_default_group_set_is_read_from_uv_toml_too(root: Path) -> None: ) (root / "uv.toml").write_text('default-groups = ["dev", "plots"]\n') assert scan_lock(root).non_default_groups == () + + +# ---- machine-level uv configuration ---------------------------------------- + + +def test_machine_config_setting_an_audited_key_is_advisory( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """uv merges user- and system-level config underneath the project's, + and list settings concatenate across levels — so a machine-level file + steers the sync while env_version sees nothing. Reported, never + hashed: machine state in the hash would make one commit answer + differently on two hosts.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text("no-build = true\n") + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + scan = scan_lock(root) + + assert scan.machine_config == (f"{user} sets no-build",) + + +def test_machine_config_without_audited_keys_stays_silent( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """cache-dir, link-mode and friends decide where bytes come from, not + what gets installed — the same line the install-settings hash draws.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text('cache-dir = "/scratch/uv"\nlink-mode = "copy"\n') + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + assert scan_lock(root).machine_config == () + + +def test_absent_or_unreadable_machine_config_stays_silent( + root: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A file uv cannot parse fails the sync loudly on its own; the + advisory adds nothing by refusing first.""" + from lightcone.engine import identity + + broken = tmp_path / "broken-uv.toml" + broken.write_text("no-build = [unclosed\n") + monkeypatch.setattr( + identity, + "_machine_config_paths", + lambda: (tmp_path / "missing" / "uv.toml", broken), + ) + + assert scan_lock(root).machine_config == () diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 13582add..84db7599 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -450,6 +450,35 @@ def test_a_lock_that_builds_from_source_is_a_warning_not_a_refusal(root: Path) - assert any("oldlib" in w for w in report.warnings) +def test_machine_level_uv_config_is_reported( + root: Path, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The config-file half of the same hole the ambient scrub closes: + env_version cannot see it, so the run says so.""" + from lightcone.engine import identity + + user = tmp_path / "user-uv.toml" + user.write_text("no-binary = true\n") + monkeypatch.setattr(identity, "_machine_config_paths", lambda: (user,)) + + report = engine.check(root, []) + + assert any("env_version cannot see it" in w and str(user) in w for w in report.warnings) + + +def test_ambient_uv_settings_are_scrubbed_and_reported( + root: Path, inline: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """The scrub protects env_version's install-settings term; the warning + is what tells a user why their variable stopped steering the sync.""" + monkeypatch.setenv("UV_NO_BINARY", "1") + + report = engine.materialize(root, []) + + assert report.ok + assert any("UV_NO_BINARY" in w for w in report.warnings) + + # ---- leaving the tree as clean as it was found ----------------------------- diff --git a/tests/test_project.py b/tests/test_project.py index 6630393a..78ab98cd 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -674,6 +674,50 @@ def test_ambient_virtualenv_is_not_passed_to_tools( assert env["LC_TEST_CANARY"] == "kept", "the rest of the environment is untouched" +def test_ambient_uv_install_settings_are_scrubbed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An ambient install setting would steer what a sync installs without + moving env_version — the identity hole the scrub closes. Plumbing (the + cache, timeouts, index credentials) survives: it decides where bytes + come from and how fast, never what gets installed.""" + import os + + from lightcone.engine.project import child_env, scrubbed_uv_vars + + for name in [k for k in os.environ if k.startswith("UV_")]: + monkeypatch.delenv(name) # the suite itself may run under `uv run` + monkeypatch.setenv("UV_NO_BINARY", "1") + monkeypatch.setenv("UV_PYTHON", "3.10") + monkeypatch.setenv("UV_INDEX_URL", "https://elsewhere.invalid/simple") + monkeypatch.setenv("UV_CACHE_DIR", "/scratch/uv") + monkeypatch.setenv("UV_INDEX_INTERNAL_PASSWORD", "hunter2") + monkeypatch.setenv("UV_OFFLINE", "1") + monkeypatch.setenv("LC_TEST_CANARY", "kept") + + env = child_env() + assert "UV_NO_BINARY" not in env + assert "UV_PYTHON" not in env + assert "UV_INDEX_URL" not in env + assert env["UV_CACHE_DIR"] == "/scratch/uv", "shared-cache plumbing survives" + assert env["UV_INDEX_INTERNAL_PASSWORD"] == "hunter2", "credentials survive" + assert env["UV_OFFLINE"] == "1", "air-gap mode survives" + assert env["LC_TEST_CANARY"] == "kept" + assert scrubbed_uv_vars() == ["UV_INDEX_URL", "UV_NO_BINARY", "UV_PYTHON"], ( + "the report names exactly what the scrub dropped" + ) + + +def test_an_empty_scrubbed_variable_is_not_reported( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An empty variable steers nothing, so warning about it is noise.""" + from lightcone.engine.project import scrubbed_uv_vars + + monkeypatch.setenv("UV_NO_BUILD", "") + assert "UV_NO_BUILD" not in scrubbed_uv_vars() + + def test_relays_uv_warnings_into_the_report( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 677497fe4e3a4cdede52c7195f6dc61e876ad87a Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:32:31 +0200 Subject: [PATCH 02/12] Network is not controlled on any mechanism: drop --network none from the OCI wrap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The containerized backend was the one mechanism restricting the network, which made "containerized" carry a promise the direct mechanisms never made — and the direction of the recorded decision is symmetry: nothing pretends to a control it does not apply. The wrap now emits no --network flag, the attestation says `allowed` like landlock and seatbelt, and the `denied` literal stays reserved for a mechanism that genuinely emits a denial flag. This also retires the Perlmutter spike item about `--network none` hanging on compute nodes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 39 ++++++++++++++------------- src/lightcone/engine/container.py | 6 ++--- src/lightcone/engine/materialize.py | 2 +- src/lightcone/engine/run.py | 4 +-- src/lightcone/engine/sandbox/model.py | 8 +++--- src/lightcone/engine/sandbox/oci.py | 8 +++--- tests/test_cli.py | 2 +- tests/test_container_smoke.py | 18 +++---------- tests/test_sandbox_oci.py | 8 +++--- 9 files changed, 44 insertions(+), 51 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6675c45c..6cc5fa8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1369,9 +1369,10 @@ stays self-describing without one. **The engine never enters the image.** The container is the *recipe's* execution world: driver, git, annex, dask and classification all stay the host's `lc`, and exactly two things ever run in-image — the environment -converge (`container.sync`, network on, project `:rw`, host uv cache -mounted, into `.lightcone/venv`) and each recipe/probe exec -(`--network none`). This deviates from spec v6.1's full-stack rule, +converge (`container.sync`, project `:rw`, host uv cache mounted, into +`.lightcone/venv`) and each recipe/probe exec (mount table only — the +converge differs by its writable mounts, not by network: the network is +uncontrolled on every mechanism). This deviates from spec v6.1's full-stack rule, recorded: v6.1's reason was the host-sync deadlock, which the in-container sync solves, and the spec's own Perlmutter row ("recipe wrap, step 3 only — never the dask worker") is this exact shape. What it @@ -1479,9 +1480,9 @@ every bind read and `:z` would relabel the user's own files. No in-container Landlock, no seccomp probe, no shim-in-image: the engine container never gets the tree `:rw`, so mounts alone express the whole policy, and the attestation (`mechanism: podman|docker`, -`fs: declared`, `network: denied`) is derived flag-for-flag from the -argv — `--network none` is the codebase's one honest `denied`, loopback -intact. One `OCIBackend`, data-parameterized: podman and docker differ +`fs: declared`, `network: allowed`) is derived flag-for-flag from the +argv — no flag touches the network, the same non-control every +mechanism attests. One `OCIBackend`, data-parameterized: podman and docker differ in spellings (`--userns=keep-id`+`--pull=never` vs `--user uid:gid`), not shape. Runtime is **host capability** — detected podman → docker, the @@ -1665,11 +1666,10 @@ analogue is a one-time archive→SIF conversion into gitignored `.lightcone/` cache keyed by the same runtime-independent config-blob id (the reason `docker-archive` stays the store format). `container.backend()` stays the single construction point; a future -world-backend is one dataclass in `sandbox/` plus one branch there. A -runtime that cannot deny network must attest `network: allowed` with no -denial flag emitted — no consumer may assume containerized ⇒ denied, -and `_sandbox_line`'s containerized prose is the one place that -assumption lives today. `boundary.run`'s exit-125 note is a +world-backend is one dataclass in `sandbox/` plus one branch there. +Since the hardening pass every mechanism attests `network: allowed` +with no denial flag emitted, so a store-less runtime has nothing to +imitate there. `boundary.run`'s exit-125 note is a podman/docker-family fact, not a `contains_prefix` fact — it becomes mechanism-keyed when a non-OCI backend lands. @@ -1680,8 +1680,7 @@ behavior inside salloc/sbatch steps; `nidXXXXXX` resolution from peer nodes (else `--interface hsn0`); `SLURM_CPUS_ON_NODE` on a CPU node (128 vs 256 hyperthreads); cold-Lustre `distributed` import vs the 120 s worker wait; `podman-hpc migrate` accepting a bare image id and -re-running cheaply; `--network none` on compute nodes (the -`network: denied` attestation hangs on it); podman-hpc `--module` +re-running cheaply; podman-hpc `--module` site-injected mounts vs the honesty of `fs: declared` (the one item that could add a flag); whether Landlock is in the SLES boot LSM list (either answer is handled — a host without it attests `fs: open` with @@ -1970,12 +1969,16 @@ unlinks before writing; a new tampering test should too. invokes the shim on lc's own interpreter, so writer and reader are the same lightcone-cli by construction, and a compatibility field would be backward-compat machinery with no consumer. -- **Network is not controlled, on either platform**, by decision. §7's +- **Network is not controlled, on any mechanism**, by decision. §7's matrix has Seatbelt record `denied`; the generated SBPL explicitly - allows network and both platforms attest `network: allowed`. Symmetric - and honest — nothing pretends to a control it does not apply. (codex - ships a seccomp denylist for this; adding one is a live option, not a - gap we are hiding.) + allows network, and since the hardening pass the OCI backend emits no + `--network` flag either (it briefly shipped `--network none`, dropped + for consistency: three mechanisms, one answer). Every mechanism + attests `network: allowed` — symmetric and honest, nothing pretends + to a control it does not apply. (codex ships a seccomp denylist for + this; adding one is a live option, not a gap we are hiding — and it, + or a runtime that genuinely denies, is what the `denied` literal in + `Attestation` is reserved for.) - **`lc run` has no rename guard and no sandbox flags.** §4's guard against `lc run ` existed only for muscle memory from the pre-rebuild CLI — backward compatibility we do not promise — and §7's diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py index 347b44a7..624405d9 100644 --- a/src/lightcone/engine/container.py +++ b/src/lightcone/engine/container.py @@ -355,9 +355,9 @@ def sync(root: Path, runtime: Runtime) -> list[str]: """Converge ``.lightcone/venv`` inside the image. The containerized twin of ``project.sync``. - The one container run that gets the network and a writable project - mount — converge once, then execute without writing to the - environment, the same discipline as direct mode. The host's uv cache + The one container run that gets a writable project mount — converge + once, then execute without writing to the environment, the same + discipline as direct mode. The host's uv cache is mounted at its identical path, so a complete environment materializes from cache hits in about a second; the cache location is ``uv cache dir``'s answer, never a guess, because that is uv's own diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index ff43ef99..2ec0ab71 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -403,7 +403,7 @@ def _sandbox_line(mode: str) -> str: """ if mode == "containerized": if runtime := container.runtime_hint(): - return f"{runtime} (fs: declared, network: denied)" + return f"{runtime} (fs: declared, network: allowed)" return "no container runtime — install podman (or docker)" from lightcone.engine import sandbox diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index e406909d..3e326e5f 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -59,8 +59,8 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: list(command), cwd=project, # The direct hop converges; the containerized one must not — - # the converge above already did, and the exec runs with the - # network denied, where a sync cannot. + # the converge above already did, into the in-image + # environment the hop is about to enter. prefix=uv_prefix(project, sync=runtime.mode == "direct"), # Same reason as convergence: this uv invocation names its # project explicitly, so an environment activated elsewhere diff --git a/src/lightcone/engine/sandbox/model.py b/src/lightcone/engine/sandbox/model.py index ea4fdb39..b8ad809b 100644 --- a/src/lightcone/engine/sandbox/model.py +++ b/src/lightcone/engine/sandbox/model.py @@ -87,10 +87,10 @@ class Attestation: """The hermeticity record for one exec. Derived from the flags actually applied, never from the mechanism - matrix's expectations. ``network`` is ``allowed`` wherever lc applied - no restriction — the direct-mode mechanisms — and ``denied`` only - where a flag actually denied it, which today is the OCI backend's - ``--network none``. + matrix's expectations. ``network`` is ``allowed`` everywhere today — + lc controls the filesystem, not the network, and every mechanism says + so identically. ``denied`` stays in the type for a mechanism that + genuinely emits a denial flag; nothing may attest it without one. """ mechanism: Literal["landlock", "seatbelt", "podman", "docker", "podman-hpc", "none"] diff --git a/src/lightcone/engine/sandbox/oci.py b/src/lightcone/engine/sandbox/oci.py index 3cd07f5c..0acbf4e9 100644 --- a/src/lightcone/engine/sandbox/oci.py +++ b/src/lightcone/engine/sandbox/oci.py @@ -84,7 +84,6 @@ def wrap(self, policy: Policy, argv: Sequence[str]) -> list[str]: overlay = [f"--env={k}={v}" for k, v in sorted(policy.env.items())] return [ self.runtime, "run", "--rm", - "--network", "none", "--entrypoint", "", # The rootfs is read-only so a write outside the declared set # is a loud denial rather than bytes vanishing with the @@ -113,9 +112,9 @@ def attest(self, policy: Policy) -> Attestation: Every value is a flag in :meth:`wrap`'s output: the mounts plus the read-only rootfs bound the filesystem to the declared set, - and ``--network none`` is a real denial — the one place in the - codebase that honestly says ``denied`` (loopback stays intact, - which is the meaning of it). + and no flag touches the network — ``allowed``, the same answer + every mechanism gives, because lc does not control the network + anywhere and the attestation says only what was enforced. Args: policy: The policy being wrapped. @@ -126,5 +125,4 @@ def attest(self, policy: Policy) -> Attestation: return Attestation( mechanism=self.runtime, fs="declared", - network="denied", ) diff --git a/tests/test_cli.py b/tests/test_cli.py index eec31b5c..0507b6a2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -530,7 +530,7 @@ def test_status_headers_answer_mode_image_and_sandbox( "state": "absent", "archive": ".datalad/environments/lc-env-0123456789abcdef/image", } - report.sandbox = "podman (fs: declared, network: denied)" + report.sandbox = "podman (fs: declared, network: allowed)" _status_stub(monkeypatch, report) output = runner.invoke(main, ["status"]).output diff --git a/tests/test_container_smoke.py b/tests/test_container_smoke.py index 76b61d4c..5010aed9 100644 --- a/tests/test_container_smoke.py +++ b/tests/test_container_smoke.py @@ -169,8 +169,8 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: `lc run` never builds: the first probe refuses naming `lc build`, and succeeding after one is the mutation check. The mounts are the mechanism: an undeclared host file simply is not there (while the - host itself reads it fine). And `--network none` is a real denial - with loopback intact — the meaning of `network: denied`.""" + host itself reads it fine). The network is not controlled here any + more than on the host mechanisms — `allowed`, symmetrically.""" with pytest.raises(ProjectError, match="lc build"): engine_run.probe(cproject, ["bc", "--version"]) @@ -178,7 +178,7 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: outcome = engine_run.probe(cproject, ["bc", "--version"]) assert outcome.returncode == 0 assert outcome.attestation.mechanism == runtime - assert outcome.attestation.network == "denied" + assert outcome.attestation.network == "allowed" assert outcome.attestation.fs == "declared" outside = Path.home() / ".lc-smoke-outside.txt" @@ -205,16 +205,6 @@ def test_the_probe_and_its_boundary(runtime: str, cproject: Path) -> None: ) assert loopback.returncode == 0 - egress = engine_run.probe( - cproject, - [ - "python", "-c", - "import socket, urllib.request; socket.setdefaulttimeout(3); " - 'urllib.request.urlopen("http://1.1.1.1")', - ], # fmt: skip - ) - assert egress.returncode != 0 - # ---- lc materialize --------------------------------------------------------- @@ -234,7 +224,7 @@ def test_materialize_end_to_end_in_the_image(runtime: str, cproject: Path) -> No manifest = assets.read(cproject / "results/baseline/sums") assert manifest is not None assert manifest.hermeticity["mechanism"] == runtime - assert manifest.hermeticity["network"] == "denied" + assert manifest.hermeticity["network"] == "allowed" assert manifest.hermeticity["fs"] == "declared" assert manifest.image is not None assert manifest.image["id"] == _inspect_id(runtime, manifest.image["id"]) diff --git a/tests/test_sandbox_oci.py b/tests/test_sandbox_oci.py index c39bda95..cf8cdccd 100644 --- a/tests/test_sandbox_oci.py +++ b/tests/test_sandbox_oci.py @@ -160,9 +160,11 @@ def test_execution_pins_the_image_by_id_never_a_tag(root: Path, policy: Policy) assert not any("lc-env-" in part for part in argv) -def test_the_network_is_denied_by_flag(root: Path, policy: Policy) -> None: +def test_no_flag_touches_the_network(root: Path, policy: Policy) -> None: + """lc does not control the network on any mechanism, and the argv is + where that has to be true for the `allowed` attestation to be honest.""" argv = _backend(root).wrap(policy, ["true"]) - assert "--network" in argv and argv[argv.index("--network") + 1] == "none" + assert "--network" not in argv def test_runtimes_differ_only_in_their_spellings(root: Path, policy: Policy) -> None: @@ -211,7 +213,7 @@ def test_the_attestation_is_derived_from_the_flags(root: Path, policy: Policy) - attested = _backend(root, runtime).attest(policy) assert attested.mechanism == runtime assert attested.fs == "declared" - assert attested.network == "denied" + assert attested.network == "allowed" assert attested.landlock_abi is None From ae1bbd15e741bebac44181175ea479f9527f3bf9 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:36:59 +0200 Subject: [PATCH 03/12] Narrow a recipe's write scope to its own output directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the cross-write residue: a concurrent task landing bytes in a sibling's output directory before the sibling hashed produced a manifest that was self-consistent and wrong — undetectable by any checksum, so prevention is the only fix. exec_policy gains one keyword (output_dir), handed down from the worker's task; a probe has no output id and keeps results/ whole, which is now the one probe/recipe asymmetry: the probe→recipe promise excludes exactly the commands that write outside their own output directory. All three mechanisms express the narrower shape natively (the same nested-writable-directory form, one level deeper), and the OCI mount table inherits it from the write set with no backend change. Integrity-answering stays data_version's job alone. The enforcement test is mutation-checked: the same cross-write through Unavailable() succeeds (verified rc 0, bytes replaced). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 79 +++++++++++++++++--------- src/lightcone/engine/container.py | 6 +- src/lightcone/engine/sandbox/policy.py | 36 ++++++------ src/lightcone/engine/worker.py | 2 +- tests/test_sandbox_enforcement.py | 24 ++++++++ tests/test_sandbox_policy.py | 44 ++++++++++++-- 6 files changed, 142 insertions(+), 49 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6cc5fa8d..82269b93 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1112,25 +1112,22 @@ one thing it does put on disk is the per-run private `$HOME` `rmtree` of that directory has one owner. (`wrap` stays pure; the impurity lives in policy construction, once.) -**There is one policy, `exec_policy`, and a recipe gets exactly what a -probe gets.** The tree is read-only apart from `results/`, for both. So -layer 5's promise is not "in the direction that matters" — it is simply -true: a command that works under `lc run` works as a recipe, with nothing -in between to reason about. - -A recipe is deliberately **not** narrowed to its own output directory. -That would be a second answer to "are these bytes what produced them", -and the manifest's `data_version` is the first — content-addressed, -checked by `lc verify`, and the only one that survives a rebuild on -another machine. Two mechanisms for one guarantee is one more than can be -kept honest, and the sandbox's is the one that cannot travel. - -The residue, recorded rather than argued away: a cross-write that lands -*before* the victim task hashes leaves a manifest that is self-consistent -and wrong, which no checksum can see. It needs concurrent tasks and a -hardcoded sibling path, and the threat model here is accidental leakage -rather than a hostile recipe — but `lc verify` will not catch that one, so -do not describe it as covered. +**There is one policy, `exec_policy`, and it differs between a recipe +and a probe by one keyword.** The tree is read-only apart from the +in-tree write scope: a recipe's is **its own output directory** +(`output_dir=`, handed down from the worker's task), a probe's is +`results/` whole, because a probe has no output id. This narrowing is +the hardening pass reversing an earlier decision (see Recorded +decisions): it exists as leak *prevention*, closing the cross-write +residue — a concurrent task landing bytes in a sibling's directory +before the sibling hashes produced a manifest that was self-consistent +and wrong, which no checksum could ever see, so prevention was the only +possible fix. The probe→recipe promise ("a command that works under +`lc run` works as a recipe") now excludes exactly the commands that +write outside their own output directory — which is the accident being +prevented, not a loophole in the promise. Integrity-answering is still +`data_version`'s job alone; the sandbox prevents the write, it does not +attest the bytes. **`cluster_for_run()` is the seam, and it is two methods wide.** `submit(fn, *args, key=…)` and `completed(handles)`. That is all the @@ -1466,7 +1463,8 @@ attested by the manifest's image id and the archive's bytes. Read from containerized `exec_policy` shape is the same policy minus the host: no OS read baseline, no stdlib root, no exec set — the image *is* those, and everything in it was declared — leaving exactly the paths that -become mounts, project `:ro`, `results/` `:rw`, declared inputs `:ro`, +become mounts, project `:ro`, the write scope `:rw` (a recipe's own +output directory; a probe's `results/`), declared inputs `:ro`, the private HOME `:rw`, `--tmpfs /tmp`, over a **`--read-only` rootfs**: without that flag a write outside the declared set *succeeds* into the container's ephemeral layer and vanishes while the run attests @@ -1947,6 +1945,30 @@ unlinks before writing; a new tampering test should too. cross-builds) is layer 7's, with the venue that makes it real. Old archives accumulate in annex history; reclaiming them is the user's `git annex unused`/`drop` — no GC verb. +- **The hardening pass** (2026-08, post-layer-8) closed the residues that + did not need Perlmutter, and re-examined two it then deliberately left: + - *A recipe is narrowed to its own output directory* — reversing the + layer-5 "not narrowed" decision. The old rationale rejected the + sandbox as a second *integrity* mechanism, and that half stands + (`data_version` remains the only answer to "are these bytes what + produced them"); what the narrowing is instead is leak *prevention*, + closing the cross-write residue, whose corruption was undetectable by + construction. A probe keeps `results/` whole (no output id), the one + probe/recipe asymmetry. + - *The network is uncontrolled everywhere*: the OCI wrap dropped + `--network none`, its one restriction, so all three mechanisms attest + `network: allowed` symmetrically and no consumer can read a promise + into "containerized". `denied` stays in the `Attestation` type for a + mechanism that genuinely emits a denial flag. + - *The forged run-record subject stays a recorded residue*, re-examined + and left: the threat model is accidental damage and shortcuts, and a + copied `[DATALAD RUNCMD]` subject is already deliberate — a + body-verifying comparator raises the forgery cost without changing + who it stops, and nothing history-based is adversary-proof anyway. + - *The crate validator floor stays pinned at five*: three entries are + publisher/affiliation metadata lc genuinely does not know, and a + `[tool.lightcone.publication]` surface was considered and rejected — + revisit when a real deposit target demands it. ### Recorded deviations from the spec @@ -1998,17 +2020,20 @@ unlinks before writing; a new tampering test should too. into the manifest's `hermeticity` field. `lc run` is still a probe with no output (§4), so there the attestation is returned and printed, never persisted. -- **`results/` is writable; the rest of the tree is not**, where §4 gives - a probe no output and therefore no in-tree write scope at all. A probe - gets the same write scope a recipe does, so a probe that works means a - recipe will — and the environment a run starts with is the one it - finishes with. +- **The in-tree write scope is writable; the rest of the tree is not**, + where §4 gives a probe no output and therefore no in-tree write scope + at all. A probe writes `results/` whole; a recipe, since the hardening + pass, writes only its own output directory (the cross-write closure — + see the layer-4 policy invariant and Recorded decisions) — and the + environment a run starts with is the one it finishes with. - **The shape was chosen because all three mechanisms express it natively.** A writable directory *nested inside* a read-only tree is the widening direction: Landlock unions rights over ancestors, SBPL restates the write tier after the guard, and podman mounts the - project `:ro` with `results` `:rw` over it — all verified by running - them. The reverse — a writable tree with `.venv` carved out — needs + project `:ro` with the write scope `:rw` over it — all verified by + running them, and the narrowing to one output directory is the same + shape one level deeper. The reverse — a writable tree with `.venv` + carved out — needs rights *subtraction*, which podman and SBPL can do and **Landlock cannot at all**. That asymmetry is the whole argument: the read-only shape is the only one direct mode and containerized mode can both diff --git a/src/lightcone/engine/container.py b/src/lightcone/engine/container.py index 624405d9..0a7b8540 100644 --- a/src/lightcone/engine/container.py +++ b/src/lightcone/engine/container.py @@ -414,7 +414,9 @@ def converge(runtime: Runtime) -> list[str]: return sync(runtime.root, runtime) -def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: +def policy_for( + runtime: Runtime, read_paths: list[Path], *, output_dir: Path | None = None +) -> sandbox.Policy: """Build the exec policy for a resolved runtime. The one place the ``env_dir``/``containerized`` pair is assembled — @@ -426,6 +428,7 @@ def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: Args: runtime: The resolved runtime. read_paths: Declared inputs, as :func:`sandbox.exec_policy` takes. + output_dir: A recipe's own output directory; absent for a probe. Returns: The policy for this world. @@ -435,6 +438,7 @@ def policy_for(runtime: Runtime, read_paths: list[Path]) -> sandbox.Policy: read_paths=read_paths, env_dir=runtime.env_dir, containerized=runtime.mode == "containerized", + output_dir=output_dir, ) diff --git a/src/lightcone/engine/sandbox/policy.py b/src/lightcone/engine/sandbox/policy.py index d3e7fbf1..feaf59d9 100644 --- a/src/lightcone/engine/sandbox/policy.py +++ b/src/lightcone/engine/sandbox/policy.py @@ -6,7 +6,8 @@ The shape it encodes is what a container gives the command, minus the container: the project and the declared inputs and the OS baseline -readable, ``results/`` and a private scratch scope writable, and only +readable, the in-tree write scope (a recipe's own output directory; a +probe's ``results/``) and a private scratch scope writable, and only the project's own environment plus a versioned utility allowlist runnable. What it catches is a command reaching *outside* that set — a tool, a library, or a data file that is on this machine and would not be @@ -180,19 +181,18 @@ def exec_policy( read_paths: Sequence[Path] = (), env_dir: Path | None = None, containerized: bool = False, + output_dir: Path | None = None, ) -> Policy: """Build what a sandboxed command may touch. - The tree is read-only apart from ``results/``, so ``lc run`` and a - recipe get the same scope: a command that works under one works under - the other. - - A recipe is not narrowed to its own output directory. Whether an - output's bytes are its own is what the manifest's ``data_version`` - answers, and that answer travels; a second mechanism for it would not. - The residue: a cross-write landing before the victim task hashes - leaves a manifest that is self-consistent and wrong, which no checksum - can see. + The tree is read-only apart from the write scope: a recipe writes its + own output directory and nothing else in the tree, so a concurrent + task cannot land bytes in a sibling's directory before that sibling + hashes — the one corruption ``data_version`` could never see, because + the manifest it produces is self-consistent and wrong. A probe has no + output, so ``lc run`` gets ``results/`` whole; the probe→recipe + promise therefore excludes exactly the commands that write outside + their own output directory, which is the accident being prevented. The containerized shape is the same policy with the host stripped out: the *image* is the OS baseline and the exec set — everything @@ -207,11 +207,14 @@ def exec_policy( ``.venv``. containerized: Build the mount-shaped policy instead of the host one. + output_dir: The one in-tree directory a recipe may write; absent + for a probe, which gets ``results/`` whole. Returns: - The policy. ``results/`` is granted only if it exists — a policy - describes, it does not prepare. Creates the per-run HOME on disk; - the caller owns removing it (see + The policy. The in-tree write scope is granted only if it exists — + a policy describes, it does not prepare (the worker resets the + output directory before building one). Creates the per-run HOME on + disk; the caller owns removing it (see :func:`~lightcone.engine.sandbox.boundary.scope`). """ env_dir = env_dir if env_dir is not None else project / ".venv" @@ -228,6 +231,7 @@ def exec_policy( for sub in _HOME_LAYOUT.values(): (tmp_home / sub).mkdir(parents=True, exist_ok=True) + in_tree_write = output_dir if output_dir is not None else project / "results" if containerized: # Declared spellings, not realpaths — the one shape that keeps # its paths unresolved. These become mount *destinations*, and a @@ -237,7 +241,7 @@ def exec_policy( # *source* side itself.) return Policy( read=_declared([project, *read_paths]), - write=_declared([tmp_home, project / "results"]), + write=_declared([tmp_home, in_tree_write]), execute=(), tmp_home=tmp_home, env=home_overlay(tmp_home, env_dir, containerized=True), @@ -247,7 +251,7 @@ def exec_policy( # EXECUTE on the interpreter *file*; READ on the install root beside # it, for the stdlib. See :func:`_venv_python` and :func:`_stdlib_root`. stdlib = _stdlib_root(python) - write = _existing([tmp_home, project / "results", *_write_roots(project)]) + write = _existing([tmp_home, in_tree_write, *_write_roots(project)]) read = _existing([project, *read_paths, *stdlib, *(Path(p) for p in _OS_READ_BASELINE)]) return Policy( diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index bbed7208..949cb0d9 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -232,7 +232,7 @@ def execute( task.output_dir.mkdir(parents=True) read_paths = [p for p in task.inputs.values() if p.exists()] - policy = container.policy_for(runtime, read_paths) + policy = container.policy_for(runtime, read_paths, output_dir=task.output_dir) started_at = _now() with sandbox.scope(policy): outcome = sandbox.run( diff --git a/tests/test_sandbox_enforcement.py b/tests/test_sandbox_enforcement.py index f673ab25..93c86c67 100644 --- a/tests/test_sandbox_enforcement.py +++ b/tests/test_sandbox_enforcement.py @@ -382,6 +382,30 @@ def test_results_can_be_written(backend: sandbox.Backend, project: Path) -> None assert (project / "results" / "out.csv").read_text() == "out" +def test_a_recipe_cannot_write_a_sibling_output_directory( + backend: sandbox.Backend, project: Path +) -> None: + """The cross-write closure, at the kernel: a recipe granted its own + output directory cannot land bytes in a sibling's — the corruption + that would otherwise enter the sibling's digest as though its recipe + wrote it. Both writes target user-owned paths, so only the boundary + can refuse the first; the second is the mutation check in-place.""" + own = project / "results" / "baseline" / "first" + sibling = project / "results" / "baseline" / "second" + own.mkdir(parents=True) + sibling.mkdir(parents=True) + (sibling / "value.txt").write_text("theirs\n") + with sandbox.scope(sandbox.exec_policy(project, output_dir=own)) as policy: + crossed = shell( + backend, policy, f"printf forged > {sibling / 'value.txt'}", cwd=project + ) + owned = shell(backend, policy, f"printf mine > {own / 'value.txt'}", cwd=project) + assert crossed.returncode != 0 + assert (sibling / "value.txt").read_text() == "theirs\n", "the file changed anyway" + assert owned.returncode == 0, owned.stderr + assert (own / "value.txt").read_text() == "mine" + + def test_a_declared_input_is_read_only( backend: sandbox.Backend, project: Path, outside: Path ) -> None: diff --git a/tests/test_sandbox_policy.py b/tests/test_sandbox_policy.py index bdb3b08d..e05a492b 100644 --- a/tests/test_sandbox_policy.py +++ b/tests/test_sandbox_policy.py @@ -46,20 +46,56 @@ def test_the_tree_is_read_only_apart_from_results( assert not built.grants(project / ".venv" / "bin" / "python", built.write) -def test_results_is_writable(tmp_path: Path) -> None: - """Output goes here, and it is the same scope a recipe gets — which - is what makes a probe a probe of the real thing. +def test_results_is_writable_for_a_probe(tmp_path: Path) -> None: + """A probe has no output id, so its write scope is `results/` whole. A writable directory nested inside a read-only tree is the shape all three mechanisms express natively: Landlock unions rights so a nested grant only widens, SBPL restates the write tier after the guard, and - podman mounts `results` `:rw` over a `:ro` project.""" + podman mounts the scope `:rw` over a `:ro` project.""" project = tmp_path / "proj" (project / "results").mkdir(parents=True) with scope(policy_module.exec_policy(project)) as built: assert built.grants(project / "results" / "out.csv", built.write) +def test_a_recipe_is_narrowed_to_its_own_output_directory(tmp_path: Path) -> None: + """The cross-write closure: a concurrent task landing bytes in a + sibling's directory before the sibling hashes produces a manifest + that is self-consistent and wrong — no checksum can ever see it, so + prevention is the only fix. Same nested-writable shape, one level + deeper.""" + project = tmp_path / "proj" + own = project / "results" / "baseline" / "first" + sibling = project / "results" / "baseline" / "second" + own.mkdir(parents=True) + sibling.mkdir(parents=True) + with scope(policy_module.exec_policy(project, output_dir=own)) as built: + assert built.grants(own / "out.csv", built.write) + assert not built.grants(sibling / "out.csv", built.write) + assert not built.grants(project / "results", built.write) + assert built.grants(sibling / "out.csv", built.read), ( + "an upstream output is still a readable input" + ) + + +def test_the_containerized_recipe_mounts_only_its_own_output_directory( + tmp_path: Path, +) -> None: + """The mount table derives from the write set, so the narrowing must + survive into the containerized shape untranslated.""" + project = tmp_path / "proj" + own = project / "results" / "baseline" / "first" + own.mkdir(parents=True) + with scope( + policy_module.exec_policy( + project, containerized=True, env_dir=project / ".lightcone/venv", output_dir=own + ) + ) as built: + assert own in built.write + assert project / "results" not in built.write + + def test_results_is_granted_only_if_it_exists(tmp_path: Path) -> None: """Convergence makes it. A policy that made directories would be a side effect nobody asked a probe for.""" From 07a3c2097311113048e7bc3617f3900a90850108 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:40:10 +0200 Subject: [PATCH 04/12] Attest uv_version in every manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine-closure decision's concrete loss was that nothing recorded which uv converged the environment — the one tool between the lock and the installed artifacts. The driver probes `uv --version` once per run and hands it down (the HEAD discipline; the rerun entry point probes its own), and the worker records it beside lc_version. Attestation, never identity: defaulted empty, outside both hashes, never read by classify, and an unparseable probe records "" rather than failing a run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 27 ++++++++++++++++++--------- src/lightcone/engine/assets.py | 6 ++++++ src/lightcone/engine/materialize.py | 4 ++++ src/lightcone/engine/project.py | 23 +++++++++++++++++++++++ src/lightcone/engine/worker.py | 16 +++++++++++++--- tests/test_assets.py | 1 + tests/test_materialize.py | 15 +++++++++++++++ tests/test_worker.py | 4 ++++ 8 files changed, 84 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 82269b93..f86d4e65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1158,12 +1158,18 @@ provides that, and for what workers do *not* need (git, git-annex). `started_at` / `finished_at` (ISO 8601 UTC, **millisecond** precision because RO-Crate consumers parse `endTime` with at most three fractional digits; attestation like `lc_version`, defaulted `""`, - never read by `classify`). Spec §3's longer list — - `uv_version`, `platform`, `worker_runtime`, `python_build`, + never read by `classify`), and since the hardening pass `uv_version` + (probed once per run by the driver — `project.uv_version` — and + handed down, the HEAD discipline; the rerun entry point probes its + own; empty on failure, attestation must not fail a run). Spec §3's + remaining list — + `platform`, `worker_runtime`, `python_build`, `dpkg_snapshot_sha256`, `sdist_built`, `env_snapshot`, `gpu_driver` — is attestation nothing here reads; it lands with the verb that reads it (`worker_runtime` is additionally derivable from - `hermeticity.mechanism`, so it may never land at all). + `hermeticity.mechanism`, so it may never land at all — the hardening + pass considered `platform` too and took only `uv_version`, the one + field the engine-closure decision named as its concrete loss). - **`env_version` has four terms.** Layer 6 added the image term — the system layer's identity document, hashed as the literal `null` for a direct project so the formula stays one formula. (The spec's @@ -1843,13 +1849,16 @@ unlinks before writing; a new tampering test should too. own image). One venue cost to remember: the `assets.Versions` memo degrades to once **per worker process**, so a declared input shared by many tasks is re-hashed per process — efficiency, not correctness. - - *The engine's dependency closure left the record entirely, and is not - replaced.* The project lock used to pin what the engine resolved — - most concretely the git-annex build that wrote the bytes. Now - `lc_version` names the engine and nothing pins what it was made of. - Accepted rather than re-provided: the alternative is hashing an + - *The engine's dependency closure left the record entirely, and is + mostly not replaced.* The project lock used to pin what the engine + resolved — most concretely the git-annex build that wrote the bytes. + Now `lc_version` names the engine, the hardening pass added + `uv_version` beside it (the one closure member that decides which + artifacts a lock installs), and nothing pins the rest. Accepted + rather than re-provided: the alternative is hashing an environment lc does not own into artifacts it does, which is the - over-sensitivity the `behind` model exists to avoid. + over-sensitivity the `behind` model exists to avoid — both fields + are attestation, never identity. - *Layer 6 resolved its note the other way:* the image gets **no engine layer at all**. The engine stays on the host in containerized mode too — the container is the recipe's world, never the engine's — diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index 92175ac0..139acbfc 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -266,6 +266,12 @@ class Manifest: #: existed, not back-compat machinery. started_at: str = "" finished_at: str = "" + #: The uv that resolved and installed the environment the recipe ran + #: in — the one tool between the lock and the installed artifacts. + #: Attestation, like ``lc_version``: outside both hashes, never a + #: rebuild signal, defaulted empty because that is the true value for + #: a manifest written before the field existed. + uv_version: str = "" #: The image the recipe ran in — ``{tag, id, archive, arch}`` — or #: ``None`` on the host. Defaulted, and that is not back-compat #: machinery: ``None`` is the *true* value for every manifest a diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 2ec0ab71..12ffcc61 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -507,6 +507,9 @@ def materialize( # manifests a commit this run created — nondeterministically, depending # on whether a recipe finished before or after the previous save. head = dataset.head(root) + # Probed once and handed down, like HEAD: attestation for every + # manifest this run writes, and empty is an answer, not a failure. + uv = project.uv_version(root) # One memo for the run, for the same reason as one HEAD read: a # declared input shared by several outputs — or by one output across # several universes — is the same bytes every time it is asked for. @@ -544,6 +547,7 @@ def materialize( refresh, foreign[key], runtime, + uv, *[pending[dep] for dep in task.depends_on], key=_name(key), ) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 4f838c99..1a22b66d 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -727,6 +727,29 @@ def scrubbed_uv_vars() -> list[str]: return sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)) +def uv_version(directory: Path) -> str: + """Ask uv its version, for the manifest's attestation. + + Read once per run by whoever owns the run — the driver, or the rerun + entry point — and handed down, the HEAD discipline. Empty on any + failure: attestation must never fail a run. + + Args: + directory: Where to run the probe. + + Returns: + The version token (``0.12.5``), or ``""``. + """ + try: + proc = _run(["uv", "--version"], cwd=directory) + except OSError: + return "" + words = str(proc.stdout or "").split() + if proc.returncode != 0 or len(words) < 2 or words[0] != "uv": + return "" + return words[1] + + def _check_call(argv: list[str], *, cwd: Path) -> list[str]: """Run a tool, surfacing a nonzero exit as :class:`ProjectError`. diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index 949cb0d9..42330acc 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -36,7 +36,7 @@ from pathlib import Path from typing import Literal -from lightcone.engine import assets, container, dataset, identity, plan, sandbox, venue +from lightcone.engine import assets, container, dataset, identity, plan, project, sandbox, venue from lightcone.engine.plan import Key, Task from lightcone.engine.project import ( ProjectError, @@ -99,6 +99,7 @@ def materialize( refresh: bool, foreign: dataset.LastWrite | None, runtime: container.Runtime, + uv_version: str, *upstream: TaskResult, ) -> TaskResult: """Make *task* if it needs making. What Dask submits, once per task. @@ -124,6 +125,8 @@ def materialize( runtime: The execution world, resolved once by the driver — the same discipline as *head*, because resolving per task could answer differently mid-run. + uv_version: The uv the run converges environments with, probed + once by the driver. Attestation only. *upstream: The results of this task's dependencies, arriving as the futures it was given — which is what makes Dask the scheduler rather than a loop here. @@ -133,7 +136,7 @@ def materialize( """ try: return _materialize( - root, task, env_version, head, versions, refresh, foreign, runtime, upstream + root, task, env_version, head, versions, refresh, foreign, runtime, uv_version, upstream ) except Exception as e: # the contract is that this function returns return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") @@ -148,6 +151,7 @@ def _materialize( refresh: bool, foreign: dataset.LastWrite | None, runtime: container.Runtime, + uv_version: str, upstream: tuple[TaskResult, ...], ) -> TaskResult: reported = {u.key: u for u in upstream if u.usable} @@ -169,7 +173,9 @@ def _materialize( foreign=foreign, ) if verdict.calls_for_a_remake(refresh=refresh): - return execute(root, task, env_version, inputs, head=head, runtime=runtime) + return execute( + root, task, env_version, inputs, head=head, runtime=runtime, uv_version=uv_version + ) # Left alone, so the bytes on disk stand. Their *recorded* digest, # never a recomputed one: on a clone that has fetched no annex content @@ -194,6 +200,7 @@ def execute( *, head: Head, runtime: container.Runtime, + uv_version: str = "", ) -> TaskResult: """Run *task*'s recipe and record what it produced. @@ -213,6 +220,7 @@ def execute( runtime: The execution world the recipe enters — the host under the platform's mechanism, or the project image behind its mount table. + uv_version: The uv that converged the environment. Attestation. Returns: ``ok`` with the output's ``data_version``, or ``failed``. Commits @@ -275,6 +283,7 @@ def execute( git_sha=sha, git_remote=remote, lc_version=lc_version(), + uv_version=uv_version, hermeticity=asdict(outcome.attestation), started_at=started_at, finished_at=finished_at, @@ -393,6 +402,7 @@ def main(argv: list[str]) -> int: _from_disk(task), head=dataset.head(root), runtime=runtime, + uv_version=project.uv_version(root), ) except ProjectError as e: print(f"error: {e}", file=sys.stderr) diff --git a/tests/test_assets.py b/tests/test_assets.py index ad6a07cd..e2ef5fed 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -40,6 +40,7 @@ def _manifest(**overrides: object) -> Manifest: "git_sha": "abc123", "git_remote": "https://example/demo.git", "lc_version": "0.4.2", + "uv_version": "0.12.5", "hermeticity": {"mechanism": "landlock", "fs": "declared", "network": "allowed"}, } return Manifest(**{**base, **overrides}) # type: ignore[arg-type] diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 84db7599..32920db5 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -179,6 +179,21 @@ def test_refresh_remakes_what_is_behind_and_commits_it(root: Path, inline: None) assert manifest.env_version == identity.env_version(root) +def test_the_manifest_records_the_uv_that_converged_the_environment( + root: Path, inline: None +) -> None: + """Probed once by the driver and handed to every task — attestation + beside lc_version, never a rebuild signal.""" + from lightcone.engine import project + + engine.materialize(root, ["first"]) + + manifest = assets.read(root / "results/baseline/first") + assert manifest is not None + assert manifest.uv_version == project.uv_version(root) + assert manifest.uv_version.count(".") >= 1, "a real version token, not prose" + + def test_check_reports_behind_without_planning_it(root: Path, inline: None) -> None: """`--check` is a gate, and `behind` must not close it — a project of curated results would never pass again.""" diff --git a/tests/test_worker.py b/tests/test_worker.py index 014b882f..f6a1b3e5 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -81,6 +81,7 @@ def _make( refresh, None, _runtime(root), + "0.0.0-test", *upstream, ) @@ -127,6 +128,9 @@ def test_the_manifest_is_complete_before_anything_is_saved(root: Path) -> None: # The engine's version is attestation, not identity: with lc outside # the project's lock, this field is the record of which engine ran. assert manifest.lc_version == worker.lc_version() + # And so is the uv that converged the environment: probed once by the + # driver, handed down, never a rebuild signal. + assert manifest.uv_version == "0.0.0-test" def test_the_recipe_runs_under_the_boundary(root: Path) -> None: From 7645f47c5d73d3e37f6f38952b9f40d074e53063 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:44:23 +0200 Subject: [PATCH 05/12] =?UTF-8?q?Per-file=20checksums=20in=20the=20crate,?= =?UTF-8?q?=20from=20annex=20keys=20=E2=80=94=20and=20no=20framed=20hash?= =?UTF-8?q?=20as=20sha256?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every file in an output directory now appears in the crate as a File under its dataset's hasPart, carrying sha256 and contentSize parsed from its SHA256E annex key — the raw digest `sha256sum` can verify after a `git archive` deposit, available with none of the bytes fetched because keys are repository state (dataset.annex_keys; `--include=*` is load-bearing — bare `find` lists only present files). A non-SHA-256 backend key yields size and no digest, never a wrong one; git-carried files (the lock, universes, manifests) hash their own bytes. The key map is injected into render like the writer, so the builder stays git-free and the render pure. This also fixes a latent honesty bug: an out-of-tree declared input's recorded input_versions digest is lc's *framed* hash, and publishing it under the workflow-run `sha256` term claimed a checksum nothing could verify. Externals now publish no digest — the manifests keep the full story, the layer's stated weaker promise. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- src/lightcone/engine/crate.py | 76 ++++++++++++++++++----- src/lightcone/engine/dataset.py | 32 ++++++++++ src/lightcone/engine/materialize.py | 1 + tests/test_crate.py | 93 +++++++++++++++++++++++++++-- tests/test_dataset.py | 29 +++++++++ 5 files changed, 211 insertions(+), 20 deletions(-) diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py index a3ead986..1400625e 100644 --- a/src/lightcone/engine/crate.py +++ b/src/lightcone/engine/crate.py @@ -26,10 +26,12 @@ from __future__ import annotations +import hashlib import json +import re import tomllib import uuid -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path from typing import Any @@ -63,6 +65,16 @@ #: data entities and every action's ``object`` list cannot drift apart. _ENVIRONMENT = ("pyproject.toml", "uv.lock", ".python-version") +#: A SHA-256-backed annex key: ``SHA256E-s--<64 hex>`` (the E +#: backend keeps the extension). The hex *is* the raw sha256 of the +#: content, which is what makes a checksum publishable with none of the +#: bytes fetched. Any other backend yields a size and no digest — never +#: a wrong one. +_SHA256_KEY = re.compile(r"^SHA256E?-s(\d+)--([0-9a-f]{64})(?:\..*)?$") + +#: Any backend key's size field, for ``contentSize`` alone. +_KEY_SIZE = re.compile(r"^\w+-s(\d+)") + def license_of(root: Path) -> str: """Read the project's declared license out of ``pyproject.toml``. @@ -98,6 +110,7 @@ def render( license: str, dsid: str, writer: Callable[[Path], LastWrite], + keys: Mapping[str, str], ) -> str: """Build the crate document for the project as it stands. @@ -115,11 +128,15 @@ def render( writer: Answers "which commit last touched this path" — :func:`dataset.last_writer` bound to the root, injected so the builder stays free of git. + keys: Each annexed file's key, repository-relative — + :func:`dataset.annex_keys`'s answer, injected for the same + reason as *writer*. SHA-256-backed keys become per-file + checksums an archive can verify with ``sha256sum``. Returns: The ``ro-crate-metadata.json`` text, trailing newline included. """ - build = _Builder(root, graph, license, dsid, writer) + build = _Builder(root, graph, license, dsid, writer, keys) return build.document() @@ -137,6 +154,7 @@ def __init__( license: str, dsid: str, writer: Callable[[Path], LastWrite], + keys: Mapping[str, str], ) -> None: from astra.helpers import load_yaml @@ -144,6 +162,7 @@ def __init__( self.graph = graph self.license = license self.writer = writer + self.keys = dict(keys) self.crate = ROCrate() self.crate.metadata.extra_contexts.append(_WORKFLOW_RUN_CONTEXT) #: Every materialized task, sorted: the one iteration order. @@ -314,8 +333,30 @@ def _file(self, name: str) -> Any: properties: dict[str, Any] = {} if fmt := _format_of(name): properties["encodingFormat"] = fmt + properties.update(self._integrity(name)) return self.crate.add_file(self.root / name, name, properties=properties) + def _integrity(self, name: str) -> dict[str, str]: + """``sha256`` and ``contentSize`` for one repository file. + + An annexed file answers from its key — the working tree may hold + only a pointer, and hashing that would publish a digest of the + wrong bytes — and a git-carried file from the bytes themselves. + Both are repository state, so the render stays pure. A file + neither annexed nor readable carries no claim at all. + """ + if key := self.keys.get(name): + if digest := _SHA256_KEY.match(key): + return {"contentSize": digest.group(1), "sha256": digest.group(2)} + if size := _KEY_SIZE.match(key): + return {"contentSize": size.group(1)} + return {} + try: + data = (self.root / name).read_bytes() + except OSError: + return {} + return {"contentSize": str(len(data)), "sha256": hashlib.sha256(data).hexdigest()} + def _dataset_id(self, key: Key) -> str: """One output directory's crate id — :func:`plan.declared_path`'s answer, never a second spelling of the results layout.""" @@ -331,6 +372,13 @@ def _dataset(self, key: Key, manifest: assets.Manifest) -> None: manifest_file = self._file(f"{dataset_id}{assets.MANIFEST_FILENAME}") manifest_file["about"] = {"@id": dataset_id} entity["subjectOf"] = {"@id": manifest_file.id} + # Every file the directory holds, each with the checksum its + # annex key already carries — the claim `sha256sum` can check + # after a `git archive` deposit, where `version` above is lc's + # own framed directory digest and deliberately is not that. + parts = [manifest_file] + parts += [self._file(name) for name in sorted(self.keys) if name.startswith(dataset_id)] + entity["hasPart"] = [{"@id": part.id} for part in parts] # ----- the runs ----- @@ -374,7 +422,7 @@ def _objects(self, key: Key, manifest: assets.Manifest) -> list[dict[str, str]]: if upstream in self.made_keys: refs.append({"@id": self._dataset_id(upstream)}) continue - refs.append({"@id": self._external(name, task.inputs[name], manifest)}) + refs.append({"@id": self._external(name, task.inputs[name])}) # The *recorded* decisions: the values the recipe actually ran # under, whatever the spec says today. A decision the workflow no # longer declares gets no exampleOfWork — there is no parameter @@ -394,33 +442,31 @@ def _objects(self, key: Key, manifest: assets.Manifest) -> list[dict[str, str]]: refs.append({"@id": value.id}) return refs - def _external(self, name: str, path: Path, manifest: assets.Manifest) -> str: + def _external(self, name: str, path: Path) -> str: """A declared input the spec points at, in or out of the tree. In-or-out is :func:`plan.declared_path`'s answer — relative inside the tree, absolute outside it — never a second spelling of that rule here: two copies of one path rule is how the first one shipped a bug. + + An in-tree input's checksum comes from its annex key, like every + other file. An out-of-tree input carries none: its recorded + ``input_versions`` digest is lc's *framed* hash, not a raw + sha256, so publishing it under the workflow-run ``sha256`` term + would be a checksum nothing can verify — the manifests keep the + full story, which is the layer's stated weaker promise. """ declared = plan.declared_path(self.root, path) in_tree = not Path(declared).is_absolute() entity_id = declared if in_tree else Path(declared).as_uri() - recorded = manifest.input_versions.get(name, "") - digest = recorded.removeprefix("sha256:") if recorded.startswith("sha256:") else "" - if (existing := self.crate.dereference(entity_id)) is not None: - # Two manifests can testify to different bytes for one input - # — a half-rebuilt project. Publishing either digest would - # contradict a manifest in the same crate, so publish - # neither; the manifests themselves keep the full story. - if digest and existing.get("sha256") not in (None, digest): - existing.pop("sha256") + if self.crate.dereference(entity_id) is not None: return entity_id properties: dict[str, Any] = {"@type": "File", "name": declared} if fmt := _format_of(declared): properties["encodingFormat"] = fmt - if digest: - properties["sha256"] = digest if in_tree: + properties.update(self._integrity(declared)) self.crate.add_file(path, declared, properties=properties) else: # Outside the repository: recorded by content, not stored diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index a59f73ad..a28d9e09 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -241,6 +241,38 @@ def last_writer(directory: Path, path: Path) -> LastWrite: return LastWrite(*out.split("\0")) +def annex_keys(directory: Path) -> dict[str, str]: + """Map every annexed file to its key, repository-relative. + + One process for the whole tree. ``--include=*`` is load-bearing: a + bare ``find`` lists only files whose *content* is present, and the + crate must answer on a clone that holds none of the bytes — the keys + are repository state, which is what keeps the render pure. + + "Cannot say" — no annex, no git, an unborn repository — is the empty + answer, never an error, the :func:`last_writer` discipline. + + Args: + directory: The project root. + + Returns: + ``{relative path: key}`` for every annexed file. + """ + argv = ["annex", "find", "--include=*", "--format=${file}\\t${key}\\n"] + try: + proc = project._run(["git", *argv], cwd=directory) + except OSError: + return {} + if proc.returncode != 0: + return {} + keys: dict[str, str] = {} + for line in str(proc.stdout or "").splitlines(): + file, sep, key = line.partition("\t") + if sep and file and key: + keys[file] = key + return keys + + def save(directory: Path, paths: Iterable[Path], message: str) -> bool: """Commit *paths*. diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 12ffcc61..bf084887 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -786,6 +786,7 @@ def _converge_crate(root: Path, report: MaterializeReport, full: Graph, dsid: st license=spdx, dsid=dsid, writer=functools.partial(dataset.last_writer, root), + keys=dataset.annex_keys(root), ) if not (path.is_file() and path.read_text() == document): path.write_text(document) diff --git a/tests/test_crate.py b/tests/test_crate.py index e25679db..6e301a6c 100644 --- a/tests/test_crate.py +++ b/tests/test_crate.py @@ -107,8 +107,15 @@ def _writer(path: Path) -> LastWrite: return LastWrite("a" * 40, "irrelevant", "Ada Lovelace", "ada@example.org", "2026-08-19") -def _render(root: Path, graph: Graph, writer: Writer = _writer) -> dict[str, Any]: - text = crate.render(root, graph, license="MIT", dsid=_DSID, writer=writer) +def _render( + root: Path, + graph: Graph, + writer: Writer = _writer, + keys: dict[str, str] | None = None, +) -> dict[str, Any]: + text = crate.render( + root, graph, license="MIT", dsid=_DSID, writer=writer, keys=keys or {} + ) loaded = json.loads(text) assert isinstance(loaded, dict) return loaded @@ -126,8 +133,9 @@ def test_rendering_twice_at_the_same_state_is_byte_identical(project: Path) -> N _made(project, "baseline", "second", git_sha="aaa111") graph = _graph(project) - first = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) - second = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer) + keys = {"results/baseline/first/out.txt": "SHA256E-s24--" + "c" * 64 + ".txt"} + first = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer, keys=keys) + second = crate.render(project, graph, license="MIT", dsid=_DSID, writer=_writer, keys=keys) assert first == second @@ -221,7 +229,6 @@ def test_an_action_chains_its_inputs_and_its_environment(project: Path) -> None: second = actions["run of `second` in universe `baseline`"] first_objects = {ref["@id"] for ref in first["object"]} assert {"uv.lock", ".python-version", "pyproject.toml", "data/catalog.csv"} <= first_objects - assert entities["data/catalog.csv"]["sha256"] == "cafe" assert "results/baseline/first/" in {ref["@id"] for ref in second["object"]} assert second["result"] == [{"@id": "results/baseline/second/"}] assert second["description"] == "make second" @@ -312,6 +319,82 @@ def test_the_license_is_a_local_entity_never_a_minted_url(project: Path) -> None } +# ---- per-file integrity ---------------------------------------------------- + + +def test_output_files_carry_checksums_from_their_annex_keys(project: Path) -> None: + """The keys are repository state, so a bytes-free clone renders the + same claims — and the hex in a SHA256E key is the raw sha256 an + archive can verify with `sha256sum` after a `git archive` deposit, + where the dataset's `version` is lc's framed digest and cannot be.""" + _made(project, "baseline", "first", git_sha="aaa111") + digest = "d" * 64 + keys = {"results/baseline/first/out.txt": f"SHA256E-s21--{digest}.txt"} + entities = _entities(_render(project, _graph(project), keys=keys)) + + part = entities["results/baseline/first/out.txt"] + assert part["sha256"] == digest + assert part["contentSize"] == "21" + parts = {ref["@id"] for ref in entities["results/baseline/first/"]["hasPart"]} + assert parts == { + "results/baseline/first/.lightcone-manifest.json", + "results/baseline/first/out.txt", + } + + +def test_a_non_sha256_key_yields_size_and_no_digest(project: Path) -> None: + """`annex.backend` is the researcher's to set, and a wrong checksum + is worse than none — the publish-neither discipline.""" + _made(project, "baseline", "first", git_sha="aaa111") + keys = {"results/baseline/first/out.txt": "MD5E-s21--" + "e" * 32 + ".txt"} + entities = _entities(_render(project, _graph(project), keys=keys)) + + part = entities["results/baseline/first/out.txt"] + assert part["contentSize"] == "21" + assert "sha256" not in part + + +def test_git_carried_files_are_hashed_by_their_bytes(project: Path) -> None: + """The lock and its companions are in git, so their working-tree + bytes are the content — repository state, and the render stays + pure.""" + import hashlib + + _made(project, "baseline", "first", git_sha="aaa111") + entities = _entities(_render(project, _graph(project))) + + body = (project / "uv.lock").read_bytes() + assert entities["uv.lock"]["sha256"] == hashlib.sha256(body).hexdigest() + assert entities["uv.lock"]["contentSize"] == str(len(body)) + + +def test_an_out_of_tree_input_publishes_no_checksum( + project: Path, tmp_path: Path +) -> None: + """Its recorded input_versions digest is lc's *framed* hash, not a + raw sha256 — publishing it under the workflow-run term would be a + checksum nothing can verify. The manifests keep the full story.""" + catalog = tmp_path / "shared" / "catalog.csv" + catalog.parent.mkdir() + catalog.write_text("a,b\n") + task = Task( + "baseline", + "first", + project / "results/baseline/first", + "make first", + {"catalog": catalog}, + {}, + {}, + "sha256:def", + ) + _made(project, "baseline", "first", git_sha="aaa111", inputs={"catalog": "sha256:cafe"}) + entities = _entities(_render(project, Graph({("baseline", "first"): task}))) + + external = entities[catalog.as_uri()] + assert "sha256" not in external + assert "contentSize" not in external + + def test_license_of_reads_every_spelling(tmp_path: Path) -> None: cases = { 'license = "MIT"': "MIT", diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 9356c057..fd0e8b57 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -459,6 +459,35 @@ def test_the_annex_executables_are_ours_to_install() -> None: # ---- who last wrote a path ------------------------------------------------- +def test_annex_keys_maps_every_annexed_file_content_present_or_not(repo: Path) -> None: + """The crate's per-file checksums come from here, and they must + answer on a clone that holds none of the bytes — `--include=*` is + what turns `find` from "present files" into "annexed files".""" + out = repo / "results" / "fit" + out.mkdir() + (out / "value.dat").write_bytes(b"x" * 300) + dataset.save(repo, [out], "make fit") + + keys = dataset.annex_keys(repo) + key = keys["results/fit/value.dat"] + assert key.startswith("SHA256E-s300--"), key + assert ".gitattributes" not in keys, "git-carried files have no key" + + clone = repo.parent / "clone" + dataset._git(["clone", "-q", str(repo), str(clone)], cwd=repo.parent) + dataset.init_annex(clone) + assert dataset.annex_keys(clone)["results/fit/value.dat"] == key, ( + "keys are repository state, bytes not required" + ) + + +def test_annex_keys_of_a_plain_directory_is_empty(tmp_path: Path, real_tools: None) -> None: + """Cannot say is empty, never an error — the last_writer discipline.""" + bare = tmp_path / "bare" + bare.mkdir() + assert dataset.annex_keys(bare) == {} + + def test_last_writer_names_the_commit_that_last_touched_a_path(repo: Path) -> None: """The foreign-write fact's whole mechanism: every output is committed, so a hand edit needs a commit, and history names it.""" From c862cd61389ff8934c62abf81e13284244ff1006 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:49:43 +0200 Subject: [PATCH 06/12] lc status places the publication view: the crate: header line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rerun-lag residue made visible. The render pins datePublished to the newest manifest finished_at, so status can read lag off the document itself — a content comparison against the manifests the walk already read, with no git call and no rocrate import (the crate stays the one materialize-only dependency on status's path; license_of and CRATE_FILENAME move to project.py for the same reason). Four states: not maintained / will be created / up to date / behind, plus an honest "unreadable" for a corrupted file — and the line lands in --json through as_dict like the rest of the header. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 17 +++++++++-- src/lightcone/cli/commands.py | 1 + src/lightcone/engine/crate.py | 32 +------------------- src/lightcone/engine/materialize.py | 46 +++++++++++++++++++++++++++-- src/lightcone/engine/project.py | 34 +++++++++++++++++++++ tests/test_cli.py | 3 ++ tests/test_crate.py | 12 -------- tests/test_materialize.py | 32 ++++++++++++++++++++ tests/test_project.py | 16 ++++++++++ 9 files changed, 144 insertions(+), 49 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f86d4e65..91f2ca37 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1526,8 +1526,9 @@ guarantees. discipline, a frozen `container.Runtime` through `worker.materialize` — because resolving per task could answer differently mid-run. The rerun entry point resolves its own, as it does HEAD: it *is* the driver of its -one-task run. `lc status` gained the three header lines (mode, image -state, sandbox) — repository facts only, no runtime and no network +one-task run. `lc status` gained the header lines (mode, image +state, sandbox — and, since the hardening pass, crate) — repository +facts only, no runtime and no network required, which is where the denial note and the runtime-missing refusal point. @@ -1704,7 +1705,17 @@ is `git archive` / `datalad export-archive` on a repository that is already a crate; nothing is copied and there is no bundle directory. The rerun entry point deliberately does not regenerate it (it is one task's executor, not the driver), so the crate lags until the next -materialize — recorded residue. +materialize — and since the hardening pass `lc status` says so: its +`crate:` line compares the document's own `datePublished` against the +newest manifest `finished_at` the status walk already read — a content +check, no git and no rocrate import (the recorded constraint that the +crate stays the one materialize-only dependency on status's path). The +`datePublished` pin is therefore load-bearing twice: it keeps the clock +out of the render *and* it is what makes the lag detectable. A results +edit that changes no manifest is invisible to the line — harmless, since +such an edit is either a foreign write (reported stale) or render- +neutral. `license_of` and `CRATE_FILENAME` moved to `project.py` so +status can ask about publication intent without the renderer's stack. **Publication intent is derived, never configured.** A `[project].license` in `pyproject.toml` turns crate maintenance on — diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 5f918509..3ef8dc0b 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -364,6 +364,7 @@ def status(as_json: bool) -> None: }[state] lines.append(f" image: {tag} — {described}") lines.append(f" sandbox: {report.sandbox}") + lines.append(f" crate: {report.crate}") lines.append("") marks = {"current": "[dim]·[/dim]", "behind": "[cyan]·[/cyan]", "stale": "[yellow]![/yellow]"} width = max((len(o.output) for o in report.outputs), default=0) diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py index 1400625e..10cd54e8 100644 --- a/src/lightcone/engine/crate.py +++ b/src/lightcone/engine/crate.py @@ -29,7 +29,6 @@ import hashlib import json import re -import tomllib import uuid from collections.abc import Callable, Mapping from pathlib import Path @@ -44,8 +43,6 @@ from lightcone.engine.plan import Graph, Key from lightcone.engine.project import SPEC_FILENAME -CRATE_FILENAME = "ro-crate-metadata.json" - #: The vocabulary the run-level facts come from. Without it in the #: ``@context``, terms like ``containerImage`` and ``sha256`` are #: undefined and silently dropped on JSON-LD expansion — typed but inert. @@ -76,33 +73,6 @@ _KEY_SIZE = re.compile(r"^\w+-s(\d+)") -def license_of(root: Path) -> str: - """Read the project's declared license out of ``pyproject.toml``. - - Presence is what turns crate maintenance on: RO-Crate requires a - license, a run must not refuse over a missing key, and inventing one - would assert terms over someone's data — so declaring - ``[project].license`` is declaring the intent to publish. - - Args: - root: The project root. - - Returns: - The license as declared — an SPDX expression, a URL, free text, - or a file path for the table forms — or empty when undeclared. - """ - try: - data = tomllib.loads((root / "pyproject.toml").read_text()) - except (OSError, tomllib.TOMLDecodeError): - return "" - declared = data.get("project", {}).get("license") - if isinstance(declared, str): - return declared - if isinstance(declared, dict): - return str(declared.get("text") or declared.get("file") or "") - return "" - - def render( root: Path, graph: Graph, @@ -122,7 +92,7 @@ def render( Args: root: The project root. graph: The full task graph — every universe, every output. - license: The declared license, from :func:`license_of`. + license: The declared license, from :func:`project.license_of`. dsid: The dataset UUID, the namespace absolute entity ids are minted under so they are stable across clones. writer: Answers "which commit last touched this path" — diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index bf084887..4b0e8975 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -317,6 +317,10 @@ class StatusReport: image: dict[str, str] | None = None #: One line naming the enforcement a run here would get. sandbox: str = "" + #: Where the publication view stands — maintained, and if so whether + #: it still reflects the outputs. Repository facts only, like the + #: rest of the header. + crate: str = "" @property def counts(self) -> dict[str, int]: @@ -336,6 +340,7 @@ def as_dict(self) -> dict[str, Any]: "mode": self.mode, "image": self.image, "sandbox": self.sandbox, + "crate": self.crate, "counts": self.counts, "outputs": [output.as_dict() for output in self.outputs], "warnings": self.warnings, @@ -367,7 +372,10 @@ def status(root: Path) -> StatusReport: if state != "direct": result.image = {"tag": tag, "state": state, "archive": archive} result.sandbox = _sandbox_line(result.mode) + stamps = [] for key, verdict, manifest, foreign in _classified(root, [], report, refresh=False): + if manifest and manifest.finished_at: + stamps.append(manifest.finished_at) result.outputs.append( OutputStatus( output=_name(key), @@ -378,10 +386,42 @@ def status(root: Path) -> StatusReport: foreign_write=foreign.sha if foreign else "", ) ) + result.crate = _crate_line(root, max(stamps, default="")) result.warnings = report.warnings return result +def _crate_line(root: Path, newest: str) -> str: + """One line placing the publication view, from repository facts alone. + + Lag is read off the document itself, not history: the render pins + ``datePublished`` to the newest manifest ``finished_at``, so a crate + whose date no longer matches the manifests was written before the + newest output — the rerun residue made visible, since a rerun never + regenerates the view. No rocrate import (the crate is the one + materialize-only dependency on status's path) and no git: the + manifests were already read by the walk. + """ + spdx = project.license_of(root) + path = root / project.CRATE_FILENAME + if not spdx: + if path.is_file(): + return "no longer maintained — pyproject.toml declares no [project].license" + return "not maintained — declare [project].license to enable it" + if not path.is_file(): + return "will be created by the next `lc materialize`" + try: + entities = json.loads(path.read_text()).get("@graph", []) + published = next( + (str(e.get("datePublished", "")) for e in entities if e.get("@id") == "./"), "" + ) + except (OSError, ValueError, AttributeError): + return "unreadable — the next `lc materialize` rewrites it" + if newest and published != newest: + return "behind — outputs changed after it was written; `lc materialize` refreshes it" + return "up to date" + + def _foreign_write(root: Path, key: Key) -> dataset.LastWrite | None: """Find the commit that last touched *key*'s directory, unless it is the output's own run record — then ``None``, the clean answer. The @@ -761,12 +801,12 @@ def _converge_crate(root: Path, report: MaterializeReport, full: Graph, dsid: st # crate is the one materialize-only dependency on their shared path. from lightcone.engine import crate - spdx = crate.license_of(root) - path = root / crate.CRATE_FILENAME + spdx = project.license_of(root) + path = root / project.CRATE_FILENAME if not spdx: report.warnings.append( f"pyproject.toml no longer declares [project].license, so " - f"{crate.CRATE_FILENAME} is no longer maintained" + f"{project.CRATE_FILENAME} is no longer maintained" if path.exists() else "no [project].license in pyproject.toml, so no RO-Crate " "publication view is maintained — declare one to enable it" diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 1a22b66d..e7f75e6e 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -20,6 +20,9 @@ SPEC_FILENAME = "astra.yaml" +#: The publication view `lc materialize` converges at the project root. +CRATE_FILENAME = "ro-crate-metadata.json" + class ProjectError(Exception): """A project cannot be read or converged.""" @@ -515,6 +518,37 @@ def mode(directory: Path) -> Literal["direct", "containerized"]: return "direct" if image._table(directory) is None else "containerized" +def license_of(directory: Path) -> str: + """Read the project's declared license out of ``pyproject.toml``. + + Presence is what turns crate maintenance on: RO-Crate requires a + license, a run must not refuse over a missing key, and inventing one + would assert terms over someone's data — so declaring + ``[project].license`` is declaring the intent to publish. The same + derived-never-configured shape as :func:`mode`, and it lives here so + ``lc status`` can ask without importing the crate renderer's stack. + + Args: + directory: The project root. + + Returns: + The license as declared — an SPDX expression, a URL, free text, + or a file path for the table forms — or empty when undeclared. + """ + import tomllib + + try: + data = tomllib.loads((directory / "pyproject.toml").read_text()) + except (OSError, tomllib.TOMLDecodeError): + return "" + declared = data.get("project", {}).get("license") + if isinstance(declared, str): + return declared + if isinstance(declared, dict): + return str(declared.get("text") or declared.get("file") or "") + return "" + + def env_dir(directory: Path) -> Path: """Locate the project environment for this project's mode. diff --git a/tests/test_cli.py b/tests/test_cli.py index 0507b6a2..9fe4f6ff 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -531,6 +531,7 @@ def test_status_headers_answer_mode_image_and_sandbox( "archive": ".datalad/environments/lc-env-0123456789abcdef/image", } report.sandbox = "podman (fs: declared, network: allowed)" + report.crate = "up to date" _status_stub(monkeypatch, report) output = runner.invoke(main, ["status"]).output @@ -539,6 +540,7 @@ def test_status_headers_answer_mode_image_and_sandbox( assert "lc-env-0123456789abcdef" in output assert "needs build" in output and "lc build" in output assert "podman" in output + assert "crate: up to date" in output def test_build_on_a_direct_project_is_an_explanatory_no_op( @@ -609,6 +611,7 @@ def test_status_json_is_machine_readable( "mode": "direct", "image": None, "sandbox": "", + "crate": "", "counts": {"current": 1, "behind": 1, "stale": 1}, "outputs": [ { diff --git a/tests/test_crate.py b/tests/test_crate.py index 6e301a6c..5fbb083e 100644 --- a/tests/test_crate.py +++ b/tests/test_crate.py @@ -395,17 +395,5 @@ def test_an_out_of_tree_input_publishes_no_checksum( assert "contentSize" not in external -def test_license_of_reads_every_spelling(tmp_path: Path) -> None: - cases = { - 'license = "MIT"': "MIT", - 'license = { text = "BSD-3-Clause" }': "BSD-3-Clause", - 'license = { file = "LICENSE" }': "LICENSE", - "": "", - } - for spelling, expected in cases.items(): - (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\n{spelling}\n') - assert crate.license_of(tmp_path) == expected, spelling - - def _as_list(value: Any) -> list[Any]: return value if isinstance(value, list) else [value] diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 32920db5..f0607194 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -1091,6 +1091,38 @@ def test_a_removed_license_stops_maintenance_but_keeps_the_file( assert any("no longer maintained" in w for w in report.warnings) +def test_status_places_the_publication_view(root: Path, inline: None) -> None: + """The `crate:` header line, through its three plain states.""" + assert engine.status(root).crate == "not maintained — declare [project].license to enable it" + + _declare_license(root) + assert engine.status(root).crate == "will be created by the next `lc materialize`" + + engine.materialize(root, []) + assert engine.status(root).crate == "up to date" + + +def test_status_sees_the_crate_lag_a_rerun_leaves(root: Path, inline: None) -> None: + """The recorded residue made visible: a rerun rewrites a manifest but + never regenerates the view. Status reads the mismatch off the + document's own datePublished against the manifests it already read — + no git, no rocrate import.""" + from dataclasses import replace + + _declare_license(root) + engine.materialize(root, []) + directory = root / "results/baseline/second" + manifest = assets.read(directory) + assert manifest is not None + assets.write(directory, replace(manifest, finished_at="2027-01-01T00:00:00.000+00:00")) + dataset.save(root, [directory], "a rerun-shaped manifest rewrite") + + assert engine.status(root).crate.startswith("behind") + + engine.materialize(root, []) + assert engine.status(root).crate == "up to date" + + def test_an_output_the_spec_dropped_is_excluded_and_named(root: Path, inline: None) -> None: _declare_license(root) engine.materialize(root, []) diff --git a/tests/test_project.py b/tests/test_project.py index 78ab98cd..503b2675 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -791,3 +791,19 @@ def test_surfaces_a_lock_failure(tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) with pytest.raises(ProjectError, match="no solution found"): converge(tmp_path / "proj") + + +def test_license_of_reads_every_spelling(tmp_path: Path) -> None: + """Publication intent, derived never configured — the crate is + maintained iff [project].license is declared, in any of its forms.""" + from lightcone.engine.project import license_of + + cases = { + 'license = "MIT"': "MIT", + 'license = { text = "BSD-3-Clause" }': "BSD-3-Clause", + 'license = { file = "LICENSE" }': "LICENSE", + "": "", + } + for spelling, expected in cases.items(): + (tmp_path / "pyproject.toml").write_text(f'[project]\nname = "x"\n{spelling}\n') + assert license_of(tmp_path) == expected, spelling From 439e22088bb9d517fae044f33c666c82a9f62a67 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 19:51:23 +0200 Subject: [PATCH 07/12] End-of-run edit warning, and the worker's console-script absence pinned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mid-run edit hole, surfaced: the dirty check runs at start of run while manifests are written per-output later, so an edit in between left manifests whose git_sha no longer described the code that ran, silently. The run now ends with one dataset.status call — the tree started clean and save/restore keeps results/ clean, so any dirt appeared mid-run — and warns with the edited paths. Still a warning, never a manifest field: the spec's git_dirty stays unwritten by decision. Also closes the recorded review item: a metadata test now pins that no [project.scripts] entry ever targets lightcone.engine or the sandbox shim — every entry point is the CLI or a mirrored git-annex executable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 17 ++++++++++------ src/lightcone/engine/materialize.py | 12 +++++++++++ tests/test_dataset.py | 16 +++++++++++++++ tests/test_materialize.py | 31 +++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 91f2ca37..2facb723 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1090,10 +1090,10 @@ design. `lc --help` advertising it would hand people a footgun, and a `uv tool install`. `lightcone/_sandbox_exec.py` is the same shape for the same reason. Keep it cheap to import — **no click, no rich** — it is on the path of every task and every rerun. -`test_the_worker_module_imports_neither_click_nor_rich` pins the imports -and `test_help_does_not_advertise_the_worker` the absence from `--help`; -nothing pins the absence of a `[project.scripts]` entry, so treat that -as a review item. +`test_the_worker_module_imports_neither_click_nor_rich` pins the imports, +`test_help_does_not_advertise_the_worker` the absence from `--help`, and +`test_the_worker_and_the_shim_are_never_console_scripts` the absence of +a `[project.scripts]` entry. **The record's format is datalad's, so it is tested through datalad.** `get_run_info` matches with a regex and returns `(None, None)` on any @@ -1145,8 +1145,13 @@ provides that, and for what workers do *not* need (git, git-annex). refusal makes it constant. The limitation that comes with that, stated: the check is at start of run while manifests are written per-output much later, so a user who edits `src/fit.py` while a long graph runs gets a - manifest whose `git_sha` no longer describes the code that ran, and - nothing records it. + manifest whose `git_sha` no longer describes the code that ran. Since + the hardening pass the run *says* so — one `dataset.status` call after + the consume loop, warning with the edited paths (the tree started + clean and save/restore keeps `results/` clean, so any dirt appeared + mid-run) — but still records nothing in the manifest: the driver does + not rewrite files the worker owns, and per-output attribution would + need a per-save probe nothing has asked for yet. - **The manifest carries what this layer can honestly fill.** `schema_version`, `output_id`, `universe_id`, `recipe`, `definition_version`, `env_version`, `data_version`, `decisions`, diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 4b0e8975..5e95ae5b 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -601,6 +601,18 @@ def materialize( # survive. for task in outstanding.values(): dataset.restore(root, [task.output_dir]) + # The tree was clean at the start-of-run refusal and save/restore + # keeps `results/` clean, so anything dirty *now* was edited while + # the graph ran — and every manifest records the starting commit, + # which no longer describes that code. A warning, not a manifest + # field: the spec's `git_dirty` stays unwritten (see the recorded + # deviation), and the driver does not rewrite files the worker owns. + if edited := dataset.status(root): + names = ", ".join(sorted(path for _, path in edited)) + report.warnings.append( + f"edited while the run was in flight: {names} — the manifests " + "record the starting commit, which no longer describes this code" + ) _converge_crate(root, report, full, dsid) return report diff --git a/tests/test_dataset.py b/tests/test_dataset.py index fd0e8b57..50f414a3 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -454,6 +454,22 @@ def test_the_annex_executables_are_ours_to_install() -> None: assert ours.get(name) == value, f"{name} is not re-declared as {value}" +def test_the_worker_and_the_shim_are_never_console_scripts() -> None: + """`python -m lightcone.engine.worker` makes an output unconditionally, + commits nothing, and leaves the tree dirty by design; the shim is the + sandbox's own plumbing. A `[project.scripts]` entry would put either + on `$PATH` through `uv tool install` — a footgun `lc --help` already + refuses to advertise. Every entry point is either the CLI or a + mirrored git-annex executable, and nothing else.""" + from importlib.metadata import distribution + + theirs = {e.value for e in distribution("git-annex").entry_points} + for entry in distribution("lightcone-cli").entry_points: + assert "lightcone.engine" not in entry.value + assert "_sandbox_exec" not in entry.value + assert entry.value.startswith("lightcone.cli") or entry.value in theirs, entry + + # ---- who last wrote a path ------------------------------------------------- diff --git a/tests/test_materialize.py b/tests/test_materialize.py index f0607194..130b6948 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -494,6 +494,37 @@ def test_ambient_uv_settings_are_scrubbed_and_reported( assert any("UV_NO_BINARY" in w for w in report.warnings) +def test_an_edit_while_the_graph_runs_is_reported( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The dirty check runs at start of run and manifests are written + per-output later, so an edit in between leaves manifests whose + git_sha no longer describes the code that ran. The run ends with one + status call and says so — the honest floor under the unwritten + `git_dirty` field.""" + + class Editing(_Inline): + def completed(self, handles: list[object]) -> Iterator[object]: + (root / "notes.md").write_text("scribbled while the graph ran\n") + yield from handles + + @contextmanager + def fake() -> Iterator[_Inline]: + yield Editing() + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + report = engine.materialize(root, []) + + assert report.ok + assert any("notes.md" in w and "in flight" in w for w in report.warnings) + + +def test_a_clean_run_reports_no_in_flight_edit(root: Path, inline: None) -> None: + report = engine.materialize(root, []) + assert not any("in flight" in w for w in report.warnings) + + # ---- leaving the tree as clean as it was found ----------------------------- From ac112599144611297e4a2fe89d2b8912decd2873 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 20:41:36 +0200 Subject: [PATCH 08/12] CLAUDE.md: record the hardening pass The uv identity holes (scrub landed, machine-config advisory), the crate's raw-digest rule and the injected key map, the status crate line, and the launcher-scrub note resolved. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 54 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2facb723..e77f499b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -691,18 +691,31 @@ two projects that install the same artifacts are one environment however they spell it. `scan_lock` reads `default-groups` through the same function, because it is asking uv's question too. -What this deliberately cannot reach is **user-level configuration** -(`~/.config/uv/uv.toml`), which uv merges in underneath the project's own -(measured). That is machine state, not project state: hashing it would +What this deliberately cannot reach is **machine-level configuration** +(user `~/.config/uv/uv.toml` and system `/etc/uv/uv.toml`), which uv +merges in underneath the project's own (measured). That is machine +state, not project state: hashing it would make one commit answer differently on two hosts, so a colleague's clone -would report every output as behind. The residue is real and unguarded — -a user-level `no-build` changes what a sync installs and nothing records -it. **There is no flag that closes it**: `--config-file` refuses a +would report every output as behind. **There is no flag that closes +it**: `--config-file` refuses a `pyproject.toml` outright, and both `--config-file ` and `--no-config` drop the project's own `[tool.uv]` along with everything else (measured; `--no-config` also drops `[tool.uv.index]`, which is the -GPU mechanism, so adopting it would break working projects). Tracked as -issue #176 with the options and the measurements; don't re-derive them. +GPU mechanism, so adopting it would break working projects). Since the +hardening pass the hole is *annotated at run time* (issue #176's +advisory option): `identity._machine_config` checks the two documented +paths per platform — those levels can only ever be a `uv.toml`, so the +probe is complete, and it tests which *keys* a file sets because list +settings concatenate across levels — and a hit lands in `scan_lock`'s +advisory tier beside `sdist_built`. Never hashed, still. The env-var +spelling of the same hole (`UV_NO_BINARY` and friends) is *closed*, not +annotated: `project.child_env` scrubs ambient `UV_*` outside a plumbing +allowlist (`_UV_KEPT` — cache dir, timeouts, TLS, air-gap, index +credentials, uv's own recursion guard), and the run verbs warn with the +names of any non-empty variable dropped, from the same predicate +(issue #179). The suite blinds itself to the host's machine config via +the autouse `machine_uv_config` fixture — `/etc/uv/uv.toml` has no +environment variable to scrub. **The git commit is recorded, never hashed, and never a signal.** It goes in the manifest so the code that produced a result stays recoverable. It @@ -1769,6 +1782,22 @@ JSON-LD drops on expansion — the pre-rebuild exporter's silent failure. The committed archive is one entity, `["File", "ContainerImage"]`, identity (`sha256` = config-blob id) and payload together. +**A published `sha256` is always a raw digest an outsider can verify** — +since the hardening pass, never lc's framed hash. Every file in an +output directory is a `File` under its dataset's `hasPart`, with +`sha256` and `contentSize` parsed from its SHA256E annex key +(`dataset.annex_keys`, one `git annex find --include=*` — the +`--include=*` is load-bearing, bare `find` lists only *present* files +and the crate must answer bytes-free; the key map is injected into +`render` like the writer, so the builder stays git-free). A non-SHA-256 +backend key yields size and no digest, never a wrong one; git-carried +files hash their own working-tree bytes. Out-of-tree declared inputs +publish *no* digest: their recorded `input_versions` value is the framed +hash, which shipped once under the `sha256` term as though `sha256sum` +could check it — the manifests keep that story. The dataset's `version` +stays lc's framed directory digest, deliberately distinct from the +per-file claims. + **The validator floor is pinned as a set, not a count.** `tests/test_crate_smoke.py` materializes a real project and runs the official `rocrate-validator` (dev dependency) against Provenance Run @@ -1887,9 +1916,10 @@ unlinks before writing; a new tampering test should too. resolved from PyPI; and recipes no longer find `lc` or `git-annex` on the sandbox PATH — the project `.venv/bin` no longer carries them, which is the boundary telling the truth. The `UV_*` ambient scrub the - launcher would have done is still worth having and is tracked - separately; it protects `env_version`'s install-settings term, not the - delegation that is gone. + launcher would have done landed in the hardening pass, in + `project.child_env` — it protects `env_version`'s install-settings + term, not the delegation that is gone (see the layer-2 residue note + for the allowlist). - **Reads stay restricted, and the OS baseline is ours to maintain.** Codex restricts reads too now, but its Linux read baseline is a *mount table*, not a path list — there is nothing to adopt there. Keeping the @@ -2112,7 +2142,7 @@ unlinks before writing; a new tampering test should too. | Change how a recipe runs | `src/lightcone/engine/worker.py` + `tests/test_worker.py` | Never raises, never writes git; mutation-check every denial test | | Change what a run commits | `src/lightcone/engine/materialize.py` + `tests/test_materialize.py` | The driver owns git alone; the tree ends as clean as it started | | Change where a run executes | `src/lightcone/engine/venue.py` + `materialize.cluster_for_run` + `tests/test_venue.py` | One detection ladder, in `cluster_for_run` alone; venues are detected, never configured; test by faking the host (env vars + a stub srun), never the code | -| Change what the crate says | `src/lightcone/engine/crate.py` + `tests/test_crate.py` | Pure builder: sorted iteration, no clock, git injected as `writer`; structure tests, never byte goldens — the one byte-level claim is render-twice-identical. The validator floor lives in `tests/test_crate_smoke.py::_FLOOR` | +| Change what the crate says | `src/lightcone/engine/crate.py` + `tests/test_crate.py` | Pure builder: sorted iteration, no clock, git injected as `writer` and the annex key map as `keys`; structure tests, never byte goldens — the one byte-level claim is render-twice-identical. The validator floor lives in `tests/test_crate_smoke.py::_FLOOR` | | Change how a foreign write is detected | `dataset.last_writer` + `materialize._foreign_write` + `tests/test_dataset.py` | History, never hashing; `datalad_run_subject` is the one spelling of the record's subject; a foreign write classifies `stale` in every verb | | Add a CLI verb | `src/lightcone/cli/commands.py` | `@main.command()`; keep logic in the engine, raise `ProjectError`, render here | | Add a sandbox mechanism | `src/lightcone/engine/sandbox/` | One module with a `Backend` (`wrap` pure, `attest` honest) + one line in `detect()`. Nothing above the seam changes | From e4648fb0421f0614afa124493dfea7ab2721bc0c Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Thu, 20 Aug 2026 21:41:11 +0200 Subject: [PATCH 09/12] Address review: partial commits, pointer guard, and the honest edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep: dataset.save now commits with the same pathspec it stages — a partial commit, built from HEAD plus its own paths alone — so work the user staged while a graph ran stays staged and warned-about, never swept into an output or crate commit. The nothing-to-commit probe is scoped the same way, and the annex per-add config rides on the commit too, since a partial commit takes the paths through the clean filter again. The stronger move — a frozen execution worktree — is recorded as deferred to the venue era, not rejected. The crate's byte fallback re-checks the annex pointer shape before hashing: annex_keys answers empty for the whole repository when git-annex cannot answer at all, and a pointer file reads perfectly well, so one failed `git annex find` would otherwise publish a well-formed digest of the pointer text for every output file. The key parse splits from the last tab (git-annex emits ${file} unescaped; keys never contain tabs), and _file stops hashing the license file twice. The scrub allowlist keeps four more plumbing variables — UV_PYTHON_INSTALL_DIR (the interpreter store has no project-level spelling, so scrubbing it shipped a remedy that does not exist), UV_LINK_MODE, UV_PYTHON_INSTALL_MIRROR, UV_KEYRING_PROVIDER — and the warning is composed once (project.uv_scrub_warning) and surfaced by every uv-acting verb: init and build now say it too, instead of leaving a corporate-mirror user with uv's raw resolution error. The machine config probe covers XDG_CONFIG_DIRS, keeping its "complete" claim true. Honesty edges: the crate status line says "up to date with the outputs" / "behind the outputs" — the exact scope of its datePublished proxy, correct in both directions (a dropped output regresses the newest stamp as a rerun advances it); the write-denial remedy names the recipe's own output directory instead of results/; the Landlock/Seatbelt divergence the narrowing exposed (`rm -rf $OUT` is EACCES on Linux, fine on macOS) is recorded with its fix — delete the redundant prelude, the worker resets the directory; the deposit-vs-consumed meaning of a File's sha256 is recorded where the old conflict rule used to be; and a comment that referenced the spec stands on its own now. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 56 +++++++++++++++++++++----- src/lightcone/cli/commands.py | 14 +++---- src/lightcone/engine/crate.py | 34 +++++++++++++--- src/lightcone/engine/dataset.py | 29 +++++++++---- src/lightcone/engine/identity.py | 7 +++- src/lightcone/engine/materialize.py | 33 ++++++++------- src/lightcone/engine/project.py | 42 +++++++++++++++++-- src/lightcone/engine/sandbox/denial.py | 8 ++-- tests/test_cli.py | 4 +- tests/test_container.py | 4 +- tests/test_crate.py | 16 ++++++++ tests/test_dataset.py | 41 +++++++++++++++++++ tests/test_materialize.py | 36 ++++++++++++++++- tests/test_project.py | 19 +++++++++ tests/test_sandbox_denial.py | 2 +- 15 files changed, 283 insertions(+), 62 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e77f499b..00f1ccf7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1349,10 +1349,20 @@ follows POSIX — unlinking a directory needs write on its *parent* — so a recipe granted write on its output directory cannot remove that directory. Seatbelt's `(allow file-write* (subpath …))` covers the directory node, and permits it. Found by CI, which is the point of -running one suite on both. Nothing depends on either answer (the worker -resets the directory anyway, and a recipe that removes it fails to record -its output), so the asymmetry is documented rather than papered over — -but a test that asserts one mechanism's answer will go red on the other. +running one suite on both. Since the hardening pass narrowed a recipe's +write scope to its own output directory, the asymmetry sits exactly on +the node a ported recipe likes to `rm -rf` as a prelude (measured on +real Landlock: `rm -rf "$OUT" && mkdir "$OUT"` is EACCES on Linux and +succeeds on macOS) — so such a recipe is green on a laptop and red on +the Linux venue. Nothing in *lc* depends on either answer: the worker +resets the directory before every execution, so the prelude is +redundant and the fix is deleting it; likewise a temp-then-rename +through a `results/` sibling now fails on both platforms — scratch +belongs in `$TMPDIR` (the private HOME), which is what the write-denial +remedy says. The asymmetry cannot be closed without granting write on +the *parent*, which is precisely the sibling-write hole the narrowing +exists to close; documented rather than papered over, and a test that +asserts one mechanism's answer will go red on the other. **SBPL is last-match-wins; Landlock unions.** This asymmetry decides where a rule can live, and it cuts both ways. A later `(deny …)` can take @@ -1729,10 +1739,15 @@ newest manifest `finished_at` the status walk already read — a content check, no git and no rocrate import (the recorded constraint that the crate stays the one materialize-only dependency on status's path). The `datePublished` pin is therefore load-bearing twice: it keeps the clock -out of the render *and* it is what makes the lag detectable. A results -edit that changes no manifest is invisible to the line — harmless, since -such an edit is either a foreign write (reported stale) or render- -neutral. `license_of` and `CRATE_FILENAME` moved to `project.py` so +out of the render *and* it is what makes the lag detectable. The line's +claim is scoped to what the proxy can see and worded to it ("up to date +**with the outputs**" / "behind **the outputs**"): a crate-affecting +edit that moves no manifest — the lock's bytes in a `File` entity, a +spec edit — is invisible here, and fine, because the next materialize +converges those anyway; a rerun's lag is the case with no other +surface, and the proxy is exact for it in both directions (a dropped +output regresses the newest stamp just as a rerun advances it). +`license_of` and `CRATE_FILENAME` moved to `project.py` so status can ask about publication intent without the renderer's stack. **Publication intent is derived, never configured.** A @@ -1796,7 +1811,13 @@ publish *no* digest: their recorded `input_versions` value is the framed hash, which shipped once under the `sha256` term as though `sha256sum` could check it — the manifests keep that story. The dataset's `version` stays lc's framed directory digest, deliberately distinct from the -per-file claims. +per-file claims. The move re-scoped what an in-tree input's `sha256` +*means*, and the old conflict rule went with it deliberately: a `File` +entity's checksum now describes the deposit — the bytes a `git archive` +carries — never what any particular run consumed, so two manifests +recording different digests for a shared input (a half-rebuilt project) +no longer suppress it; which bytes a *run* consumed is its own +manifest's `input_versions`, in the crate as that manifest `File`. **The validator floor is pinned as a set, not a count.** `tests/test_crate_smoke.py` materializes a real project and runs the @@ -2024,6 +2045,20 @@ unlinks before writing; a new tampering test should too. publisher/affiliation metadata lc genuinely does not know, and a `[tool.lightcone.publication]` surface was considered and rejected — revisit when a real deposit target demands it. + - *lc's commits are partial commits, and a frozen execution worktree + is deferred, not rejected.* `dataset.save` commits with the same + pathspec it stages, so each save is built from HEAD plus its own + paths alone — work the user staged while a graph ran stays staged + and is named by the end-of-run warning, never swept into an output + or crate commit. The stronger move — executing in a dedicated + `git worktree` at the starting commit, which would also freeze the + *code* a long run reads — was considered and deferred to the venue + era: the branch dance under a live checkout, result propagation + back into it, and a second environment/mid-run-gate story outweigh + a hazard the warning now names precisely, and the case that makes + it genuinely worth it (editing on a login node while an `sbatch` + materialize runs for hours) arrives with the submission-model + venue. ### Recorded deviations from the spec @@ -2065,7 +2100,8 @@ unlinks before writing; a new tampering test should too. - **The denial's remedies are only what works today** — `uv add` for a Python package, the system layer (`apt-install` + the containerize note, real since layer 6), the ASTRA input declaration for data, and - "output goes in `results/`" plus `tempfile.mkdtemp()` for a write + "a recipe writes only its own output directory; a probe writes + `results/`" plus `tempfile.mkdtemp()` for a write denial. Nothing in a denial message names a verb, flag, or declaration that does not exist. - **`Attestation` has no serializer of its own.** An earlier draft diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 3ef8dc0b..89182677 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -165,15 +165,11 @@ def run(command: tuple[str, ...]) -> None: """Run COMMAND in the project environment, under isolation. """ from lightcone.engine import run as engine_run - from lightcone.engine.project import current_project, scrubbed_uv_vars + from lightcone.engine.project import current_project, uv_scrub_warning directory = current_project() - if dropped := scrubbed_uv_vars(): - click.echo( - f"ignored ambient {', '.join(dropped)} — an install setting is " - "the project's to declare (pyproject.toml)", - err=True, - ) + if warning := uv_scrub_warning(): + click.echo(warning, err=True) outcome = engine_run.probe(directory, command) if outcome.notes: click.echo("\n".join(["", *outcome.notes]), err=True) @@ -207,9 +203,11 @@ def build(as_json: bool) -> None: alone. """ from lightcone.engine import container as engine_container - from lightcone.engine.project import current_project + from lightcone.engine.project import current_project, uv_scrub_warning root = current_project() + if (warning := uv_scrub_warning()) and not as_json: + click.echo(warning, err=True) state, tag, _ = engine_container.image_state(root) if state == "direct": if as_json: diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py index 10cd54e8..9bcf1dd1 100644 --- a/src/lightcone/engine/crate.py +++ b/src/lightcone/engine/crate.py @@ -300,6 +300,10 @@ def _environment_files(self) -> None: readme["about"] = {"@id": "./"} def _file(self, name: str) -> Any: + # Idempotent by id, so the second asker (the license file is + # asked for by the workflow and the root) does not hash again. + if (existing := self.crate.dereference(name)) is not None: + return existing properties: dict[str, Any] = {} if fmt := _format_of(name): properties["encodingFormat"] = fmt @@ -314,6 +318,13 @@ def _integrity(self, name: str) -> dict[str, str]: wrong bytes — and a git-carried file from the bytes themselves. Both are repository state, so the render stays pure. A file neither annexed nor readable carries no claim at all. + + The byte path re-checks the pointer shape rather than trusting + the key map's absence: ``annex_keys`` answers empty for a whole + repository whenever git-annex cannot answer at all, and a + pointer file reads perfectly well — so without the guard, one + failed ``git annex find`` would publish a well-formed digest of + the pointer text for every annexed file, silently. """ if key := self.keys.get(name): if digest := _SHA256_KEY.match(key): @@ -321,8 +332,14 @@ def _integrity(self, name: str) -> dict[str, str]: if size := _KEY_SIZE.match(key): return {"contentSize": size.group(1)} return {} + path = self.root / name try: - data = (self.root / name).read_bytes() + if path.is_symlink() or ( + path.stat().st_size <= assets._POINTER_MAX_BYTES + and path.read_bytes().startswith(assets._POINTER_PREFIX) + ): + return {} + data = path.read_bytes() except OSError: return {} return {"contentSize": str(len(data)), "sha256": hashlib.sha256(data).hexdigest()} @@ -421,11 +438,16 @@ def _external(self, name: str, path: Path) -> str: one shipped a bug. An in-tree input's checksum comes from its annex key, like every - other file. An out-of-tree input carries none: its recorded - ``input_versions`` digest is lc's *framed* hash, not a raw - sha256, so publishing it under the workflow-run ``sha256`` term - would be a checksum nothing can verify — the manifests keep the - full story, which is the layer's stated weaker promise. + other file: a ``File`` entity's ``sha256`` describes the + *deposit* — the bytes a ``git archive`` carries — never what any + particular run consumed, so manifests disagreeing about a shared + input (a half-rebuilt project) do not suppress it; which bytes a + run consumed is its own manifest's ``input_versions``. An + out-of-tree input carries none: its recorded digest is lc's + *framed* hash, not a raw sha256, so publishing it under the + workflow-run ``sha256`` term would be a checksum nothing can + verify — the manifests keep the full story, which is the layer's + stated weaker promise. """ declared = plan.declared_path(self.root, path) in_tree = not Path(declared).is_absolute() diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index a28d9e09..ca69180b 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -267,7 +267,11 @@ def annex_keys(directory: Path) -> dict[str, str]: return {} keys: dict[str, str] = {} for line in str(proc.stdout or "").splitlines(): - file, sep, key = line.partition("\t") + # From the *last* tab: git-annex emits ${file} unescaped, so a + # tab in a filename would otherwise split inside the path and + # hand back a truncated file with a corrupted key. Keys never + # contain tabs, so the rightmost split is always the real one. + file, sep, key = line.rpartition("\t") if sep and file and key: keys[file] = key return keys @@ -302,6 +306,14 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: Per-add and never written to the repository's config, so a user's own ``git add`` keeps git-annex's stock behavior. + The commit carries the same pathspec as the add, making it a + *partial* commit: git builds it from HEAD plus these paths alone and + leaves anything else in the index staged and untouched. Without the + pathspec, ``git commit`` commits the whole index — so a file the + user staged while a graph was running would be swept, silently, + into whichever save landed next. The end-of-run warning names such + edits; this is what keeps lc's commits from eating them. + Args: directory: The repository root. paths: What to stage, absolute or repository-relative. @@ -311,13 +323,16 @@ def save(directory: Path, paths: Iterable[Path], message: str) -> bool: False if there was nothing to commit. """ relative = [_rel(directory, p) for p in paths] - _git( - ["-c", "annex.thin=true", "-c", "annex.dotfiles=true", "add", "-A", "--", *relative], - cwd=directory, - ) - if _git_ok(["diff", "--cached", "--quiet"], cwd=directory): + annex = ["-c", "annex.thin=true", "-c", "annex.dotfiles=true"] + _git([*annex, "add", "-A", "--", *relative], cwd=directory) + # Scoped like the commit: foreign staged content must neither count + # as "something to commit" here nor be committed below. + if _git_ok(["diff", "--cached", "--quiet", "--", *relative], cwd=directory): return False - _git(["commit", "-q", "-m", message], cwd=directory) + # The annex config rides on the commit too: a partial commit takes + # the paths' content through the clean filter again, and without the + # flags that pass would route the bytes by stock rules. + _git([*annex, "commit", "-q", "-m", message, "--", *relative], cwd=directory) return True diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py index 3bba9d00..bbc857ce 100644 --- a/src/lightcone/engine/identity.py +++ b/src/lightcone/engine/identity.py @@ -248,7 +248,12 @@ def _machine_config_paths() -> tuple[Path, ...]: if var in os.environ ) config_home = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config") - return (config_home / "uv" / "uv.toml", Path("/etc/uv/uv.toml")) + config_dirs = os.environ.get("XDG_CONFIG_DIRS") or "/etc/xdg" + return ( + config_home / "uv" / "uv.toml", + *(Path(d) / "uv" / "uv.toml" for d in config_dirs.split(":") if d), + Path("/etc/uv/uv.toml"), + ) def _machine_config() -> tuple[str, ...]: diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index 5e95ae5b..d72a6062 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -395,12 +395,16 @@ def _crate_line(root: Path, newest: str) -> str: """One line placing the publication view, from repository facts alone. Lag is read off the document itself, not history: the render pins - ``datePublished`` to the newest manifest ``finished_at``, so a crate - whose date no longer matches the manifests was written before the - newest output — the rerun residue made visible, since a rerun never - regenerates the view. No rocrate import (the crate is the one - materialize-only dependency on status's path) and no git: the - manifests were already read by the walk. + ``datePublished`` to the newest manifest ``finished_at``, so a date + that no longer matches the manifests — in either direction, a rerun + adds an output the view predates and a dropped output regresses the + newest — means the view no longer describes the outputs. That is the + line's whole claim, and it is worded to it: a crate-affecting edit + that moves no manifest (the lock's bytes, the spec) is invisible + here, and fine — the next materialize converges those anyway, where + a rerun's lag has no other surface. No rocrate import (the crate is + the one materialize-only dependency on status's path) and no git: + the manifests were already read by the walk. """ spdx = project.license_of(root) path = root / project.CRATE_FILENAME @@ -418,8 +422,8 @@ def _crate_line(root: Path, newest: str) -> str: except (OSError, ValueError, AttributeError): return "unreadable — the next `lc materialize` rewrites it" if newest and published != newest: - return "behind — outputs changed after it was written; `lc materialize` refreshes it" - return "up to date" + return "behind the outputs — `lc materialize` refreshes it" + return "up to date with the outputs" def _foreign_write(root: Path, key: Key) -> dataset.LastWrite | None: @@ -489,12 +493,8 @@ def materialize( project.require_git_annex() dataset.require_committer(root) report = MaterializeReport() - if dropped := project.scrubbed_uv_vars(): - report.warnings.append( - f"ignored ambient {', '.join(dropped)} — an install setting is " - "the project's to declare (pyproject.toml), and an ambient one " - "would steer the sync without moving env_version" - ) + if warning := project.uv_scrub_warning(): + report.warnings.append(warning) # The dirty check comes before anything that writes: the image # converge below *commits*, and `dataset.save` stages scoped but # commits the whole index — on a dirty tree the user's staged edits @@ -604,9 +604,8 @@ def materialize( # The tree was clean at the start-of-run refusal and save/restore # keeps `results/` clean, so anything dirty *now* was edited while # the graph ran — and every manifest records the starting commit, - # which no longer describes that code. A warning, not a manifest - # field: the spec's `git_dirty` stays unwritten (see the recorded - # deviation), and the driver does not rewrite files the worker owns. + # which no longer describes that code. A warning, never a manifest + # field: the driver does not rewrite files the worker owns. if edited := dataset.status(root): names = ", ".join(sorted(path for _, path in edited)) report.warnings.append( diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index e7f75e6e..3f328518 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -198,6 +198,8 @@ def converge(directory: Path, *, write: bool = True) -> ConvergenceReport: require_git_annex() c = _Converger(write=write) + if warning := uv_scrub_warning(): + c.warn(warning) if write: directory.mkdir(parents=True, exist_ok=True) @@ -705,6 +707,20 @@ def _run(argv: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: "UV_NATIVE_TLS", "UV_INSECURE_HOST", "UV_OFFLINE", + # How package content lands (hardlink/copy/symlink) — the same + # line the install-settings hash draws: `link-mode` is not an + # audited setting either. + "UV_LINK_MODE", + # The managed-interpreter store — the shared-filesystem story + # `UV_CACHE_DIR` is kept for, and *which* interpreter is pinned + # by `.python-version`, not by where its bytes live. There is no + # project-level spelling for this one, so scrubbing it would + # come with a remedy that does not exist. + "UV_PYTHON_INSTALL_DIR", + # Where the pinned interpreter downloads from, not which one. + "UV_PYTHON_INSTALL_MIRROR", + # Auth plumbing, the credentials family. + "UV_KEYRING_PROVIDER", # uv's own recursion guard, set on every `uv run` child — lc # itself frequently *is* one. Dropping it disables the guard and # makes the scrub report uv's variable as the user's. @@ -751,9 +767,8 @@ def child_env() -> dict[str, str]: def scrubbed_uv_vars() -> list[str]: """Name the ambient non-empty ``UV_*`` variables the scrub drops. - The run verbs surface these as a warning: a user whose ``UV_PYTHON`` - stopped steering a sync deserves a pointer to why. One predicate with - :func:`child_env`, so the report can never disagree with the scrub. + One predicate with :func:`child_env`, so the report can never + disagree with the scrub. Returns: Sorted variable names, set and non-empty in this process. @@ -761,6 +776,27 @@ def scrubbed_uv_vars() -> list[str]: return sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)) +def uv_scrub_warning() -> str: + """Compose the dropped-ambient-variables warning, once for every verb. + + A user whose ``UV_PYTHON`` or ``UV_INDEX_URL`` stopped steering uv + deserves a pointer to why on *whichever* verb they hit first — + ``lc init`` resolving against the wrong index fails with uv's raw + error otherwise. One spelling here, so the verbs cannot drift from + each other or from the scrub. + + Returns: + The warning, or ``""`` when nothing non-empty was dropped. + """ + if dropped := scrubbed_uv_vars(): + return ( + f"ignored ambient {', '.join(dropped)} — an install setting is " + "the project's to declare (pyproject.toml), and an ambient one " + "would steer uv without moving env_version" + ) + return "" + + def uv_version(directory: Path) -> str: """Ask uv its version, for the manifest's attestation. diff --git a/src/lightcone/engine/sandbox/denial.py b/src/lightcone/engine/sandbox/denial.py index 713a8b3e..c19325ec 100644 --- a/src/lightcone/engine/sandbox/denial.py +++ b/src/lightcone/engine/sandbox/denial.py @@ -184,9 +184,11 @@ def _render_write(path: Path) -> list[str]: return _message( f"cannot write {path}", [ - " output goes in results/ — the rest of the tree is read-only, so", - " the environment a run starts with is the one it ends with. For", - " anything that is not output, write somewhere scratch:", + " a recipe writes only its own output directory ({output} in the", + " recipe); a probe writes results/. The rest of the tree is", + " read-only, so the environment a run starts with is the one it", + " ends with. For anything that is not output, write somewhere", + " scratch:", " import tempfile; tempfile.mkdtemp() # or $TMPDIR", ], ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fe4f6ff..ca71de6f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -531,7 +531,7 @@ def test_status_headers_answer_mode_image_and_sandbox( "archive": ".datalad/environments/lc-env-0123456789abcdef/image", } report.sandbox = "podman (fs: declared, network: allowed)" - report.crate = "up to date" + report.crate = "up to date with the outputs" _status_stub(monkeypatch, report) output = runner.invoke(main, ["status"]).output @@ -540,7 +540,7 @@ def test_status_headers_answer_mode_image_and_sandbox( assert "lc-env-0123456789abcdef" in output assert "needs build" in output and "lc build" in output assert "podman" in output - assert "crate: up to date" in output + assert "crate: up to date with the outputs" in output def test_build_on_a_direct_project_is_an_explanatory_no_op( diff --git a/tests/test_container.py b/tests/test_container.py index ad7a8a1b..22e4d4a6 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -178,7 +178,7 @@ def test_a_missing_archive_refuses_unless_the_caller_may_build( with pytest.raises(ProjectError, match="lc build"): container.runtime_for_run(root, build=False) assert _argvs(fake, "podman", "build") == [] - assert _argvs(fake, "git", "commit") == [] + assert [c for c in fake if c[0] == "git" and "commit" in c] == [] runtime = container.runtime_for_run(root, build=True) @@ -268,7 +268,7 @@ def test_the_build_saves_and_commits_the_archive(root: Path, fake: list[list[str assert f".datalad/environments/{runtime.image_tag}" in " ".join(add) # The dot-path routing: without it the archive is a full blob in git. assert "annex.dotfiles=true" in add - assert len(_argvs(fake, "git", "commit")) == 1 + assert len([c for c in fake if c[0] == "git" and "commit" in c]) == 1 def test_an_unrouted_archive_refuses_before_building( diff --git a/tests/test_crate.py b/tests/test_crate.py index 5fbb083e..99a4872c 100644 --- a/tests/test_crate.py +++ b/tests/test_crate.py @@ -354,6 +354,22 @@ def test_a_non_sha256_key_yields_size_and_no_digest(project: Path) -> None: assert "sha256" not in part +def test_pointer_shaped_bytes_never_become_a_checksum(project: Path) -> None: + """`annex_keys` answers empty for the whole repository whenever + git-annex cannot answer at all, and an unlocked pointer file reads + perfectly well — so the byte fallback re-checks the pointer shape + instead of publishing a well-formed digest of the pointer text.""" + _made(project, "baseline", "first", git_sha="aaa111") + pointer = "/annex/objects/SHA256E-s300--" + "a" * 64 + ".csv\n" + (project / "data" / "catalog.csv").write_text(pointer) + + entities = _entities(_render(project, _graph(project), keys={})) + + catalog = entities["data/catalog.csv"] + assert "sha256" not in catalog + assert "contentSize" not in catalog + + def test_git_carried_files_are_hashed_by_their_bytes(project: Path) -> None: """The lock and its companions are in git, so their working-tree bytes are the content — repository state, and the render stays diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 50f414a3..de1a3401 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -231,6 +231,35 @@ def test_a_locked_file_without_its_content_is_refused_too(repo: Path) -> None: # ---- committing ------------------------------------------------------------ +def test_save_leaves_foreign_staged_work_staged_and_uncommitted(repo: Path) -> None: + """The user can `git add` while a graph runs; the next save must not + sweep it. The commit is a partial commit — built from HEAD plus the + saved paths alone — so their work stays exactly where they left it: + staged, and in no commit of lc's.""" + (repo / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=repo) + out = repo / "results" / "fit" + out.mkdir() + (out / "value.txt").write_text("42\n") + + assert dataset.save(repo, [out], "make fit") + + committed = dataset._git(["show", "--name-only", "--format=", "HEAD"], cwd=repo).split() + assert "notes.py" not in committed + assert sorted(committed) == ["results/fit/value.txt"] + staged = dataset._git(["diff", "--cached", "--name-only"], cwd=repo).split() + assert staged == ["notes.py"], "still staged, exactly as the user left it" + + +def test_save_sees_nothing_to_commit_past_foreign_staged_work(repo: Path) -> None: + """The nothing-to-commit probe is scoped like the commit, or foreign + staged content would make an empty save attempt a commit and fail.""" + (repo / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=repo) + + assert not dataset.save(repo, [repo / "results"], "nothing here") + + def test_save_reports_when_there_was_nothing_to_commit(repo: Path) -> None: """`lc materialize` may not leave an empty commit behind for an output that produced nothing new.""" @@ -497,6 +526,18 @@ def test_annex_keys_maps_every_annexed_file_content_present_or_not(repo: Path) - ) +def test_annex_keys_survives_a_tab_in_a_filename(repo: Path) -> None: + """git-annex emits ${file} unescaped, so the parse splits from the + *last* tab — keys never contain one, filenames legally can.""" + out = repo / "results" / "fit" + out.mkdir() + (out / "run\t1.dat").write_bytes(b"x" * 300) + dataset.save(repo, [out], "make fit") + + keys = dataset.annex_keys(repo) + assert keys["results/fit/run\t1.dat"].startswith("SHA256E-s300--") + + def test_annex_keys_of_a_plain_directory_is_empty(tmp_path: Path, real_tools: None) -> None: """Cannot say is empty, never an error — the last_writer discipline.""" bare = tmp_path / "bare" diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 130b6948..71714bbb 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -520,6 +520,38 @@ def fake() -> Iterator[_Inline]: assert any("notes.md" in w and "in flight" in w for w in report.warnings) +def test_a_mid_run_stage_is_not_swept_into_lcs_commits( + root: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`dataset.save` stages scoped and commits scoped — a partial + commit — so work the user staged while the graph ran ends the run + exactly where they left it: staged, warned about, and in none of + lc's commits (the per-output saves and the trailing crate commit + alike).""" + _declare_license(root) + + class Staging(_Inline): + def completed(self, handles: list[object]) -> Iterator[object]: + (root / "notes.py").write_text("draft = True\n") + dataset._git(["add", "--", "notes.py"], cwd=root) + yield from handles + + @contextmanager + def fake() -> Iterator[_Inline]: + yield Staging() + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + report = engine.materialize(root, []) + + assert report.ok + assert any("notes.py" in w and "in flight" in w for w in report.warnings) + staged = dataset._git(["diff", "--cached", "--name-only"], cwd=root).split() + assert staged == ["notes.py"] + ever_committed = dataset._git(["log", "--name-only", "--format="], cwd=root).split() + assert "notes.py" not in ever_committed + + def test_a_clean_run_reports_no_in_flight_edit(root: Path, inline: None) -> None: report = engine.materialize(root, []) assert not any("in flight" in w for w in report.warnings) @@ -1130,7 +1162,7 @@ def test_status_places_the_publication_view(root: Path, inline: None) -> None: assert engine.status(root).crate == "will be created by the next `lc materialize`" engine.materialize(root, []) - assert engine.status(root).crate == "up to date" + assert engine.status(root).crate == "up to date with the outputs" def test_status_sees_the_crate_lag_a_rerun_leaves(root: Path, inline: None) -> None: @@ -1151,7 +1183,7 @@ def test_status_sees_the_crate_lag_a_rerun_leaves(root: Path, inline: None) -> N assert engine.status(root).crate.startswith("behind") engine.materialize(root, []) - assert engine.status(root).crate == "up to date" + assert engine.status(root).crate == "up to date with the outputs" def test_an_output_the_spec_dropped_is_excluded_and_named(root: Path, inline: None) -> None: diff --git a/tests/test_project.py b/tests/test_project.py index 503b2675..023db170 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -693,6 +693,8 @@ def test_ambient_uv_install_settings_are_scrubbed( monkeypatch.setenv("UV_CACHE_DIR", "/scratch/uv") monkeypatch.setenv("UV_INDEX_INTERNAL_PASSWORD", "hunter2") monkeypatch.setenv("UV_OFFLINE", "1") + monkeypatch.setenv("UV_PYTHON_INSTALL_DIR", "/scratch/uv/python") + monkeypatch.setenv("UV_LINK_MODE", "copy") monkeypatch.setenv("LC_TEST_CANARY", "kept") env = child_env() @@ -702,12 +704,29 @@ def test_ambient_uv_install_settings_are_scrubbed( assert env["UV_CACHE_DIR"] == "/scratch/uv", "shared-cache plumbing survives" assert env["UV_INDEX_INTERNAL_PASSWORD"] == "hunter2", "credentials survive" assert env["UV_OFFLINE"] == "1", "air-gap mode survives" + assert env["UV_PYTHON_INSTALL_DIR"] == "/scratch/uv/python", ( + "the interpreter store is plumbing, and it has no project-level spelling" + ) + assert env["UV_LINK_MODE"] == "copy", "link-mode is not an audited setting either" assert env["LC_TEST_CANARY"] == "kept" assert scrubbed_uv_vars() == ["UV_INDEX_URL", "UV_NO_BINARY", "UV_PYTHON"], ( "the report names exactly what the scrub dropped" ) +def test_converge_reports_the_uv_scrub( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`lc init` resolves and syncs, so a user whose ambient UV_INDEX_URL + was dropped must hear it here — not only on the verbs they have not + reached when resolution fails with uv's raw error.""" + monkeypatch.setenv("UV_INDEX_URL", "https://mirror.invalid/simple") + + report = converge(tmp_path / "proj") + + assert any("UV_INDEX_URL" in w for w in report.warnings) + + def test_an_empty_scrubbed_variable_is_not_reported( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_sandbox_denial.py b/tests/test_sandbox_denial.py index f38dfbbf..02ef15b2 100644 --- a/tests/test_sandbox_denial.py +++ b/tests/test_sandbox_denial.py @@ -97,7 +97,7 @@ def test_an_in_tree_write_is_its_own_kind_of_denial(policy: Policy, project: Pat stderr = "PermissionError: [Errno 13] Permission denied: 'astra.yaml'\n" joined = "\n".join(denial.explain(stderr, policy, cwd=project)) assert "cannot write" in joined - assert "output goes in results/" in joined + assert "its own output directory" in joined assert "inputs:" not in joined From f23ae92ac8a7f60a0f3b494f4d85499c2d049ad1 Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 09:49:55 +0200 Subject: [PATCH 10/12] Simplify after review: one pointer rule, bisected keys, one warning path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse: the annex pointer-shape test is assets.is_pointer now — one spelling of git-annex's isPointerFile rule, used by require_fetched and the crate's byte fallback alike, ending the cross-module reach into private constants. dataset gains _ask, the one seam for git reads where "cannot say" must be an answer (last_writer and annex_keys shared the shape verbatim). Altitude: the uv-scrub warning has one composer and only engine surfaces — convergence's report, materialize's warnings, and the probe's outcome notes — so the CLI never composes engine facts and lc run's --json-less echo asymmetry is gone. The lc build echo is deleted outright: the image's uv runs inside the container build, where the host environment does not reach. The driver-resolved run facts (env_version, HEAD, the versions memo, runtime, uv_version) are one frozen RunContext handed to every task, so the next attestation field is one line instead of an edit to five signatures. Simplification: scrubbed_uv_vars folded into uv_scrub_warning (its only caller); the test-only defaults on execute's uv_version and LockScan.machine_config are gone — a real caller can no longer forget either silently. Efficiency: _dataset selects an output's files by bisecting one sorted key list instead of re-sorting and rescanning the whole map per output, and _integrity reads a file once. Tests: one _git_calls helper in test_container, one _cluster helper for the custom-scheduler tests in test_materialize. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- CLAUDE.md | 14 ++-- src/lightcone/cli/commands.py | 11 +-- src/lightcone/engine/assets.py | 29 ++++++-- src/lightcone/engine/crate.py | 14 ++-- src/lightcone/engine/dataset.py | 37 ++++++---- src/lightcone/engine/identity.py | 2 +- src/lightcone/engine/materialize.py | 31 ++++---- src/lightcone/engine/project.py | 20 ++---- src/lightcone/engine/run.py | 16 ++++- src/lightcone/engine/worker.py | 107 ++++++++++++++-------------- tests/test_container.py | 12 +++- tests/test_identity.py | 4 +- tests/test_materialize.py | 28 ++++---- tests/test_project.py | 10 +-- tests/test_run.py | 21 ++++++ tests/test_worker.py | 31 ++++---- 16 files changed, 222 insertions(+), 165 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 00f1ccf7..7d22459b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -710,10 +710,16 @@ settings concatenate across levels — and a hit lands in `scan_lock`'s advisory tier beside `sdist_built`. Never hashed, still. The env-var spelling of the same hole (`UV_NO_BINARY` and friends) is *closed*, not annotated: `project.child_env` scrubs ambient `UV_*` outside a plumbing -allowlist (`_UV_KEPT` — cache dir, timeouts, TLS, air-gap, index -credentials, uv's own recursion guard), and the run verbs warn with the -names of any non-empty variable dropped, from the same predicate -(issue #179). The suite blinds itself to the host's machine config via +allowlist (`_UV_KEPT` — cache dir, link mode, the managed-interpreter +store and its mirror, timeouts, TLS, air-gap, credentials, uv's own +recursion guard), and every uv-acting verb names the non-empty +variables dropped through `project.uv_scrub_warning` — one composer, +one predicate with the scrub: convergence puts it in the report (so +`lc init` says it), materialize in its warnings, and the probe in its +outcome's notes, which is why the CLI never composes it (issue #179). +`lc build` deliberately says nothing: the image's uv runs inside the +container build, where the host environment does not reach. The suite +blinds itself to the host's machine config via the autouse `machine_uv_config` fixture — `/etc/uv/uv.toml` has no environment variable to scrub. diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 89182677..4e10f248 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -165,12 +165,9 @@ def run(command: tuple[str, ...]) -> None: """Run COMMAND in the project environment, under isolation. """ from lightcone.engine import run as engine_run - from lightcone.engine.project import current_project, uv_scrub_warning + from lightcone.engine.project import current_project - directory = current_project() - if warning := uv_scrub_warning(): - click.echo(warning, err=True) - outcome = engine_run.probe(directory, command) + outcome = engine_run.probe(current_project(), command) if outcome.notes: click.echo("\n".join(["", *outcome.notes]), err=True) # `Popen.returncode` is negative for a signal, and `sys.exit(-9)` @@ -203,11 +200,9 @@ def build(as_json: bool) -> None: alone. """ from lightcone.engine import container as engine_container - from lightcone.engine.project import current_project, uv_scrub_warning + from lightcone.engine.project import current_project root = current_project() - if (warning := uv_scrub_warning()) and not as_json: - click.echo(warning, err=True) state, tag, _ = engine_container.image_state(root) if state == "direct": if as_json: diff --git a/src/lightcone/engine/assets.py b/src/lightcone/engine/assets.py index 139acbfc..40bdfd37 100644 --- a/src/lightcone/engine/assets.py +++ b/src/lightcone/engine/assets.py @@ -201,11 +201,8 @@ def require_fetched(path: Path) -> None: """ if path.is_symlink() and not path.exists(): unfetched = _ANNEX_OBJECTS in path.readlink().as_posix() - elif path.stat().st_size > _POINTER_MAX_BYTES: - unfetched = False else: - with path.open("rb") as f: - unfetched = f.read(len(_POINTER_PREFIX)) == _POINTER_PREFIX + unfetched = is_pointer(path) if unfetched: raise ContentNotFetchedError( f"{path}: the content is not in this clone — git-annex holds a " @@ -213,6 +210,30 @@ def require_fetched(path: Path) -> None: ) +def is_pointer(path: Path) -> bool: + """Test whether a regular file holds an annex pointer, not content. + + git-annex's own ``isPointerFile`` rule, spelled once: a file no + larger than 32 KiB whose bytes begin ``/annex/objects/``. The locked + shape — a symlink into the object store — is a separate question the + callers ask themselves, because what a symlink means differs by + caller. + + Args: + path: An existing regular file. + + Returns: + Whether it is a pointer. + + Raises: + OSError: If the file cannot be read. + """ + if path.stat().st_size > _POINTER_MAX_BYTES: + return False + with path.open("rb") as f: + return f.read(len(_POINTER_PREFIX)) == _POINTER_PREFIX + + def _feed(h: hashlib._Hash, path: Path) -> None: """Stream *path* into *h* — outputs are not assumed to fit in memory.""" with path.open("rb") as f: diff --git a/src/lightcone/engine/crate.py b/src/lightcone/engine/crate.py index 9bcf1dd1..78c10c9b 100644 --- a/src/lightcone/engine/crate.py +++ b/src/lightcone/engine/crate.py @@ -26,6 +26,7 @@ from __future__ import annotations +import bisect import hashlib import json import re @@ -133,6 +134,10 @@ def __init__( self.license = license self.writer = writer self.keys = dict(keys) + #: Sorted once: each output selects its files by bisecting this, + #: not by rescanning the whole map — the map holds every annexed + #: file in the repository, data/ included. + self.sorted_keys = sorted(self.keys) self.crate = ROCrate() self.crate.metadata.extra_contexts.append(_WORKFLOW_RUN_CONTEXT) #: Every materialized task, sorted: the one iteration order. @@ -334,10 +339,7 @@ def _integrity(self, name: str) -> dict[str, str]: return {} path = self.root / name try: - if path.is_symlink() or ( - path.stat().st_size <= assets._POINTER_MAX_BYTES - and path.read_bytes().startswith(assets._POINTER_PREFIX) - ): + if path.is_symlink() or assets.is_pointer(path): return {} data = path.read_bytes() except OSError: @@ -364,7 +366,9 @@ def _dataset(self, key: Key, manifest: assets.Manifest) -> None: # after a `git archive` deposit, where `version` above is lc's # own framed directory digest and deliberately is not that. parts = [manifest_file] - parts += [self._file(name) for name in sorted(self.keys) if name.startswith(dataset_id)] + lo = bisect.bisect_left(self.sorted_keys, dataset_id) + hi = bisect.bisect_left(self.sorted_keys, dataset_id + "\uffff") + parts += [self._file(name) for name in self.sorted_keys[lo:hi]] entity["hasPart"] = [{"@id": part.id} for part in parts] # ----- the runs ----- diff --git a/src/lightcone/engine/dataset.py b/src/lightcone/engine/dataset.py index ca69180b..def066fb 100644 --- a/src/lightcone/engine/dataset.py +++ b/src/lightcone/engine/dataset.py @@ -231,12 +231,8 @@ def last_writer(directory: Path, path: Path) -> LastWrite: git cannot answer at all. """ argv = ["log", "-1", "--format=%H%x00%s%x00%an%x00%ae%x00%as", "--", _rel(directory, path)] - try: - proc = project._run(["git", *argv], cwd=directory) - except OSError: - return LastWrite() - out = str(proc.stdout or "").strip("\n") - if proc.returncode != 0 or not out: + out = _ask(argv, cwd=directory) + if not (out := (out or "").strip("\n")): return LastWrite() return LastWrite(*out.split("\0")) @@ -258,15 +254,9 @@ def annex_keys(directory: Path) -> dict[str, str]: Returns: ``{relative path: key}`` for every annexed file. """ - argv = ["annex", "find", "--include=*", "--format=${file}\\t${key}\\n"] - try: - proc = project._run(["git", *argv], cwd=directory) - except OSError: - return {} - if proc.returncode != 0: - return {} + out = _ask(["annex", "find", "--include=*", "--format=${file}\\t${key}\\n"], cwd=directory) keys: dict[str, str] = {} - for line in str(proc.stdout or "").splitlines(): + for line in (out or "").splitlines(): # From the *last* tab: git-annex emits ${file} unescaped, so a # tab in a filename would otherwise split inside the path and # hand back a truncated file with a corrupted key. Keys never @@ -360,6 +350,25 @@ def restore(directory: Path, paths: Iterable[Path]) -> None: # ============================================================================= +def _ask(argv: list[str], *, cwd: Path) -> str | None: + """Run git where "cannot say" must be an answer, never an error. + + The read-only-verbs discipline, as one seam: an unborn HEAD, a + stripped ``.git``, a host without git — states a project can really + be in — come back as ``None``, and the caller renders its own empty. + + Returns: + git's stdout, or ``None`` when git cannot answer. + """ + try: + proc = project._run(["git", *argv], cwd=cwd) + except OSError: + return None + if proc.returncode != 0: + return None + return str(proc.stdout or "") + + def _git(argv: list[str], *, cwd: Path) -> str: """Run git in *cwd*, returning its stdout; a nonzero exit raises.""" proc = project._run(["git", *argv], cwd=cwd) diff --git a/src/lightcone/engine/identity.py b/src/lightcone/engine/identity.py index bbc857ce..68567443 100644 --- a/src/lightcone/engine/identity.py +++ b/src/lightcone/engine/identity.py @@ -178,7 +178,7 @@ class LockScan: #: Machine-level uv config files setting audited install settings. #: Advisory: they steer the sync underneath the project's own #: settings, and ``env_version`` deliberately cannot see them. - machine_config: tuple[str, ...] = () + machine_config: tuple[str, ...] def scan_lock(root: Path) -> LockScan: diff --git a/src/lightcone/engine/materialize.py b/src/lightcone/engine/materialize.py index d72a6062..331d7b37 100644 --- a/src/lightcone/engine/materialize.py +++ b/src/lightcone/engine/materialize.py @@ -542,18 +542,19 @@ def materialize( # rerun does not come through here; its entry point converges too.) report.warnings.extend(f"uv: {w}" for w in container.converge(runtime)) - # Read once, for every task: the driver commits each output as it lands, - # so HEAD moves during the run, and a per-task read would give later - # manifests a commit this run created — nondeterministically, depending - # on whether a recipe finished before or after the previous save. - head = dataset.head(root) - # Probed once and handed down, like HEAD: attestation for every - # manifest this run writes, and empty is an answer, not a failure. - uv = project.uv_version(root) - # One memo for the run, for the same reason as one HEAD read: a - # declared input shared by several outputs — or by one output across - # several universes — is the same bytes every time it is asked for. - versions = assets.Versions() + # The run's driver-resolved facts, each read once: HEAD because the + # driver commits as outputs land and a per-task read would stamp + # later manifests with a commit this run created; the uv probe + # because attestation is a fact about the run (and empty is an + # answer, not a failure); one content-hash memo because a declared + # input shared by several outputs is the same bytes every time. + context = worker.RunContext( + env_version=env_version, + head=dataset.head(root), + versions=assets.Versions(), + runtime=runtime, + uv_version=project.uv_version(root), + ) # The history question is the driver's to answer — workers have no # git, by design — so each task is told up front whether its # directory was last written by something other than its own run @@ -581,13 +582,9 @@ def materialize( worker.materialize, root, task, - env_version, - head, - versions, + context, refresh, foreign[key], - runtime, - uv, *[pending[dep] for dep in task.depends_on], key=_name(key), ) diff --git a/src/lightcone/engine/project.py b/src/lightcone/engine/project.py index 3f328518..836ca691 100644 --- a/src/lightcone/engine/project.py +++ b/src/lightcone/engine/project.py @@ -764,31 +764,21 @@ def child_env() -> dict[str, str]: } -def scrubbed_uv_vars() -> list[str]: - """Name the ambient non-empty ``UV_*`` variables the scrub drops. - - One predicate with :func:`child_env`, so the report can never - disagree with the scrub. - - Returns: - Sorted variable names, set and non-empty in this process. - """ - return sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)) - - def uv_scrub_warning() -> str: """Compose the dropped-ambient-variables warning, once for every verb. A user whose ``UV_PYTHON`` or ``UV_INDEX_URL`` stopped steering uv deserves a pointer to why on *whichever* verb they hit first — ``lc init`` resolving against the wrong index fails with uv's raw - error otherwise. One spelling here, so the verbs cannot drift from - each other or from the scrub. + error otherwise. One spelling here, and one predicate with + :func:`child_env`, so the verbs cannot drift from each other or the + report from the scrub. Empty variables steer nothing and are not + reported. Returns: The warning, or ``""`` when nothing non-empty was dropped. """ - if dropped := scrubbed_uv_vars(): + if dropped := sorted(k for k, v in os.environ.items() if v and _uv_scrubbed(k)): return ( f"ignored ambient {', '.join(dropped)} — an install setting is " "the project's to declare (pyproject.toml), and an ambient one " diff --git a/src/lightcone/engine/run.py b/src/lightcone/engine/run.py index 3e326e5f..766ec59e 100644 --- a/src/lightcone/engine/run.py +++ b/src/lightcone/engine/run.py @@ -15,11 +15,18 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import replace from pathlib import Path from typing import Any from lightcone.engine import container, sandbox -from lightcone.engine.project import SPEC_FILENAME, child_env, require_uv, uv_prefix +from lightcone.engine.project import ( + SPEC_FILENAME, + child_env, + require_uv, + uv_prefix, + uv_scrub_warning, +) def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: @@ -53,7 +60,7 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: built = container.policy_for(runtime, input_paths(project, spec)) with sandbox.scope(built) as policy: - return sandbox.run( + outcome = sandbox.run( container.backend(runtime), policy, list(command), @@ -68,6 +75,11 @@ def probe(project: Path, command: Sequence[str]) -> sandbox.Outcome: # the middle of the probe's own output. env=child_env(), ) + # The probe is what called `child_env`, so the probe's outcome is + # where the scrub's fact belongs — the caller prints notes verbatim. + if warning := uv_scrub_warning(): + outcome = replace(outcome, notes=(warning, *outcome.notes)) + return outcome def read_spec(project: Path) -> dict[str, Any]: diff --git a/src/lightcone/engine/worker.py b/src/lightcone/engine/worker.py index 42330acc..5d019224 100644 --- a/src/lightcone/engine/worker.py +++ b/src/lightcone/engine/worker.py @@ -85,6 +85,32 @@ def usable(self) -> bool: return self.status in ("ok", "current", "behind") +@dataclass(frozen=True) +class RunContext: + """The driver-resolved facts of one run, handed to every task. + + Each field is read or resolved exactly once, by whoever owns the run + — the driver, or the rerun entry point — because a per-task read + could answer differently mid-run: HEAD moves as the driver commits, + a runtime resolved twice could disagree, and a provenance field that + depends on task timing is worse than either answer. Frozen and + picklable, so it crosses to workers by value; one object, so the + next attestation field is one line here rather than an edit to five + signatures. + """ + + #: The run's environment identity, checked either side of each recipe. + env_version: str + #: The run's ``(commit sha, origin URL)``. + head: Head + #: The run's content-hash memo for declared inputs. + versions: assets.Versions + #: The execution world — the host mechanism, or the project image. + runtime: container.Runtime + #: The uv that converges environments this run. Attestation only. + uv_version: str + + # ============================================================================= # The Dask unit: decide, then execute # ============================================================================= @@ -93,13 +119,9 @@ def usable(self) -> bool: def materialize( root: Path, task: Task, - env_version: str, - head: Head, - versions: assets.Versions, + context: RunContext, refresh: bool, foreign: dataset.LastWrite | None, - runtime: container.Runtime, - uv_version: str, *upstream: TaskResult, ) -> TaskResult: """Make *task* if it needs making. What Dask submits, once per task. @@ -112,21 +134,12 @@ def materialize( Args: root: The project root. task: The output to make. - env_version: The run's environment identity, checked either side - of the recipe. - head: The run's ``(commit sha, origin URL)``, read once by the - driver because it commits as outputs land and HEAD moves. - versions: The run's content-hash memo for declared inputs. + context: The run's driver-resolved facts. refresh: Whether to remake an output that is merely behind. foreign: The commit that last wrote the output's directory in place of its own run record, or ``None`` — answered by the driver, because history is git's and workers have no git; handed to the one classification rule, where it is `stale`. - runtime: The execution world, resolved once by the driver — the - same discipline as *head*, because resolving per task could - answer differently mid-run. - uv_version: The uv the run converges environments with, probed - once by the driver. Attestation only. *upstream: The results of this task's dependencies, arriving as the futures it was given — which is what makes Dask the scheduler rather than a loop here. @@ -135,9 +148,7 @@ def materialize( What happened. Never raises. """ try: - return _materialize( - root, task, env_version, head, versions, refresh, foreign, runtime, uv_version, upstream - ) + return _materialize(root, task, context, refresh, foreign, upstream) except Exception as e: # the contract is that this function returns return TaskResult(task.key, "failed", reason=f"{type(e).__name__}: {e}") @@ -145,13 +156,9 @@ def materialize( def _materialize( root: Path, task: Task, - env_version: str, - head: Head, - versions: assets.Versions, + context: RunContext, refresh: bool, foreign: dataset.LastWrite | None, - runtime: container.Runtime, - uv_version: str, upstream: tuple[TaskResult, ...], ) -> TaskResult: reported = {u.key: u for u in upstream if u.usable} @@ -161,21 +168,19 @@ def _materialize( live = {key: u.data_version for key, u in reported.items()} inputs = { - name: live[key] if (key := task.produced_by.get(name)) else versions.of(path) + name: live[key] if (key := task.produced_by.get(name)) else context.versions.of(path) for name, path in task.inputs.items() } manifest = assets.read(task.output_dir) verdict = assets.classify( definition_version=task.definition_version, - env_version=env_version, + env_version=context.env_version, manifest=manifest, inputs=inputs, foreign=foreign, ) if verdict.calls_for_a_remake(refresh=refresh): - return execute( - root, task, env_version, inputs, head=head, runtime=runtime, uv_version=uv_version - ) + return execute(root, task, inputs, context) # Left alone, so the bytes on disk stand. Their *recorded* digest, # never a recomputed one: on a clone that has fetched no annex content @@ -195,38 +200,29 @@ def _materialize( def execute( root: Path, task: Task, - env_version: str, input_versions: Mapping[str, str], - *, - head: Head, - runtime: container.Runtime, - uv_version: str = "", + context: RunContext, ) -> TaskResult: """Run *task*'s recipe and record what it produced. The output directory is reset first: the recipe owns it, and a file left from a previous run would otherwise enter the content hash and be - committed as part of an output that never produced it. + committed as part of an output that never produced it. The context's + ``env_version`` is checked either side of the recipe, so a mid-run + lock edit cannot be recorded as if it had been in force. Args: root: The project root. task: The output to make. - env_version: The run's environment identity, checked either side - of the recipe so a mid-run lock edit cannot be recorded as if - it had been in force. input_versions: Each declared input's content identity, recorded in the manifest as the chain. - head: The run's ``(commit sha, origin URL)``. - runtime: The execution world the recipe enters — the host under - the platform's mechanism, or the project image behind its - mount table. - uv_version: The uv that converged the environment. Attestation. + context: The run's driver-resolved facts. Returns: ``ok`` with the output's ``data_version``, or ``failed``. Commits nothing and never touches git beyond reading HEAD. """ - if moved := _gate(root, env_version): + if moved := _gate(root, context.env_version): return TaskResult(task.key, "failed", reason=moved) # The whole directory, not a list of expected files: a recipe declares @@ -240,11 +236,11 @@ def execute( task.output_dir.mkdir(parents=True) read_paths = [p for p in task.inputs.values() if p.exists()] - policy = container.policy_for(runtime, read_paths, output_dir=task.output_dir) + policy = container.policy_for(context.runtime, read_paths, output_dir=task.output_dir) started_at = _now() with sandbox.scope(policy): outcome = sandbox.run( - container.backend(runtime), + container.backend(context.runtime), policy, [_SHELL, "-c", task.recipe], cwd=root, @@ -260,14 +256,14 @@ def execute( reason=f"the recipe exited {outcome.returncode}", notes=outcome.notes, ) - if moved := _gate(root, env_version): + if moved := _gate(root, context.env_version): return TaskResult(task.key, "failed", reason=moved, notes=outcome.notes) # Guarded separately from the boundary catch above it, because these # two failures deserve different words: "your recipe failed" and "your # recipe worked and we could not record it" are different problems. try: - sha, remote = head + sha, remote = context.head data_version = assets.data_version(task.output_dir) assets.write( task.output_dir, @@ -276,18 +272,18 @@ def execute( universe_id=task.universe_id, recipe=task.recipe, definition_version=task.definition_version, - env_version=env_version, + env_version=context.env_version, data_version=data_version, decisions=dict(task.decisions), input_versions=dict(input_versions), git_sha=sha, git_remote=remote, lc_version=lc_version(), - uv_version=uv_version, + uv_version=context.uv_version, hermeticity=asdict(outcome.attestation), started_at=started_at, finished_at=finished_at, - image=runtime.manifest_image(), + image=context.runtime.manifest_image(), ), ) except (OSError, ProjectError) as e: @@ -398,11 +394,14 @@ def main(argv: list[str]) -> int: result = execute( root, task, - identity.env_version(root), _from_disk(task), - head=dataset.head(root), - runtime=runtime, - uv_version=project.uv_version(root), + RunContext( + env_version=identity.env_version(root), + head=dataset.head(root), + versions=assets.Versions(), + runtime=runtime, + uv_version=project.uv_version(root), + ), ) except ProjectError as e: print(f"error: {e}", file=sys.stderr) diff --git a/tests/test_container.py b/tests/test_container.py index 22e4d4a6..dc7a8f6a 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -123,6 +123,12 @@ def _argvs(calls: list[list[str]], *head: str) -> list[list[str]]: return [c for c in calls if c[: len(head)] == list(head)] +def _git_calls(calls: list[list[str]], sub: str) -> list[list[str]]: + """git calls carrying *sub* anywhere — `-c key=val` pairs may precede + the subcommand, so a prefix match misses them.""" + return [c for c in calls if c[0] == "git" and sub in c] + + # ---- runtime detection ------------------------------------------------------ @@ -178,7 +184,7 @@ def test_a_missing_archive_refuses_unless_the_caller_may_build( with pytest.raises(ProjectError, match="lc build"): container.runtime_for_run(root, build=False) assert _argvs(fake, "podman", "build") == [] - assert [c for c in fake if c[0] == "git" and "commit" in c] == [] + assert _git_calls(fake, "commit") == [] runtime = container.runtime_for_run(root, build=True) @@ -264,11 +270,11 @@ def test_the_build_saves_and_commits_the_archive(root: Path, fake: list[list[str configured = {c[-2] for c in _argvs(fake, "git", "config", "-f", ".datalad/config")} assert f"datalad.containers.{runtime.image_tag}.image" in configured assert f"datalad.containers.{runtime.image_tag}.cmdexec" in configured - (add,) = [c for c in fake if c[0] == "git" and "add" in c] + (add,) = _git_calls(fake, "add") assert f".datalad/environments/{runtime.image_tag}" in " ".join(add) # The dot-path routing: without it the archive is a full blob in git. assert "annex.dotfiles=true" in add - assert len([c for c in fake if c[0] == "git" and "commit" in c]) == 1 + assert len(_git_calls(fake, "commit")) == 1 def test_an_unrouted_archive_refuses_before_building( diff --git a/tests/test_identity.py b/tests/test_identity.py index 88cc0ac6..594703c9 100644 --- a/tests/test_identity.py +++ b/tests/test_identity.py @@ -201,7 +201,9 @@ def test_the_environment_is_not_part_of_what_an_output_is(root: Path) -> None: def test_a_clean_lock_scans_clean(root: Path) -> None: - assert scan_lock(root) == LockScan(refusals=(), sdist_built=(), non_default_groups=()) + assert scan_lock(root) == LockScan( + refusals=(), sdist_built=(), non_default_groups=(), machine_config=() + ) def test_a_path_dependency_is_refused(root: Path) -> None: diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 71714bbb..5ff96e8d 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -71,6 +71,16 @@ def _commits(root: Path) -> int: return len(dataset._git(["log", "--oneline"], cwd=root).splitlines()) +def _cluster(monkeypatch: pytest.MonkeyPatch, scheduler: _Inline) -> None: + """Point the run at a custom scheduler — the one monkeypatch point.""" + + @contextmanager + def fake() -> Iterator[_Inline]: + yield scheduler + + monkeypatch.setattr(engine, "cluster_for_run", fake) + + # ---- a run, end to end ----------------------------------------------------- @@ -508,11 +518,7 @@ def completed(self, handles: list[object]) -> Iterator[object]: (root / "notes.md").write_text("scribbled while the graph ran\n") yield from handles - @contextmanager - def fake() -> Iterator[_Inline]: - yield Editing() - - monkeypatch.setattr(engine, "cluster_for_run", fake) + _cluster(monkeypatch, Editing()) report = engine.materialize(root, []) @@ -536,11 +542,7 @@ def completed(self, handles: list[object]) -> Iterator[object]: dataset._git(["add", "--", "notes.py"], cwd=root) yield from handles - @contextmanager - def fake() -> Iterator[_Inline]: - yield Staging() - - monkeypatch.setattr(engine, "cluster_for_run", fake) + _cluster(monkeypatch, Staging()) report = engine.materialize(root, []) @@ -629,11 +631,7 @@ def completed(self, handles: list[Any]) -> Iterator[TaskResult]: yield handles[0] raise KeyboardInterrupt - @contextmanager - def fake() -> Iterator[_Interrupted]: - yield _Interrupted() - - monkeypatch.setattr(engine, "cluster_for_run", fake) + _cluster(monkeypatch, _Interrupted()) with pytest.raises(KeyboardInterrupt): engine.materialize(root, []) diff --git a/tests/test_project.py b/tests/test_project.py index 023db170..463af2eb 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -683,7 +683,7 @@ def test_ambient_uv_install_settings_are_scrubbed( come from and how fast, never what gets installed.""" import os - from lightcone.engine.project import child_env, scrubbed_uv_vars + from lightcone.engine.project import child_env, uv_scrub_warning for name in [k for k in os.environ if k.startswith("UV_")]: monkeypatch.delenv(name) # the suite itself may run under `uv run` @@ -709,8 +709,8 @@ def test_ambient_uv_install_settings_are_scrubbed( ) assert env["UV_LINK_MODE"] == "copy", "link-mode is not an audited setting either" assert env["LC_TEST_CANARY"] == "kept" - assert scrubbed_uv_vars() == ["UV_INDEX_URL", "UV_NO_BINARY", "UV_PYTHON"], ( - "the report names exactly what the scrub dropped" + assert "UV_INDEX_URL, UV_NO_BINARY, UV_PYTHON" in uv_scrub_warning(), ( + "the warning names exactly what the scrub dropped" ) @@ -731,10 +731,10 @@ def test_an_empty_scrubbed_variable_is_not_reported( monkeypatch: pytest.MonkeyPatch, ) -> None: """An empty variable steers nothing, so warning about it is noise.""" - from lightcone.engine.project import scrubbed_uv_vars + from lightcone.engine.project import uv_scrub_warning monkeypatch.setenv("UV_NO_BUILD", "") - assert "UV_NO_BUILD" not in scrubbed_uv_vars() + assert "UV_NO_BUILD" not in uv_scrub_warning() def test_relays_uv_warnings_into_the_report( diff --git a/tests/test_run.py b/tests/test_run.py index 8334c2d6..d80d6377 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -196,3 +196,24 @@ def test_a_recipe_does_not_sync_where_a_probe_does(project: Path) -> None: concurrent worker writes the same `.venv`.""" assert "--no-sync" in uv_prefix(project, sync=False) assert "--exact" not in uv_prefix(project, sync=False) + + +def test_the_probe_reports_the_uv_scrub_in_its_notes( + project: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The probe is what builds the child environment, so the scrub's + fact rides its outcome — the caller prints notes verbatim, and no + verb has to remember to ask.""" + from lightcone.engine import run as engine_run + from lightcone.engine import sandbox + from lightcone.engine.sandbox.model import Attestation + + monkeypatch.setenv("UV_NO_BINARY", "1") + outcome = sandbox.Outcome( + returncode=0, attestation=Attestation(mechanism="none", fs="open") + ) + monkeypatch.setattr(sandbox, "run", lambda *a, **k: outcome) + + outcome = engine_run.probe(project, ["true"]) + + assert any("UV_NO_BINARY" in note for note in outcome.notes) diff --git a/tests/test_worker.py b/tests/test_worker.py index f6a1b3e5..ad715c86 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -67,22 +67,23 @@ def _runtime(root: Path) -> container.Runtime: return container.runtime_for_run(root, build=False) +def _context(root: Path, env_version: str | None = None) -> worker.RunContext: + """The driver-resolved facts a real run would hand down.""" + return worker.RunContext( + env_version=env_version if env_version is not None else identity.env_version(root), + head=_HEAD, + versions=assets.Versions(), + runtime=_runtime(root), + uv_version="0.0.0-test", + ) + + def _make( root: Path, output_id: str, *upstream: TaskResult, refresh: bool = False ) -> TaskResult: """Run one task the way Dask would, handed its upstream results.""" - task = _task(root, output_id) return worker.materialize( - root, - task, - identity.env_version(root), - _HEAD, - assets.Versions(), - refresh, - None, - _runtime(root), - "0.0.0-test", - *upstream, + root, _task(root, output_id), _context(root), refresh, None, *upstream ) @@ -253,10 +254,7 @@ def test_a_stale_file_does_not_survive_a_rebuild(root: Path) -> None: output = root / "results/baseline/first" (output / "leftover.txt").write_text("from a previous run\n") - worker.execute( - root, _task(root, "first"), identity.env_version(root), {}, head=_HEAD, - runtime=_runtime(root), - ) + worker.execute(root, _task(root, "first"), {}, _context(root)) assert not (output / "leftover.txt").exists() assert (output / "value.txt").exists() @@ -315,8 +313,7 @@ def test_an_environment_that_moved_under_the_run_is_refused(root: Path) -> None: """A manifest may not claim an environment that had already been edited by the time the recipe ran.""" result = worker.execute( - root, _task(root, "first"), "sha256:from-another-run", {}, head=_HEAD, - runtime=_runtime(root), + root, _task(root, "first"), {}, _context(root, env_version="sha256:from-another-run") ) assert result.status == "failed" From d71de5b47ae2819ff9af007bce8454afe2ef608d Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:17:59 +0200 Subject: [PATCH 11/12] Make the suite hermetic against CI's own environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two host facts the runners exposed: CI pins its matrix interpreter through an ambient UV_PYTHON, which the scrub correctly drops and reports — so every converge grew the warning and warnings == [] depended on the host; an autouse fixture now strips scrubbable UV_* suite-wide, derived from the scrub's own predicate. And the annex_keys clone test ran `git annex init` in a clone with no git identity — a clone inherits no local config, and init in a clone has remote git-annex branch state to commit, where a fresh repo's init tolerates the absence; the test sets identity the way the repo fixture does. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- tests/conftest.py | 20 ++++++++++++++++++++ tests/test_dataset.py | 5 +++++ 2 files changed, 25 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index c3fa42f9..51536f3c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,26 @@ def venue_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv(name, raising=False) +@pytest.fixture(autouse=True) +def ambient_uv(monkeypatch: pytest.MonkeyPatch) -> None: + """Strip scrubbable ``UV_*`` out of the suite's environment. + + CI pins its matrix interpreter through an ambient ``UV_PYTHON``, + which the scrub correctly drops and reports — so without this every + converge in the suite carries the warning and every ``warnings == + []`` assertion depends on the host. Derived from the scrub's own + predicate, so a variable the allowlist later admits stops being + stripped here for free; the scrub tests set their own variables + back deliberately. + """ + import os + + from lightcone.engine.project import _uv_scrubbed + + for name in [k for k in os.environ if _uv_scrubbed(k)]: + monkeypatch.delenv(name) + + @pytest.fixture(autouse=True) def machine_uv_config(monkeypatch: pytest.MonkeyPatch) -> None: """Blind the suite to the host's machine-level uv configuration. diff --git a/tests/test_dataset.py b/tests/test_dataset.py index de1a3401..c21e6491 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -520,6 +520,11 @@ def test_annex_keys_maps_every_annexed_file_content_present_or_not(repo: Path) - clone = repo.parent / "clone" dataset._git(["clone", "-q", str(repo), str(clone)], cwd=repo.parent) + # A clone inherits no local config, and annex init in a *clone* has + # remote git-annex branch state to commit — identity required, where + # a fresh repo's init tolerates its absence. + for key_, value in (("user.email", "t@example.com"), ("user.name", "Test")): + dataset._git(["config", key_, value], cwd=clone) dataset.init_annex(clone) assert dataset.annex_keys(clone)["results/fit/value.dat"] == key, ( "keys are repository state, bytes not required" From 73343788574d2dd6cb55429faf22019f96091aae Mon Sep 17 00:00:00 2001 From: Francois Lanusse Date: Fri, 21 Aug 2026 10:29:24 +0200 Subject: [PATCH 12/12] Trigger the eval on any ready-for-review PR Widens #183's label gate: the eval runs on every non-draft PR, with `ready_for_review` beside the default types so flipping a draft to ready triggers the run the draft guard skips. The per-PR concurrency group already cancels superseded runs, so a push train costs one eval. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016CRERrB5zWdVXD6uPv2BRa --- .github/workflows/eval.yml | 6 ++++-- CLAUDE.md | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 5b24e23e..4befd2ba 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -8,7 +8,9 @@ name: Eval on: workflow_dispatch: pull_request: - types: [labeled] + # `ready_for_review` beside the defaults, so flipping a draft to + # ready triggers the run the draft guard below skips. + types: [opened, synchronize, reopened, ready_for_review] # Only one eval per PR at a time — cancel in-progress runs concurrency: @@ -21,7 +23,7 @@ permissions: jobs: eval: - if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-eval' + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false runs-on: ubuntu-latest # Step-level timeout on the agent step (below) is what actually # bounds the run: a job-level timeout would cancel the always() diff --git a/CLAUDE.md b/CLAUDE.md index 7d22459b..f160e694 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -202,8 +202,9 @@ Test, lint and type-check are the whole loop, and they are what `.github/workflows/{tests,lint}.yml` run. There is deliberately no task runner in between — the pre-rebuild `justfile` was 90 lines of wrappers around them plus recipes for the frozen docs and the dormant eval. The -other workflows are `eval.yml` (the agentic eval, on dispatch or PR -label), `pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. +other workflows are `eval.yml` (the agentic eval, on dispatch or any +non-draft PR — flipping a draft to ready triggers it), +`pypi-publish.yaml`, and `docs-deploy.yml` for the frozen docs. ## Key Invariants (layer 1)