From d70a537f1e3f94d9dcf21b3a237572e56f8eaa1a Mon Sep 17 00:00:00 2001 From: Marius1311 Date: Fri, 21 Aug 2026 15:12:07 +0200 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20analysis-task=20conventions=20?= =?UTF-8?q?=E2=80=94=20task=5Fpaths,=20task=20template,=20AGENTS.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the working conventions for agent-run analysis tasks, so a repo made from this template starts consistent instead of growing its own layout. A task is one session, one worktree, one branch, one directory. Its outputs split by durability rather than by kind: tracked `results/`/`reports/` stay in the calling checkout and ride the PR, while untracked `figures/`/`outputs/`/`logs/` are anchored to the main checkout so they survive the worktree being removed. `task_paths(__file__)` resolves both. The anchoring is why `main_checkout()` uses `git rev-parse --git-common-dir` rather than `FilePaths.ROOT`: `_find_root` walks up for a `.git` marker, and a worktree's `.git` is a file that exists, so it stops at the worktree. Untracked output would then be written somewhere that later disappears — silently, since worktrees are gitignored and git never warns about them. Also: - `.gitignore` gains `outputs/` and `logs/` (unanchored, matching at any depth) plus negations so a task's small evidence tables and HTML report stay tracked — they are what a reviewer actually reads on GitHub. - `AGENTS.md` carries the conventions; `CLAUDE.md` is a pointer to it, and `copilot-instructions.md` defers to it rather than restating. The top-level `figures/` keeps its role for curated output (talks, papers), which is distinct from task output. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VYsKhgGuNCFtxqFhSLGtCK --- .github/copilot-instructions.md | 10 ++- .gitignore | 11 ++- AGENTS.md | 96 +++++++++++++++++++++++ CLAUDE.md | 3 + analysis/_task_template/README.md | 39 ++++++++++ analysis/_task_template/_common.py | 32 ++++++++ analysis/_task_template/slurm/job.sbatch | 21 +++++ analysis/_task_template/slurm/submit.sh | 25 ++++++ src/myanalysis/__init__.py | 4 +- src/myanalysis/_constants.py | 98 ++++++++++++++++++++++++ 10 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 analysis/_task_template/README.md create mode 100644 analysis/_task_template/_common.py create mode 100644 analysis/_task_template/slurm/job.sbatch create mode 100644 analysis/_task_template/slurm/submit.sh diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9c80755..3f98a82 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -14,9 +14,15 @@ See the project README for details about the project goal, datasets, and structu | Add PyPI package | `pixi add --pypi ` | ## Project structure + +Canonical guidance lives in `AGENTS.md` — analysis tasks, where outputs go, the data rules and +environments. If this file conflicts with it, `AGENTS.md` wins. + - **Notebooks**: `analysis/[INITIALS]-[YYYY]-[MM]-[DD]_description.ipynb` +- **Agent tasks**: one directory each, `analysis//...//` — copy + `analysis/_task_template/`. Output paths come from `task_paths(__file__)`, never a bare + relative path. - **Data**: `data//{raw,processed,resources,results}/` - **Paths**: Use `from import FilePaths` (edit `_constants.py` for datasets) - **Deps**: All in `pixi.toml` (not pyproject.toml) -- pyproject.toml exists mainly for package metadata and testing -- Run `pixi install` after pulling changes that update `pixi.toml` +- Run `pixi install` after pulling changes that update `pixi.toml` — in the main checkout diff --git a/.gitignore b/.gitignore index 54dd2f0..402991b 100644 --- a/.gitignore +++ b/.gitignore @@ -150,8 +150,17 @@ dmypy.json *.gmt *.gmx -# Directories to ignore +# Directories to ignore. Unanchored, so they match at any depth: these are the +# untracked half of an analysis task (see AGENTS.md "Analysis tasks"). figures/ +outputs/ +logs/ + +# ...but a task's results/ and reports/ are tracked on purpose: small evidence +# tables and the HTML report are what a reviewer actually reads on GitHub. +!analysis/**/results/*.csv +!analysis/**/results/*.json +!analysis/**/reports/*.html # OS specifics **.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..01013a4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,96 @@ +# AGENTS.md — analysis conventions + +This file owns the working conventions. `README.md` is the user-facing overview; anything +documented elsewhere is referenced from here, never restated. + +## Analysis tasks + +A **task** is one agent session, one git worktree, one branch, and however many PRs it takes. +It gets one directory, `analysis//...//`, carrying a version suffix. The name is +chosen once and never changed, because the path is what links artifacts back to the analysis +that produced them. + +Copy `analysis/_task_template/` to start one. + +Humans also work in `analysis/` in notebooks, in the main checkout — the same conventions apply, +and notebooks keep their `[INITIALS]-[YYYY]-[MM]-[DD]_description.ipynb` naming. + +### Where outputs go + +Placement is decided by *sharing*, on either side: anything another task or a human reads or +writes is central (`data//`); everything else is task-local. That call is made at +end-of-session sign-off rather than at write time — you cannot know at write time whether +something will be reused — so until then artifacts sit in `outputs/`. + +Task-local splits by durability, not by kind: + +| dir | tracked | lives in | holds | +| --- | --- | --- | --- | +| `results/` | yes | this checkout | small evidence tables — csv, json | +| `reports/` | yes | this checkout | HTML, figures embedded as base64 so they render on GitHub | +| `figures/` | no | the **main** checkout | pdf, png | +| `outputs/` | no | the **main** checkout | data artifacts, checkpoints | +| `logs/` | no | the **main** checkout | batch job output | + +The tracked two stay in the worktree so they ride the PR; the untracked three are anchored to +the main checkout so they survive the worktree being removed. Both come from +`task_paths(__file__)` — **never write a bare relative output path.** From a worktree that lands +in the worktree, and since worktrees are gitignored, nothing will warn you. + +Batch jobs are the usual way this bites: `--output` resolves against the *submit* directory, so +pass it absolutely (`slurm/submit.sh` does). + +The top-level `figures/` is for **curated** output — figures chosen for a talk or a paper — not +for task output. + +### The README contract + +Every task delivers a report, however small, and a `README.md` naming the date, the inputs it +consumed, the outputs it wrote and where write-back landed. The version suffix appears in the +write-back key names too, so grepping `analysis/**/README.md` gets you from an `obs` column back +to the task that made it. That reverse index is what keeps central storage from being anonymous. + +## Data + +- Datasets live in `data//{raw,processed,resources,results}/`, gitignored. Note that + `data//results/` is *central and untracked* — not the same thing as a task's tracked + `results/`. +- Use `from import FilePaths`; edit `_constants.py` to add a dataset. Never hardcode a + dataset path. +- **Accumulate by addition.** Adding new keys to a freshly re-read object is commutative, so + concurrent sessions cannot lose each other's work whatever the write order. Removing something + is not — that means a new dated copy, keeping the old one so old scripts still run. +- **Never write your in-memory object back over a shared one.** By the time an analysis + finishes, its copy is stale shared state. Re-read from disk, apply your named additions, write + a temp file in the same directory and `os.replace` it. +- Which object is current is stated by the human per session and recorded in the task README — + never hardcoded in a helper or a config, which is how a config ends up pointing at an object + retired months ago. +- Writing back to a shared object needs explicit sign-off on that specific diff. That gate is + also what serializes concurrent sessions, which is why no file lock is needed. + +## Environments + +Dependencies live in `pixi.toml`. **Reusability decides where a package goes, not just whether it +resolves:** one likely to be carried forward is worth making work in the root environment even +when it resists; a stack of packages for a benchmark where at most the winner survives belongs in +an isolated task-local `pixi.toml`, promoted later if it earns it. Only the manifest and lock +persist — the environment is derived, dies with the worktree, and rebuilds with +`pixi install --frozen`. + +Run `pixi install` **in the main checkout only**. If the root manifest declares the package as an +editable `path = "."` dependency, installing from a worktree rebases the package — and every +resolved data path with it — into the worktree. + +| Task | Command | +| --- | --- | +| Run Python | `pixi run python script.py` | +| Run tests | `pixi run test` | +| Add conda package | `pixi add ` | +| Add PyPI package | `pixi add --pypi ` | + +## Sessions + +One task, one session, one worktree. Exit with `/exit` and answer *remove*; push before walking +away, since a worktree with unpushed commits is the only unrecoverable state. Don't exit while +batch jobs are still queued — they reference scripts by path inside the worktree. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..5ddc5bb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +# Analysis Agent Entry Point + +@AGENTS.md diff --git a/analysis/_task_template/README.md b/analysis/_task_template/README.md new file mode 100644 index 0000000..569ca7f --- /dev/null +++ b/analysis/_task_template/README.md @@ -0,0 +1,39 @@ +# + +**Date:** + +One or two sentences on what this task set out to answer. + +## Inputs + +Name the actual files, not concepts — this is what makes the task re-runnable. + +- Working object: `data//processed/.h5ad` +- Other artifacts consumed: `analysis//outputs/`, `data//results/` + +## Outputs + +| what | where | +| --- | --- | +| evidence tables | `results/` (tracked) | +| report | `reports/` (tracked) | +| figures | `figures/` (gitignored, in the main checkout) | +| data artifacts | `outputs/` (gitignored, in the main checkout) | + +## Write-back + +What went into the working object, and under which keys. The version identifier in this +directory's name must appear in the key names, so an `obs` column can be traced back here +by grepping `analysis/**/README.md`. + +- `obs["_"]` — one line on what it holds +- `obsm["X__"]` — likewise + +Artifacts too large to embed stay in `outputs/` and are listed above instead. Anything +promoted to central storage (`data//...`) at sign-off is recorded here with its +final path. + +## Notes + +Decisions a reader would otherwise have to reverse-engineer: what was tried and rejected, +which parameters are load-bearing, what is still provisional. diff --git a/analysis/_task_template/_common.py b/analysis/_task_template/_common.py new file mode 100644 index 0000000..0b4425b --- /dev/null +++ b/analysis/_task_template/_common.py @@ -0,0 +1,32 @@ +"""Shared paths and constants for this task. Copy with the template; edit in place. + +Every output path in the task comes from here, so nothing is a bare relative path. That +matters because a relative write from inside a git worktree lands in the worktree and +disappears when it is removed — and since worktrees are gitignored, git will not warn you. + +``task_paths`` splits the task's directories by durability, not by kind: + + results/ reports/ tracked -> stay in this checkout, ride the pull request + figures/ outputs/ logs/ ignored -> anchored to the MAIN checkout, survive teardown + +Nothing is created at import time. Call ``PATHS.ensure()`` in the writer, so a dry run +stays dry. +""" + +from myanalysis import task_paths + +PATHS = task_paths(__file__) + +#: Small, reviewable evidence tables (csv/json) and the task's HTML report. +RESULTS = PATHS.results +REPORTS = PATHS.reports + +#: Heavy or noisy: figures, data artifacts, Slurm logs. Gitignored, anchored to MAIN. +FIGURES = PATHS.figures +OUTPUTS = PATHS.outputs +LOGS = PATHS.logs + +#: The object this task reads. Supplied by the human at session start and recorded in +#: README.md — never hardcoded here, because which object is current changes over time. +#: Set it from an environment variable, a CLI argument, or edit this line for the task. +WORKING_OBJECT = None diff --git a/analysis/_task_template/slurm/job.sbatch b/analysis/_task_template/slurm/job.sbatch new file mode 100644 index 0000000..32fd141 --- /dev/null +++ b/analysis/_task_template/slurm/job.sbatch @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Submit with ./slurm/submit.sh slurm/job.sbatch — it supplies --output/--error as +# absolute paths in the MAIN checkout. Do not add --output here; a relative one would +# write into the worktree and be lost at teardown. +#SBATCH --job-name=task-step +#SBATCH --time=04:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem-per-cpu=8G +set -euo pipefail + +TASK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MAIN="$(dirname "$(git -C "$TASK_DIR" rev-parse --path-format=absolute --git-common-dir)")" + +# Record what this ran against, so the log identifies the code even after a merge. +echo "task: $TASK_DIR" +echo "commit: $(git -C "$TASK_DIR" rev-parse --short HEAD)" + +# Resolve the interpreter once: concurrent `pixi run` calls serialize on the env prefix. +PY="$MAIN/.pixi/envs/default/bin/python" + +"$PY" "$TASK_DIR/scripts/step.py" "$@" diff --git a/analysis/_task_template/slurm/submit.sh b/analysis/_task_template/slurm/submit.sh new file mode 100644 index 0000000..e416a53 --- /dev/null +++ b/analysis/_task_template/slurm/submit.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Submit a job from this task with its log directory resolved absolutely. +# +# `#SBATCH --output=logs/...` is relative to the submit directory, so from a git worktree +# it writes into the worktree and the logs die when the worktree is removed. This resolves +# the task's log directory through `task_paths`, which anchors it to the MAIN checkout, and +# creates it before submitting (sbatch fails silently if the directory is missing). +# +# ./slurm/submit.sh slurm/job.sbatch [sbatch args...] +set -euo pipefail + +TASK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MAIN="$(git -C "$TASK_DIR" rev-parse --path-format=absolute --git-common-dir)" +MAIN="$(dirname "$MAIN")" +PY="$MAIN/.pixi/envs/default/bin/python" + +LOGS="$("$PY" -c " +from myanalysis import task_paths +p = task_paths('$TASK_DIR/_common.py') +p.ensure() +print(p.logs) +")" + +echo "logs -> $LOGS" +sbatch --output="$LOGS/%x-%j.out" --error="$LOGS/%x-%j.err" "$@" diff --git a/src/myanalysis/__init__.py b/src/myanalysis/__init__.py index a336f31..46d15d7 100644 --- a/src/myanalysis/__init__.py +++ b/src/myanalysis/__init__.py @@ -1,7 +1,7 @@ from importlib.metadata import version -from ._constants import DatasetPaths, FilePaths +from ._constants import DatasetPaths, FilePaths, TaskPaths, main_checkout, task_paths from .plotting import qc_violin -__all__ = ["DatasetPaths", "FilePaths", "qc_violin"] +__all__ = ["DatasetPaths", "FilePaths", "TaskPaths", "main_checkout", "qc_violin", "task_paths"] __version__ = version("myanalysis") diff --git a/src/myanalysis/_constants.py b/src/myanalysis/_constants.py index ac69cd6..3e4ed34 100644 --- a/src/myanalysis/_constants.py +++ b/src/myanalysis/_constants.py @@ -2,7 +2,9 @@ from __future__ import annotations +import subprocess from dataclasses import dataclass +from functools import lru_cache from pathlib import Path # Files that mark the repository root, searched for upward from this module. @@ -75,3 +77,99 @@ def dataset(cls, name: str) -> DatasetPaths: >>> paths.processed / "adata.h5ad" # doctest: +SKIP """ return DatasetPaths(cls.DATA / name) + + +# --------------------------------------------------------------------------- # +# Analysis tasks # +# --------------------------------------------------------------------------- # + +#: Task subdirectories that git tracks: small, reviewable, they ride the PR. +TRACKED_TASK_DIRS = ("results", "reports") + +#: Task subdirectories git ignores: heavy or noisy, anchored to the main checkout. +UNTRACKED_TASK_DIRS = ("figures", "outputs", "logs") + +#: Directory names that are *inside* a task rather than a task themselves. +_RESERVED_TASK_SUBDIRS = frozenset({*TRACKED_TASK_DIRS, *UNTRACKED_TASK_DIRS, "scripts", "slurm", "notebooks", "docs"}) + + +@lru_cache(maxsize=1) +def main_checkout() -> Path: + """Absolute path of the *main* checkout, even when called from a git worktree. + + Resolved from git rather than from where this package happens to be installed. + ``--git-common-dir`` points at the main checkout's ``.git`` from any worktree, + whereas :data:`FilePaths.ROOT` walks up for a marker and so stops at the worktree + (a worktree's ``.git`` is a file, but it still exists). + """ + try: + out = subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], + cwd=Path(__file__).resolve().parent, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, OSError): + return FilePaths.ROOT + return Path(out).parent + + +@dataclass(frozen=True) +class TaskPaths: + """Where one analysis task's outputs go, split by durability rather than by kind. + + ``results`` and ``reports`` stay in the checkout the calling file lives in, so they + ride the pull request. ``figures``, ``outputs`` and ``logs`` are anchored to the main + checkout, so they survive a worktree being removed — a bare relative path written from + a worktree is lost silently, because worktrees are gitignored and git will not warn you. + """ + + task: Path + results: Path + reports: Path + figures: Path + outputs: Path + logs: Path + + def ensure(self) -> TaskPaths: + """Create the directories. Call this from the writer, never at import time.""" + for name in (*TRACKED_TASK_DIRS, *UNTRACKED_TASK_DIRS): + getattr(self, name).mkdir(parents=True, exist_ok=True) + return self + + +def task_paths(file: str | Path) -> TaskPaths: + """Resolve the output directories for the task that ``file`` belongs to. + + Pass ``__file__``. The task directory is the nearest ancestor under ``analysis/`` + whose name is not a known task subdirectory, so both ``/_common.py`` and + ``/scripts/step.py`` resolve to ````. + + Examples + -------- + >>> paths = task_paths(__file__).ensure() # doctest: +SKIP + >>> paths.results / "markers.csv" # tracked # doctest: +SKIP + >>> paths.outputs / "embedding.h5ad" # gitignored, in the main checkout # doctest: +SKIP + """ + path = Path(file).resolve() + parts = path.parts + if "analysis" not in parts: + raise ValueError(f"{path} is not under an 'analysis/' directory") + checkout = Path(*parts[: parts.index("analysis")]) + + task = path.parent + while task.name in _RESERVED_TASK_SUBDIRS: + task = task.parent + if task in (checkout / "analysis", checkout): + raise ValueError(f"{path} is not inside a task directory under 'analysis/'") + + main_task = main_checkout() / "analysis" / task.relative_to(checkout / "analysis") + return TaskPaths( + task=task, + results=task / "results", + reports=task / "reports", + figures=main_task / "figures", + outputs=main_task / "outputs", + logs=main_task / "logs", + ) From 8c3ae82751048eb6b5335870f77995ff14ede97c Mon Sep 17 00:00:00 2001 From: Marius1311 Date: Sat, 22 Aug 2026 16:13:14 +0200 Subject: [PATCH 2/3] refactor(template): drop the slurm folder, keep the template scheduler-agnostic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The template shipped an Euler-flavoured slurm/ pair that restated four facts owned by the euler-slurm skill (--output semantics, the missing-log-dir requeue loop, sbatch idiom, MAIN resolution) — two of them already drifted: OMP_NUM_THREADS was dropped while requesting 8 CPUs, and the requeue loop was described as a silent failure. The one fact the template owns is a path fact, not a scheduler fact: log paths come from task_paths().logs and are passed absolutely. That is now stated once, without naming a scheduler. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VYsKhgGuNCFtxqFhSLGtCK --- AGENTS.md | 5 +++-- analysis/_task_template/_common.py | 2 +- analysis/_task_template/slurm/job.sbatch | 21 -------------------- analysis/_task_template/slurm/submit.sh | 25 ------------------------ 4 files changed, 4 insertions(+), 49 deletions(-) delete mode 100644 analysis/_task_template/slurm/job.sbatch delete mode 100644 analysis/_task_template/slurm/submit.sh diff --git a/AGENTS.md b/AGENTS.md index 01013a4..d8bd7f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,8 +37,9 @@ the main checkout so they survive the worktree being removed. Both come from `task_paths(__file__)` — **never write a bare relative output path.** From a worktree that lands in the worktree, and since worktrees are gitignored, nothing will warn you. -Batch jobs are the usual way this bites: `--output` resolves against the *submit* directory, so -pass it absolutely (`slurm/submit.sh` does). +Batch jobs are the usual way this bites: schedulers resolve a relative log path against the +*submit* directory. Take the path from `task_paths(__file__).logs` and pass it absolutely. How +your scheduler spells that is its own business, not this template's. The top-level `figures/` is for **curated** output — figures chosen for a talk or a paper — not for task output. diff --git a/analysis/_task_template/_common.py b/analysis/_task_template/_common.py index 0b4425b..5fcf7c7 100644 --- a/analysis/_task_template/_common.py +++ b/analysis/_task_template/_common.py @@ -21,7 +21,7 @@ RESULTS = PATHS.results REPORTS = PATHS.reports -#: Heavy or noisy: figures, data artifacts, Slurm logs. Gitignored, anchored to MAIN. +#: Heavy or noisy: figures, data artifacts, batch-job logs. Gitignored, anchored to MAIN. FIGURES = PATHS.figures OUTPUTS = PATHS.outputs LOGS = PATHS.logs diff --git a/analysis/_task_template/slurm/job.sbatch b/analysis/_task_template/slurm/job.sbatch deleted file mode 100644 index 32fd141..0000000 --- a/analysis/_task_template/slurm/job.sbatch +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Submit with ./slurm/submit.sh slurm/job.sbatch — it supplies --output/--error as -# absolute paths in the MAIN checkout. Do not add --output here; a relative one would -# write into the worktree and be lost at teardown. -#SBATCH --job-name=task-step -#SBATCH --time=04:00:00 -#SBATCH --cpus-per-task=8 -#SBATCH --mem-per-cpu=8G -set -euo pipefail - -TASK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MAIN="$(dirname "$(git -C "$TASK_DIR" rev-parse --path-format=absolute --git-common-dir)")" - -# Record what this ran against, so the log identifies the code even after a merge. -echo "task: $TASK_DIR" -echo "commit: $(git -C "$TASK_DIR" rev-parse --short HEAD)" - -# Resolve the interpreter once: concurrent `pixi run` calls serialize on the env prefix. -PY="$MAIN/.pixi/envs/default/bin/python" - -"$PY" "$TASK_DIR/scripts/step.py" "$@" diff --git a/analysis/_task_template/slurm/submit.sh b/analysis/_task_template/slurm/submit.sh deleted file mode 100644 index e416a53..0000000 --- a/analysis/_task_template/slurm/submit.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Submit a job from this task with its log directory resolved absolutely. -# -# `#SBATCH --output=logs/...` is relative to the submit directory, so from a git worktree -# it writes into the worktree and the logs die when the worktree is removed. This resolves -# the task's log directory through `task_paths`, which anchors it to the MAIN checkout, and -# creates it before submitting (sbatch fails silently if the directory is missing). -# -# ./slurm/submit.sh slurm/job.sbatch [sbatch args...] -set -euo pipefail - -TASK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MAIN="$(git -C "$TASK_DIR" rev-parse --path-format=absolute --git-common-dir)" -MAIN="$(dirname "$MAIN")" -PY="$MAIN/.pixi/envs/default/bin/python" - -LOGS="$("$PY" -c " -from myanalysis import task_paths -p = task_paths('$TASK_DIR/_common.py') -p.ensure() -print(p.logs) -")" - -echo "logs -> $LOGS" -sbatch --output="$LOGS/%x-%j.out" --error="$LOGS/%x-%j.err" "$@" From 9219ea63655be8cf9fc6efe09aef4503c768a67f Mon Sep 17 00:00:00 2001 From: Marius1311 Date: Sat, 22 Aug 2026 16:21:07 +0200 Subject: [PATCH 3/3] refactor: one root resolver, one directory-set pattern; agent-neutral instructions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paths had two parallel mechanisms. `_find_root` walked up for a marker while `main_checkout` asked git, and `FilePaths.ROOT` used the former — so central data would resolve into a worktree if the package were ever installed from one, the hazard AGENTS.md had to warn about. There is now one resolver, git-first with the marker walk as fallback, and `FilePaths.ROOT` is it. `DatasetPaths` and `TaskPaths` were the same idea twice with different spellings (properties vs fields, `create()` vs `ensure()`). Both are now field-only dataclasses over a shared base with a single `create()`, and both are built by expanding the directory-name tuples, so a directory name is written once. `EXAMPLE_DATASET` is gone: `FilePaths.dataset()` is the one way to reach a dataset. The copilot instructions restated AGENTS.md; they are now a pointer, matching CLAUDE.md, and the session rule no longer names a specific agent's exit command. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VYsKhgGuNCFtxqFhSLGtCK --- .github/copilot-instructions.md | 30 +-- AGENTS.md | 12 +- .../ML-2026-01-27_demo_scRNA_workflow.ipynb | 4 +- analysis/_task_template/_common.py | 24 +-- src/myanalysis/_constants.py | 176 +++++++----------- 5 files changed, 88 insertions(+), 158 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 3f98a82..62a2930 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,28 +1,6 @@ -# Copilot Instructions for Analysis Template +# Copilot Instructions -## Project context +Canonical repo guidance lives in `AGENTS.md` — analysis tasks, where outputs go, the data +rules, environments and commands. `README.md` is the user-facing overview. -See the project README for details about the project goal, datasets, and structure. - -## Quick reference - -| Task | Command | -|------|---------| -| Run Python | `pixi run python script.py` | -| Run tests | `pixi run test` | -| Add conda package | `pixi add ` | -| Add PyPI package | `pixi add --pypi ` | - -## Project structure - -Canonical guidance lives in `AGENTS.md` — analysis tasks, where outputs go, the data rules and -environments. If this file conflicts with it, `AGENTS.md` wins. - -- **Notebooks**: `analysis/[INITIALS]-[YYYY]-[MM]-[DD]_description.ipynb` -- **Agent tasks**: one directory each, `analysis//...//` — copy - `analysis/_task_template/`. Output paths come from `task_paths(__file__)`, never a bare - relative path. -- **Data**: `data//{raw,processed,resources,results}/` -- **Paths**: Use `from import FilePaths` (edit `_constants.py` for datasets) -- **Deps**: All in `pixi.toml` (not pyproject.toml) -- Run `pixi install` after pulling changes that update `pixi.toml` — in the main checkout +If this file conflicts with them, they win. diff --git a/AGENTS.md b/AGENTS.md index d8bd7f4..1c8d036 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,8 +56,10 @@ to the task that made it. That reverse index is what keeps central storage from - Datasets live in `data//{raw,processed,resources,results}/`, gitignored. Note that `data//results/` is *central and untracked* — not the same thing as a task's tracked `results/`. -- Use `from import FilePaths`; edit `_constants.py` to add a dataset. Never hardcode a - dataset path. +- Reach a dataset with `FilePaths.dataset("")`, which gives `.raw`, `.processed`, + `.resources`, `.results` and a `.create()`. Never hardcode a dataset path. `FilePaths.ROOT` is + the main checkout even when called from a worktree, so central data resolves the same either + way. Task outputs come from `task_paths(__file__)`, the same shape with a `.create()`. - **Accumulate by addition.** Adding new keys to a freshly re-read object is commutative, so concurrent sessions cannot lose each other's work whatever the write order. Removing something is not — that means a new dated copy, keeping the old one so old scripts still run. @@ -92,6 +94,6 @@ resolved data path with it — into the worktree. ## Sessions -One task, one session, one worktree. Exit with `/exit` and answer *remove*; push before walking -away, since a worktree with unpushed commits is the only unrecoverable state. Don't exit while -batch jobs are still queued — they reference scripts by path inside the worktree. +One task, one session, one worktree. Remove the worktree when you end the session, and push +first — a worktree with unpushed commits is the only unrecoverable state. Don't tear one down +while batch jobs are still queued: they reference scripts by path inside it. diff --git a/analysis/ML-2026-01-27_demo_scRNA_workflow.ipynb b/analysis/ML-2026-01-27_demo_scRNA_workflow.ipynb index e93671b..cccbafd 100644 --- a/analysis/ML-2026-01-27_demo_scRNA_workflow.ipynb +++ b/analysis/ML-2026-01-27_demo_scRNA_workflow.ipynb @@ -95,7 +95,7 @@ "outputs": [], "source": [ "sc.settings.verbosity = 2\n", - "sc.settings.datasetdir = FilePaths.EXAMPLE_DATASET / \"raw\"\n", + "sc.settings.datasetdir = FilePaths.dataset(\"example_dataset\").raw\n", "sc.settings.set_figure_params(dpi=100, frameon=False)\n", "sc.settings.figdir = FilePaths.FIGURES / \"example_dataset\"\n", "\n", @@ -498,7 +498,7 @@ "metadata": {}, "outputs": [], "source": [ - "output_path = FilePaths.EXAMPLE_DATASET / \"processed\" / \"pbmc3k_processed.h5ad\"\n", + "output_path = FilePaths.dataset(\"example_dataset\").processed / \"pbmc3k_processed.h5ad\"\n", "adata.write(output_path)\n", "print(f\"Saved to: {output_path}\")" ] diff --git a/analysis/_task_template/_common.py b/analysis/_task_template/_common.py index 5fcf7c7..7721e78 100644 --- a/analysis/_task_template/_common.py +++ b/analysis/_task_template/_common.py @@ -1,32 +1,22 @@ -"""Shared paths and constants for this task. Copy with the template; edit in place. +"""Shared paths for this task. Copy with the template; edit in place. -Every output path in the task comes from here, so nothing is a bare relative path. That -matters because a relative write from inside a git worktree lands in the worktree and -disappears when it is removed — and since worktrees are gitignored, git will not warn you. - -``task_paths`` splits the task's directories by durability, not by kind: - - results/ reports/ tracked -> stay in this checkout, ride the pull request - figures/ outputs/ logs/ ignored -> anchored to the MAIN checkout, survive teardown - -Nothing is created at import time. Call ``PATHS.ensure()`` in the writer, so a dry run -stays dry. +Every output path in the task comes from here, so nothing is a bare relative path. +Nothing is created at import: call ``PATHS.create()`` in the writer, so a dry run stays dry. """ from myanalysis import task_paths PATHS = task_paths(__file__) -#: Small, reviewable evidence tables (csv/json) and the task's HTML report. +#: Tracked: small evidence tables and the report. Ride the pull request. RESULTS = PATHS.results REPORTS = PATHS.reports -#: Heavy or noisy: figures, data artifacts, batch-job logs. Gitignored, anchored to MAIN. +#: Gitignored, anchored to the main checkout so they survive worktree teardown. FIGURES = PATHS.figures OUTPUTS = PATHS.outputs LOGS = PATHS.logs -#: The object this task reads. Supplied by the human at session start and recorded in -#: README.md — never hardcoded here, because which object is current changes over time. -#: Set it from an environment variable, a CLI argument, or edit this line for the task. +#: The object this task reads. Supplied per session and recorded in README.md — never +#: hardcoded, because which object is current changes over time. WORKING_OBJECT = None diff --git a/src/myanalysis/_constants.py b/src/myanalysis/_constants.py index 3e4ed34..b06125c 100644 --- a/src/myanalysis/_constants.py +++ b/src/myanalysis/_constants.py @@ -1,128 +1,78 @@ -"""Project-wide path constants for notebooks and scripts.""" +"""Project-wide paths for notebooks, scripts and analysis tasks.""" from __future__ import annotations import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, fields from functools import lru_cache from pathlib import Path +from typing import Self -# Files that mark the repository root, searched for upward from this module. -_ROOT_MARKERS = ("pixi.toml", ".git") +#: Standard subfolders of ``data//``. +DATASET_DIRS = ("raw", "processed", "resources", "results") - -def _find_root(start: Path) -> Path: - """Locate the repo root by walking upward until a marker file is found. - - Falls back to the fixed ``src//`` layout (three levels up) when no - marker is present, e.g. for a non-editable installed copy. - """ - for parent in (start, *start.parents): - if any((parent / marker).exists() for marker in _ROOT_MARKERS): - return parent - return start.parents[2] - - -@dataclass(frozen=True) -class DatasetPaths: - """Standard subfolders for a single dataset (``data//``).""" - - root: Path - - @property - def raw(self) -> Path: - """Original, unmodified input data.""" - return self.root / "raw" - - @property - def processed(self) -> Path: - """Preprocessed / intermediate data.""" - return self.root / "processed" - - @property - def resources(self) -> Path: - """Reference data, gene sets, annotations.""" - return self.root / "resources" - - @property - def results(self) -> Path: - """Analysis outputs (tables, exported objects).""" - return self.root / "results" - - def create(self) -> DatasetPaths: - """Create all standard subfolders (idempotent). Returns ``self``.""" - for path in (self.raw, self.processed, self.resources, self.results): - path.mkdir(parents=True, exist_ok=True) - return self - - -class FilePaths: - """Project-wide paths for notebooks and scripts.""" - - ROOT = _find_root(Path(__file__).resolve()) - - DATA = ROOT / "data" - FIGURES = ROOT / "figures" - - # The bundled example dataset; customize / add your own via `dataset()`. - EXAMPLE_DATASET = DATA / "example_dataset" - - @classmethod - def dataset(cls, name: str) -> DatasetPaths: - """Return the standard raw/processed/resources/results paths for a dataset. - - Examples - -------- - >>> paths = FilePaths.dataset("pbmc3k").create() - >>> paths.processed / "adata.h5ad" # doctest: +SKIP - """ - return DatasetPaths(cls.DATA / name) - - -# --------------------------------------------------------------------------- # -# Analysis tasks # -# --------------------------------------------------------------------------- # - -#: Task subdirectories that git tracks: small, reviewable, they ride the PR. +#: Task subdirectories git tracks: small, reviewable, they ride the pull request. TRACKED_TASK_DIRS = ("results", "reports") #: Task subdirectories git ignores: heavy or noisy, anchored to the main checkout. UNTRACKED_TASK_DIRS = ("figures", "outputs", "logs") -#: Directory names that are *inside* a task rather than a task themselves. +#: Directory names that sit *inside* a task rather than being one. _RESERVED_TASK_SUBDIRS = frozenset({*TRACKED_TASK_DIRS, *UNTRACKED_TASK_DIRS, "scripts", "slurm", "notebooks", "docs"}) @lru_cache(maxsize=1) def main_checkout() -> Path: - """Absolute path of the *main* checkout, even when called from a git worktree. + """The main checkout, resolved from git so a worktree still points at it. - Resolved from git rather than from where this package happens to be installed. - ``--git-common-dir`` points at the main checkout's ``.git`` from any worktree, - whereas :data:`FilePaths.ROOT` walks up for a marker and so stops at the worktree - (a worktree's ``.git`` is a file, but it still exists). + Falls back to walking up for ``pixi.toml``/``.git`` outside a repository. """ try: - out = subprocess.run( + git_dir = subprocess.run( ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], cwd=Path(__file__).resolve().parent, capture_output=True, text=True, check=True, ).stdout.strip() + return Path(git_dir).parent except (subprocess.CalledProcessError, OSError): - return FilePaths.ROOT - return Path(out).parent + here = Path(__file__).resolve() + for parent in (here, *here.parents): + if (parent / "pixi.toml").exists() or (parent / ".git").exists(): + return parent + return here.parents[2] + + +@dataclass(frozen=True) +class _Dirs: + """A named set of directories.""" + + def create(self) -> Self: + """Create them all, idempotently. Call from the writer, never at import.""" + for field in fields(self): + getattr(self, field.name).mkdir(parents=True, exist_ok=True) + return self + + +@dataclass(frozen=True) +class DatasetPaths(_Dirs): + """``data//`` and its standard subfolders.""" + + root: Path + raw: Path + processed: Path + resources: Path + results: Path @dataclass(frozen=True) -class TaskPaths: - """Where one analysis task's outputs go, split by durability rather than by kind. +class TaskPaths(_Dirs): + """One task's directories, split by durability rather than by kind. - ``results`` and ``reports`` stay in the checkout the calling file lives in, so they - ride the pull request. ``figures``, ``outputs`` and ``logs`` are anchored to the main - checkout, so they survive a worktree being removed — a bare relative path written from - a worktree is lost silently, because worktrees are gitignored and git will not warn you. + ``results``/``reports`` stay in the calling checkout so they ride the pull request; + ``figures``/``outputs``/``logs`` are anchored to the main checkout so they survive the + worktree being removed. """ task: Path @@ -132,23 +82,36 @@ class TaskPaths: outputs: Path logs: Path - def ensure(self) -> TaskPaths: - """Create the directories. Call this from the writer, never at import time.""" - for name in (*TRACKED_TASK_DIRS, *UNTRACKED_TASK_DIRS): - getattr(self, name).mkdir(parents=True, exist_ok=True) - return self + +class FilePaths: + """Project-wide paths. Add datasets here; never hardcode one.""" + + ROOT = main_checkout() + DATA = ROOT / "data" + FIGURES = ROOT / "figures" + + @classmethod + def dataset(cls, name: str) -> DatasetPaths: + """Standard paths for ``data//``. + + Examples + -------- + >>> FilePaths.dataset("pbmc3k").create().processed # doctest: +SKIP + """ + root = cls.DATA / name + return DatasetPaths(root=root, **{d: root / d for d in DATASET_DIRS}) def task_paths(file: str | Path) -> TaskPaths: - """Resolve the output directories for the task that ``file`` belongs to. + """Output directories for the task ``file`` belongs to. Pass ``__file__``. - Pass ``__file__``. The task directory is the nearest ancestor under ``analysis/`` - whose name is not a known task subdirectory, so both ``/_common.py`` and - ``/scripts/step.py`` resolve to ````. + The task is the nearest ancestor under ``analysis/`` that is not a known task + subdirectory, so ``/_common.py`` and ``/scripts/step.py`` both resolve + to ````. Examples -------- - >>> paths = task_paths(__file__).ensure() # doctest: +SKIP + >>> paths = task_paths(__file__).create() # doctest: +SKIP >>> paths.results / "markers.csv" # tracked # doctest: +SKIP >>> paths.outputs / "embedding.h5ad" # gitignored, in the main checkout # doctest: +SKIP """ @@ -167,9 +130,6 @@ def task_paths(file: str | Path) -> TaskPaths: main_task = main_checkout() / "analysis" / task.relative_to(checkout / "analysis") return TaskPaths( task=task, - results=task / "results", - reports=task / "reports", - figures=main_task / "figures", - outputs=main_task / "outputs", - logs=main_task / "logs", + **{d: task / d for d in TRACKED_TASK_DIRS}, + **{d: main_task / d for d in UNTRACKED_TASK_DIRS}, )