Feat/codebase cache

docs/guides/codebase.md

@@ -68,6 +68,24 @@ Each dispatch gets its own private environment holding:
 An eval that declares no `codebase` still gets a Git repository, initialized on `work`, exactly as
 it always has.
 
+## One checkout per iteration
+
+Every environment a run provisions — each `(eval, condition, run)` cell — is built from one cached
+checkout per distinct codebase, materialized once per iteration under `iteration-N/.codebase/`
+when the run prepares.
+Environments are provisioned from that cache with `git clone --local`: Git hard-links the object
+store instead of copying it and checks out a fresh working tree, so `--runs 10` against a large
+repository costs one clone plus a working tree per environment, not a full copy of the tree and
+its history per environment.
+
+Shared objects are content-addressed and immutable — Git never rewrites an object once written —
+so each environment is still an independent working tree with an independent history. Commits,
+branches, and edits in one environment are invisible to the others and to the cache.
+
+Where hard-linking is unavailable (the cache and the environments on different filesystems) or the
+source carries no Git history to clone, environments fall back to a plain copy of the cache. The
+result is the same tree, provisioned more slowly.
+
 ## `files` is an overlay
 
 `files` and `files_root` still work, and are applied *on top* of the codebase at their declared

src/cli/run/orchestrate/mod.rs

@@ -308,6 +308,21 @@ fn print_run_plan(ctx: &RunContext, opts: &RunOptions, r: &Resolved) {
         "  skill source: {}{revision}",
         source.resolved_path.as_deref().unwrap_or(&source.source)
     );
+    // The codebases the environments are built from, in the same shape as the
+    // skill source line — and the one-checkout-per-iteration fact the caching
+    // makes true.
+    for codebase in &r.codebases {
+        let source = &codebase.source;
+        let revision = source
+            .revision
+            .as_deref()
+            .map(|sha| format!(" ({})", &sha[..7.min(sha.len())]))
+            .unwrap_or_default();
+        println!(
+            "  codebase: {}{revision} — materialized once per iteration",
+            source.resolved_path.as_deref().unwrap_or(&source.source)
+        );
+    }
     if r.selected_evals.len() != r.total_evals {
         let (flag, ids) = match (opts.only, opts.skip) {
             (Some(ids), _) => ("--only", ids),

src/cli/run/orchestrate/stage.rs

@@ -110,13 +110,17 @@ pub(super) fn stage_conditions(
             // agent left behind. Start from nothing instead.
             fs::remove_dir_all(&target.root)?;
         }
-        fs::create_dir_all(&target.root)?;
-
         // The codebase goes down first: staged skills and the `files` overlay are
-        // both applied *on top* of it.
+        // both applied *on top* of it. Every environment is provisioned from the
+        // iteration's single cached materialization of it — a local clone while
+        // the host allows the hard link, a plain copy otherwise — so `--runs 10`
+        // against a real repository costs one checkout, not ten copies.
         if let Some(codebase) = codebase {
             let source_tree = materialize_codebase(&r.iteration_dir, codebase, &mut materialized)?;
-            copy_entry_materialized(&source_tree, &target.root)?;
+            crate::source::provision_env(&codebase.source, &source_tree, &target.root)
+                .map_err(|error| RunError::msg(error.to_string()))?;
+        } else {
+            fs::create_dir_all(&target.root)?;
         }
 
         if !opts.no_stage {
@@ -234,9 +238,11 @@ fn copy_skill_dir(source: &Path, dest: &Path, root: &Path) -> Result<(), RunErro
 
 /// The materialized tree for `codebase`, creating it on first use.
 ///
-/// One materialization per distinct codebase per iteration; each environment is
-/// then provisioned from it by copy. Cloning per environment instead would mean
-/// one network round trip per `(group, condition, run)` cell.
+/// One materialization per distinct codebase per iteration; every environment
+/// is then provisioned from it ([`crate::source::provision_env`] — a local
+/// clone while the host allows the hard link, a copy otherwise). Materializing
+/// per environment instead would mean one network round trip per
+/// `(group, condition, run)` cell.
 fn materialize_codebase(
     iteration_dir: &Path,
     codebase: &super::RunCodebase,

src/core/fs.rs

@@ -183,6 +183,31 @@ pub fn copy_entry_materialized(source: &Path, destination: &Path) -> io::Result<
     Ok(())
 }
 
+/// Whether a file created under `from` can be hard-linked into `to` — the
+/// capability `git clone --local` relies on to share an object store instead
+/// of copying it.
+///
+/// Two directories a run owns can sit on different filesystems (a workspace
+/// on a mounted volume, a cache on tmpfs), and `link(2)` is what says so.
+/// Any failure reads as unavailable, so the caller falls back to copying rather
+/// than provisioning wrong.
+pub fn hardlinks_available(from: &Path, to: &Path) -> bool {
+    let Ok(probe) = tempfile::NamedTempFile::new_in(from) else {
+        return false;
+    };
+    let Some(name) = probe.path().file_name() else {
+        return false;
+    };
+    let target = to.join(name);
+    match fs::hard_link(probe.path(), &target) {
+        Ok(()) => {
+            let _ = fs::remove_file(&target);
+            true
+        }
+        Err(_) => false,
+    }
+}
+
 /// Create a symlink at `link` pointing at `target`.
 ///
 /// `to_directory` is consulted only on Windows, which has separate file and
@@ -605,4 +630,42 @@ mod tests {
 
         assert_eq!(err.kind(), io::ErrorKind::NotFound);
     }
+
+    /// The probe both succeeds and cleans up after itself: it runs inside the
+    /// per-iteration codebase cache, where a leftover file would ship into the
+    /// next environment built from it.
+    #[test]
+    fn hardlinks_available_is_true_between_directories_on_one_filesystem() {
+        let tmp = TempDir::new().unwrap();
+        let from = tmp.path().join("from");
+        let to = tmp.path().join("to");
+        fs::create_dir_all(&from).unwrap();
+        fs::create_dir_all(&to).unwrap();
+
+        assert!(hardlinks_available(&from, &to));
+
+        assert_eq!(
+            fs::read_dir(&from).unwrap().count(),
+            0,
+            "no probe residue in from"
+        );
+        assert_eq!(
+            fs::read_dir(&to).unwrap().count(),
+            0,
+            "no probe residue in to"
+        );
+    }
+
+    /// Any failure — a missing directory on either side, a filesystem that
+    /// refuses the link — reads as "unavailable", so callers fall back to
+    /// copying rather than provisioning wrong.
+    #[test]
+    fn hardlinks_available_is_false_when_a_directory_is_missing() {
+        let tmp = TempDir::new().unwrap();
+        let present = tmp.path().join("present");
+        fs::create_dir_all(&present).unwrap();
+
+        assert!(!hardlinks_available(&tmp.path().join("absent"), &present));
+        assert!(!hardlinks_available(&present, &tmp.path().join("absent")));
+    }
 }

src/source/mod.rs

@@ -1,9 +1,12 @@
-//! Resolving a declared source to a revision, and materializing it as a tree.
+//! Resolving a declared source to a revision, materializing it as a tree, and
+//! provisioning task environments from that tree.
 //!
-//! Two phases, deliberately split. [`resolve`] is read-only: it answers "what
-//! exactly does this declaration point at?" without creating a directory, so a
-//! run can fail on an unreachable repository or a ref that does not exist before
-//! it has built any part of a workspace.
+//! Three operations, deliberately split. [`resolve`] is read-only: it answers
+//! "what exactly does this declaration point at?" without creating a directory,
+//! so a run can fail on an unreachable repository or a ref that does not exist
+//! before it has built any part of a workspace. [`materialize`] creates the one
+//! cached checkout an iteration shares; [`provision_env`] turns that cache into
+//! each individual task environment.
 //!
 //! Nothing here knows what a codebase is. A caller hands it a [`SourceSpec`] and
 //! gets back a [`ResolvedSource`]; the eval config's `codebase` block is one
@@ -303,6 +306,96 @@ fn clone_repository(
     Ok(())
 }
 
+/// How [`provision_env`] produced an environment.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EnvProvisioning {
+    /// `git clone --local` from the cache: Git hard-links the object store and
+    /// checks out a fresh working tree, so the history arrives intact while the
+    /// cache's bytes are paid for once per iteration, not once per environment.
+    LocalClone,
+    /// A plain materialized copy of the cache: taken when the host cannot
+    /// hard-link between the two directories, or the cache carries no commits
+    /// to check out.
+    PlainCopy,
+}
+
+/// Produce one task environment at `dest` from the cached materialization of
+/// `resolved` that [`materialize`] left at `cache`.
+///
+/// A run materializes each distinct codebase once per iteration and provisions
+/// every `(group, condition, run)` environment from that single checkout. A
+/// local clone is the fast path: Git hard-links the object store instead of
+/// copying it, and the clone's history is the checkout's history. A local
+/// clone also names its source as an `origin` remote — removed here, so no
+/// environment retains a path back to the cache. The plain copy stands in
+/// wherever cloning could not deliver: a cache without commits (an empty
+/// repository clones to an empty working tree) or a host that refuses the
+/// hard link (a cache and an environment on different filesystems).
+///
+/// `dest` must not already exist. Returns how the environment was provisioned.
+pub fn provision_env(
+    resolved: &ResolvedSource,
+    cache: &Path,
+    dest: &Path,
+) -> Result<EnvProvisioning, SourceError> {
+    if resolved.revision.is_none() {
+        copy_from_cache(cache, dest)?;
+        return Ok(EnvProvisioning::PlainCopy);
+    }
+    let git = IsolatedGit::new().map_err(SourceError::msg)?;
+    let parent = dest.parent().unwrap_or(dest);
+    std::fs::create_dir_all(parent).map_err(|error| {
+        SourceError::msg(format!("could not create {}: {error}", parent.display()))
+    })?;
+    // Probing `cache/.git`, not the working tree: a probe file in the tree
+    // would be a leftover in the cache even after deletion racing a clone,
+    // while `.git` is metadata no checkout ever reads.
+    if !crate::core::fs::hardlinks_available(&cache.join(".git"), parent) {
+        copy_from_cache(cache, dest)?;
+        return Ok(EnvProvisioning::PlainCopy);
+    }
+    checked(
+        &git,
+        Path::new("."),
+        &[
+            "clone",
+            "--quiet",
+            "--local",
+            "--template",
+            &git.template_dir().to_string_lossy(),
+            &cache.to_string_lossy(),
+            &dest.to_string_lossy(),
+        ],
+        "clone the cached codebase checkout",
+    )?;
+    checked(
+        &git,
+        dest,
+        &["remote", "remove", "origin"],
+        "remove the cache as a remote",
+    )?;
+    Ok(EnvProvisioning::LocalClone)
+}
+
+/// The fallback provisioning: a materialized copy of the whole cache, for a
+/// host that cannot hard-link between the two directories or a cache with no
+/// commits to check out.
+fn copy_from_cache(cache: &Path, dest: &Path) -> Result<(), SourceError> {
+    std::fs::create_dir_all(dest).map_err(|error| {
+        SourceError::msg(format!(
+            "could not create environment {}: {error}",
+            dest.display()
+        ))
+    })?;
+    crate::core::fs::copy_entry_materialized(cache, dest).map_err(|error| {
+        SourceError::msg(format!(
+            "could not copy cached codebase {} into {}: {error}",
+            cache.display(),
+            dest.display()
+        ))
+    })
+}
+
 /// Run git in `cwd`, turning a non-zero exit into an error naming the intent.
 fn checked(git: &IsolatedGit, cwd: &Path, args: &[&str], intent: &str) -> Result<(), SourceError> {
     let output = git.run(cwd, args);