docs(cookbooks): add Daytona RL cookbook

cookbooks/daytona-rl/.gitignore

@@ -0,0 +1 @@
+/train_run.json

cookbooks/daytona-rl/Dockerfile.hud

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+# Not uv.lock* — Docker lets that glob match nothing, Daytona's builder errors
+# with "Path does not exist: uv.lock*". The lock is committed, so copy it by name.
+COPY pyproject.toml uv.lock ./
+RUN pip install uv && uv sync --frozen --no-dev
+COPY env.py tasks.py bugs.py ./
+
+EXPOSE 8765
+CMD ["uv", "run", "hud", "serve", "env:env", "--host", "0.0.0.0", "--port", "8765"]

cookbooks/daytona-rl/README.md

@@ -0,0 +1,79 @@
+# RL on Daytona sandboxes
+
+Run an agent evaluation where every rollout gets its own fresh
+[Daytona](https://daytona.io) sandbox — then train a model on the graded
+rollouts that same evaluation produced. One `env.py`, no second pipeline.
+
+The task: a small Python file with seeded bugs, graded by whether `pytest`
+passes. Ten GRPO steps at 128 parallel rollouts took a Qwen3.5 4B fork from
+**35.9% to 81.2%** pass rate on held-out bug variants it never trained on.
+
+![training curve](bench/train_curve.svg)
+
+The full measured walkthrough — spin-up ladders to 256 concurrent, warm pools,
+and sizing rules — is [the guide on Daytona's docs](DAYTONA_GUIDE_URL). The
+condensed HUD-side walkthrough is
+[on the HUD docs](https://docs.hud.ai/v6/cookbooks/daytona-rl).
+
+| File | Purpose |
+|------|---------|
+| `env.py` | The environment: a workspace, one task, a pytest grader |
+| `bugs.py` | 24 deterministic bug variants (4/5/6 broken functions) |
+| `tasks.py` | Task list for `hud eval` |
+| `train.py` | The published 10-step GRPO run |
+| `pool.py` | Warm-pool helpers (create, wait until really full, drop) |
+| `snapshot.py` | Content-addressed snapshot naming |
+| `bench/` | Repro receipts behind the guide's numbers |
+
+## Setup
+
+```bash
+uv sync
+hud set HUD_API_KEY=...        # https://hud.ai/project/api-keys
+export DAYTONA_API_KEY=...     # https://app.daytona.io
+```
+
+`DAYTONA_API_KEY` has to be a real `export`: the Daytona SDK reads the process
+environment, not `~/.hud/.env`.
+
+## Run
+
+```bash
+hud eval tasks.py claude                                   # one local rollout, no Daytona
+uv run train.py --steps 10 --group 8 --concurrent 128      # the published training run
+PYTHONPATH=. uv run bench/reap.py --delete                 # clean up stray sandboxes
+```
+
+`train.py` builds the snapshot from `Dockerfile.hud` on first run, keeps a warm
+pool the width of the batch, and appends per-step metrics to `train_run.json`.
+Variants 0-15 train; 16-23 are held out for the before/after measurement.
+
+The snapshot name is a hash of `env.py`, `bugs.py`, `Dockerfile.hud` and
+`pyproject.toml` (`snapshot.py`), so editing the environment mints a new one
+instead of silently running the old image.
+
+## `bench/`
+
+Run from this directory with `PYTHONPATH=.`.
+
+| File | Purpose |
+|------|---------|
+| `baseline.py` | Pass rate with no optimizer — the 35.9% before and 81.2% after |
+| `run_daytona.py` | One rollout on Daytona, with the spin-up/agent time split |
+| `ladder.py`, `ladder_reps.csv` | Concurrency ladder, N up to 256 |
+| `warmpool.py`, `warmpool_repro.py` | Warm-pool A/B |
+| `train_run.json`, `train_curve.svg` | The published training curve |
+| `reap.py` | Delete stray sandboxes and stale snapshots |
+
+## Two things that will bite you
+
+**A warm pool says it's full before it is.** Its `current_size` reports the target
+about 12 seconds early. `pool.py` counts actual sandboxes instead — and that
+count needs `include_warm=True`, because unclaimed pool members are excluded from
+`list_sandboxes` by default. Count the obvious way and you get 0 forever.
+
+**An interrupted run can strand a pool.** `train.py` drops its pool on exit and
+on SIGINT/SIGTERM, but a hard kill leaves one parked. `bench/reap.py` will not
+catch it — it calls `AsyncDaytona.list()`, blind to unclaimed pool sandboxes for
+the same `include_warm` reason, and it does not delete pools at all. Check after
+any interrupted run.

cookbooks/daytona-rl/bench/baseline.py

@@ -0,0 +1,106 @@
+"""Probe the task's difficulty before committing to a training run.
+
+    uv run baseline.py                       # 24 variants, 1 rollout each
+    uv run baseline.py --variants 24 --group 2
+
+Runs the same agent config training would use, with no optimizer. The decision
+this informs: a ~20-40% pass rate leaves room for GRPO to learn. Near 0% or near
+100% means retune the task instead of spending hours discovering the gradient was
+dead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import time
+from collections import Counter
+
+from daytona import Image
+from hud.agents import create_agent
+from hud.eval import DaytonaRuntime, Job, Taskset
+
+import bugs
+from env import fix_calc
+from snapshot import snapshot_name
+
+MODEL = "daytona-calc-3"
+
+
+async def main(*, variants: int, group: int, max_concurrent: int, start: int, model: str) -> None:
+    agent = create_agent(
+        model,
+        completion_kwargs={"max_tokens": 2048, "extra_body": {"return_token_ids": True}},
+    )
+    variant_ids = list(range(start, start + variants))
+    taskset = Taskset("calc", [fix_calc(variant=v) for v in variant_ids])
+    runtime = DaytonaRuntime(snapshot_name(), image=Image.from_dockerfile("Dockerfile.hud"))
+
+    print(
+        f"model={model} variants={variant_ids} group={group} rollouts={len(variant_ids) * group} tests={bugs.test_count()}"
+    )
+    session = await Job.start(f"calc-baseline-{start}-{start + variants - 1}", group=group)
+    t0 = time.perf_counter()
+    await taskset.run(agent, runtime=runtime, job=session, max_concurrent=max_concurrent)
+    wall = time.perf_counter() - t0
+
+    runs = session.runs
+    rewards = [r.reward for r in runs]
+    solved = sum(1 for r in rewards if r == 1.0)
+
+    by_k: Counter = Counter()
+    solved_by_k: Counter = Counter()
+    unattributed = 0
+    for run in runs:
+        variant = getattr(run, "_args", {}).get("variant")
+        if variant is None:
+            unattributed += 1
+            continue
+        k = len(bugs.broken_for(variant))
+        by_k[k] += 1
+        if run.reward == 1.0:
+            solved_by_k[k] += 1
+
+    launched = len(runs) - unattributed
+    print(f"\nwall {wall:.1f}s | runs {len(runs)} | launched {launched}")
+
+    if unattributed:
+        print(
+            f"\n{unattributed}/{len(runs)} rollouts never launched — no difficulty "
+            f"signal here. Check the env starts in the sandbox (a missing file in "
+            f"Dockerfile.hud shows up as \"env closed connection during 'hello'\")."
+        )
+        return
+
+    print(f"pass rate: {solved}/{launched} = {solved / launched:.1%}")
+    for k in sorted(by_k):
+        print(f"  k={k} bugs: {solved_by_k[k]}/{by_k[k]} = {solved_by_k[k] / by_k[k]:.1%}")
+
+    rate = solved / launched
+    verdict = (
+        "in range — a training run is justified"
+        if 0.15 <= rate <= 0.55
+        else "too easy — add bugs or harder ones"
+        if rate > 0.55
+        else "too hard — reduce k or drop the subtlest bugs"
+    )
+    print(f"\nverdict: {rate:.1%} {verdict}")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("--variants", type=int, default=24)
+    p.add_argument("--group", type=int, default=1)
+    p.add_argument("--max-concurrent", type=int, default=12)
+    p.add_argument("--start", type=int, default=0)
+    p.add_argument("--model", default=MODEL)
+    a = p.parse_args()
+    asyncio.run(
+        main(
+            variants=a.variants,
+            group=a.group,
+            max_concurrent=a.max_concurrent,
+            start=a.start,
+            model=a.model,
+        )
+    )

cookbooks/daytona-rl/bench/ladder.py

@@ -0,0 +1,187 @@
+"""Concurrency ladder for Daytona sandboxes: N in {1, 8, 32, 128}.
+
+    uv run ladder.py                      # infra mode, no LLM spend
+    uv run ladder.py --mode rollout       # real Claude rollouts (costs tokens)
+    uv run ladder.py --levels 1,8         # override the ladder
+
+Infra mode measures the thing we actually doubt: how sandbox spin-up and the
+one-SSH-connection-per-sandbox transport (runtime.py:780) behave under fan-out.
+Each worker acquires a sandbox, opens a *fresh control connection* to it, does
+one real handshake (`client.manifest()`), and exits — no agent, no tokens.
+
+Writes ladder.csv and ladder.svg.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import csv
+import resource
+import statistics
+import time
+from pathlib import Path
+
+from daytona import Image
+from hud.clients import connect
+from hud.eval import DaytonaRuntime
+
+from env import fix_calc
+from snapshot import snapshot_name
+from timing import TimedProvider
+
+SNAPSHOT = snapshot_name()
+OUT_CSV = Path("ladder.csv")
+OUT_SVG = Path("ladder.svg")
+
+
+async def one_infra(provider, idx: int) -> dict:
+    """Acquire a sandbox, open a fresh control connection, handshake, tear down."""
+    t0 = time.perf_counter()
+    row = {"idx": idx, "spin_up_s": None, "connect_s": None, "ok": False, "error": ""}
+    try:
+        async with provider(fix_calc()) as rt:
+            row["spin_up_s"] = time.perf_counter() - t0
+            t1 = time.perf_counter()
+            async with connect(rt) as client:
+                assert client.manifest is not None
+                row["connect_s"] = time.perf_counter() - t1
+            row["ok"] = True
+    except Exception as exc:
+        row["error"] = f"{type(exc).__name__}: {exc}"
+    row["total_s"] = time.perf_counter() - t0
+    return row
+
+
+async def one_rollout(provider, idx: int) -> dict:
+    from hud.agents.claude import ClaudeAgent
+
+    timed = TimedProvider(provider)
+    t0 = time.perf_counter()
+    row = {"idx": idx, "spin_up_s": None, "connect_s": None, "ok": False, "error": ""}
+    try:
+        job = await fix_calc().run(ClaudeAgent(), runtime=timed)
+        row["ok"] = True
+        row["reward"] = job.reward
+        row["spin_up_s"] = timed.spin_ups[0] if timed.spin_ups else None
+    except Exception as exc:
+        row["error"] = f"{type(exc).__name__}: {exc}"
+    row["total_s"] = time.perf_counter() - t0
+    return row
+
+
+async def run_level(n: int, mode: str) -> dict:
+    provider = DaytonaRuntime(SNAPSHOT, image=Image.from_dockerfile("Dockerfile.hud"))
+    worker = one_infra if mode == "infra" else one_rollout
+
+    t0 = time.perf_counter()
+    rows = await asyncio.gather(*(worker(provider, i) for i in range(n)))
+    wall = time.perf_counter() - t0
+
+    ok = [r for r in rows if r["ok"]]
+    spin = sorted(r["spin_up_s"] for r in ok if r["spin_up_s"] is not None)
+    errors = [r["error"] for r in rows if r["error"]]
+
+    def pct(p: float) -> float | None:
+        if not spin:
+            return None
+        return spin[min(len(spin) - 1, int(p * len(spin)))]
+
+    summary = {
+        "n": n,
+        "mode": mode,
+        "wall_s": round(wall, 2),
+        "ok": len(ok),
+        "failed": n - len(ok),
+        "per_min": round(len(ok) / wall * 60, 1) if wall else 0.0,
+        "spin_min_s": round(spin[0], 2) if spin else None,
+        "spin_med_s": round(statistics.median(spin), 2) if spin else None,
+        "spin_p90_s": round(pct(0.9), 2) if spin else None,
+        "spin_max_s": round(spin[-1], 2) if spin else None,
+    }
+    print(f"[N={n:>3}] {summary}")
+    for e in dict.fromkeys(errors):
+        print(f"        error: {e[:200]}")
+    return summary
+
+
+def chart(rows: list[dict]) -> None:
+    """Minimal hand-rolled SVG: throughput bars + median spin-up line."""
+    w, h, pad = 640, 320, 56
+    plot_w, plot_h = w - 2 * pad, h - 2 * pad
+    max_rate = max((r["per_min"] or 0) for r in rows) or 1
+    max_spin = max((r["spin_max_s"] or 0) for r in rows) or 1
+    bar_w = plot_w / max(len(rows), 1) * 0.55
+
+    parts = [
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" '
+        f'viewBox="0 0 {w} {h}" font-family="ui-sans-serif,system-ui,sans-serif">',
+        f'<rect width="{w}" height="{h}" fill="#fff"/>',
+        f'<text x="{pad}" y="28" font-size="14" font-weight="600">'
+        f"Daytona concurrency ladder — {rows[0]['mode']} mode</text>",
+        f'<line x1="{pad}" y1="{pad + plot_h}" x2="{pad + plot_w}" y2="{pad + plot_h}" '
+        f'stroke="#999"/>',
+    ]
+    pts = []
+    for i, r in enumerate(rows):
+        cx = pad + plot_w * (i + 0.5) / len(rows)
+        bh = plot_h * (r["per_min"] or 0) / max_rate
+        parts.append(
+            f'<rect x="{cx - bar_w / 2:.1f}" y="{pad + plot_h - bh:.1f}" '
+            f'width="{bar_w:.1f}" height="{bh:.1f}" fill="#6366f1" opacity="0.85"/>'
+        )
+        parts.append(
+            f'<text x="{cx:.1f}" y="{pad + plot_h + 16:.1f}" font-size="11" '
+            f'text-anchor="middle">N={r["n"]}</text>'
+        )
+        parts.append(
+            f'<text x="{cx:.1f}" y="{pad + plot_h - bh - 6:.1f}" font-size="10" '
+            f'text-anchor="middle" fill="#4338ca">{r["per_min"]}/min</text>'
+        )
+        sy = pad + plot_h - plot_h * (r["spin_med_s"] or 0) / max_spin
+        pts.append(f"{cx:.1f},{sy:.1f}")
+        parts.append(f'<circle cx="{cx:.1f}" cy="{sy:.1f}" r="3.5" fill="#ef4444"/>')
+        if r["failed"]:
+            parts.append(
+                f'<text x="{cx:.1f}" y="{pad - 8:.1f}" font-size="10" '
+                f'text-anchor="middle" fill="#dc2626">{r["failed"]} failed</text>'
+            )
+    parts.append(
+        f'<polyline points="{" ".join(pts)}" fill="none" stroke="#ef4444" stroke-width="2"/>'
+    )
+    parts.append(
+        f'<text x="{pad}" y="{h - 14}" font-size="11" fill="#4338ca">bars: completions/min</text>'
+        f'<text x="{pad + 190}" y="{h - 14}" font-size="11" fill="#ef4444">'
+        f"line: median spin-up (max {max_spin}s)</text>"
+    )
+    parts.append("</svg>")
+    OUT_SVG.write_text("\n".join(parts))
+
+
+async def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--mode", choices=("infra", "rollout"), default="infra")
+    ap.add_argument("--levels", default="1,8,32,128")
+    args = ap.parse_args()
+
+    levels = [int(x) for x in args.levels.split(",")]
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    want = max(levels) * 16 + 256
+    if soft < want:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (min(want, hard), hard))
+    print(f"fd limit: {soft} -> {resource.getrlimit(resource.RLIMIT_NOFILE)[0]} (hard {hard})")
+
+    rows = []
+    for n in levels:
+        rows.append(await run_level(n, args.mode))
+
+    with OUT_CSV.open("w", newline="") as fh:
+        writer = csv.DictWriter(fh, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+    chart(rows)
+    print(f"\nwrote {OUT_CSV} and {OUT_SVG}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())